tf-idf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import gensim\n",
    "\n",
    "pd.set_option('display.max_colwidth', None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>QA</th>\n",
       "      <th>Atypical_Annot</th>\n",
       "      <th>Aty</th>\n",
       "      <th>method</th>\n",
       "      <th>obj1</th>\n",
       "      <th>obj2</th>\n",
       "      <th>Atypical_Annot2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>110991.jpg</td>\n",
       "      <td>I should eat Oreos because I can snack on Oreos during the basketball game</td>\n",
       "      <td>Liquid Deformed Object: The liquid which has been deformed is &lt;b&gt;Milk&lt;/b&gt;</td>\n",
       "      <td>milk</td>\n",
       "      <td>Liquid Deformed Object</td>\n",
       "      <td>milk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Liquid Deformed Object: The liquid which has been deformed is Milk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>110991.jpg</td>\n",
       "      <td>I should eat Oreos because I can snack on Oreos during the basketball game</td>\n",
       "      <td>Liquid Deformed Object: The liquid which has been deformed is &lt;b&gt;milk&lt;/b&gt;</td>\n",
       "      <td>milk</td>\n",
       "      <td>Liquid Deformed Object</td>\n",
       "      <td>milk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Liquid Deformed Object: The liquid which has been deformed is milk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>110991.jpg</td>\n",
       "      <td>I should eat Oreos because I can snack on Oreos during the basketball game</td>\n",
       "      <td>Object Replacement: The object which is placed in the context of another object is &lt;b&gt;cookie- milk&lt;/b&gt; and the object which is replaced by another object (expected object) &lt;b&gt;person- body- hoop/net&lt;/b&gt;</td>\n",
       "      <td>cookie milk person body hoopnet</td>\n",
       "      <td>Object Replacement</td>\n",
       "      <td>cookie- milk</td>\n",
       "      <td>person- body- hoop/net</td>\n",
       "      <td>Object Replacement: The object which is placed in the context of another object is cookie- milk and the object which is replaced by another object (expected object) person- body- hoop/net</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>110991.jpg</td>\n",
       "      <td>I should eat Oreo's because Oreo fun to dunk Oreo.</td>\n",
       "      <td>Liquid Deformed Object: The liquid which has been deformed is &lt;b&gt;Milk&lt;/b&gt;</td>\n",
       "      <td>milk</td>\n",
       "      <td>Liquid Deformed Object</td>\n",
       "      <td>milk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Liquid Deformed Object: The liquid which has been deformed is Milk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>110991.jpg</td>\n",
       "      <td>I should eat Oreo's because Oreo fun to dunk Oreo.</td>\n",
       "      <td>Liquid Deformed Object: The liquid which has been deformed is &lt;b&gt;milk&lt;/b&gt;</td>\n",
       "      <td>milk</td>\n",
       "      <td>Liquid Deformed Object</td>\n",
       "      <td>milk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Liquid Deformed Object: The liquid which has been deformed is milk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>13080.jpg</td>\n",
       "      <td>I should buy Absolut vodka because Absolut makes you hungry</td>\n",
       "      <td>Object Replacement: The object which is placed in the context of another object is &lt;b&gt;chocolate box spaces- void in fork- bottle arranged&lt;/b&gt; and the object which is replaced by another object (expected object) &lt;b&gt;vodka bottle&lt;/b&gt;</td>\n",
       "      <td>vodka bottle</td>\n",
       "      <td>Object Replacement</td>\n",
       "      <td>vodka bottle</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Object Replacement: The object which is placed in the context of another object is chocolate box spaces- void in fork- bottle arranged and the object which is replaced by another object (expected object) vodka bottle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>13080.jpg</td>\n",
       "      <td>i should get some buy some absolut vodka because some absolut vodka goes with everything</td>\n",
       "      <td>Texture Replacement 2: The object which has a new texture is &lt;b&gt;bottle&lt;/b&gt; and the objects that have created the texture are &lt;b&gt;forks- chocolate- food&lt;/b&gt;</td>\n",
       "      <td>bottle forks chocolate food</td>\n",
       "      <td>Texture Replacement 2</td>\n",
       "      <td>bottle</td>\n",
       "      <td>forks- chocolate- food</td>\n",
       "      <td>Texture Replacement 2: The object which has a new texture is bottle and the objects that have created the texture are forks- chocolate- food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>13080.jpg</td>\n",
       "      <td>i should get some buy some absolut vodka because some absolut vodka goes with everything</td>\n",
       "      <td>Object Replacement: The object which is placed in the context of another object is &lt;b&gt;chocolate box spaces- void in fork- bottle arranged&lt;/b&gt; and the object which is replaced by another object (expected object) &lt;b&gt;vodka bottle&lt;/b&gt;</td>\n",
       "      <td>vodka bottle</td>\n",
       "      <td>Object Replacement</td>\n",
       "      <td>vodka bottle</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Object Replacement: The object which is placed in the context of another object is chocolate box spaces- void in fork- bottle arranged and the object which is replaced by another object (expected object) vodka bottle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>13080.jpg</td>\n",
       "      <td>I should go out and buy groceries because each image in the ad makes me think of food and beverages.</td>\n",
       "      <td>Texture Replacement 2: The object which has a new texture is &lt;b&gt;bottle&lt;/b&gt; and the objects that have created the texture are &lt;b&gt;forks- chocolate- food&lt;/b&gt;</td>\n",
       "      <td>bottle forks chocolate food</td>\n",
       "      <td>Texture Replacement 2</td>\n",
       "      <td>bottle</td>\n",
       "      <td>forks- chocolate- food</td>\n",
       "      <td>Texture Replacement 2: The object which has a new texture is bottle and the objects that have created the texture are forks- chocolate- food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>13080.jpg</td>\n",
       "      <td>I should go out and buy groceries because each image in the ad makes me think of food and beverages.</td>\n",
       "      <td>Object Replacement: The object which is placed in the context of another object is &lt;b&gt;chocolate box spaces- void in fork- bottle arranged&lt;/b&gt; and the object which is replaced by another object (expected object) &lt;b&gt;vodka bottle&lt;/b&gt;</td>\n",
       "      <td>vodka bottle</td>\n",
       "      <td>Object Replacement</td>\n",
       "      <td>vodka bottle</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Object Replacement: The object which is placed in the context of another object is chocolate box spaces- void in fork- bottle arranged and the object which is replaced by another object (expected object) vodka bottle</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Id  \\\n",
       "0   110991.jpg   \n",
       "1   110991.jpg   \n",
       "2   110991.jpg   \n",
       "3   110991.jpg   \n",
       "4   110991.jpg   \n",
       "..         ...   \n",
       "95   13080.jpg   \n",
       "96   13080.jpg   \n",
       "97   13080.jpg   \n",
       "98   13080.jpg   \n",
       "99   13080.jpg   \n",
       "\n",
       "                                                                                                      QA  \\\n",
       "0                             I should eat Oreos because I can snack on Oreos during the basketball game   \n",
       "1                             I should eat Oreos because I can snack on Oreos during the basketball game   \n",
       "2                             I should eat Oreos because I can snack on Oreos during the basketball game   \n",
       "3                                                     I should eat Oreo's because Oreo fun to dunk Oreo.   \n",
       "4                                                     I should eat Oreo's because Oreo fun to dunk Oreo.   \n",
       "..                                                                                                   ...   \n",
       "95                                           I should buy Absolut vodka because Absolut makes you hungry   \n",
       "96              i should get some buy some absolut vodka because some absolut vodka goes with everything   \n",
       "97              i should get some buy some absolut vodka because some absolut vodka goes with everything   \n",
       "98  I should go out and buy groceries because each image in the ad makes me think of food and beverages.   \n",
       "99  I should go out and buy groceries because each image in the ad makes me think of food and beverages.   \n",
       "\n",
       "                                                                                                                                                                                                                            Atypical_Annot  \\\n",
       "0                                                                                                                                                                Liquid Deformed Object: The liquid which has been deformed is <b>Milk</b>   \n",
       "1                                                                                                                                                                Liquid Deformed Object: The liquid which has been deformed is <b>milk</b>   \n",
       "2                                Object Replacement: The object which is placed in the context of another object is <b>cookie- milk</b> and the object which is replaced by another object (expected object) <b>person- body- hoop/net</b>   \n",
       "3                                                                                                                                                                Liquid Deformed Object: The liquid which has been deformed is <b>Milk</b>   \n",
       "4                                                                                                                                                                Liquid Deformed Object: The liquid which has been deformed is <b>milk</b>   \n",
       "..                                                                                                                                                                                                                                     ...   \n",
       "95  Object Replacement: The object which is placed in the context of another object is <b>chocolate box spaces- void in fork- bottle arranged</b> and the object which is replaced by another object (expected object) <b>vodka bottle</b>   \n",
       "96                                                                              Texture Replacement 2: The object which has a new texture is <b>bottle</b> and the objects that have created the texture are <b>forks- chocolate- food</b>   \n",
       "97  Object Replacement: The object which is placed in the context of another object is <b>chocolate box spaces- void in fork- bottle arranged</b> and the object which is replaced by another object (expected object) <b>vodka bottle</b>   \n",
       "98                                                                              Texture Replacement 2: The object which has a new texture is <b>bottle</b> and the objects that have created the texture are <b>forks- chocolate- food</b>   \n",
       "99  Object Replacement: The object which is placed in the context of another object is <b>chocolate box spaces- void in fork- bottle arranged</b> and the object which is replaced by another object (expected object) <b>vodka bottle</b>   \n",
       "\n",
       "                                Aty                  method          obj1  \\\n",
       "0                              milk  Liquid Deformed Object          milk   \n",
       "1                              milk  Liquid Deformed Object          milk   \n",
       "2   cookie milk person body hoopnet      Object Replacement  cookie- milk   \n",
       "3                              milk  Liquid Deformed Object          milk   \n",
       "4                              milk  Liquid Deformed Object          milk   \n",
       "..                              ...                     ...           ...   \n",
       "95                     vodka bottle      Object Replacement  vodka bottle   \n",
       "96      bottle forks chocolate food   Texture Replacement 2        bottle   \n",
       "97                     vodka bottle      Object Replacement  vodka bottle   \n",
       "98      bottle forks chocolate food   Texture Replacement 2        bottle   \n",
       "99                     vodka bottle      Object Replacement  vodka bottle   \n",
       "\n",
       "                      obj2  \\\n",
       "0                      NaN   \n",
       "1                      NaN   \n",
       "2   person- body- hoop/net   \n",
       "3                      NaN   \n",
       "4                      NaN   \n",
       "..                     ...   \n",
       "95                     NaN   \n",
       "96  forks- chocolate- food   \n",
       "97                     NaN   \n",
       "98  forks- chocolate- food   \n",
       "99                     NaN   \n",
       "\n",
       "                                                                                                                                                                                                             Atypical_Annot2  \n",
       "0                                                                                                                                                         Liquid Deformed Object: The liquid which has been deformed is Milk  \n",
       "1                                                                                                                                                         Liquid Deformed Object: The liquid which has been deformed is milk  \n",
       "2                                Object Replacement: The object which is placed in the context of another object is cookie- milk and the object which is replaced by another object (expected object) person- body- hoop/net  \n",
       "3                                                                                                                                                         Liquid Deformed Object: The liquid which has been deformed is Milk  \n",
       "4                                                                                                                                                         Liquid Deformed Object: The liquid which has been deformed is milk  \n",
       "..                                                                                                                                                                                                                       ...  \n",
       "95  Object Replacement: The object which is placed in the context of another object is chocolate box spaces- void in fork- bottle arranged and the object which is replaced by another object (expected object) vodka bottle  \n",
       "96                                                                              Texture Replacement 2: The object which has a new texture is bottle and the objects that have created the texture are forks- chocolate- food  \n",
       "97  Object Replacement: The object which is placed in the context of another object is chocolate box spaces- void in fork- bottle arranged and the object which is replaced by another object (expected object) vodka bottle  \n",
       "98                                                                              Texture Replacement 2: The object which has a new texture is bottle and the objects that have created the texture are forks- chocolate- food  \n",
       "99  Object Replacement: The object which is placed in the context of another object is chocolate box spaces- void in fork- bottle arranged and the object which is replaced by another object (expected object) vodka bottle  \n",
       "\n",
       "[100 rows x 8 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('aty.csv', index_col=0)\n",
    "df=df[:100]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>QA</th>\n",
       "      <th>keywords</th>\n",
       "      <th>Atypical_Annot</th>\n",
       "      <th>key_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I should eat Oreos because I can snack on Oreos during the basketball game</td>\n",
       "      <td>{'oreos': 0.686, 'snack': 0.395, 'game': 0.395, 'basketball': 0.395, 'eat': 0.245}</td>\n",
       "      <td>Liquid Deformed Object: The liquid which has been deformed is Milk</td>\n",
       "      <td>{'liquid': 0.709, 'deformed': 0.6, 'milk': 0.37}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I should eat Oreos because I can snack on Oreos during the basketball game</td>\n",
       "      <td>{'oreos': 0.686, 'snack': 0.395, 'game': 0.395, 'basketball': 0.395, 'eat': 0.245}</td>\n",
       "      <td>Liquid Deformed Object: The liquid which has been deformed is milk</td>\n",
       "      <td>{'liquid': 0.709, 'deformed': 0.6, 'milk': 0.37}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I should eat Oreos because I can snack on Oreos during the basketball game</td>\n",
       "      <td>{'oreos': 0.686, 'snack': 0.395, 'game': 0.395, 'basketball': 0.395, 'eat': 0.245}</td>\n",
       "      <td>Object Replacement: The object which is placed in the context of another object is cookie- milk and the object which is replaced by another object (expected object) person- body- hoop/net</td>\n",
       "      <td>{'person': 0.374, 'net': 0.374, 'hoop': 0.374, 'cookie': 0.374, 'body': 0.374, 'milk': 0.257, 'replaced': 0.227, 'placed': 0.227, 'expected': 0.227, 'context': 0.227}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I should eat Oreo's because Oreo fun to dunk Oreo.</td>\n",
       "      <td>{'oreo': 0.913, 'dunk': 0.264, 'fun': 0.246, 'eat': 0.189}</td>\n",
       "      <td>Liquid Deformed Object: The liquid which has been deformed is Milk</td>\n",
       "      <td>{'liquid': 0.709, 'deformed': 0.6, 'milk': 0.37}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I should eat Oreo's because Oreo fun to dunk Oreo.</td>\n",
       "      <td>{'oreo': 0.913, 'dunk': 0.264, 'fun': 0.246, 'eat': 0.189}</td>\n",
       "      <td>Liquid Deformed Object: The liquid which has been deformed is milk</td>\n",
       "      <td>{'liquid': 0.709, 'deformed': 0.6, 'milk': 0.37}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>I should buy Absolut vodka because Absolut makes you hungry</td>\n",
       "      <td>{'absolut': 0.739, 'hungry': 0.417, 'vodka': 0.37, 'makes': 0.326, 'buy': 0.193}</td>\n",
       "      <td>Object Replacement: The object which is placed in the context of another object is chocolate box spaces- void in fork- bottle arranged and the object which is replaced by another object (expected object) vodka bottle</td>\n",
       "      <td>{'bottle': 0.503, 'void': 0.295, 'vodka': 0.295, 'spaces': 0.295, 'fork': 0.295, 'box': 0.295, 'arranged': 0.295, 'chocolate': 0.252, 'replaced': 0.189, 'placed': 0.189}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>i should get some buy some absolut vodka because some absolut vodka goes with everything</td>\n",
       "      <td>{'vodka': 0.647, 'absolut': 0.647, 'goes': 0.365, 'buy': 0.169}</td>\n",
       "      <td>Texture Replacement 2: The object which has a new texture is bottle and the objects that have created the texture are forks- chocolate- food</td>\n",
       "      <td>{'texture': 0.665, 'forks': 0.358, 'food': 0.358, 'chocolate': 0.306, 'bottle': 0.306, 'created': 0.192, 'new': 0.177, 'replacement': 0.166, 'objects': 0.137}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>i should get some buy some absolut vodka because some absolut vodka goes with everything</td>\n",
       "      <td>{'vodka': 0.647, 'absolut': 0.647, 'goes': 0.365, 'buy': 0.169}</td>\n",
       "      <td>Object Replacement: The object which is placed in the context of another object is chocolate box spaces- void in fork- bottle arranged and the object which is replaced by another object (expected object) vodka bottle</td>\n",
       "      <td>{'bottle': 0.503, 'void': 0.295, 'vodka': 0.295, 'spaces': 0.295, 'fork': 0.295, 'box': 0.295, 'arranged': 0.295, 'chocolate': 0.252, 'replaced': 0.189, 'placed': 0.189}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>I should go out and buy groceries because each image in the ad makes me think of food and beverages.</td>\n",
       "      <td>{'think': 0.393, 'image': 0.393, 'groceries': 0.393, 'food': 0.393, 'beverages': 0.393, 'ad': 0.319, 'makes': 0.307, 'buy': 0.182}</td>\n",
       "      <td>Texture Replacement 2: The object which has a new texture is bottle and the objects that have created the texture are forks- chocolate- food</td>\n",
       "      <td>{'texture': 0.665, 'forks': 0.358, 'food': 0.358, 'chocolate': 0.306, 'bottle': 0.306, 'created': 0.192, 'new': 0.177, 'replacement': 0.166, 'objects': 0.137}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>I should go out and buy groceries because each image in the ad makes me think of food and beverages.</td>\n",
       "      <td>{'think': 0.393, 'image': 0.393, 'groceries': 0.393, 'food': 0.393, 'beverages': 0.393, 'ad': 0.319, 'makes': 0.307, 'buy': 0.182}</td>\n",
       "      <td>Object Replacement: The object which is placed in the context of another object is chocolate box spaces- void in fork- bottle arranged and the object which is replaced by another object (expected object) vodka bottle</td>\n",
       "      <td>{'bottle': 0.503, 'void': 0.295, 'vodka': 0.295, 'spaces': 0.295, 'fork': 0.295, 'box': 0.295, 'arranged': 0.295, 'chocolate': 0.252, 'replaced': 0.189, 'placed': 0.189}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                      QA  \\\n",
       "0                             I should eat Oreos because I can snack on Oreos during the basketball game   \n",
       "1                             I should eat Oreos because I can snack on Oreos during the basketball game   \n",
       "2                             I should eat Oreos because I can snack on Oreos during the basketball game   \n",
       "3                                                     I should eat Oreo's because Oreo fun to dunk Oreo.   \n",
       "4                                                     I should eat Oreo's because Oreo fun to dunk Oreo.   \n",
       "..                                                                                                   ...   \n",
       "95                                           I should buy Absolut vodka because Absolut makes you hungry   \n",
       "96              i should get some buy some absolut vodka because some absolut vodka goes with everything   \n",
       "97              i should get some buy some absolut vodka because some absolut vodka goes with everything   \n",
       "98  I should go out and buy groceries because each image in the ad makes me think of food and beverages.   \n",
       "99  I should go out and buy groceries because each image in the ad makes me think of food and beverages.   \n",
       "\n",
       "                                                                                                                              keywords  \\\n",
       "0                                                   {'oreos': 0.686, 'snack': 0.395, 'game': 0.395, 'basketball': 0.395, 'eat': 0.245}   \n",
       "1                                                   {'oreos': 0.686, 'snack': 0.395, 'game': 0.395, 'basketball': 0.395, 'eat': 0.245}   \n",
       "2                                                   {'oreos': 0.686, 'snack': 0.395, 'game': 0.395, 'basketball': 0.395, 'eat': 0.245}   \n",
       "3                                                                           {'oreo': 0.913, 'dunk': 0.264, 'fun': 0.246, 'eat': 0.189}   \n",
       "4                                                                           {'oreo': 0.913, 'dunk': 0.264, 'fun': 0.246, 'eat': 0.189}   \n",
       "..                                                                                                                                 ...   \n",
       "95                                                    {'absolut': 0.739, 'hungry': 0.417, 'vodka': 0.37, 'makes': 0.326, 'buy': 0.193}   \n",
       "96                                                                     {'vodka': 0.647, 'absolut': 0.647, 'goes': 0.365, 'buy': 0.169}   \n",
       "97                                                                     {'vodka': 0.647, 'absolut': 0.647, 'goes': 0.365, 'buy': 0.169}   \n",
       "98  {'think': 0.393, 'image': 0.393, 'groceries': 0.393, 'food': 0.393, 'beverages': 0.393, 'ad': 0.319, 'makes': 0.307, 'buy': 0.182}   \n",
       "99  {'think': 0.393, 'image': 0.393, 'groceries': 0.393, 'food': 0.393, 'beverages': 0.393, 'ad': 0.319, 'makes': 0.307, 'buy': 0.182}   \n",
       "\n",
       "                                                                                                                                                                                                              Atypical_Annot  \\\n",
       "0                                                                                                                                                         Liquid Deformed Object: The liquid which has been deformed is Milk   \n",
       "1                                                                                                                                                         Liquid Deformed Object: The liquid which has been deformed is milk   \n",
       "2                                Object Replacement: The object which is placed in the context of another object is cookie- milk and the object which is replaced by another object (expected object) person- body- hoop/net   \n",
       "3                                                                                                                                                         Liquid Deformed Object: The liquid which has been deformed is Milk   \n",
       "4                                                                                                                                                         Liquid Deformed Object: The liquid which has been deformed is milk   \n",
       "..                                                                                                                                                                                                                       ...   \n",
       "95  Object Replacement: The object which is placed in the context of another object is chocolate box spaces- void in fork- bottle arranged and the object which is replaced by another object (expected object) vodka bottle   \n",
       "96                                                                              Texture Replacement 2: The object which has a new texture is bottle and the objects that have created the texture are forks- chocolate- food   \n",
       "97  Object Replacement: The object which is placed in the context of another object is chocolate box spaces- void in fork- bottle arranged and the object which is replaced by another object (expected object) vodka bottle   \n",
       "98                                                                              Texture Replacement 2: The object which has a new texture is bottle and the objects that have created the texture are forks- chocolate- food   \n",
       "99  Object Replacement: The object which is placed in the context of another object is chocolate box spaces- void in fork- bottle arranged and the object which is replaced by another object (expected object) vodka bottle   \n",
       "\n",
       "                                                                                                                                                                    key_words  \n",
       "0                                                                                                                            {'liquid': 0.709, 'deformed': 0.6, 'milk': 0.37}  \n",
       "1                                                                                                                            {'liquid': 0.709, 'deformed': 0.6, 'milk': 0.37}  \n",
       "2      {'person': 0.374, 'net': 0.374, 'hoop': 0.374, 'cookie': 0.374, 'body': 0.374, 'milk': 0.257, 'replaced': 0.227, 'placed': 0.227, 'expected': 0.227, 'context': 0.227}  \n",
       "3                                                                                                                            {'liquid': 0.709, 'deformed': 0.6, 'milk': 0.37}  \n",
       "4                                                                                                                            {'liquid': 0.709, 'deformed': 0.6, 'milk': 0.37}  \n",
       "..                                                                                                                                                                        ...  \n",
       "95  {'bottle': 0.503, 'void': 0.295, 'vodka': 0.295, 'spaces': 0.295, 'fork': 0.295, 'box': 0.295, 'arranged': 0.295, 'chocolate': 0.252, 'replaced': 0.189, 'placed': 0.189}  \n",
       "96             {'texture': 0.665, 'forks': 0.358, 'food': 0.358, 'chocolate': 0.306, 'bottle': 0.306, 'created': 0.192, 'new': 0.177, 'replacement': 0.166, 'objects': 0.137}  \n",
       "97  {'bottle': 0.503, 'void': 0.295, 'vodka': 0.295, 'spaces': 0.295, 'fork': 0.295, 'box': 0.295, 'arranged': 0.295, 'chocolate': 0.252, 'replaced': 0.189, 'placed': 0.189}  \n",
       "98             {'texture': 0.665, 'forks': 0.358, 'food': 0.358, 'chocolate': 0.306, 'bottle': 0.306, 'created': 0.192, 'new': 0.177, 'replacement': 0.166, 'objects': 0.137}  \n",
       "99  {'bottle': 0.503, 'void': 0.295, 'vodka': 0.295, 'spaces': 0.295, 'fork': 0.295, 'box': 0.295, 'arranged': 0.295, 'chocolate': 0.252, 'replaced': 0.189, 'placed': 0.189}  \n",
       "\n",
       "[100 rows x 4 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_keywords(docs):  \n",
    "\n",
    "    all_stopwords = gensim.parsing.preprocessing.STOPWORDS\n",
    "    cv=CountVectorizer(max_df=0.85,stop_words=all_stopwords) #,max_features=10000)\n",
    "    word_count_vector=cv.fit_transform(docs)\n",
    "\n",
    "    from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "    tfidf_transformer=TfidfTransformer(smooth_idf=True,use_idf=True)\n",
    "    tfidf_transformer.fit(word_count_vector)\n",
    "    feature_names=cv.get_feature_names()\n",
    "\n",
    "    def sort_coo(coo_matrix):\n",
    "        tuples = zip(coo_matrix.col, coo_matrix.data)\n",
    "        return sorted(tuples, key=lambda x: (x[1], x[0]), reverse=True)\n",
    "\n",
    "    def extract_topn_from_vector(feature_names, sorted_items, topn=10):\n",
    "        \"\"\"get the feature names and tf-idf score of top n items\"\"\"\n",
    "\n",
    "        #use only topn items from vector\n",
    "        sorted_items = sorted_items[:topn]\n",
    "\n",
    "        score_vals = []\n",
    "        feature_vals = []\n",
    "\n",
    "        for idx, score in sorted_items:\n",
    "            fname = feature_names[idx]\n",
    "\n",
    "            #keep track of feature name and its corresponding score\n",
    "            score_vals.append(round(score, 3))\n",
    "            feature_vals.append(feature_names[idx])\n",
    "\n",
    "        #create a tuples of feature,score\n",
    "        #results = zip(feature_vals,score_vals)\n",
    "        results= {}\n",
    "        for idx in range(len(feature_vals)):\n",
    "            results[feature_vals[idx]]=score_vals[idx]\n",
    "\n",
    "        return results\n",
    "\n",
    "    #generate tf-idf for all documents in your list. docs_test has 500 documents\n",
    "    tf_idf_vector=tfidf_transformer.transform(cv.transform(docs))\n",
    "\n",
    "    results=[]\n",
    "    for i in range(tf_idf_vector.shape[0]):\n",
    "\n",
    "        # get vector for a single document\n",
    "        curr_vector=tf_idf_vector[i]\n",
    "\n",
    "        #sort the tf-idf vector by descending order of scores\n",
    "        sorted_items=sort_coo(curr_vector.tocoo())\n",
    "\n",
    "        #extract only the top n; n here is 10\n",
    "\n",
    "        keywords=extract_topn_from_vector(feature_names,sorted_items,10)\n",
    "\n",
    "\n",
    "        results.append(keywords)\n",
    "    \n",
    "    return results\n",
    "\n",
    "\n",
    "\n",
    "result1 = get_keywords(df['QA'].tolist())\n",
    "result2 = get_keywords(df['Atypical_Annot2'].tolist())\n",
    "\n",
    "    \n",
    "results = pd.DataFrame(zip(df['QA'],result1, df['Atypical_Annot2'], result2),columns=['QA','keywords', 'Atypical_Annot', 'key_words'])\n",
    "results"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}