Miscellaneous/gensim_word2vec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gensim Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import spacy\n",
    "import logging\n",
    "import pandas as pd\n",
    "from time import time\n",
    "from collections import defaultdict\n",
    "import io\n",
    "import requests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dataset preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = requests.get('https://raw.githubusercontent.com/harshildarji/DataScienceLab/master/data/train_data.csv').content\n",
    "df = pd.read_csv(io.StringIO(url.decode('utf-8')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>desc</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>ahrc esrc project gcrf investigating local res...</td>\n",
       "      <td>argues for greater engagement with faith value...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>learn how to do things alone</td>\n",
       "      <td>indonesia why unhcr amp iom in indonesia keep ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>multiple sclerosis sag film tv medical psych a...</td>\n",
       "      <td>trump s election fraud committee found no vote...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>l a filmmaker american nationalist i support o...</td>\n",
       "      <td>they need to go home it is not the job of isra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>screenplay writer producer nice guy with a big...</td>\n",
       "      <td>ice won t deport the last nazi war criminal in...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  desc  \\\n",
       "143  ahrc esrc project gcrf investigating local res...   \n",
       "144                       learn how to do things alone   \n",
       "145  multiple sclerosis sag film tv medical psych a...   \n",
       "146  l a filmmaker american nationalist i support o...   \n",
       "147  screenplay writer producer nice guy with a big...   \n",
       "\n",
       "                                                 tweet  \n",
       "143  argues for greater engagement with faith value...  \n",
       "144  indonesia why unhcr amp iom in indonesia keep ...  \n",
       "145  trump s election fraud committee found no vote...  \n",
       "146  they need to go home it is not the job of isra...  \n",
       "147  ice won t deport the last nazi war criminal in...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[143:148]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "desc     27\n",
       "tweet     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check if null values exist in the dataframe\n",
    "\n",
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "desc     0\n",
       "tweet    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.dropna().reset_index(drop=True)\n",
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>none mariotti did your ancestors have to apply...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>usmc veteran love god wife family dog and coun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>none rt japanmissionun thank you unicef unhcr ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>we ve been referring people to law firms since...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>i love life i love my family i love people wal...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                data\n",
       "0  none mariotti did your ancestors have to apply...\n",
       "1  usmc veteran love god wife family dog and coun...\n",
       "2  none rt japanmissionun thank you unicef unhcr ...\n",
       "3  we ve been referring people to law firms since...\n",
       "4  i love life i love my family i love people wal..."
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# display first few lines of dataframe\n",
    "\n",
    "df['data'] = df['desc'] + ' ' + df['tweet']\n",
    "df = df.drop(['desc', 'tweet'], axis = 1)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3929, 1)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cleaning 🧹"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lemmatization and removing stopwords\n",
    "# in case of OSError, run 'python -m spacy download en' in terminal and try again!\n",
    "\n",
    "nlp = spacy.load('en', disable=['ner', 'parser'])\n",
    "def cleaning(doc):\n",
    "    txt = [token.lemma_ for token in doc if not token.is_stop]\n",
    "    if len(txt) > 2:\n",
    "        return ' '.join(txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# removing non-alphanumeric characters\n",
    "\n",
    "brief_cleaning = (re.sub(\"[^A-Za-z']+\", ' ', str(row)).lower() for row in df['data'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 12.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# using NLP pipelines to speed-up cleaning (Read more at: https://spacy.io/usage/processing-pipelines)\n",
    "\n",
    "txt = [cleaning(doc) for doc in nlp.pipe(brief_cleaning, batch_size=5000, n_threads=-1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3291, 1)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create dataframe of cleaned data\n",
    "\n",
    "df_clean = pd.DataFrame({'clean': txt})\n",
    "df_clean = df_clean.dropna().drop_duplicates()\n",
    "df_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_clean.to_csv('new.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Common phrases and most frequent words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  using gensim phrases package to automatically detect common phrases to build a list of sentences\n",
    "\n",
    "from gensim.models.phrases import Phrases, Phraser\n",
    "\n",
    "sent = [row.split() for row in df_clean['clean']]\n",
    "bigram = Phrases(sent, min_count=30)\n",
    "sentences = bigram[sent]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10046"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_freq = defaultdict(int)\n",
    "for sent in sentences:\n",
    "    for i in sent:\n",
    "        word_freq[i] += 1\n",
    "len(word_freq)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import multiprocessing\n",
    "from gensim.models import Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of cores: 4\n"
     ]
    }
   ],
   "source": [
    "cores = multiprocessing.cpu_count()\n",
    "print('Number of cores: {}'.format(cores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "# initialize the model (Read more about parameters: https://www.kaggle.com/pierremegret/gensim-word2vec-tutorial#The-parameters:)\n",
    "\n",
    "w2v_model = Word2Vec(min_count=20,\n",
    "                     window=2,\n",
    "                     size=300,\n",
    "                     sample=6e-5, \n",
    "                     alpha=0.03, \n",
    "                     min_alpha=0.0007, \n",
    "                     negative=20,\n",
    "                     workers=cores-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1.08 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# building the vocabulary table\n",
    "\n",
    "w2v_model.build_vocab(sentences, progress_per=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 6min 6s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(177557, 1661130)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# model training\n",
    "\n",
    "w2v_model.train(sentences, total_examples = w2v_model.corpus_count, epochs = 30, report_delay = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# precompute L2-normalized vectors\n",
    "# Note: we cannot continue training after doing a replace. The model becomes read-only, means we can call most_similar, similarity etc., but not train.\n",
    "\n",
    "w2v_model.init_sims(replace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the trained model\n",
    "\n",
    "w2v_model.wv.save_word2vec_format('model.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9998005"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v_model.wv.similarity('immigration', 'refugee')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}