glove_embedding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy import spatial\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.manifold import TSNE\n",
    "import pandas as pd\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "from nltk import pos_tag ,pos_tag_sents\n",
    "import matplotlib.pyplot as plt\n",
    "from nltk import pos_tag\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from sklearn.model_selection import train_test_split, validation_curve\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from scipy.sparse import vstack\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn import metrics\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from collections import Counter\n",
    "from sklearn.model_selection import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words=set(stopwords.words(\"english\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('glove.twitter.27B.50d.txt', 'r' ,encoding='UTF-8') as f:\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        emb =np.asarray(values[1:], \"float32\")\n",
    "        embeddings_dict[word] = emb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'is' in embeddings_dict.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def remove_link(content):\n",
    "    '''remove links (using map) and hashtags and mentions'''\n",
    "    a = content[:content.find('http' or 'https')]\n",
    "    a = re.sub('#\\w+', '', a)\n",
    "    a = re.sub('@\\w+','',a)\n",
    "    return a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LemmaTokenizer(object):\n",
    "    '''lemmatize,tokenize,lower'''\n",
    "    def __init__(self):\n",
    "        self.wnl = WordNetLemmatizer()\n",
    "        self.token = RegexpTokenizer(r'[a-zA-Z]+')\n",
    "    def __call__(self, articles):\n",
    "        return [self.wnl.lemmatize(t).lower() for t in self.token.tokenize(articles) if t.lower() not in stopwords.words('english')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "def spell_correctin(sentence):\n",
    "    correct = {\"what's\": \"what is \",\n",
    "              \"i'm\": 'i am',\n",
    "              \"'ve\": \" have \",\n",
    "              \"can't\": \"cannot \",\n",
    "               \"n't\": \" not \",\n",
    "               \"i'm\":\"i am \",\n",
    "               \"couldn't\": \"are not\",\n",
    "               \"can't\": \"could not\",\n",
    "               \"didn't\": \"did not\",\n",
    "               \"doesn't\": \"does not\",\n",
    "               \"don't\": \"do not\",\n",
    "               \"hadn't\": \"had not\",\n",
    "               \"hasn't\": \"has not\",\n",
    "               \"haven't\": \"have not\",\n",
    "               \"isn't\": \"I would\",\n",
    "               \"i'll\": \"I will\",\n",
    "               \"i'd\": \"is not\",\n",
    "               \"i'm\": \"I am\",\n",
    "               \"it's\": \"it is\",\n",
    "               \"it'll\": \"it will\",\n",
    "               \"i've\": \"I have\",\n",
    "               \"mightn't\": \"might not\",\n",
    "               \"mustn't\": \"must not\",\n",
    "               \"shan't\": \"shall not\",\n",
    "               \"shouldn't\": \"should not\",\n",
    "               \"that's\": \"that is\",\n",
    "               \"there's\": \"there is\",\n",
    "               \"they'd\": \"they would\",\n",
    "               \"we'd\": \"we would\",\n",
    "               \"we're\": \"we are\",\n",
    "               \"weren't\": \"were not\",\n",
    "               \"we've\": \"we have\",\n",
    "               \"what'll\": \"what will\",\n",
    "               \"what're\": \"what are\",\n",
    "               \"what's\": \"what is\",\n",
    "               \"what've\": \"what have\",\n",
    "               \"where's\": \"where is\",\n",
    "               \"won't\": \"will not\",\n",
    "               \"wouldn't\": \"would not\",\n",
    "               \"wasn't\": \"was not\",\n",
    "               \"didn't\": \"did not\",\n",
    "               \"&lt;3\": \"good\",\n",
    "               \":d\": \"good\",\n",
    "               \":dd\": \"good\",\n",
    "               \":p\": \"good\",\n",
    "               \"yay!\": \"good\",\n",
    "               \"yay\": \"good\",\n",
    "               \"yaay\": \"good\",\n",
    "               \":/\": \"bad\",\n",
    "               \":&gt;\": \"bad\",\n",
    "               \":s\": \"bad\",\n",
    "               \"&lt;3\": \"heart\",\n",
    "               \":d\": \"smile\",\n",
    "               \":p\":\"smile\",\n",
    "               \":/\": \"worry\",\n",
    "               \":&gt;\": \"angry\",\n",
    "               r\"\\bhaha\\b\": \"ha\",\n",
    "               \"hahaha\": \"ha\",\n",
    "               \"its\": \"it is\"\n",
    "              }\n",
    "    t = sentence.split()\n",
    "    res = [correct[token] if token in correct.keys() else token for token in t]\n",
    "    return ' '.join(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>619950566786113536</td>\n",
       "      <td>neutral</td>\n",
       "      <td>Picturehouse's, Pink Floyd's, 'Roger Waters: T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>619969366986235905</td>\n",
       "      <td>neutral</td>\n",
       "      <td>Order Go Set a Watchman in store or through ou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>619971047195045888</td>\n",
       "      <td>negative</td>\n",
       "      <td>If these runway renovations at the airport pre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>619974445185302528</td>\n",
       "      <td>neutral</td>\n",
       "      <td>If you could ask an onstage interview question...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>619987808317407232</td>\n",
       "      <td>positive</td>\n",
       "      <td>A portion of book sales from our Harper Lee/Go...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id     label  \\\n",
       "0  619950566786113536   neutral   \n",
       "1  619969366986235905   neutral   \n",
       "2  619971047195045888  negative   \n",
       "3  619974445185302528   neutral   \n",
       "4  619987808317407232  positive   \n",
       "\n",
       "                                             content  \n",
       "0  Picturehouse's, Pink Floyd's, 'Roger Waters: T...  \n",
       "1  Order Go Set a Watchman in store or through ou...  \n",
       "2  If these runway renovations at the airport pre...  \n",
       "3  If you could ask an onstage interview question...  \n",
       "4  A portion of book sales from our Harper Lee/Go...  "
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('SemEval2017-task4-dev.subtask-A.english.INPUT.txt', names = ['id', 'label', 'content'], sep ='\\t', index_col=False)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.content = data.content.map(remove_link)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tokenize, lemmatize, lower case,spell correction, puncuation and number remove each content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.content = data.content.map(spell_correctin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "lt = LemmaTokenizer()\n",
    "data['token'] = data.content.map(lt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>content</th>\n",
       "      <th>token</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>619950566786113536</td>\n",
       "      <td>neutral</td>\n",
       "      <td>Picturehouse's, Pink Floyd's, 'Roger Waters: T...</td>\n",
       "      <td>[picturehouse, pink, floyd, roger, waters, wal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>619969366986235905</td>\n",
       "      <td>neutral</td>\n",
       "      <td>Order Go Set a Watchman in store or through ou...</td>\n",
       "      <td>[order, go, set, watchman, store, website, tue...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>619971047195045888</td>\n",
       "      <td>negative</td>\n",
       "      <td>If these runway renovations at the airport pre...</td>\n",
       "      <td>[runway, renovation, airport, prevent, seeing,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>619974445185302528</td>\n",
       "      <td>neutral</td>\n",
       "      <td>If you could ask an onstage interview question...</td>\n",
       "      <td>[could, ask, onstage, interview, question, mis...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>619987808317407232</td>\n",
       "      <td>positive</td>\n",
       "      <td>A portion of book sales from our Harper Lee/Go...</td>\n",
       "      <td>[portion, book, sale, harper, lee, go, set, wa...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id     label  \\\n",
       "0  619950566786113536   neutral   \n",
       "1  619969366986235905   neutral   \n",
       "2  619971047195045888  negative   \n",
       "3  619974445185302528   neutral   \n",
       "4  619987808317407232  positive   \n",
       "\n",
       "                                             content  \\\n",
       "0  Picturehouse's, Pink Floyd's, 'Roger Waters: T...   \n",
       "1  Order Go Set a Watchman in store or through ou...   \n",
       "2  If these runway renovations at the airport pre...   \n",
       "3  If you could ask an onstage interview question...   \n",
       "4  A portion of book sales from our Harper Lee/Go...   \n",
       "\n",
       "                                               token  \n",
       "0  [picturehouse, pink, floyd, roger, waters, wal...  \n",
       "1  [order, go, set, watchman, store, website, tue...  \n",
       "2  [runway, renovation, airport, prevent, seeing,...  \n",
       "3  [could, ask, onstage, interview, question, mis...  \n",
       "4  [portion, book, sale, harper, lee, go, set, wa...  "
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "def embedder(x):\n",
    "    '''embed each token with glove (if it exists in DS), also make the vector size 20'''\n",
    "    t =[]\n",
    "    for y in x:\n",
    "        try:\n",
    "            t.append(embeddings_dict[y])\n",
    "        except:\n",
    "            pass\n",
    "    while len(t)<20:\n",
    "        t.append([0]*50)\n",
    "    \n",
    "    return np.array(t[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['vector']=data.token.map(embedder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>content</th>\n",
       "      <th>token</th>\n",
       "      <th>vector</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>619950566786113536</td>\n",
       "      <td>neutral</td>\n",
       "      <td>Picturehouse's, Pink Floyd's, 'Roger Waters: T...</td>\n",
       "      <td>[picturehouse, pink, floyd, roger, waters, wal...</td>\n",
       "      <td>[[0.16812999546527863, 0.2084600031375885, 0.3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>619969366986235905</td>\n",
       "      <td>neutral</td>\n",
       "      <td>Order Go Set a Watchman in store or through ou...</td>\n",
       "      <td>[order, go, set, watchman, store, website, tue...</td>\n",
       "      <td>[[-0.5772899985313416, -0.24309000372886658, -...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>619971047195045888</td>\n",
       "      <td>negative</td>\n",
       "      <td>If these runway renovations at the airport pre...</td>\n",
       "      <td>[runway, renovation, airport, prevent, seeing,...</td>\n",
       "      <td>[[-1.4539999961853027, -0.14899000525474548, 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>619974445185302528</td>\n",
       "      <td>neutral</td>\n",
       "      <td>If you could ask an onstage interview question...</td>\n",
       "      <td>[could, ask, onstage, interview, question, mis...</td>\n",
       "      <td>[[0.01691400073468685, 0.480679988861084, -0.3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>619987808317407232</td>\n",
       "      <td>positive</td>\n",
       "      <td>A portion of book sales from our Harper Lee/Go...</td>\n",
       "      <td>[portion, book, sale, harper, lee, go, set, wa...</td>\n",
       "      <td>[[-0.07840699702501297, -0.4659999907016754, 0...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id     label  \\\n",
       "0  619950566786113536   neutral   \n",
       "1  619969366986235905   neutral   \n",
       "2  619971047195045888  negative   \n",
       "3  619974445185302528   neutral   \n",
       "4  619987808317407232  positive   \n",
       "\n",
       "                                             content  \\\n",
       "0  Picturehouse's, Pink Floyd's, 'Roger Waters: T...   \n",
       "1  Order Go Set a Watchman in store or through ou...   \n",
       "2  If these runway renovations at the airport pre...   \n",
       "3  If you could ask an onstage interview question...   \n",
       "4  A portion of book sales from our Harper Lee/Go...   \n",
       "\n",
       "                                               token  \\\n",
       "0  [picturehouse, pink, floyd, roger, waters, wal...   \n",
       "1  [order, go, set, watchman, store, website, tue...   \n",
       "2  [runway, renovation, airport, prevent, seeing,...   \n",
       "3  [could, ask, onstage, interview, question, mis...   \n",
       "4  [portion, book, sale, harper, lee, go, set, wa...   \n",
       "\n",
       "                                              vector  \n",
       "0  [[0.16812999546527863, 0.2084600031375885, 0.3...  \n",
       "1  [[-0.5772899985313416, -0.24309000372886658, -...  \n",
       "2  [[-1.4539999961853027, -0.14899000525474548, 0...  \n",
       "3  [[0.01691400073468685, 0.480679988861084, -0.3...  \n",
       "4  [[-0.07840699702501297, -0.4659999907016754, 0...  "
      ]
     },
     "execution_count": 221,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### find number of unique words count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_index= []\n",
    "for token in data.token:\n",
    "    for t in token:\n",
    "        word_index.append(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "full words count:  225069\n"
     ]
    }
   ],
   "source": [
    "print('full words count: ',len(word_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of unique words  20665\n"
     ]
    }
   ],
   "source": [
    "word_index_len = len(list(set(word_index)))\n",
    "print('number of unique words ',word_index_len)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## convert tokens to integer indexes\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "counts = Counter(word_index)\n",
    "vocab = sorted(counts, key=counts.get, reverse=True)\n",
    "vocab_to_int = {word_index: ii for ii, word_index in enumerate(vocab, 1)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "def token2integer(token):\n",
    "    '''converts word token array to integer token array '''\n",
    "    x= [vocab_to_int[t] for t in token if t in embeddings_dict.keys() ]\n",
    "    while len(x)<20:\n",
    "        x.append(0)\n",
    "    return x[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['integer_token'] = data.token.map(token2integer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "for i in data.integer_token:\n",
    "    X.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20632, 20)"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X= np.array(X)\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20665"
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab_to_int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## embedding matrix: it contains embedding for each word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_matrix = np.zeros([len(vocab_to_int)+1 , 50])\n",
    "for word, i in vocab_to_int.items():\n",
    "    try:\n",
    "        embedding_matrix[i] = embeddings_dict[word]\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20666, 50)"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "data['encoded_label'] = le.fit_transform(data.label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test train split\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, data.encoded_label.values, test_size=0.3, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11553 samples\n",
      "Epoch 1/6\n",
      "11553/11553 [==============================] - 11s 986us/sample - loss: 0.9000 - accuracy: 0.5690\n",
      "Epoch 2/6\n",
      "11553/11553 [==============================] - 7s 616us/sample - loss: 0.8132 - accuracy: 0.6221\n",
      "Epoch 3/6\n",
      "11553/11553 [==============================] - 7s 645us/sample - loss: 0.7847 - accuracy: 0.6394\n",
      "Epoch 4/6\n",
      "11553/11553 [==============================] - 8s 691us/sample - loss: 0.7699 - accuracy: 0.6421 - loss: 0.7697 \n",
      "Epoch 5/6\n",
      "11553/11553 [==============================] - 8s 665us/sample - loss: 0.7602 - accuracy: 0.6508 - loss: 0.7608 - accuracy\n",
      "Epoch 6/6\n",
      "11553/11553 [==============================] - 7s 626us/sample - loss: 0.7348 - accuracy: 0.6661\n",
      "fold 0 acc: 0.6535133\n",
      "Train on 11553 samples\n",
      "Epoch 1/6\n",
      "11553/11553 [==============================] - 11s 950us/sample - loss: 0.9151 - accuracy: 0.5631\n",
      "Epoch 2/6\n",
      "11553/11553 [==============================] - 7s 620us/sample - loss: 0.8119 - accuracy: 0.6239\n",
      "Epoch 3/6\n",
      "11553/11553 [==============================] - 7s 635us/sample - loss: 0.7841 - accuracy: 0.6408\n",
      "Epoch 4/6\n",
      "11553/11553 [==============================] - 7s 625us/sample - loss: 0.7745 - accuracy: 0.6439\n",
      "Epoch 5/6\n",
      "11553/11553 [==============================] - 7s 629us/sample - loss: 0.7539 - accuracy: 0.6495\n",
      "Epoch 6/6\n",
      "11553/11553 [==============================] - 7s 625us/sample - loss: 0.7359 - accuracy: 0.6608\n",
      "fold 1 acc: 0.63586015\n",
      "Train on 11554 samples\n",
      "Epoch 1/6\n",
      "11554/11554 [==============================] - 11s 947us/sample - loss: 0.9108 - accuracy: 0.5538\n",
      "Epoch 2/6\n",
      "11554/11554 [==============================] - 7s 628us/sample - loss: 0.8170 - accuracy: 0.6180\n",
      "Epoch 3/6\n",
      "11554/11554 [==============================] - 7s 639us/sample - loss: 0.7966 - accuracy: 0.6266\n",
      "Epoch 4/6\n",
      "11554/11554 [==============================] - 7s 636us/sample - loss: 0.7830 - accuracy: 0.6369\n",
      "Epoch 5/6\n",
      "11554/11554 [==============================] - 7s 632us/sample - loss: 0.7599 - accuracy: 0.6503\n",
      "Epoch 6/6\n",
      "11554/11554 [==============================] - 7s 634us/sample - loss: 0.7435 - accuracy: 0.6578\n",
      "fold 2 acc: 0.6450831\n",
      "Train on 11554 samples\n",
      "Epoch 1/6\n",
      "11554/11554 [==============================] - 13s 1ms/sample - loss: 0.8998 - accuracy: 0.5607\n",
      "Epoch 2/6\n",
      "11554/11554 [==============================] - 7s 623us/sample - loss: 0.8161 - accuracy: 0.6203\n",
      "Epoch 3/6\n",
      "11554/11554 [==============================] - 7s 629us/sample - loss: 0.7874 - accuracy: 0.6370\n",
      "Epoch 4/6\n",
      "11554/11554 [==============================] - 7s 630us/sample - loss: 0.7723 - accuracy: 0.6490\n",
      "Epoch 5/6\n",
      "11554/11554 [==============================] - 7s 623us/sample - loss: 0.7599 - accuracy: 0.6548\n",
      "Epoch 6/6\n",
      "11554/11554 [==============================] - 7s 634us/sample - loss: 0.7405 - accuracy: 0.6624\n",
      "fold 3 acc: 0.63504153\n",
      "Train on 11554 samples\n",
      "Epoch 1/6\n",
      "11554/11554 [==============================] - 11s 952us/sample - loss: 0.8962 - accuracy: 0.5617\n",
      "Epoch 2/6\n",
      "11554/11554 [==============================] - 7s 638us/sample - loss: 0.8118 - accuracy: 0.6259\n",
      "Epoch 3/6\n",
      "11554/11554 [==============================] - 8s 651us/sample - loss: 0.7909 - accuracy: 0.6327\n",
      "Epoch 4/6\n",
      "11554/11554 [==============================] - 7s 629us/sample - loss: 0.7702 - accuracy: 0.6477\n",
      "Epoch 5/6\n",
      "11554/11554 [==============================] - 7s 637us/sample - loss: 0.7574 - accuracy: 0.6494\n",
      "Epoch 6/6\n",
      "11554/11554 [==============================] - 7s 632us/sample - loss: 0.7342 - accuracy: 0.6638\n",
      "fold 4 acc: 0.6599723\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.layers import Embedding, LSTM, Dense, Dropout, Input\n",
    "acc_per_fold = []\n",
    "loss_per_fold = []\n",
    "fold_num = 0\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)\n",
    "model = [None] * 5\n",
    "for train, test in kfold.split(X_train,y_train):\n",
    "    model[fold_num]=Sequential()\n",
    "    model[fold_num].add(Embedding(word_index_len + 1,\n",
    "                                50,\n",
    "                                weights=[embedding_matrix],\n",
    "                                input_length=20,\n",
    "                                trainable=False))\n",
    "    model[fold_num].add(LSTM(128)) # model.add(LSTM(64,return_sequences=True))\n",
    "\n",
    "    model[fold_num].add(Dense(3, activation='sigmoid'))\n",
    "    model[fold_num].compile(loss='sparse_categorical_crossentropy', \n",
    "             optimizer='adam', \n",
    "             metrics=['accuracy'],learning_rate= 0.01)\n",
    "    num_epochs = 6\n",
    "\n",
    "    model[fold_num].fit(X_train[train],y_train[train], batch_size=batch_size, epochs=num_epochs,)\n",
    "    scores = model[fold_num].evaluate(X_train[test],y_train[test], verbose=0)\n",
    "    print('fold',fold_num, 'acc:',scores[1])\n",
    "    fold_num += 1\n",
    "    acc_per_fold.append(scores[1])\n",
    "    loss_per_fold.append(scores[0])\n",
    "#print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.6535133, 0.63586015, 0.6450831, 0.63504153, 0.6599723]"
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc_per_fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy mean: 0.64589405\n"
     ]
    }
   ],
   "source": [
    "print('accuracy mean:',np.mean(acc_per_fold))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss mean: 0.776854051817742\n"
     ]
    }
   ],
   "source": [
    "print('loss mean:',np.mean(loss_per_fold))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### examine all dataset with each model, and do Bagging from Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.37      0.67      0.47       540\n",
      "           1       0.75      0.65      0.70      3557\n",
      "           2       0.63      0.63      0.63      2093\n",
      "\n",
      "    accuracy                           0.65      6190\n",
      "   macro avg       0.58      0.65      0.60      6190\n",
      "weighted avg       0.68      0.65      0.66      6190\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "results=[model[t].predict_classes(X_test) for t in range(2) ]\n",
    "most_common = []\n",
    "for i in range(results[0].shape[0]):\n",
    "    temp = [results[t][i] for t in range(len(results))]\n",
    "    most_common.append(Counter(temp).most_common()[0][0])\n",
    "    \n",
    "print(metrics.classification_report(most_common,y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 20, 50)            1033300   \n",
      "_________________________________________________________________\n",
      "lstm (LSTM)                  (None, 128)               91648     \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 3)                 387       \n",
      "=================================================================\n",
      "Total params: 1,125,335\n",
      "Trainable params: 92,035\n",
      "Non-trainable params: 1,033,300\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model[0].summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}