jobguy_sentiment_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Steps:\n",
    "* Using countvectorize of words\n",
    "* Using Bayes models\n",
    "* Using Hazm package\n",
    "* Using StanforlNLP package"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import hazm\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from collections import Counter\n",
    "from itertools import chain\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split, validation_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('scrapy_project/jubguy_ir/jubguy_ir/spiders/jobguy_pros_cons.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.tell_friend = data.tell_friend.map(lambda x: 'NNN' if type(x)== float else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>company_slug</th>\n",
       "      <th>company</th>\n",
       "      <th>title</th>\n",
       "      <th>pros</th>\n",
       "      <th>cons</th>\n",
       "      <th>mean_salary</th>\n",
       "      <th>comment</th>\n",
       "      <th>url</th>\n",
       "      <th>tell_friend</th>\n",
       "      <th>like</th>\n",
       "      <th>dislike</th>\n",
       "      <th>f1</th>\n",
       "      <th>f2</th>\n",
       "      <th>f3</th>\n",
       "      <th>f4</th>\n",
       "      <th>f5</th>\n",
       "      <th>f6</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>tadbir-kish-vira</td>\n",
       "      <td>تدبیر کیش ویرا</td>\n",
       "      <td>\\n            تجربه بسیار بد کارآموزی\\n</td>\n",
       "      <td>['NaN']</td>\n",
       "      <td>['عدم دریافت پاداش', 'پایه حقوق کم', 'مدیریت ض...</td>\n",
       "      <td>3.5</td>\n",
       "      <td>رفتم مصاحبه شرکت تدبیر کیش ویرا و 2 مرحله باها...</td>\n",
       "      <td>https://jobguy.ir/review/300/تجربه-بسیار-بد-کا...</td>\n",
       "      <td>خیر</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>irantic</td>\n",
       "      <td>ایرانتیک</td>\n",
       "      <td>\\n            مدیران نالایق\\n</td>\n",
       "      <td>['اصلا ندارد']</td>\n",
       "      <td>['عدم دریافت پاداش', 'مدیریت بی ثبات', 'مدیریت...</td>\n",
       "      <td>4.3</td>\n",
       "      <td>با تمام وجود متنفرم از این سازمان. کار بیش از ...</td>\n",
       "      <td>https://jobguy.ir/review/563/مدیران-نالایق</td>\n",
       "      <td>خیر</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>hermes-sharif</td>\n",
       "      <td>ایده های هوشمند هرمس شریف</td>\n",
       "      <td>\\n            اگه دنبال یه شرکت دهه هفتادی هست...</td>\n",
       "      <td>['NaN']</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.3</td>\n",
       "      <td>محیط انعطاف پذیر</td>\n",
       "      <td>https://jobguy.ir/review/74/اگه-دنبال-یه-شرکت-...</td>\n",
       "      <td>بله</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>hermes-sharif</td>\n",
       "      <td>ایده های هوشمند هرمس شریف</td>\n",
       "      <td>\\n            طراح محصول\\n</td>\n",
       "      <td>['تیم جوان و با انگیزه', 'کار با آخرین تکنولوژ...</td>\n",
       "      <td>['عدم دریافت پاداش', 'حقوق ناکافی', 'عدم پرداخ...</td>\n",
       "      <td>3.3</td>\n",
       "      <td>محیط بسیار دوستانه و پویا، با کارمند جوان و پر...</td>\n",
       "      <td>https://jobguy.ir/review/339/طراح-محصول</td>\n",
       "      <td>بله</td>\n",
       "      <td>['4']</td>\n",
       "      <td>['1']</td>\n",
       "      <td>3.4</td>\n",
       "      <td>4</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>irantic</td>\n",
       "      <td>ایرانتیک</td>\n",
       "      <td>\\n            مدیریت بسیار بد\\n</td>\n",
       "      <td>['تعهد بالای پرسنل به کار', 'تیم صمیمی']</td>\n",
       "      <td>['مدیریت بی ثبات', 'مدیریت ضعیف', 'عدم امنیت ش...</td>\n",
       "      <td>4.3</td>\n",
       "      <td>از همه منزجر کننده تر در این شرکت سخن...</td>\n",
       "      <td>https://jobguy.ir/review/381/مدیریت-بسیار-بد</td>\n",
       "      <td>خیر</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>irantic</td>\n",
       "      <td>ایرانتیک</td>\n",
       "      <td>\\n            انصاف در زندگی چیز خوبیست\\n     ...</td>\n",
       "      <td>['NaN']</td>\n",
       "      <td>[]</td>\n",
       "      <td>4.3</td>\n",
       "      <td>من بعنوان یک فرد تازه کار در سمت backend استخد...</td>\n",
       "      <td>https://jobguy.ir/review/607/انصاف-در-زندگی-چی...</td>\n",
       "      <td>بله</td>\n",
       "      <td>['1']</td>\n",
       "      <td>['2']</td>\n",
       "      <td>4.8</td>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>zhaket</td>\n",
       "      <td>ژاکت</td>\n",
       "      <td>\\n            تخصص شما پیر خواهد شد\\n</td>\n",
       "      <td>['NaN']</td>\n",
       "      <td>['مدیریت بی ثبات', 'مدیریت ضعیف', 'حقوق ناکافی...</td>\n",
       "      <td>5.7</td>\n",
       "      <td>ژاکت از نطر من ۳ مشکل اساسی داره که همه مشکلات...</td>\n",
       "      <td>https://jobguy.ir/review/562/تخصص-شما-پیر-خواه...</td>\n",
       "      <td>خیر</td>\n",
       "      <td>['3']</td>\n",
       "      <td>['0']</td>\n",
       "      <td>1.6</td>\n",
       "      <td>3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>tadbir-kish-vira</td>\n",
       "      <td>تدبیر کیش ویرا</td>\n",
       "      <td>\\n            برنامه نویس Front End\\n</td>\n",
       "      <td>['NaN']</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.5</td>\n",
       "      <td>من تازه از خدمت سربازی تمام شده بود و قبلا به ...</td>\n",
       "      <td>https://jobguy.ir/review/428/برنامه-نویس-front...</td>\n",
       "      <td>بله</td>\n",
       "      <td>['1']</td>\n",
       "      <td>['0']</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>snapp-trip</td>\n",
       "      <td>اسنپ تریپ</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['کار با آخرین تکنولوژی ها', 'تیم صمیمی']</td>\n",
       "      <td>['مدیریت ضعیف', 'بد قول', 'شرکت کوچک']</td>\n",
       "      <td>15.2</td>\n",
       "      <td>اولا من رزومه براشون ارسال نکردم اونا خودشون ب...</td>\n",
       "      <td>https://jobguy.ir/interview/24/مصاحبه-کاری</td>\n",
       "      <td>NNN</td>\n",
       "      <td>['0']</td>\n",
       "      <td>['0']</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>interview</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>kte</td>\n",
       "      <td>عصر فناوری دانش</td>\n",
       "      <td>\\n            شرکتی با کمترین درجه اهمیت به کا...</td>\n",
       "      <td>['بیمه تکمیلی', 'بیمه', 'اهمیت به دیزاین']</td>\n",
       "      <td>['عدم دریافت پاداش', 'مدیریت بی ثبات', 'پایه ح...</td>\n",
       "      <td>3.2</td>\n",
       "      <td>محیط بسیار  کارمندان که بدون هیچ دلیلی با افرا...</td>\n",
       "      <td>https://jobguy.ir/review/564/شرکتی-با-کمترین-د...</td>\n",
       "      <td>خیر</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>kte</td>\n",
       "      <td>عصر فناوری دانش</td>\n",
       "      <td>\\n            تبعیض میان کارکنان بخصوص برنامه ...</td>\n",
       "      <td>['بیمه تکمیلی', 'تیم صمیمی']</td>\n",
       "      <td>['پایه حقوق کم', 'مدیریت ضعیف', 'حقوق ناکافی',...</td>\n",
       "      <td>3.2</td>\n",
       "      <td>شرکت نسبتا کوچیکه . حقوق هم خوب کلا کم میدن . ...</td>\n",
       "      <td>https://jobguy.ir/review/629/تبعیض-میان-کارکنا...</td>\n",
       "      <td>خیر</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0</td>\n",
       "      <td>fartech</td>\n",
       "      <td>فرتک</td>\n",
       "      <td>\\n            سابقه همکاری نسبتا طولانی\\n     ...</td>\n",
       "      <td>['تیم جوان و با انگیزه', 'نوشیدنی رایگان', 'زم...</td>\n",
       "      <td>['شرکت کوچک']</td>\n",
       "      <td>1.2</td>\n",
       "      <td>من چندسالی در این شرکت کار میکردم به دلیل یکسر...</td>\n",
       "      <td>https://jobguy.ir/review/131/سابقه-همکاری-نسبت...</td>\n",
       "      <td>بله</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>review</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0      company_slug                    company  \\\n",
       "0            0  tadbir-kish-vira             تدبیر کیش ویرا   \n",
       "1            0           irantic                   ایرانتیک   \n",
       "2            0     hermes-sharif  ایده های هوشمند هرمس شریف   \n",
       "3            0     hermes-sharif  ایده های هوشمند هرمس شریف   \n",
       "4            0           irantic                   ایرانتیک   \n",
       "5            0           irantic                   ایرانتیک   \n",
       "6            0            zhaket                       ژاکت   \n",
       "7            0  tadbir-kish-vira             تدبیر کیش ویرا   \n",
       "8            0        snapp-trip                  اسنپ تریپ   \n",
       "9            0               kte            عصر فناوری دانش   \n",
       "10           0               kte            عصر فناوری دانش   \n",
       "11           0           fartech                       فرتک   \n",
       "\n",
       "                                                title  \\\n",
       "0   \\n            تجربه بسیار بد کارآموزی\\n             \n",
       "1             \\n            مدیران نالایق\\n             \n",
       "2   \\n            اگه دنبال یه شرکت دهه هفتادی هست...   \n",
       "3                \\n            طراح محصول\\n             \n",
       "4           \\n            مدیریت بسیار بد\\n             \n",
       "5   \\n            انصاف در زندگی چیز خوبیست\\n     ...   \n",
       "6     \\n            تخصص شما پیر خواهد شد\\n             \n",
       "7     \\n            برنامه نویس Front End\\n             \n",
       "8                                                 NaN   \n",
       "9   \\n            شرکتی با کمترین درجه اهمیت به کا...   \n",
       "10  \\n            تبعیض میان کارکنان بخصوص برنامه ...   \n",
       "11  \\n            سابقه همکاری نسبتا طولانی\\n     ...   \n",
       "\n",
       "                                                 pros  \\\n",
       "0                                             ['NaN']   \n",
       "1                                      ['اصلا ندارد']   \n",
       "2                                             ['NaN']   \n",
       "3   ['تیم جوان و با انگیزه', 'کار با آخرین تکنولوژ...   \n",
       "4            ['تعهد بالای پرسنل به کار', 'تیم صمیمی']   \n",
       "5                                             ['NaN']   \n",
       "6                                             ['NaN']   \n",
       "7                                             ['NaN']   \n",
       "8           ['کار با آخرین تکنولوژی ها', 'تیم صمیمی']   \n",
       "9          ['بیمه تکمیلی', 'بیمه', 'اهمیت به دیزاین']   \n",
       "10                       ['بیمه تکمیلی', 'تیم صمیمی']   \n",
       "11  ['تیم جوان و با انگیزه', 'نوشیدنی رایگان', 'زم...   \n",
       "\n",
       "                                                 cons  mean_salary  \\\n",
       "0   ['عدم دریافت پاداش', 'پایه حقوق کم', 'مدیریت ض...          3.5   \n",
       "1   ['عدم دریافت پاداش', 'مدیریت بی ثبات', 'مدیریت...          4.3   \n",
       "2                                                  []          3.3   \n",
       "3   ['عدم دریافت پاداش', 'حقوق ناکافی', 'عدم پرداخ...          3.3   \n",
       "4   ['مدیریت بی ثبات', 'مدیریت ضعیف', 'عدم امنیت ش...          4.3   \n",
       "5                                                  []          4.3   \n",
       "6   ['مدیریت بی ثبات', 'مدیریت ضعیف', 'حقوق ناکافی...          5.7   \n",
       "7                                                  []          3.5   \n",
       "8              ['مدیریت ضعیف', 'بد قول', 'شرکت کوچک']         15.2   \n",
       "9   ['عدم دریافت پاداش', 'مدیریت بی ثبات', 'پایه ح...          3.2   \n",
       "10  ['پایه حقوق کم', 'مدیریت ضعیف', 'حقوق ناکافی',...          3.2   \n",
       "11                                      ['شرکت کوچک']          1.2   \n",
       "\n",
       "                                              comment  \\\n",
       "0   رفتم مصاحبه شرکت تدبیر کیش ویرا و 2 مرحله باها...   \n",
       "1   با تمام وجود متنفرم از این سازمان. کار بیش از ...   \n",
       "2                                    محیط انعطاف پذیر   \n",
       "3   محیط بسیار دوستانه و پویا، با کارمند جوان و پر...   \n",
       "4            از همه منزجر کننده تر در این شرکت سخن...   \n",
       "5   من بعنوان یک فرد تازه کار در سمت backend استخد...   \n",
       "6   ژاکت از نطر من ۳ مشکل اساسی داره که همه مشکلات...   \n",
       "7   من تازه از خدمت سربازی تمام شده بود و قبلا به ...   \n",
       "8   اولا من رزومه براشون ارسال نکردم اونا خودشون ب...   \n",
       "9   محیط بسیار  کارمندان که بدون هیچ دلیلی با افرا...   \n",
       "10  شرکت نسبتا کوچیکه . حقوق هم خوب کلا کم میدن . ...   \n",
       "11  من چندسالی در این شرکت کار میکردم به دلیل یکسر...   \n",
       "\n",
       "                                                  url tell_friend   like  \\\n",
       "0   https://jobguy.ir/review/300/تجربه-بسیار-بد-کا...         خیر     []   \n",
       "1          https://jobguy.ir/review/563/مدیران-نالایق         خیر     []   \n",
       "2   https://jobguy.ir/review/74/اگه-دنبال-یه-شرکت-...         بله     []   \n",
       "3             https://jobguy.ir/review/339/طراح-محصول         بله  ['4']   \n",
       "4        https://jobguy.ir/review/381/مدیریت-بسیار-بد         خیر     []   \n",
       "5   https://jobguy.ir/review/607/انصاف-در-زندگی-چی...         بله  ['1']   \n",
       "6   https://jobguy.ir/review/562/تخصص-شما-پیر-خواه...         خیر  ['3']   \n",
       "7   https://jobguy.ir/review/428/برنامه-نویس-front...         بله  ['1']   \n",
       "8          https://jobguy.ir/interview/24/مصاحبه-کاری         NNN  ['0']   \n",
       "9   https://jobguy.ir/review/564/شرکتی-با-کمترین-د...         خیر     []   \n",
       "10  https://jobguy.ir/review/629/تبعیض-میان-کارکنا...         خیر     []   \n",
       "11  https://jobguy.ir/review/131/سابقه-همکاری-نسبت...         بله     []   \n",
       "\n",
       "   dislike   f1  f2   f3   f4   f5   f6       type  \n",
       "0       []  0.0   0  0.0  0.0  0.0  0.0     review  \n",
       "1       []  0.0   0  0.0  0.0  0.0  0.0     review  \n",
       "2       []  0.0   0  0.0  0.0  0.0  0.0     review  \n",
       "3    ['1']  3.4   4  2.0  4.0  3.0  4.0     review  \n",
       "4       []  0.0   0  0.0  0.0  0.0  0.0     review  \n",
       "5    ['2']  4.8   4  5.0  5.0  5.0  5.0     review  \n",
       "6    ['0']  1.6   3  2.0  1.0  1.0  1.0     review  \n",
       "7    ['0']  5.0   5  5.0  5.0  5.0  5.0     review  \n",
       "8    ['0']  3.0   2  NaN  NaN  NaN  NaN  interview  \n",
       "9       []  0.0   0  0.0  0.0  0.0  0.0     review  \n",
       "10      []  0.0   0  0.0  0.0  0.0  0.0     review  \n",
       "11      []  0.0   0  0.0  0.0  0.0  0.0     review  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_review = data[['comment', 'tell_friend']][data.type == 'review']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_review['tell_fried_labeled'] = le.fit_transform(data_review.tell_friend.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment</th>\n",
       "      <th>tell_friend</th>\n",
       "      <th>tell_fried_labeled</th>\n",
       "      <th>comment_tokenized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>رفتم مصاحبه شرکت تدبیر کیش ویرا و 2 مرحله باها...</td>\n",
       "      <td>خیر</td>\n",
       "      <td>1</td>\n",
       "      <td>[رفتم, مصاحبه, شرکت, تدبیر, کیش, ویرا, و, 2, م...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>با تمام وجود متنفرم از این سازمان. کار بیش از ...</td>\n",
       "      <td>خیر</td>\n",
       "      <td>1</td>\n",
       "      <td>[با, تمام, وجود, متنفرم, از, این, سازمان., کار...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>محیط انعطاف پذیر</td>\n",
       "      <td>بله</td>\n",
       "      <td>0</td>\n",
       "      <td>[محیط, انعطاف, پذیر]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>محیط بسیار دوستانه و پویا، با کارمند جوان و پر...</td>\n",
       "      <td>بله</td>\n",
       "      <td>0</td>\n",
       "      <td>[محیط, بسیار, دوستانه, و, پویا،, با, کارمند, ج...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>از همه منزجر کننده تر در این شرکت سخن...</td>\n",
       "      <td>خیر</td>\n",
       "      <td>1</td>\n",
       "      <td>[از, همه, منزجر, کننده, تر, در, این, شرکت, سخن...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             comment tell_friend  \\\n",
       "0  رفتم مصاحبه شرکت تدبیر کیش ویرا و 2 مرحله باها...         خیر   \n",
       "1  با تمام وجود متنفرم از این سازمان. کار بیش از ...         خیر   \n",
       "2                                   محیط انعطاف پذیر         بله   \n",
       "3  محیط بسیار دوستانه و پویا، با کارمند جوان و پر...         بله   \n",
       "4           از همه منزجر کننده تر در این شرکت سخن...         خیر   \n",
       "\n",
       "   tell_fried_labeled                                  comment_tokenized  \n",
       "0                   1  [رفتم, مصاحبه, شرکت, تدبیر, کیش, ویرا, و, 2, م...  \n",
       "1                   1  [با, تمام, وجود, متنفرم, از, این, سازمان., کار...  \n",
       "2                   0                               [محیط, انعطاف, پذیر]  \n",
       "3                   0  [محیط, بسیار, دوستانه, و, پویا،, با, کارمند, ج...  \n",
       "4                   1  [از, همه, منزجر, کننده, تر, در, این, شرکت, سخن...  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_review.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_review.comment = data_review.comment.map(lambda x: 'NNN' if type(x)==float else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_review['comment_tokenized'] = data_review.comment.map(lambda x: x.split())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## create vocabulary of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_comments_token =  list(chain(*data_review.comment_tokenized.values))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "counts = Counter(full_comments_token)\n",
    "vocab = sorted(counts, key=counts.get, reverse=True)\n",
    "vocab_to_int = {full_comments_token: ii for ii, full_comments_token in enumerate(vocab, 1)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def token2integer(token):\n",
    "    '''converts word token array to integer token array '''\n",
    "    x= [vocab_to_int[t] for t in token]\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_review['integer_token'] = data_review.comment_tokenized.map(token2integer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_review['summary_token'] = data_review.integer_token.map(lambda x: x[:80] if len(x)>=80 else x+[0]*(80 - len(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7813"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab_to_int.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment</th>\n",
       "      <th>tell_friend</th>\n",
       "      <th>tell_fried_labeled</th>\n",
       "      <th>comment_tokenized</th>\n",
       "      <th>integer_token</th>\n",
       "      <th>summary_token</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>رفتم مصاحبه شرکت تدبیر کیش ویرا و 2 مرحله باها...</td>\n",
       "      <td>خیر</td>\n",
       "      <td>1</td>\n",
       "      <td>[رفتم, مصاحبه, شرکت, تدبیر, کیش, ویرا, و, 2, م...</td>\n",
       "      <td>[477, 168, 6, 1732, 2236, 2237, 1, 281, 1078, ...</td>\n",
       "      <td>[477, 168, 6, 1732, 2236, 2237, 1, 281, 1078, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>با تمام وجود متنفرم از این سازمان. کار بیش از ...</td>\n",
       "      <td>خیر</td>\n",
       "      <td>1</td>\n",
       "      <td>[با, تمام, وجود, متنفرم, از, این, سازمان., کار...</td>\n",
       "      <td>[7, 139, 43, 3311, 4, 8, 3312, 9, 335, 4, 3313...</td>\n",
       "      <td>[7, 139, 43, 3311, 4, 8, 3312, 9, 335, 4, 3313...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>محیط انعطاف پذیر</td>\n",
       "      <td>بله</td>\n",
       "      <td>0</td>\n",
       "      <td>[محیط, انعطاف, پذیر]</td>\n",
       "      <td>[42, 1741, 1742]</td>\n",
       "      <td>[42, 1741, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>محیط بسیار دوستانه و پویا، با کارمند جوان و پر...</td>\n",
       "      <td>بله</td>\n",
       "      <td>0</td>\n",
       "      <td>[محیط, بسیار, دوستانه, و, پویا،, با, کارمند, ج...</td>\n",
       "      <td>[42, 27, 158, 1, 3338, 7, 173, 370, 1, 258, 33...</td>\n",
       "      <td>[42, 27, 158, 1, 3338, 7, 173, 370, 1, 258, 33...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>از همه منزجر کننده تر در این شرکت سخن...</td>\n",
       "      <td>خیر</td>\n",
       "      <td>1</td>\n",
       "      <td>[از, همه, منزجر, کننده, تر, در, این, شرکت, سخن...</td>\n",
       "      <td>[4, 37, 3343, 527, 320, 5, 8, 6, 3344, 3345, 5...</td>\n",
       "      <td>[4, 37, 3343, 527, 320, 5, 8, 6, 3344, 3345, 5...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             comment tell_friend  \\\n",
       "0  رفتم مصاحبه شرکت تدبیر کیش ویرا و 2 مرحله باها...         خیر   \n",
       "1  با تمام وجود متنفرم از این سازمان. کار بیش از ...         خیر   \n",
       "2                                   محیط انعطاف پذیر         بله   \n",
       "3  محیط بسیار دوستانه و پویا، با کارمند جوان و پر...         بله   \n",
       "4           از همه منزجر کننده تر در این شرکت سخن...         خیر   \n",
       "\n",
       "   tell_fried_labeled                                  comment_tokenized  \\\n",
       "0                   1  [رفتم, مصاحبه, شرکت, تدبیر, کیش, ویرا, و, 2, م...   \n",
       "1                   1  [با, تمام, وجود, متنفرم, از, این, سازمان., کار...   \n",
       "2                   0                               [محیط, انعطاف, پذیر]   \n",
       "3                   0  [محیط, بسیار, دوستانه, و, پویا،, با, کارمند, ج...   \n",
       "4                   1  [از, همه, منزجر, کننده, تر, در, این, شرکت, سخن...   \n",
       "\n",
       "                                       integer_token  \\\n",
       "0  [477, 168, 6, 1732, 2236, 2237, 1, 281, 1078, ...   \n",
       "1  [7, 139, 43, 3311, 4, 8, 3312, 9, 335, 4, 3313...   \n",
       "2                                   [42, 1741, 1742]   \n",
       "3  [42, 27, 158, 1, 3338, 7, 173, 370, 1, 258, 33...   \n",
       "4  [4, 37, 3343, 527, 320, 5, 8, 6, 3344, 3345, 5...   \n",
       "\n",
       "                                       summary_token  \n",
       "0  [477, 168, 6, 1732, 2236, 2237, 1, 281, 1078, ...  \n",
       "1  [7, 139, 43, 3311, 4, 8, 3312, 9, 335, 4, 3313...  \n",
       "2  [42, 1741, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...  \n",
       "3  [42, 27, 158, 1, 3338, 7, 173, 370, 1, 258, 33...  \n",
       "4  [4, 37, 3343, 527, 320, 5, 8, 6, 3344, 3345, 5...  "
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_review.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(data_review.summary_token, data_review.tell_fried_labeled, test_size= 0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = [np.array(x) for x in X_train]\n",
    "X_train = np.array(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = [np.array(x) for x in X_test]\n",
    "X_test = np.array(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "rnd_forrest = RandomForestClassifier(n_estimators=120, max_depth=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "                       max_depth=200, max_features='auto', max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=120,\n",
       "                       n_jobs=None, oob_score=False, random_state=None,\n",
       "                       verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnd_forrest.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6645161290322581"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnd_forrest.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.layers import Embedding, LSTM, Dense, Dropout, Input, Conv1D, MaxPool1D, Dropout,Flatten\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 361 samples, validate on 155 samples\n",
      "Epoch 1/30\n",
      "361/361 [==============================] - 2s 6ms/sample - loss: 0.6763 - accuracy: 0.5983 - val_loss: 0.6759 - val_accuracy: 0.5742\n",
      "Epoch 2/30\n",
      "361/361 [==============================] - 1s 2ms/sample - loss: 0.6640 - accuracy: 0.5983 - val_loss: 0.6627 - val_accuracy: 0.5742\n",
      "Epoch 3/30\n",
      "361/361 [==============================] - 1s 2ms/sample - loss: 0.6427 - accuracy: 0.6205 - val_loss: 0.6448 - val_accuracy: 0.5806\n",
      "Epoch 4/30\n",
      "361/361 [==============================] - 1s 2ms/sample - loss: 0.6134 - accuracy: 0.6427 - val_loss: 0.6310 - val_accuracy: 0.6194\n",
      "Epoch 5/30\n",
      "361/361 [==============================] - 1s 2ms/sample - loss: 0.5767 - accuracy: 0.6427 - val_loss: 0.6225 - val_accuracy: 0.6387\n",
      "Epoch 6/30\n",
      "361/361 [==============================] - 1s 2ms/sample - loss: 0.5211 - accuracy: 0.6814 - val_loss: 0.6036 - val_accuracy: 0.6516\n",
      "Epoch 7/30\n",
      "361/361 [==============================] - 1s 2ms/sample - loss: 0.4372 - accuracy: 0.7978 - val_loss: 0.5702 - val_accuracy: 0.6774\n",
      "Epoch 8/30\n",
      "361/361 [==============================] - 1s 2ms/sample - loss: 0.3465 - accuracy: 0.8864 - val_loss: 0.5688 - val_accuracy: 0.6323\n",
      "Epoch 9/30\n",
      "361/361 [==============================] - 1s 2ms/sample - loss: 0.2586 - accuracy: 0.9363 - val_loss: 0.5276 - val_accuracy: 0.7161\n",
      "Epoch 10/30\n",
      "361/361 [==============================] - 1s 2ms/sample - loss: 0.1821 - accuracy: 0.9474 - val_loss: 0.5262 - val_accuracy: 0.7355\n",
      "Epoch 11/30\n",
      "361/361 [==============================] - 1s 2ms/sample - loss: 0.1231 - accuracy: 0.9584 - val_loss: 0.6249 - val_accuracy: 0.7097\n",
      "Epoch 12/30\n",
      "361/361 [==============================] - 1s 2ms/sample - loss: 0.0934 - accuracy: 0.9640 - val_loss: 0.6086 - val_accuracy: 0.7161\n",
      "Epoch 13/30\n",
      "361/361 [==============================] - 1s 2ms/sample - loss: 0.0777 - accuracy: 0.9668 - val_loss: 0.5419 - val_accuracy: 0.7484\n",
      "Epoch 14/30\n",
      "361/361 [==============================] - 1s 2ms/sample - loss: 0.0656 - accuracy: 0.9723 - val_loss: 0.5290 - val_accuracy: 0.8065\n",
      "Epoch 15/30\n",
      "361/361 [==============================] - 1s 2ms/sample - loss: 0.0614 - accuracy: 0.9695 - val_loss: 0.5477 - val_accuracy: 0.8000\n",
      "Epoch 00015: early stopping\n",
      "accuracy:0.800000011920929\n"
     ]
    }
   ],
   "source": [
    "model_cnn=Sequential()\n",
    "model_cnn.add(Embedding(7813+1,50, input_length=80))\n",
    "model_cnn.add(Conv1D(64,3,activation='relu' ,padding='valid'))\n",
    "model_cnn.add(MaxPool1D(2))\n",
    "model_cnn.add(Conv1D(128,3,activation='relu' ,padding='valid'))\n",
    "model_cnn.add(MaxPool1D(2))\n",
    "\n",
    "model_cnn.add(Flatten())\n",
    "model_cnn.add(Dense(512, activation='sigmoid'))\n",
    "model_cnn.add(Dense(2, activation='sigmoid'))\n",
    "model_cnn.compile(loss='sparse_categorical_crossentropy', \n",
    "         optimizer='adam', \n",
    "         metrics=['accuracy'],learning_rate= 0.001)\n",
    "num_epochs = 30\n",
    "\n",
    "es = EarlyStopping(monitor='val_loss', mode='min', verbose=1,patience=5)\n",
    "model_cnn.fit(X_train,y_train.values,\n",
    "                        batch_size=128, epochs=num_epochs,\n",
    "                        validation_data=(X_test,y_test.values),callbacks=[es])\n",
    "scores =  model_cnn.evaluate(X_test,y_test.values, verbose=0)\n",
    "print('accuracy:{}'.format(scores[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_8\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_8 (Embedding)      (None, 80, 50)            390700    \n",
      "_________________________________________________________________\n",
      "conv1d_14 (Conv1D)           (None, 78, 64)            9664      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_14 (MaxPooling (None, 39, 64)            0         \n",
      "_________________________________________________________________\n",
      "conv1d_15 (Conv1D)           (None, 37, 128)           24704     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_15 (MaxPooling (None, 18, 128)           0         \n",
      "_________________________________________________________________\n",
      "flatten_7 (Flatten)          (None, 2304)              0         \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 256)               590080    \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 2)                 514       \n",
      "=================================================================\n",
      "Total params: 1,015,662\n",
      "Trainable params: 1,015,662\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_cnn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.7806930961147431, 0.7741935]"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 361 samples, validate on 155 samples\n",
      "Epoch 1/12\n",
      "256/361 [====================>.........] - ETA: 1s - loss: 0.6917 - accuracy: 0.5586\n",
      "Epoch 00001: val_accuracy improved from -inf to 0.60645, saving model to lstm_best_model.h5\n",
      "361/361 [==============================] - 6s 17ms/sample - loss: 0.6902 - accuracy: 0.5956 - val_loss: 0.6865 - val_accuracy: 0.6065\n",
      "Epoch 2/12\n",
      "256/361 [====================>.........] - ETA: 0s - loss: 0.6777 - accuracy: 0.6484\n",
      "Epoch 00002: val_accuracy did not improve from 0.60645\n",
      "361/361 [==============================] - 2s 5ms/sample - loss: 0.6728 - accuracy: 0.6399 - val_loss: 0.6687 - val_accuracy: 0.5806\n",
      "Epoch 3/12\n",
      "256/361 [====================>.........] - ETA: 0s - loss: 0.7743 - accuracy: 0.6367\n",
      "Epoch 00003: val_accuracy improved from 0.60645 to 0.61290, saving model to lstm_best_model.h5\n",
      "361/361 [==============================] - 2s 5ms/sample - loss: 0.7388 - accuracy: 0.6343 - val_loss: 0.6693 - val_accuracy: 0.6129\n",
      "Epoch 4/12\n",
      "256/361 [====================>.........] - ETA: 0s - loss: 0.6459 - accuracy: 0.6367\n",
      "Epoch 00004: val_accuracy improved from 0.61290 to 0.61935, saving model to lstm_best_model.h5\n",
      "361/361 [==============================] - 2s 5ms/sample - loss: 0.6504 - accuracy: 0.6399 - val_loss: 0.6793 - val_accuracy: 0.6194\n",
      "Epoch 5/12\n",
      "256/361 [====================>.........] - ETA: 0s - loss: 0.6666 - accuracy: 0.6484\n",
      "Epoch 00005: val_accuracy did not improve from 0.61935\n",
      "361/361 [==============================] - 2s 5ms/sample - loss: 0.6642 - accuracy: 0.6593 - val_loss: 0.6811 - val_accuracy: 0.6065\n",
      "Epoch 6/12\n",
      "256/361 [====================>.........] - ETA: 0s - loss: 0.6696 - accuracy: 0.6406\n",
      "Epoch 00006: val_accuracy did not improve from 0.61935\n",
      "361/361 [==============================] - 2s 5ms/sample - loss: 0.6658 - accuracy: 0.6593 - val_loss: 0.6809 - val_accuracy: 0.6065\n",
      "Epoch 7/12\n",
      "256/361 [====================>.........] - ETA: 0s - loss: 0.6579 - accuracy: 0.6836\n",
      "Epoch 00007: val_accuracy did not improve from 0.61935\n",
      "361/361 [==============================] - 2s 5ms/sample - loss: 0.6637 - accuracy: 0.6593 - val_loss: 0.6797 - val_accuracy: 0.6065\n",
      "Epoch 00007: early stopping\n",
      "accuracy:0.6193548440933228\n"
     ]
    }
   ],
   "source": [
    "mc = ModelCheckpoint('lstm_best_model.h5', monitor='val_accuracy', mode='max', verbose=1, save_best_only=True)\n",
    "es = EarlyStopping(monitor='val_loss', mode='min', verbose=1,patience=5)\n",
    "model=Sequential()\n",
    "model.add(Embedding(7813+1,50, input_length=80))\n",
    "model.add(LSTM(256)) # model.add(LSTM(64,return_sequences=True))\n",
    "\n",
    "model.add(Dense(2, activation='sigmoid'))\n",
    "model.compile(loss='sparse_categorical_crossentropy', \n",
    "         optimizer='adam', \n",
    "         metrics=['accuracy'],learning_rate= 0.01)\n",
    "num_epochs = 12\n",
    "\n",
    "model.fit(X_train,y_train.values, batch_size=128, validation_data= (X_test,y_test.values), epochs=num_epochs,callbacks=[es,mc])\n",
    "model.load_weights('lstm_best_model.h5')\n",
    "scores = model.evaluate(X_test,y_test.values, verbose=0)\n",
    "print('accuracy:{}'.format(scores[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}