kaggle_competition.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f9a6094f",
   "metadata": {},
   "source": [
    "# 1. Read Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5b9aa53",
   "metadata": {},
   "source": [
    "    dataset: crawled from Twitter(w. labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e2a930f",
   "metadata": {},
   "source": [
    "    匯入tweets_DM.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b5b969b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1867535\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "# 由於文件中有多行，直接讀取會出現錯誤，因此一行一行讀取\n",
    "file = open(\"tweets_DM.json\", 'r', encoding='utf-8')\n",
    "papers = []\n",
    "for line in file.readlines():\n",
    "    dic = json.loads(line)\n",
    "    papers.append(dic)\n",
    "\n",
    "\n",
    "print(len(papers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2c737b12",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'_score': 391,\n",
       "  '_index': 'hashtag_tweets',\n",
       "  '_source': {'tweet': {'hashtags': ['Snapchat'],\n",
       "    'tweet_id': '0x376b20',\n",
       "    'text': 'People who post \"add me on #Snapchat\" must be dehydrated. Cuz man.... that\\'s <LH>'}},\n",
       "  '_crawldate': '2015-05-23 11:42:47',\n",
       "  '_type': 'tweets'},\n",
       " {'_score': 433,\n",
       "  '_index': 'hashtag_tweets',\n",
       "  '_source': {'tweet': {'hashtags': ['freepress', 'TrumpLegacy', 'CNN'],\n",
       "    'tweet_id': '0x2d5350',\n",
       "    'text': '@brianklaas As we see, Trump is dangerous to #freepress around the world. What a <LH> <LH> #TrumpLegacy.  #CNN'}},\n",
       "  '_crawldate': '2016-01-28 04:52:09',\n",
       "  '_type': 'tweets'}]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "papers[:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cce1f19d",
   "metadata": {},
   "source": [
    "    把jason 轉成data frame "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e709ace6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pandas import json_normalize\n",
    "df = json_normalize(papers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c0e68abe",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns = ['score','index', 'crawldate', 'type', 'hashtags', 'id', 'text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7270e58c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "      <th>index</th>\n",
       "      <th>crawldate</th>\n",
       "      <th>type</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>391</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2015-05-23 11:42:47</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[Snapchat]</td>\n",
       "      <td>0x376b20</td>\n",
       "      <td>People who post \"add me on #Snapchat\" must be ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>433</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2016-01-28 04:52:09</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[freepress, TrumpLegacy, CNN]</td>\n",
       "      <td>0x2d5350</td>\n",
       "      <td>@brianklaas As we see, Trump is dangerous to #...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>232</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2017-12-25 04:39:20</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[bibleverse]</td>\n",
       "      <td>0x28b412</td>\n",
       "      <td>Confident of your obedience, I write to you, k...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>376</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2016-01-24 23:53:05</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[]</td>\n",
       "      <td>0x1cd5b0</td>\n",
       "      <td>Now ISSA is stalking Tasha 😂😂😂 &lt;LH&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>989</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2016-01-08 17:18:59</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[]</td>\n",
       "      <td>0x2de201</td>\n",
       "      <td>\"Trust is not the same as faith. A friend is s...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   score           index            crawldate    type  \\\n",
       "0    391  hashtag_tweets  2015-05-23 11:42:47  tweets   \n",
       "1    433  hashtag_tweets  2016-01-28 04:52:09  tweets   \n",
       "2    232  hashtag_tweets  2017-12-25 04:39:20  tweets   \n",
       "3    376  hashtag_tweets  2016-01-24 23:53:05  tweets   \n",
       "4    989  hashtag_tweets  2016-01-08 17:18:59  tweets   \n",
       "\n",
       "                        hashtags        id  \\\n",
       "0                     [Snapchat]  0x376b20   \n",
       "1  [freepress, TrumpLegacy, CNN]  0x2d5350   \n",
       "2                   [bibleverse]  0x28b412   \n",
       "3                             []  0x1cd5b0   \n",
       "4                             []  0x2de201   \n",
       "\n",
       "                                                text  \n",
       "0  People who post \"add me on #Snapchat\" must be ...  \n",
       "1  @brianklaas As we see, Trump is dangerous to #...  \n",
       "2  Confident of your obedience, I write to you, k...  \n",
       "3                Now ISSA is stalking Tasha 😂😂😂 <LH>  \n",
       "4  \"Trust is not the same as faith. A friend is s...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ce483c55",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "      <th>index</th>\n",
       "      <th>crawldate</th>\n",
       "      <th>type</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>391</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2015-05-23 11:42:47</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[Snapchat]</td>\n",
       "      <td>0x376b20</td>\n",
       "      <td>People who post \"add me on #Snapchat\" must be ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>433</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2016-01-28 04:52:09</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[freepress, TrumpLegacy, CNN]</td>\n",
       "      <td>0x2d5350</td>\n",
       "      <td>@brianklaas As we see, Trump is dangerous to #...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>232</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2017-12-25 04:39:20</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[bibleverse]</td>\n",
       "      <td>0x28b412</td>\n",
       "      <td>Confident of your obedience, I write to you, k...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>376</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2016-01-24 23:53:05</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[]</td>\n",
       "      <td>0x1cd5b0</td>\n",
       "      <td>Now ISSA is stalking Tasha 😂😂😂 &lt;LH&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>989</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2016-01-08 17:18:59</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[]</td>\n",
       "      <td>0x2de201</td>\n",
       "      <td>\"Trust is not the same as faith. A friend is s...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   score           index            crawldate    type  \\\n",
       "0    391  hashtag_tweets  2015-05-23 11:42:47  tweets   \n",
       "1    433  hashtag_tweets  2016-01-28 04:52:09  tweets   \n",
       "2    232  hashtag_tweets  2017-12-25 04:39:20  tweets   \n",
       "3    376  hashtag_tweets  2016-01-24 23:53:05  tweets   \n",
       "4    989  hashtag_tweets  2016-01-08 17:18:59  tweets   \n",
       "\n",
       "                        hashtags        id  \\\n",
       "0                     [Snapchat]  0x376b20   \n",
       "1  [freepress, TrumpLegacy, CNN]  0x2d5350   \n",
       "2                   [bibleverse]  0x28b412   \n",
       "3                             []  0x1cd5b0   \n",
       "4                             []  0x2de201   \n",
       "\n",
       "                                                text  \n",
       "0  People who post \"add me on #Snapchat\" must be ...  \n",
       "1  @brianklaas As we see, Trump is dangerous to #...  \n",
       "2  Confident of your obedience, I write to you, k...  \n",
       "3                Now ISSA is stalking Tasha 😂😂😂 <LH>  \n",
       "4  \"Trust is not the same as faith. A friend is s...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b0c041c8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of df:  (1867535, 7)\n"
     ]
    }
   ],
   "source": [
    "print(\"Shape of df: \", df.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ee6b496",
   "metadata": {},
   "source": [
    "---\n",
    "# Use model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c3ae857",
   "metadata": {},
   "source": [
    "---\n",
    "# 2. Preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "63cf3a70",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'game is on fire fire'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import emoji\n",
    "text = \"game is on 🔥 🔥\"\n",
    "emoji.demojize(text, delimiters=(\"\", \"\"))  # 'game is on fire fire'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e6d8ceb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def deemoji(a):\n",
    "    return emoji.demojize(a, delimiters=(\"\", \"\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "98f5687f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'game is on fire fire'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "deemoji(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cacf52d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0          People who post \"add me on #Snapchat\" must be ...\n",
       "1          @brianklaas As we see, Trump is dangerous to #...\n",
       "2          Confident of your obedience, I write to you, k...\n",
       "3                        Now ISSA is stalking Tasha 😂😂😂 <LH>\n",
       "4          \"Trust is not the same as faith. A friend is s...\n",
       "                                 ...                        \n",
       "1867530    When you buy the last 2 tickets remaining for ...\n",
       "1867531    I swear all this hard work gone pay off one da...\n",
       "1867532    @Parcel2Go no card left when I wasn't in so I ...\n",
       "1867533    Ah, corporate life, where you can date <LH> us...\n",
       "1867534               Blessed to be living #Sundayvibes <LH>\n",
       "Name: text, Length: 1867535, dtype: object"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a1eb6f90",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['text_2'] = df['text'].map(deemoji)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "02d77e78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    People who post \"add me on #Snapchat\" must be ...\n",
       "1    @brianklaas As we see, Trump is dangerous to #...\n",
       "2    Confident of your obedience, I write to you, k...\n",
       "3    Now ISSA is stalking Tasha face_with_tears_of_...\n",
       "4    \"Trust is not the same as faith. A friend is s...\n",
       "Name: text_2, dtype: object"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['text_2'][:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efc9a607",
   "metadata": {},
   "source": [
    "    寫入test_train "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3fa034a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "test_train = pd.read_csv(\"data_identification.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "07951af3",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_train.columns = ['id', 'identification']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a9f80af9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>identification</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0x28cc61</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0x29e452</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0x2b3819</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0x2db41f</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0x2a2acc</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id identification\n",
       "0  0x28cc61           test\n",
       "1  0x29e452          train\n",
       "2  0x2b3819          train\n",
       "3  0x2db41f           test\n",
       "4  0x2a2acc          train"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_train[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4aa6c363",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2 = pd.merge(df, test_train, on='id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "204279e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "      <th>index</th>\n",
       "      <th>crawldate</th>\n",
       "      <th>type</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>text_2</th>\n",
       "      <th>identification</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>391</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2015-05-23 11:42:47</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[Snapchat]</td>\n",
       "      <td>0x376b20</td>\n",
       "      <td>People who post \"add me on #Snapchat\" must be ...</td>\n",
       "      <td>People who post \"add me on #Snapchat\" must be ...</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>433</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2016-01-28 04:52:09</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[freepress, TrumpLegacy, CNN]</td>\n",
       "      <td>0x2d5350</td>\n",
       "      <td>@brianklaas As we see, Trump is dangerous to #...</td>\n",
       "      <td>@brianklaas As we see, Trump is dangerous to #...</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>232</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2017-12-25 04:39:20</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[bibleverse]</td>\n",
       "      <td>0x28b412</td>\n",
       "      <td>Confident of your obedience, I write to you, k...</td>\n",
       "      <td>Confident of your obedience, I write to you, k...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>376</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2016-01-24 23:53:05</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[]</td>\n",
       "      <td>0x1cd5b0</td>\n",
       "      <td>Now ISSA is stalking Tasha 😂😂😂 &lt;LH&gt;</td>\n",
       "      <td>Now ISSA is stalking Tasha face_with_tears_of_...</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>989</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2016-01-08 17:18:59</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[]</td>\n",
       "      <td>0x2de201</td>\n",
       "      <td>\"Trust is not the same as faith. A friend is s...</td>\n",
       "      <td>\"Trust is not the same as faith. A friend is s...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   score           index            crawldate    type  \\\n",
       "0    391  hashtag_tweets  2015-05-23 11:42:47  tweets   \n",
       "1    433  hashtag_tweets  2016-01-28 04:52:09  tweets   \n",
       "2    232  hashtag_tweets  2017-12-25 04:39:20  tweets   \n",
       "3    376  hashtag_tweets  2016-01-24 23:53:05  tweets   \n",
       "4    989  hashtag_tweets  2016-01-08 17:18:59  tweets   \n",
       "\n",
       "                        hashtags        id  \\\n",
       "0                     [Snapchat]  0x376b20   \n",
       "1  [freepress, TrumpLegacy, CNN]  0x2d5350   \n",
       "2                   [bibleverse]  0x28b412   \n",
       "3                             []  0x1cd5b0   \n",
       "4                             []  0x2de201   \n",
       "\n",
       "                                                text  \\\n",
       "0  People who post \"add me on #Snapchat\" must be ...   \n",
       "1  @brianklaas As we see, Trump is dangerous to #...   \n",
       "2  Confident of your obedience, I write to you, k...   \n",
       "3                Now ISSA is stalking Tasha 😂😂😂 <LH>   \n",
       "4  \"Trust is not the same as faith. A friend is s...   \n",
       "\n",
       "                                              text_2 identification  \n",
       "0  People who post \"add me on #Snapchat\" must be ...          train  \n",
       "1  @brianklaas As we see, Trump is dangerous to #...          train  \n",
       "2  Confident of your obedience, I write to you, k...           test  \n",
       "3  Now ISSA is stalking Tasha face_with_tears_of_...          train  \n",
       "4  \"Trust is not the same as faith. A friend is s...           test  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "a7ec1c1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "hw_test_pre = df_2[df_2[\"identification\"] == \"test\"]\n",
    "hw_test_pre = pd.DataFrame(hw_test_pre, columns=['id', 'text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "19e41d99",
   "metadata": {},
   "outputs": [],
   "source": [
    "label = pd.read_csv(\"emotion.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "98cd8144",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>emotion</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0x3140b1</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0x368b73</td>\n",
       "      <td>disgust</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0x296183</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0x2bd6e1</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0x2ee1dd</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id       emotion\n",
       "0  0x3140b1       sadness\n",
       "1  0x368b73       disgust\n",
       "2  0x296183  anticipation\n",
       "3  0x2bd6e1           joy\n",
       "4  0x2ee1dd  anticipation"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "7ffd00ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "label.columns = ['id', 'emotion']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "43247adf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.merge(df_2, label, on='id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ca45437f",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "      <th>index</th>\n",
       "      <th>crawldate</th>\n",
       "      <th>type</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>text_2</th>\n",
       "      <th>identification</th>\n",
       "      <th>emotion</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>391</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2015-05-23 11:42:47</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[Snapchat]</td>\n",
       "      <td>0x376b20</td>\n",
       "      <td>People who post \"add me on #Snapchat\" must be ...</td>\n",
       "      <td>People who post \"add me on #Snapchat\" must be ...</td>\n",
       "      <td>train</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>433</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2016-01-28 04:52:09</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[freepress, TrumpLegacy, CNN]</td>\n",
       "      <td>0x2d5350</td>\n",
       "      <td>@brianklaas As we see, Trump is dangerous to #...</td>\n",
       "      <td>@brianklaas As we see, Trump is dangerous to #...</td>\n",
       "      <td>train</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>376</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2016-01-24 23:53:05</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[]</td>\n",
       "      <td>0x1cd5b0</td>\n",
       "      <td>Now ISSA is stalking Tasha 😂😂😂 &lt;LH&gt;</td>\n",
       "      <td>Now ISSA is stalking Tasha face_with_tears_of_...</td>\n",
       "      <td>train</td>\n",
       "      <td>fear</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>120</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2015-06-11 04:44:05</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[authentic, LaughOutLoud]</td>\n",
       "      <td>0x1d755c</td>\n",
       "      <td>@RISKshow @TheKevinAllison Thx for the BEST TI...</td>\n",
       "      <td>@RISKshow @TheKevinAllison Thx for the BEST TI...</td>\n",
       "      <td>train</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1021</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2015-08-18 02:30:07</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[]</td>\n",
       "      <td>0x2c91a8</td>\n",
       "      <td>Still waiting on those supplies Liscus. &lt;LH&gt;</td>\n",
       "      <td>Still waiting on those supplies Liscus. &lt;LH&gt;</td>\n",
       "      <td>train</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   score           index            crawldate    type  \\\n",
       "0    391  hashtag_tweets  2015-05-23 11:42:47  tweets   \n",
       "1    433  hashtag_tweets  2016-01-28 04:52:09  tweets   \n",
       "2    376  hashtag_tweets  2016-01-24 23:53:05  tweets   \n",
       "3    120  hashtag_tweets  2015-06-11 04:44:05  tweets   \n",
       "4   1021  hashtag_tweets  2015-08-18 02:30:07  tweets   \n",
       "\n",
       "                        hashtags        id  \\\n",
       "0                     [Snapchat]  0x376b20   \n",
       "1  [freepress, TrumpLegacy, CNN]  0x2d5350   \n",
       "2                             []  0x1cd5b0   \n",
       "3      [authentic, LaughOutLoud]  0x1d755c   \n",
       "4                             []  0x2c91a8   \n",
       "\n",
       "                                                text  \\\n",
       "0  People who post \"add me on #Snapchat\" must be ...   \n",
       "1  @brianklaas As we see, Trump is dangerous to #...   \n",
       "2                Now ISSA is stalking Tasha 😂😂😂 <LH>   \n",
       "3  @RISKshow @TheKevinAllison Thx for the BEST TI...   \n",
       "4       Still waiting on those supplies Liscus. <LH>   \n",
       "\n",
       "                                              text_2 identification  \\\n",
       "0  People who post \"add me on #Snapchat\" must be ...          train   \n",
       "1  @brianklaas As we see, Trump is dangerous to #...          train   \n",
       "2  Now ISSA is stalking Tasha face_with_tears_of_...          train   \n",
       "3  @RISKshow @TheKevinAllison Thx for the BEST TI...          train   \n",
       "4       Still waiting on those supplies Liscus. <LH>          train   \n",
       "\n",
       "        emotion  \n",
       "0  anticipation  \n",
       "1       sadness  \n",
       "2          fear  \n",
       "3           joy  \n",
       "4  anticipation  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45396fbf",
   "metadata": {},
   "source": [
    "---\n",
    "# 3. Feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "fc7db420",
   "metadata": {},
   "outputs": [],
   "source": [
    "hw_train = df_train.drop([\"score\", \"index\", \"crawldate\", \"type\", \"hashtags\", \"id\", \"text\", \"identification\"], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "8d065ed3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1455563"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(hw_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "0d57ef31",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_2</th>\n",
       "      <th>emotion</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>People who post \"add me on #Snapchat\" must be ...</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>@brianklaas As we see, Trump is dangerous to #...</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Now ISSA is stalking Tasha face_with_tears_of_...</td>\n",
       "      <td>fear</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>@RISKshow @TheKevinAllison Thx for the BEST TI...</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Still waiting on those supplies Liscus. &lt;LH&gt;</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1455558</th>\n",
       "      <td>I'm SO HAPPY!!! #NoWonder the name of this sho...</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1455559</th>\n",
       "      <td>In every circumtance I'd like to be thankful t...</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1455560</th>\n",
       "      <td>there's currently two girls walking around the...</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1455561</th>\n",
       "      <td>Ah, corporate life, where you can date &lt;LH&gt; us...</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1455562</th>\n",
       "      <td>Blessed to be living #Sundayvibes &lt;LH&gt;</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1455563 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text_2       emotion\n",
       "0        People who post \"add me on #Snapchat\" must be ...  anticipation\n",
       "1        @brianklaas As we see, Trump is dangerous to #...       sadness\n",
       "2        Now ISSA is stalking Tasha face_with_tears_of_...          fear\n",
       "3        @RISKshow @TheKevinAllison Thx for the BEST TI...           joy\n",
       "4             Still waiting on those supplies Liscus. <LH>  anticipation\n",
       "...                                                    ...           ...\n",
       "1455558  I'm SO HAPPY!!! #NoWonder the name of this sho...           joy\n",
       "1455559  In every circumtance I'd like to be thankful t...           joy\n",
       "1455560  there's currently two girls walking around the...           joy\n",
       "1455561  Ah, corporate life, where you can date <LH> us...           joy\n",
       "1455562             Blessed to be living #Sundayvibes <LH>           joy\n",
       "\n",
       "[1455563 rows x 2 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hw_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "5d96bf46",
   "metadata": {},
   "outputs": [],
   "source": [
    "hw_test = hw_test_pre.drop([\"id\"], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "52e890ca",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "411972"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(hw_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "341c941e",
   "metadata": {},
   "outputs": [],
   "source": [
    "hw_train.to_csv(\"colab_train.csv\",index=False)\n",
    "hw_test.to_csv(\"colab_test.csv\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfce9a31",
   "metadata": {},
   "source": [
    "---\n",
    "# 4.Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01c87dae",
   "metadata": {},
   "source": [
    "## RoBERTa"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52f27aec",
   "metadata": {},
   "source": [
    "    Text Classification with RoBERTa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "9ac90893",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import regex as re\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import statistics\n",
    "import math\n",
    "import os\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras.backend as K\n",
    "import tokenizers\n",
    "from transformers import RobertaTokenizer, TFRobertaModel\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d90b387b",
   "metadata": {},
   "source": [
    "    Tokenize & encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "649e7a3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0          People who post \"add me on #Snapchat\" must be ...\n",
       "1          @brianklaas As we see, Trump is dangerous to #...\n",
       "2                        Now ISSA is stalking Tasha 😂😂😂 <LH>\n",
       "3          @RISKshow @TheKevinAllison Thx for the BEST TI...\n",
       "4               Still waiting on those supplies Liscus. <LH>\n",
       "                                 ...                        \n",
       "1455558    I'm SO HAPPY!!! #NoWonder the name of this sho...\n",
       "1455559    In every circumtance I'd like to be thankful t...\n",
       "1455560    there's currently two girls walking around the...\n",
       "1455561    Ah, corporate life, where you can date <LH> us...\n",
       "1455562               Blessed to be living #Sundayvibes <LH>\n",
       "Name: text, Length: 1455563, dtype: object"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "11fbbf77",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_data = df_train[['text']].to_numpy().reshape(-1)\n",
    "y_data = df_train[['emotion']].to_numpy().reshape(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "dfde4113",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_data = df_train[['text']].to_numpy().reshape(-1)\n",
    "y_data = df_train[['emotion']].to_numpy().reshape(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "a2ed00fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['anticipation', 'sadness', 'fear', ..., 'joy', 'joy', 'joy'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "110a5fc5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'anticipation',\n",
       " 1: 'sadness',\n",
       " 2: 'fear',\n",
       " 3: 'joy',\n",
       " 4: 'anger',\n",
       " 5: 'trust',\n",
       " 6: 'disgust',\n",
       " 7: 'surprise'}"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Transform categories into numbers\n",
    "categories = df_train[['emotion']].values.reshape(-1)\n",
    "\n",
    "category_to_id_ = {}\n",
    "category_to_name_ = {}\n",
    "\n",
    "for index, c in enumerate(y_data):\n",
    "    if c in category_to_id_:\n",
    "        category_id_ = category_to_id_[c]\n",
    "    else:\n",
    "        category_id_ = len(category_to_id_)\n",
    "        category_to_id_[c] = category_id_\n",
    "        category_to_name_[category_id_] = c\n",
    "    \n",
    "    y_data[index] = category_id_\n",
    "\n",
    "# Display dictionary\n",
    "category_to_name_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "553e247e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split into train and test datasets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_data, y_data, test_size=0.3, random_state=777) # random_state to reproduce results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "7fdbcddf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "efb081f9d5a6478a8ed8fecc8053a548",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/878k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f87ff4ca7aa14fed947bcf207f7b0bfa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/446k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "19f232c816dd45d980152b0b3e9e9236",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/1.29M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ceed0466045d459886dace063889ea7a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/481 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "MODEL_NAME = 'roberta-base'\n",
    "tokenizer = RobertaTokenizer.from_pretrained(MODEL_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8280fecd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def roberta_encode(texts, tokenizer):\n",
    "    ct = len(texts)\n",
    "    input_ids = np.ones((ct, MAX_LEN), dtype='int32')\n",
    "    attention_mask = np.zeros((ct, MAX_LEN), dtype='int32')\n",
    "    token_type_ids = np.zeros((ct, MAX_LEN), dtype='int32') # Not used in text classification\n",
    "\n",
    "    for k, text in enumerate(texts):\n",
    "        # Tokenize\n",
    "        tok_text = tokenizer.tokenize(text)\n",
    "        \n",
    "        # Truncate and convert tokens to numerical IDs\n",
    "        enc_text = tokenizer.convert_tokens_to_ids(tok_text[:(MAX_LEN-2)])\n",
    "        \n",
    "        input_length = len(enc_text) + 2\n",
    "        input_length = input_length if input_length < MAX_LEN else MAX_LEN\n",
    "        \n",
    "        # Add tokens [CLS] and [SEP] at the beginning and the end\n",
    "        input_ids[k,:input_length] = np.asarray([0] + enc_text + [2], dtype='int32')\n",
    "        \n",
    "        # Set to 1s in the attention input\n",
    "        attention_mask[k,:input_length] = 1\n",
    "    return {\n",
    "        'input_word_ids': input_ids,\n",
    "        'input_mask': attention_mask,\n",
    "        'input_type_ids': token_type_ids\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "dfa242a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LEN = 256\n",
    "X_train = roberta_encode(X_train, tokenizer)\n",
    "X_test = roberta_encode(X_test, tokenizer)\n",
    "\n",
    "y_train = np.asarray(y_train, dtype='int32')\n",
    "y_test = np.asarray(y_test, dtype='int32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "f63f176b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(n_categories):\n",
    "    input_word_ids = tf.keras.Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_word_ids')\n",
    "    input_mask = tf.keras.Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_mask')\n",
    "    input_type_ids = tf.keras.Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_type_ids')\n",
    "\n",
    "    # Import RoBERTa model from HuggingFace\n",
    "    roberta_model = TFRobertaModel.from_pretrained(MODEL_NAME)\n",
    "    x = roberta_model(input_word_ids, attention_mask=input_mask, token_type_ids=input_type_ids)\n",
    "\n",
    "    # Huggingface transformers have multiple outputs, embeddings are the first one,\n",
    "    # so let's slice out the first position\n",
    "    x = x[0]\n",
    "\n",
    "    x = tf.keras.layers.Dropout(0.1)(x)\n",
    "    x = tf.keras.layers.Flatten()(x)\n",
    "    x = tf.keras.layers.Dense(256, activation='relu')(x)\n",
    "    x = tf.keras.layers.Dense(n_categories, activation='softmax')(x)\n",
    "\n",
    "    model = tf.keras.Model(inputs=[input_word_ids, input_mask, input_type_ids], outputs=x)\n",
    "    model.compile(\n",
    "        optimizer=tf.keras.optimizers.Adam(lr=1e-5),\n",
    "        loss='sparse_categorical_crossentropy',\n",
    "        metrics=['accuracy'])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "a6b78d8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b3f82df7ae7c4838b63e91ca8853f8b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/627M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at roberta-base were not used when initializing TFRobertaModel: ['lm_head']\n",
      "- This IS expected if you are initializing TFRobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFRobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFRobertaModel were initialized from the model checkpoint at roberta-base.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaModel for predictions without further training.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_word_ids (InputLayer)    [(None, 256)]        0           []                               \n",
      "                                                                                                  \n",
      " input_mask (InputLayer)        [(None, 256)]        0           []                               \n",
      "                                                                                                  \n",
      " input_type_ids (InputLayer)    [(None, 256)]        0           []                               \n",
      "                                                                                                  \n",
      " tf_roberta_model (TFRobertaMod  TFBaseModelOutputWi  124645632  ['input_word_ids[0][0]',         \n",
      " el)                            thPoolingAndCrossAt               'input_mask[0][0]',             \n",
      "                                tentions(last_hidde               'input_type_ids[0][0]']         \n",
      "                                n_state=(None, 256,                                               \n",
      "                                 768),                                                            \n",
      "                                 pooler_output=(Non                                               \n",
      "                                e, 768),                                                          \n",
      "                                 past_key_values=No                                               \n",
      "                                ne, hidden_states=N                                               \n",
      "                                one, attentions=Non                                               \n",
      "                                e, cross_attentions                                               \n",
      "                                =None)                                                            \n",
      "                                                                                                  \n",
      " dropout_37 (Dropout)           (None, 256, 768)     0           ['tf_roberta_model[0][0]']       \n",
      "                                                                                                  \n",
      " flatten (Flatten)              (None, 196608)       0           ['dropout_37[0][0]']             \n",
      "                                                                                                  \n",
      " dense (Dense)                  (None, 256)          50331904    ['flatten[0][0]']                \n",
      "                                                                                                  \n",
      " dense_1 (Dense)                (None, 8)            2056        ['dense[0][0]']                  \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 174,979,592\n",
      "Trainable params: 174,979,592\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "categories = df_train['emotion'].unique()\n",
    "n_categories = len(categories)\n",
    "model = build_model(n_categories)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "f212a9b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of replicas: 1\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    # TPU detection. No parameters necessary if TPU_NAME environment variable is set (always set in Kaggle)\n",
    "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
    "    tf.config.experimental_connect_to_cluster(tpu)\n",
    "    tf.tpu.experimental.initialize_tpu_system(tpu)\n",
    "    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
    "    print('Running on TPU ', tpu.master())\n",
    "except ValueError:\n",
    "    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n",
    "    strategy = tf.distribute.get_strategy()\n",
    "\n",
    "print('Number of replicas:', strategy.num_replicas_in_sync)\n",
    "MODEL_NAME = 'roberta-base'\n",
    "MAX_LEN = 256\n",
    "ARTIFACTS_PATH = '../artifacts/'\n",
    "\n",
    "BATCH_SIZE = 8 * strategy.num_replicas_in_sync\n",
    "EPOCHS = 3\n",
    "\n",
    "if not os.path.exists(ARTIFACTS_PATH):\n",
    "    os.makedirs(ARTIFACTS_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "a6bfaa22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training...\n",
      "Epoch 1/2\n"
     ]
    },
    {
     "ename": "ResourceExhaustedError",
     "evalue": " OOM when allocating tensor with shape[64,256,3072] and type float on /job:localhost/replica:0/task:0/device:CPU:0 by allocator cpu\n\t [[node model/tf_roberta_model/roberta/encoder/layer_._1/intermediate/Gelu/mul\n (defined at C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\activations.py:351)\n]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n [Op:__inference_train_function_27306]\n\nErrors may have originated from an input operation.\nInput Source operations connected to node model/tf_roberta_model/roberta/encoder/layer_._1/intermediate/Gelu/mul:\nIn[0] model/tf_roberta_model/roberta/encoder/layer_._1/intermediate/Gelu/mul/x:\t\nIn[1] model/tf_roberta_model/roberta/encoder/layer_._1/intermediate/dense/BiasAdd (defined at C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\layers\\core\\dense.py:210)\n\nOperation defined at: (most recent call last)\n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\runpy.py\", line 194, in _run_module_as_main\n>>>     return _run_code(code, main_globals, None,\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\runpy.py\", line 87, in _run_code\n>>>     exec(code, run_globals)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n>>>     app.launch_new_instance()\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\traitlets\\config\\application.py\", line 846, in launch_instance\n>>>     app.start()\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 677, in start\n>>>     self.io_loop.start()\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 199, in start\n>>>     self.asyncio_loop.run_forever()\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\asyncio\\base_events.py\", line 570, in run_forever\n>>>     self._run_once()\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\asyncio\\base_events.py\", line 1859, in _run_once\n>>>     handle._run()\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\asyncio\\events.py\", line 81, in _run\n>>>     self._context.run(self._callback, *self._args)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 457, in dispatch_queue\n>>>     await self.process_one()\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 446, in process_one\n>>>     await dispatch(*args)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 353, in dispatch_shell\n>>>     await result\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 648, in execute_request\n>>>     reply_content = await reply_content\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 353, in do_execute\n>>>     res = shell.run_cell(code, store_history=store_history, silent=silent)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 533, in run_cell\n>>>     return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2901, in run_cell\n>>>     result = self._run_cell(\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2947, in _run_cell\n>>>     return runner(coro)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\IPython\\core\\async_helpers.py\", line 68, in _pseudo_sync_runner\n>>>     coro.send(None)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3172, in run_cell_async\n>>>     has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3364, in run_ast_nodes\n>>>     if (await self.run_code(code, result,  async_=asy)):\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3444, in run_code\n>>>     exec(code_obj, self.user_global_ns, self.user_ns)\n>>> \n>>>   File \"C:\\Users\\user\\AppData\\Local\\Temp/ipykernel_21096/1228480384.py\", line 4, in <module>\n>>>     history = model.fit(X_train,\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 64, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\training.py\", line 1216, in fit\n>>>     tmp_logs = self.train_function(iterator)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\training.py\", line 878, in train_function\n>>>     return step_function(self, iterator)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\training.py\", line 867, in step_function\n>>>     outputs = model.distribute_strategy.run(run_step, args=(data,))\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\training.py\", line 860, in run_step\n>>>     outputs = model.train_step(data)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\training.py\", line 808, in train_step\n>>>     y_pred = self(x, training=True)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 64, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\base_layer.py\", line 1083, in __call__\n>>>     outputs = call_fn(inputs, *args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 92, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\functional.py\", line 451, in call\n>>>     return self._run_internal_graph(\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\functional.py\", line 589, in _run_internal_graph\n>>>     outputs = node.layer(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 64, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\base_layer.py\", line 1083, in __call__\n>>>     outputs = call_fn(inputs, *args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 92, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\transformers\\models\\roberta\\modeling_tf_roberta.py\", line 998, in call\n>>>     outputs = self.roberta(\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 64, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\base_layer.py\", line 1083, in __call__\n>>>     outputs = call_fn(inputs, *args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 92, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\transformers\\models\\roberta\\modeling_tf_roberta.py\", line 757, in call\n>>>     encoder_outputs = self.encoder(\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 64, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\base_layer.py\", line 1083, in __call__\n>>>     outputs = call_fn(inputs, *args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 92, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\transformers\\models\\roberta\\modeling_tf_roberta.py\", line 530, in call\n>>>     for i, layer_module in enumerate(self.layer):\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\transformers\\models\\roberta\\modeling_tf_roberta.py\", line 536, in call\n>>>     layer_outputs = layer_module(\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 64, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\base_layer.py\", line 1083, in __call__\n>>>     outputs = call_fn(inputs, *args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 92, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\transformers\\models\\roberta\\modeling_tf_roberta.py\", line 491, in call\n>>>     intermediate_output = self.intermediate(hidden_states=attention_output)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 64, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\base_layer.py\", line 1083, in __call__\n>>>     outputs = call_fn(inputs, *args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 92, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\transformers\\models\\roberta\\modeling_tf_roberta.py\", line 393, in call\n>>>     hidden_states = self.intermediate_act_fn(hidden_states)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\activations.py\", line 351, in gelu\n>>>     return tf.nn.gelu(x, approximate)\n>>> ",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_21096/2361935929.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Training...'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m history = model.fit(X_train,\n\u001b[0m\u001b[0;32m      3\u001b[0m                     \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m                     \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mEPOCHS\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m                     \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     65\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# pylint: disable=broad-except\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     66\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 67\u001b[1;33m       \u001b[1;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfiltered_tb\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     68\u001b[0m     \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     69\u001b[0m       \u001b[1;32mdel\u001b[0m \u001b[0mfiltered_tb\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\tensorflow\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     56\u001b[0m   \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     57\u001b[0m     \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 58\u001b[1;33m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[0;32m     59\u001b[0m                                         inputs, attrs, num_outputs)\n\u001b[0;32m     60\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mResourceExhaustedError\u001b[0m:  OOM when allocating tensor with shape[64,256,3072] and type float on /job:localhost/replica:0/task:0/device:CPU:0 by allocator cpu\n\t [[node model/tf_roberta_model/roberta/encoder/layer_._1/intermediate/Gelu/mul\n (defined at C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\activations.py:351)\n]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n [Op:__inference_train_function_27306]\n\nErrors may have originated from an input operation.\nInput Source operations connected to node model/tf_roberta_model/roberta/encoder/layer_._1/intermediate/Gelu/mul:\nIn[0] model/tf_roberta_model/roberta/encoder/layer_._1/intermediate/Gelu/mul/x:\t\nIn[1] model/tf_roberta_model/roberta/encoder/layer_._1/intermediate/dense/BiasAdd (defined at C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\layers\\core\\dense.py:210)\n\nOperation defined at: (most recent call last)\n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\runpy.py\", line 194, in _run_module_as_main\n>>>     return _run_code(code, main_globals, None,\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\runpy.py\", line 87, in _run_code\n>>>     exec(code, run_globals)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n>>>     app.launch_new_instance()\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\traitlets\\config\\application.py\", line 846, in launch_instance\n>>>     app.start()\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 677, in start\n>>>     self.io_loop.start()\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 199, in start\n>>>     self.asyncio_loop.run_forever()\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\asyncio\\base_events.py\", line 570, in run_forever\n>>>     self._run_once()\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\asyncio\\base_events.py\", line 1859, in _run_once\n>>>     handle._run()\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\asyncio\\events.py\", line 81, in _run\n>>>     self._context.run(self._callback, *self._args)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 457, in dispatch_queue\n>>>     await self.process_one()\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 446, in process_one\n>>>     await dispatch(*args)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 353, in dispatch_shell\n>>>     await result\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 648, in execute_request\n>>>     reply_content = await reply_content\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 353, in do_execute\n>>>     res = shell.run_cell(code, store_history=store_history, silent=silent)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 533, in run_cell\n>>>     return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2901, in run_cell\n>>>     result = self._run_cell(\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2947, in _run_cell\n>>>     return runner(coro)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\IPython\\core\\async_helpers.py\", line 68, in _pseudo_sync_runner\n>>>     coro.send(None)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3172, in run_cell_async\n>>>     has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3364, in run_ast_nodes\n>>>     if (await self.run_code(code, result,  async_=asy)):\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3444, in run_code\n>>>     exec(code_obj, self.user_global_ns, self.user_ns)\n>>> \n>>>   File \"C:\\Users\\user\\AppData\\Local\\Temp/ipykernel_21096/1228480384.py\", line 4, in <module>\n>>>     history = model.fit(X_train,\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 64, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\training.py\", line 1216, in fit\n>>>     tmp_logs = self.train_function(iterator)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\training.py\", line 878, in train_function\n>>>     return step_function(self, iterator)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\training.py\", line 867, in step_function\n>>>     outputs = model.distribute_strategy.run(run_step, args=(data,))\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\training.py\", line 860, in run_step\n>>>     outputs = model.train_step(data)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\training.py\", line 808, in train_step\n>>>     y_pred = self(x, training=True)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 64, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\base_layer.py\", line 1083, in __call__\n>>>     outputs = call_fn(inputs, *args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 92, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\functional.py\", line 451, in call\n>>>     return self._run_internal_graph(\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\functional.py\", line 589, in _run_internal_graph\n>>>     outputs = node.layer(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 64, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\base_layer.py\", line 1083, in __call__\n>>>     outputs = call_fn(inputs, *args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 92, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\transformers\\models\\roberta\\modeling_tf_roberta.py\", line 998, in call\n>>>     outputs = self.roberta(\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 64, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\base_layer.py\", line 1083, in __call__\n>>>     outputs = call_fn(inputs, *args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 92, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\transformers\\models\\roberta\\modeling_tf_roberta.py\", line 757, in call\n>>>     encoder_outputs = self.encoder(\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 64, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\base_layer.py\", line 1083, in __call__\n>>>     outputs = call_fn(inputs, *args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 92, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\transformers\\models\\roberta\\modeling_tf_roberta.py\", line 530, in call\n>>>     for i, layer_module in enumerate(self.layer):\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\transformers\\models\\roberta\\modeling_tf_roberta.py\", line 536, in call\n>>>     layer_outputs = layer_module(\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 64, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\base_layer.py\", line 1083, in __call__\n>>>     outputs = call_fn(inputs, *args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 92, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\transformers\\models\\roberta\\modeling_tf_roberta.py\", line 491, in call\n>>>     intermediate_output = self.intermediate(hidden_states=attention_output)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 64, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\engine\\base_layer.py\", line 1083, in __call__\n>>>     outputs = call_fn(inputs, *args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py\", line 92, in error_handler\n>>>     return fn(*args, **kwargs)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\transformers\\models\\roberta\\modeling_tf_roberta.py\", line 393, in call\n>>>     hidden_states = self.intermediate_act_fn(hidden_states)\n>>> \n>>>   File \"C:\\Users\\user\\anaconda3\\lib\\site-packages\\keras\\activations.py\", line 351, in gelu\n>>>     return tf.nn.gelu(x, approximate)\n>>> "
     ]
    }
   ],
   "source": [
    "print('Training...')\n",
    "history = model.fit(X_train,\n",
    "                    y_train,\n",
    "                    epochs=EPOCHS,\n",
    "                    batch_size=BATCH_SIZE,\n",
    "                    verbose=1,\n",
    "                    validation_data=(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "edc4422b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This plot will look much better if we train models with more epochs, but anyway here is\n",
    "plt.figure(figsize=(10, 10))\n",
    "plt.title('Accuracy')\n",
    "\n",
    "xaxis = np.arange(len(history.history['accuracy']))\n",
    "plt.plot(xaxis, history.history['accuracy'], label='Train set')\n",
    "plt.plot(xaxis, history.history['val_accuracy'], label='Validation set')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "919f0cc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = model.evaluate(X_test, y_test, verbose=0)\n",
    "print(\"Accuracy: %.2f%%\" % (scores[1] * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cce7e12a",
   "metadata": {},
   "source": [
    "---\n",
    "# Not use model "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d890a83",
   "metadata": {},
   "source": [
    "---\n",
    "# 2.Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf350a50",
   "metadata": {},
   "source": [
    "## A. Bag of Words"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd8293ef",
   "metadata": {},
   "source": [
    "    First, preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "72de12d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "emotion\n",
       "anger            39867\n",
       "anticipation    248935\n",
       "disgust         139101\n",
       "fear             63999\n",
       "joy             516017\n",
       "sadness         193437\n",
       "surprise         48729\n",
       "trust           205478\n",
       "Name: text, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.groupby(['emotion']).count()['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "36fc8ad1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "hashtag_tweets    1455563\n",
       "Name: text, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.groupby(['index']).count()['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "580327f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "type\n",
       "tweets    1455563\n",
       "Name: text, dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.groupby(['type']).count()['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "bd30b931",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "identification\n",
       "train    1455563\n",
       "Name: text, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.groupby(['identification']).count()['text']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4cfffc78",
   "metadata": {},
   "source": [
    "    只留下有label的=> train"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65d7eb40",
   "metadata": {},
   "source": [
    "    看一下text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b821d35b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0          People who post \"add me on #Snapchat\" must be ...\n",
       "1          @brianklaas As we see, Trump is dangerous to #...\n",
       "2                        Now ISSA is stalking Tasha 😂😂😂 <LH>\n",
       "3          @RISKshow @TheKevinAllison Thx for the BEST TI...\n",
       "4               Still waiting on those supplies Liscus. <LH>\n",
       "                                 ...                        \n",
       "1455558    I'm SO HAPPY!!! #NoWonder the name of this sho...\n",
       "1455559    In every circumtance I'd like to be thankful t...\n",
       "1455560    there's currently two girls walking around the...\n",
       "1455561    Ah, corporate life, where you can date <LH> us...\n",
       "1455562               Blessed to be living #Sundayvibes <LH>\n",
       "Name: text, Length: 1455563, dtype: object"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['text']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39add52b",
   "metadata": {},
   "source": [
    "    then...apply feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "39eee5b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "beeb2f57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer()"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 學習所有text中的tokens\n",
    "BOW_vectorizer = CountVectorizer() \n",
    "BOW_vectorizer.fit(df_train['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "4256b2d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 轉成document-term matrix\n",
    "train_data_BOW_features = BOW_vectorizer.transform(df_train['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "34637bc0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1455563x794247 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 18849355 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 檢查結果\n",
    "train_data_BOW_features\n",
    "#可看出有1455563篇文章, 794247個voc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "7906beb7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "scipy.sparse.csr.csr_matrix"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(train_data_BOW_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "864006b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 跑不動\n",
    "# # add .toarray() to show=> sparse!!\n",
    "# import numpy as np\n",
    "# train_data_BOW_features.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "a4a29ccb",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\user\\anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['005', '00533321', '00575', '00578', '005796', '005e5n', '006', '0060', '007', '00786mujahid']\n",
      "794247\n"
     ]
    }
   ],
   "source": [
    "feature_names = BOW_vectorizer.get_feature_names()\n",
    "print(feature_names[100:110])\n",
    "print(len(feature_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "c7d74179",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"😂\" in feature_names"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0305db06",
   "metadata": {},
   "source": [
    "    試著另一種tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "dc2178e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\user\\anaconda3\\lib\\site-packages\\sklearn\\feature_extraction\\text.py:516: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1455563, 500)"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "# 只用前250筆features, 使用nltk的tokenize資料庫\n",
    "BOW_250 = CountVectorizer(max_features=500, tokenizer=nltk.word_tokenize) \n",
    "\n",
    "# apply analyzer to training data\n",
    "BOW_250.fit(df_train['text'])\n",
    "train_data_BOW_features_250 = BOW_250.transform(df_train['text'])\n",
    "\n",
    "## check dimension\n",
    "train_data_BOW_features_250.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "id": "dc2e579c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 1, 0, ..., 0, 0, 0],\n",
       "       [0, 3, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 1, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 1, 0, ..., 0, 0, 0]], dtype=int64)"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 降低dimensionality\n",
    "train_data_BOW_features_250.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "1e11e50e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\user\\anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['005', '00533321', '00575', '00578', '005796', '005e5n', '006', '0060', '007', '00786mujahid']\n",
      "794247\n"
     ]
    }
   ],
   "source": [
    "feature_names = BOW_vectorizer.get_feature_names()\n",
    "print(feature_names[100:110])\n",
    "print(len(feature_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "4431febd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"😂\" in feature_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "id": "b3fdd5c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1455563x500 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 18707204 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_BOW_features_250"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "860d940f",
   "metadata": {},
   "source": [
    "## c. TD-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56d536fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "vectorizer = TfidfVectorizer(max_features=500)\n",
    "vectors = vectorizer.fit_transform(df_train['text'])\n",
    "feature_names = vectorizer.get_feature_names()\n",
    "dense = vectors.todense()\n",
    "denselist = dense.tolist()\n",
    "df_tfidf = pd.DataFrame(denselist, columns=feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "214705ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_tfidf[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3dc0d7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# feature_names = TfidfVectorizer.get_feature_names()\n",
    "# print(feature_names[100:110])\n",
    "# print(len(feature_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "069a8384",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "b0077fc8",
   "metadata": {},
   "source": [
    "---\n",
    "# 3.predict the emotion(three output files)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8872183",
   "metadata": {},
   "source": [
    "---\n",
    "**<font color = green size=3>a. \"BOW_result.csv\"(Use dicision tree) </font>**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "id": "f8c14529",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "X_train = train_data_BOW_features_250\n",
    "y_train = df_train['emotion']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "id": "bf2865cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train.shape:  (1455563, 500)\n",
      "y_train.shape:  (1455563,)\n"
     ]
    }
   ],
   "source": [
    "print('X_train.shape: ', X_train.shape)\n",
    "print('y_train.shape: ', y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12697a45",
   "metadata": {},
   "outputs": [],
   "source": [
    "DT_model = DecisionTreeClassifier(random_state=0)\n",
    "DT_model = DT_model.fit(X_train, y_train)\n",
    "y_train_pred = DT_model.predict(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7c0b5b1",
   "metadata": {},
   "source": [
    "**<font color = red>run all of night. I found this way take too much time and doesn't have good accuracy. </font>**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "80eb1399",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "      <th>index</th>\n",
       "      <th>crawldate</th>\n",
       "      <th>type</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>identification</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>232</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2017-12-25 04:39:20</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[bibleverse]</td>\n",
       "      <td>0x28b412</td>\n",
       "      <td>Confident of your obedience, I write to you, k...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>989</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2016-01-08 17:18:59</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[]</td>\n",
       "      <td>0x2de201</td>\n",
       "      <td>\"Trust is not the same as faith. A friend is s...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>66</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2015-09-09 09:22:55</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[materialism, money, possessions]</td>\n",
       "      <td>0x218443</td>\n",
       "      <td>When do you have enough ? When are you satisfi...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>104</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2015-10-10 14:33:26</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[GodsPlan, GodsWork]</td>\n",
       "      <td>0x2939d5</td>\n",
       "      <td>God woke you up, now chase the day #GodsPlan #...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>310</td>\n",
       "      <td>hashtag_tweets</td>\n",
       "      <td>2016-10-23 08:49:50</td>\n",
       "      <td>tweets</td>\n",
       "      <td>[]</td>\n",
       "      <td>0x26289a</td>\n",
       "      <td>In these tough times, who do YOU turn to as yo...</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    score           index            crawldate    type  \\\n",
       "2     232  hashtag_tweets  2017-12-25 04:39:20  tweets   \n",
       "4     989  hashtag_tweets  2016-01-08 17:18:59  tweets   \n",
       "9      66  hashtag_tweets  2015-09-09 09:22:55  tweets   \n",
       "30    104  hashtag_tweets  2015-10-10 14:33:26  tweets   \n",
       "33    310  hashtag_tweets  2016-10-23 08:49:50  tweets   \n",
       "\n",
       "                             hashtags        id  \\\n",
       "2                        [bibleverse]  0x28b412   \n",
       "4                                  []  0x2de201   \n",
       "9   [materialism, money, possessions]  0x218443   \n",
       "30               [GodsPlan, GodsWork]  0x2939d5   \n",
       "33                                 []  0x26289a   \n",
       "\n",
       "                                                 text identification  \n",
       "2   Confident of your obedience, I write to you, k...           test  \n",
       "4   \"Trust is not the same as faith. A friend is s...           test  \n",
       "9   When do you have enough ? When are you satisfi...           test  \n",
       "30  God woke you up, now chase the day #GodsPlan #...           test  \n",
       "33  In these tough times, who do YOU turn to as yo...           test  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7488e57a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "X_test = BOW_250.transform(df_test['text'])\n",
    "print('X_test.shape: ', X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "791967ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_pred = DT_model.predict(X_test)\n",
    "y_test_pred[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d48759e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "result1_1 = pd.DataFrame( df_test['id'], columns=['id'])\n",
    "result1_1.reset_index(drop=True, inplace=True)\n",
    "result1_2 = pd.DataFrame( y_test_pred, columns=['emotion'])\n",
    "result1_2.reset_index(drop=True, inplace=True)\n",
    "result1 = pd.concat([result1_1, result1_2], axis=1, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72654e19",
   "metadata": {},
   "outputs": [],
   "source": [
    "result1.columns=['id', 'emotion']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35a3cf56",
   "metadata": {},
   "outputs": [],
   "source": [
    "result1[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9037f00",
   "metadata": {},
   "outputs": [],
   "source": [
    "result1.to_csv('BOW_result.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b24211f",
   "metadata": {},
   "source": [
    "---\n",
    "**<font color = green size=3>b. \"Naive Bayes_result.csv\"(use naive bayes)</font>**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "c05d8335",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "\n",
    "naive_bayes_classifier = MultinomialNB()\n",
    "NB_model = naive_bayes_classifier.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "b38a2401",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['anticipation', 'anticipation', 'disgust', 'anticipation',\n",
       "       'anticipation', 'sadness', 'disgust', 'joy', 'sadness', 'joy'],\n",
       "      dtype='<U12')"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NB_y_train_pred = NB_model.predict(X_test)\n",
    "NB_y_train_pred[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "f7d11524",
   "metadata": {},
   "outputs": [],
   "source": [
    "result2_1 = pd.DataFrame( df_test['id'], columns=['id'])\n",
    "result2_1.reset_index(drop=True, inplace=True)\n",
    "result2_2 = pd.DataFrame( NB_y_train_pred, columns=['emotion'])\n",
    "result2_2.reset_index(drop=True, inplace=True)\n",
    "result2 = pd.concat([result2_1, result2_2], axis=1, ignore_index=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "a10fe44c",
   "metadata": {},
   "outputs": [],
   "source": [
    "result2.columns=['id', 'emotion']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "f78224df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>emotion</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0x28b412</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0x2de201</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0x218443</td>\n",
       "      <td>disgust</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0x2939d5</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0x26289a</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id       emotion\n",
       "0  0x28b412  anticipation\n",
       "1  0x2de201  anticipation\n",
       "2  0x218443       disgust\n",
       "3  0x2939d5  anticipation\n",
       "4  0x26289a  anticipation"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result2[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "d002b8ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "result2.to_csv('Naive Bayes_result.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "945ec471",
   "metadata": {},
   "source": [
    "---\n",
    "**<font color = green size=3>c. \"model_result.csv\"(use deeper learning)</font>**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "bc4c1dfc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LabelEncoder()"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import keras\n",
    "from keras.utils import np_utils\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "label_encoder = LabelEncoder()\n",
    "label_encoder.fit(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "c938ff7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train's shape:\n",
      "(975227, 250)\n",
      "X_validation's shape:\n",
      "(480336, 250)\n",
      "y_train's shape:\n",
      "(975227,)\n",
      "y_validation's shape:\n",
      "(480336,)\n",
      "check label:  ['anger' 'anticipation' 'disgust' 'fear' 'joy' 'sadness' 'surprise'\n",
      " 'trust']\n",
      "\n",
      "## Before convert\n",
      "y_train[0:4]:\n",
      " 193200     trust\n",
      "1219141      joy\n",
      "508254       joy\n",
      "455552       joy\n",
      "Name: emotion, dtype: object\n",
      "\n",
      "y_train.shape:  (975227,)\n",
      "y_test.shape:  (480336,)\n",
      "\n",
      "\n",
      "## After convert\n",
      "y_train[0:4]:\n",
      " [[0. 0. 0. 0. 0. 0. 0. 1.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0.]]\n",
      "\n",
      "y_train.shape:  (975227, 8)\n",
      "y_test.shape:  (480336, 8)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = train_data_BOW_features_250\n",
    "y = df_train['emotion'] \n",
    "X_train, X_validation, y_train, y_validation = train_test_split(X, y, test_size = 0.33, random_state = 42)\n",
    "\n",
    "print('X_train\\'s shape:')\n",
    "print(X_train.shape)\n",
    "\n",
    "print('X_validation\\'s shape:')\n",
    "print(X_validation.shape)\n",
    "\n",
    "print('y_train\\'s shape:')\n",
    "print(y_train.shape)\n",
    "\n",
    "print('y_validation\\'s shape:')\n",
    "print(y_validation.shape)\n",
    "\n",
    "# -----------------------------------------------------\n",
    "label_encoder = LabelEncoder()\n",
    "label_encoder.fit(y_train)\n",
    "print('check label: ', label_encoder.classes_)\n",
    "print('\\n## Before convert')\n",
    "print('y_train[0:4]:\\n', y_train[0:4])\n",
    "print('\\ny_train.shape: ', y_train.shape)\n",
    "print('y_test.shape: ', y_validation.shape)\n",
    "\n",
    "def label_encode(le, labels):\n",
    "    enc = le.transform(labels)\n",
    "    return np_utils.to_categorical(enc)\n",
    "\n",
    "def label_decode(le, one_hot_label):\n",
    "    dec = np.argmax(one_hot_label, axis=1)\n",
    "    return le.inverse_transform(dec)\n",
    "\n",
    "y_train = label_encode(label_encoder, y_train)\n",
    "y_validation = label_encode(label_encoder, y_validation)\n",
    "\n",
    "print('\\n\\n## After convert')\n",
    "print('y_train[0:4]:\\n', y_train[0:4])\n",
    "print('\\ny_train.shape: ', y_train.shape)\n",
    "print('y_test.shape: ', y_validation.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "2fed82a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_shape:  250\n",
      "output_shape:  8\n"
     ]
    }
   ],
   "source": [
    "# 建模型\n",
    "input_shape = X_train.shape[1]\n",
    "print('input_shape: ', input_shape) #250 features\n",
    "\n",
    "output_shape = len(label_encoder.classes_)\n",
    "print('output_shape: ', output_shape) #4個選項"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "8ae3f939",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_3 (InputLayer)        [(None, 250)]             0         \n",
      "                                                                 \n",
      " dense_6 (Dense)             (None, 64)                16064     \n",
      "                                                                 \n",
      " re_lu_4 (ReLU)              (None, 64)                0         \n",
      "                                                                 \n",
      " dense_7 (Dense)             (None, 64)                4160      \n",
      "                                                                 \n",
      " re_lu_5 (ReLU)              (None, 64)                0         \n",
      "                                                                 \n",
      " dense_8 (Dense)             (None, 8)                 520       \n",
      "                                                                 \n",
      " softmax_2 (Softmax)         (None, 8)                 0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 20,744\n",
      "Trainable params: 20,744\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers import Input, Dense\n",
    "from keras.layers import ReLU, Softmax\n",
    "\n",
    "# input layer\n",
    "model_input = Input(shape=(input_shape, ))  # 500\n",
    "X = model_input\n",
    "\n",
    "# 1st hidden layer\n",
    "X_W1 = Dense(units=64)(X)  # 64\n",
    "H1 = ReLU()(X_W1)\n",
    "\n",
    "# 2nd hidden layer\n",
    "H1_W2 = Dense(units=64)(H1)  # 64\n",
    "H2 = ReLU()(H1_W2)\n",
    "\n",
    "# output layer\n",
    "H2_W3 = Dense(units=output_shape)(H2)  # 4\n",
    "H3 = Softmax()(H2_W3)\n",
    "\n",
    "model_output = H3\n",
    "\n",
    "# create model\n",
    "model = Model(inputs=[model_input], outputs=[model_output])\n",
    "\n",
    "# loss function & optimizer\n",
    "#去compile創出來的model\n",
    "model.compile(optimizer='adam',\n",
    "              loss='categorical_crossentropy', #分類用的\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "# show model construction\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "a9a9ae6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/12\n",
      "976/976 [==============================] - 17s 16ms/step - loss: 1.4509 - accuracy: 0.4709 - val_loss: 1.4571 - val_accuracy: 0.4679\n",
      "Epoch 2/12\n",
      "976/976 [==============================] - 18s 16ms/step - loss: 1.4436 - accuracy: 0.4737 - val_loss: 1.4545 - val_accuracy: 0.4694\n",
      "Epoch 3/12\n",
      "976/976 [==============================] - 19s 17ms/step - loss: 1.4391 - accuracy: 0.4751 - val_loss: 1.4510 - val_accuracy: 0.4703\n",
      "Epoch 4/12\n",
      "976/976 [==============================] - 19s 18ms/step - loss: 1.4362 - accuracy: 0.4761 - val_loss: 1.4508 - val_accuracy: 0.4711\n",
      "Epoch 5/12\n",
      "976/976 [==============================] - 19s 18ms/step - loss: 1.4338 - accuracy: 0.4769 - val_loss: 1.4491 - val_accuracy: 0.4715\n",
      "Epoch 6/12\n",
      "976/976 [==============================] - 19s 18ms/step - loss: 1.4317 - accuracy: 0.4777 - val_loss: 1.4488 - val_accuracy: 0.4717\n",
      "Epoch 7/12\n",
      "976/976 [==============================] - 19s 18ms/step - loss: 1.4299 - accuracy: 0.4787 - val_loss: 1.4485 - val_accuracy: 0.4719\n",
      "Epoch 8/12\n",
      "976/976 [==============================] - 21s 19ms/step - loss: 1.4283 - accuracy: 0.4791 - val_loss: 1.4487 - val_accuracy: 0.4719\n",
      "Epoch 9/12\n",
      "976/976 [==============================] - 22s 20ms/step - loss: 1.4271 - accuracy: 0.4796 - val_loss: 1.4454 - val_accuracy: 0.4732\n",
      "Epoch 10/12\n",
      "976/976 [==============================] - 26s 25ms/step - loss: 1.4259 - accuracy: 0.4800 - val_loss: 1.4454 - val_accuracy: 0.4726\n",
      "Epoch 11/12\n",
      "976/976 [==============================] - 28s 27ms/step - loss: 1.4249 - accuracy: 0.4806 - val_loss: 1.4455 - val_accuracy: 0.4731\n",
      "Epoch 12/12\n",
      "976/976 [==============================] - 27s 25ms/step - loss: 1.4240 - accuracy: 0.4806 - val_loss: 1.4447 - val_accuracy: 0.4727\n",
      "training finish\n"
     ]
    }
   ],
   "source": [
    "# from keras.callbacks import CSVLogger\n",
    "\n",
    "# #寫入log 中, 可看狀況\n",
    "# csv_logger = CSVLogger('logs/training_log.csv')\n",
    "\n",
    "# training setting\n",
    "epochs = 12\n",
    "batch_size = 1000\n",
    "\n",
    "# training!\n",
    "history = model.fit(X_train, y_train, \n",
    "                    epochs=epochs, \n",
    "                    batch_size=batch_size, \n",
    "#                     callbacks=[csv_logger],\n",
    "                    validation_data = (X_validation, y_validation))\n",
    "print('training finish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "795eeb31",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5.0283647e-03, 4.8826003e-01, 1.5703300e-02, 7.8620715e-03,\n",
       "        3.7351695e-01, 3.4232240e-02, 4.7139465e-03, 7.0683129e-02],\n",
       "       [4.6472996e-05, 8.8574868e-01, 4.1562966e-05, 1.7615786e-03,\n",
       "        7.2314784e-02, 3.5414842e-04, 7.7907229e-05, 3.9654791e-02],\n",
       "       [6.2634230e-02, 2.7879038e-01, 1.1789428e-01, 3.6730025e-02,\n",
       "        1.6678727e-01, 2.0753177e-01, 6.2202346e-02, 6.7429706e-02],\n",
       "       [8.5441470e-03, 2.4227495e-01, 3.1003997e-02, 1.4254169e-02,\n",
       "        4.1231608e-01, 3.6306467e-02, 7.1128388e-03, 2.4818727e-01],\n",
       "       [4.9585784e-03, 3.9388892e-01, 3.9818663e-02, 2.0854916e-02,\n",
       "        1.6977066e-01, 2.4696309e-02, 7.5390632e-03, 3.3847299e-01]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## predict\n",
    "pred_result = model.predict(X_test, batch_size=128)\n",
    "pred_result[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "id": "b82aac9c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['anticipation', 'anticipation', 'anticipation', 'joy',\n",
       "       'anticipation'], dtype=object)"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "pred_result = label_decode(label_encoder, pred_result)\n",
    "pred_result[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "id": "bc1f1094",
   "metadata": {},
   "outputs": [],
   "source": [
    "result3_1 = pd.DataFrame( df_test['id'], columns=['id'])\n",
    "result3_1.reset_index(drop=True, inplace=True)\n",
    "result3_2 = pd.DataFrame( pred_result, columns=['emotion'])\n",
    "result3_2.reset_index(drop=True, inplace=True)\n",
    "result3 = pd.concat([result3_1, result3_2], axis=1, ignore_index=True)\n",
    "result3.columns=['id', 'emotion']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "id": "e7a6f892",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "411972"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(result3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "61d7d583",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>emotion</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0x28b412</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0x2de201</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0x218443</td>\n",
       "      <td>disgust</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0x2939d5</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0x26289a</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id       emotion\n",
       "0  0x28b412  anticipation\n",
       "1  0x2de201  anticipation\n",
       "2  0x218443       disgust\n",
       "3  0x2939d5  anticipation\n",
       "4  0x26289a  anticipation"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result2[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "id": "8d347e63",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>emotion</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0x28b412</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0x2de201</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0x218443</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0x2939d5</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0x26289a</td>\n",
       "      <td>anticipation</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id       emotion\n",
       "0  0x28b412  anticipation\n",
       "1  0x2de201  anticipation\n",
       "2  0x218443  anticipation\n",
       "3  0x2939d5           joy\n",
       "4  0x26289a  anticipation"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result3[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "9879074f",
   "metadata": {},
   "outputs": [],
   "source": [
    "result3.to_csv('model_result.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14c2b38a",
   "metadata": {},
   "source": [
    "---\n",
    "**<font color = green size=3>d. RoBERTa(use deeper learning)</font>**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6696f165",
   "metadata": {},
   "source": [
    "    see \"kaggle_RoBERTa.ipynb\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8359767e",
   "metadata": {},
   "source": [
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}