TFIDF representation with Fully Connected Neural Network(FCNN).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "id": "uTKy9BRpJnqo",
    "outputId": "083f3d93-c8c3-48b9-fe21-76b2b0b46e7c"
   },
   "source": [
    "## Experimenting with an Fully Connected Neural Network (FCNN) Classifier:\n",
    "\n",
    "* input layer consist of 1024 neurons, with activation function of ReLU and droup out with probability of 40%\n",
    "* 4-hidden layer deep neural network, with activation functions of ReLU, and layers with 512, 256, 256, and 256 neurons.\n",
    "* output layer consist of 2 neurons, with activation function of sigmoid\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_X1dJopWKBe2"
   },
   "outputs": [],
   "source": [
    "import os \n",
    "datasets = [\"testdata.manual.2009.06.14.csv\", \"training.1600000.processed.noemoticon.csv\"]\n",
    "train_path = os.path.join(\"dataset\", datasets[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "f7aiONiPDpEA"
   },
   "source": [
    "**Loading Data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wF9kDXGtKVtC"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from tqdm import trange\n",
    "from dataloader import DataLoader\n",
    "\n",
    "data_loader = DataLoader()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 145
    },
    "colab_type": "code",
    "id": "tFtv0FOwKZ8X",
    "outputId": "5cfc6110-bbaa-4837-e94a-36c050640cb5"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sentiment</th>\n",
       "      <th>ID</th>\n",
       "      <th>Date</th>\n",
       "      <th>Query</th>\n",
       "      <th>UserID</th>\n",
       "      <th>Tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810369</td>\n",
       "      <td>Mon Apr 06 22:19:45 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>_TheSpecialOne_</td>\n",
       "      <td>@switchfoot http://twitpic.com/2y1zl - Awww, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810672</td>\n",
       "      <td>Mon Apr 06 22:19:49 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>scotthamilton</td>\n",
       "      <td>is upset that he can't update his Facebook by ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Sentiment  ...                                              Tweet\n",
       "0          0  ...  @switchfoot http://twitpic.com/2y1zl - Awww, t...\n",
       "1          0  ...  is upset that he can't update his Facebook by ...\n",
       "\n",
       "[2 rows x 6 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data_loader.read_df(train_path, \n",
    "                           df_type='csv', encoding='latin-1',\n",
    "                           names=[\"Sentiment\", \"ID\", \"Date\", \"Query\",\"UserID\",\"Tweet\"])\n",
    "data.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "00mncymVEF-P"
   },
   "source": [
    "**preprocessing using texthero**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 179
    },
    "colab_type": "code",
    "id": "bFVkO-9FLIBL",
    "outputId": "d560d63f-0466-48ab-c366-880655a3a2ea"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sentiment</th>\n",
       "      <th>ID</th>\n",
       "      <th>Date</th>\n",
       "      <th>Query</th>\n",
       "      <th>UserID</th>\n",
       "      <th>Tweet</th>\n",
       "      <th>CleanTweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810369</td>\n",
       "      <td>Mon Apr 06 22:19:45 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>_TheSpecialOne_</td>\n",
       "      <td>@switchfoot http://twitpic.com/2y1zl - Awww, t...</td>\n",
       "      <td>switchfoot http twitpic com 2y1zl awww bummer ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810672</td>\n",
       "      <td>Mon Apr 06 22:19:49 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>scotthamilton</td>\n",
       "      <td>is upset that he can't update his Facebook by ...</td>\n",
       "      <td>upset update facebook texting might cry result...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Sentiment  ...                                         CleanTweet\n",
       "0          0  ...  switchfoot http twitpic com 2y1zl awww bummer ...\n",
       "1          0  ...  upset update facebook texting might cry result...\n",
       "\n",
       "[2 rows x 7 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import texthero as hero\n",
    "\n",
    "data['CleanTweet'] = data['Tweet'].pipe(hero.clean)\n",
    "\n",
    "data.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_elxGoYpZepe"
   },
   "source": [
    "**Extracting top words for TF-IDF representation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "u97ekW5ta02g",
    "outputId": "7627f8a6-7612-44d0-93b2-e6ab777dc745"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ft:5, vocab lenght:51758\n"
     ]
    }
   ],
   "source": [
    "def get_vocabs(pos_words_dic, neg_words_dic, ft):\n",
    "    vocabs = []\n",
    "    for word,freq in pos_words_dic.items():\n",
    "        if freq > ft and word not in vocabs:\n",
    "            vocabs.append(word)\n",
    "\n",
    "    for word,freq in neg_words_dic.items():\n",
    "        if freq > ft and word not in vocabs:\n",
    "            vocabs.append(word)\n",
    "    return vocabs\n",
    "    \n",
    "ft = 5\n",
    "top_pos_words = hero.top_words(data[data['Sentiment'] == 4]['CleanTweet'])\n",
    "top_neg_words = hero.top_words(data[data['Sentiment'] == 0]['CleanTweet'])\n",
    "vocabs5 = get_vocabs(top_pos_words.to_dict(), top_neg_words.to_dict(), ft=ft)\n",
    "print( \"ft:{}, vocab lenght:{}\".format(ft, len(vocabs5)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1kZUR8dRqGTH"
   },
   "source": [
    "**Train test split**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9DPS-no1gJOb"
   },
   "outputs": [],
   "source": [
    "def transform_labels(label):\n",
    "    if label==4:\n",
    "        return 1\n",
    "    return label\n",
    "\n",
    "data['Sentiment'] = data['Sentiment'].apply(lambda x:transform_labels(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "KCi0IbftgWkF",
    "outputId": "f3d4fc74-f96a-45ab-dae4-0f541d559134"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train size: 1120000\n",
      "Test size: 480000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(data['CleanTweet'].tolist(), \n",
    "                                                    data['Sentiment'].tolist(),\n",
    "                                                    test_size=0.3, random_state=40)\n",
    "\n",
    "print(\"Train size:\", len(x_train))\n",
    "print(\"Test size:\", len(x_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "09CfpnXvgm8i"
   },
   "source": [
    "## TFIDFFCNN Model\n",
    "\n",
    "TF-IDF representation with Fully Connected Neural Network model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 173
    },
    "colab_type": "code",
    "id": "znIviv3CbEtz",
    "outputId": "cf7b7fc4-02a0-4b08-c6ed-bb95e473424c"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1008000 samples, validate on 112000 samples\n",
      "Epoch 1/3\n",
      "1008000/1008000 [==============================] - 1924s 2ms/step - loss: 0.4806 - accuracy: 0.7679 - val_loss: 0.4602 - val_accuracy: 0.7806\n",
      "Epoch 2/3\n",
      "1008000/1008000 [==============================] - 1911s 2ms/step - loss: 0.4272 - accuracy: 0.8004 - val_loss: 0.4677 - val_accuracy: 0.7826\n",
      "Epoch 3/3\n",
      "1008000/1008000 [==============================] - 1906s 2ms/step - loss: 0.3846 - accuracy: 0.8244 - val_loss: 0.4721 - val_accuracy: 0.7847\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from model import ModelPipeline\n",
    "from fcnn import FCNN\n",
    "\n",
    "model=ModelPipeline(estimator=FCNN(input_dim=len(vocabs5), \n",
    "                                   nb_classes=2, \n",
    "                                   best_model=\"fcnn\", \n",
    "                                   epoch=3,\n",
    "                                   batch_size=1024,\n",
    "                                   verbose=1,\n",
    "                                   validation_split=0.1),\n",
    "                    transformer=TfidfVectorizer(vocabulary=vocabs5) )\n",
    "\n",
    "model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 170
    },
    "colab_type": "code",
    "id": "NO1U08klgItt",
    "outputId": "9dc742ca-84c5-41d5-b330-bd3ee20a8689"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.77      0.78    239943\n",
      "           1       0.78      0.80      0.79    240057\n",
      "\n",
      "    accuracy                           0.79    480000\n",
      "   macro avg       0.79      0.79      0.79    480000\n",
      "weighted avg       0.79      0.79      0.79    480000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "y_pred = model.predict(x_test)\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EfBV-jIPqymQ"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "TPU",
  "colab": {
   "collapsed_sections": [],
   "name": "TFIDF representation with Fully Connected Neural Network.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}