submit.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /Users/Alexandre/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tweet</th>\n",
       "      <th>subtask_a</th>\n",
       "      <th>subtask_b</th>\n",
       "      <th>subtask_c</th>\n",
       "      <th>clean_tweets</th>\n",
       "      <th>tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>86426</td>\n",
       "      <td>@USER She should ask a few native Americans wh...</td>\n",
       "      <td>OFF</td>\n",
       "      <td>UNT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ask nativ american</td>\n",
       "      <td>[ask, nativ, american]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>90194</td>\n",
       "      <td>@USER @USER Go home you’re drunk!!! @USER #MAG...</td>\n",
       "      <td>OFF</td>\n",
       "      <td>TIN</td>\n",
       "      <td>IND</td>\n",
       "      <td>home drunk manga trump</td>\n",
       "      <td>[home, drunk, manga, trump]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16820</td>\n",
       "      <td>Amazon is investigating Chinese employees who ...</td>\n",
       "      <td>NOT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>amazon investig chines employe sell intern dat...</td>\n",
       "      <td>[amazon, investig, chines, employe, sell, inte...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>62688</td>\n",
       "      <td>@USER Someone should'veTaken\" this piece of sh...</td>\n",
       "      <td>OFF</td>\n",
       "      <td>UNT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>someon retaken piec shit volcano</td>\n",
       "      <td>[someon, retaken, piec, shit, volcano]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>43605</td>\n",
       "      <td>@USER @USER Obama wanted liberals &amp;amp; illega...</td>\n",
       "      <td>NOT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>obama want liber amp illeg move red state</td>\n",
       "      <td>[obama, want, liber, amp, illeg, move, red, st...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id                                              tweet subtask_a  \\\n",
       "0  86426  @USER She should ask a few native Americans wh...       OFF   \n",
       "1  90194  @USER @USER Go home you’re drunk!!! @USER #MAG...       OFF   \n",
       "2  16820  Amazon is investigating Chinese employees who ...       NOT   \n",
       "3  62688  @USER Someone should'veTaken\" this piece of sh...       OFF   \n",
       "4  43605  @USER @USER Obama wanted liberals &amp; illega...       NOT   \n",
       "\n",
       "  subtask_b subtask_c                                       clean_tweets  \\\n",
       "0       UNT       NaN                                 ask nativ american   \n",
       "1       TIN       IND                             home drunk manga trump   \n",
       "2       NaN       NaN  amazon investig chines employe sell intern dat...   \n",
       "3       UNT       NaN                   someon retaken piec shit volcano   \n",
       "4       NaN       NaN          obama want liber amp illeg move red state   \n",
       "\n",
       "                                              tokens  \n",
       "0                             [ask, nativ, american]  \n",
       "1                        [home, drunk, manga, trump]  \n",
       "2  [amazon, investig, chines, employe, sell, inte...  \n",
       "3             [someon, retaken, piec, shit, volcano]  \n",
       "4  [obama, want, liber, amp, illeg, move, red, st...  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd \n",
    "import utils.preprocessing\n",
    "from utils.test_model import *\n",
    "from utils.utils import make_submission\n",
    "import numpy as np\n",
    "\n",
    "df = pd.read_csv('data/raw/offenseval-training-v1.tsv', sep='\\t')\n",
    "utils.preprocessing.clean(df)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tweet</th>\n",
       "      <th>clean_tweets</th>\n",
       "      <th>tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15923</td>\n",
       "      <td>#WhoIsQ #WheresTheServer #DumpNike #DECLASFISA...</td>\n",
       "      <td>whose wherestheserv dumpsit declasfisa democra...</td>\n",
       "      <td>[whose, wherestheserv, dumpsit, declasfisa, de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>27014</td>\n",
       "      <td>#ConstitutionDay is revered by Conservatives, ...</td>\n",
       "      <td>constitut rever conserv hate progress socialis...</td>\n",
       "      <td>[constitut, rever, conserv, hate, progress, so...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>30530</td>\n",
       "      <td>#FOXNews #NRA #MAGA #POTUS #TRUMP #2ndAmendmen...</td>\n",
       "      <td>fox era manga lotu trump amend inc us veteran ...</td>\n",
       "      <td>[fox, era, manga, lotu, trump, amend, inc, us,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13876</td>\n",
       "      <td>#Watching #Boomer getting the news that she is...</td>\n",
       "      <td>watch boomer get news parol alway make smile w...</td>\n",
       "      <td>[watch, boomer, get, news, parol, alway, make,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60133</td>\n",
       "      <td>#NoPasaran: Unity demo to oppose the far-right...</td>\n",
       "      <td>nopasaran uniti demo oppos far right london an...</td>\n",
       "      <td>[nopasaran, uniti, demo, oppos, far, right, lo...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id                                              tweet  \\\n",
       "0  15923  #WhoIsQ #WheresTheServer #DumpNike #DECLASFISA...   \n",
       "1  27014  #ConstitutionDay is revered by Conservatives, ...   \n",
       "2  30530  #FOXNews #NRA #MAGA #POTUS #TRUMP #2ndAmendmen...   \n",
       "3  13876  #Watching #Boomer getting the news that she is...   \n",
       "4  60133  #NoPasaran: Unity demo to oppose the far-right...   \n",
       "\n",
       "                                        clean_tweets  \\\n",
       "0  whose wherestheserv dumpsit declasfisa democra...   \n",
       "1  constitut rever conserv hate progress socialis...   \n",
       "2  fox era manga lotu trump amend inc us veteran ...   \n",
       "3  watch boomer get news parol alway make smile w...   \n",
       "4  nopasaran uniti demo oppos far right london an...   \n",
       "\n",
       "                                              tokens  \n",
       "0  [whose, wherestheserv, dumpsit, declasfisa, de...  \n",
       "1  [constitut, rever, conserv, hate, progress, so...  \n",
       "2  [fox, era, manga, lotu, trump, amend, inc, us,...  \n",
       "3  [watch, boomer, get, news, parol, alway, make,...  \n",
       "4  [nopasaran, uniti, demo, oppos, far, right, lo...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "X = df[\"clean_tweets\"]\n",
    "\n",
    "vec = CountVectorizer(ngram_range=(1,3), \n",
    "                      stop_words='english', \n",
    "                      max_features=39650)\n",
    "\n",
    "df_test = pd.read_csv('data/test/task_a/testset-taska.tsv', sep='\\t')\n",
    "utils.preprocessing.clean(df_test)\n",
    "df_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Subtask A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Category split in cleaned dataset:\n",
      "\tOFF: 33.23%\n",
      "\tNOT: 66.77%\n"
     ]
    }
   ],
   "source": [
    "X_train = vec.fit_transform(X)\n",
    "X_test = vec.transform(df_test[\"clean_tweets\"])\n",
    "y_train = df[\"subtask_a\"].map({\"OFF\":0, \"NOT\":1})\n",
    "\n",
    "print('Category split in cleaned dataset:')\n",
    "print('\\tOFF: {}%'.format(100* round(y_train.value_counts()[0] / y_train.count(), 4)))\n",
    "print('\\tNOT: {}%'.format(100* round(y_train.value_counts()[1] / y_train.count(), 4)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=2, dual=True)\n",
    "clf.fit(X_train, y_train)\n",
    "y_pred = clf.predict(X_test)\n",
    "make_submission(y_pred, {0:\"OFF\", 1:\"NOT\"}, df_test, \"submissions/taska_logreg.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import hstack\n",
    "\n",
    "word_vec = CountVectorizer(ngram_range=(1,3), stop_words='english',analyzer='word', max_features=32500)\n",
    "train_word_features = word_vec.fit_transform(X)\n",
    "test_word_features = word_vec.fit_transform(df_test[\"clean_tweets\"])\n",
    "\n",
    "char_vec = CountVectorizer(ngram_range=(1,3), stop_words='english',analyzer='char', max_features=32500)\n",
    "train_char_features = char_vec.fit_transform(X)\n",
    "test_char_features = char_vec.fit_transform(df_test[\"clean_tweets\"])\n",
    "\n",
    "train_features = hstack([train_char_features, train_word_features])\n",
    "test_features = hstack([test_char_features, test_word_features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier(bootstrap=True, class_weight='balanced',\n",
      "            criterion='gini', max_depth=100, max_features=1000,\n",
      "            max_leaf_nodes=50, min_impurity_decrease=0.0,\n",
      "            min_impurity_split=None, min_samples_leaf=3,\n",
      "            min_samples_split=10, min_weight_fraction_leaf=0.0,\n",
      "            n_estimators=10, n_jobs=1, oob_score=False, random_state=None,\n",
      "            verbose=0, warm_start=False)\n",
      "=================================================================\n",
      "=========================== Training ============================\n",
      "========================== Evaluating ===========================\n",
      "Accuracy:\t75.4154\n",
      "========================== Validating ===========================\n",
      "Mean F1 score (3-fold CV):\t65.9116%\n",
      "Mean Accuracy (3-fold CV):\t69.5544%\n"
     ]
    }
   ],
   "source": [
    "rfc = RandomForestClassifier(bootstrap=True, class_weight='balanced', criterion='gini',\n",
    "            max_depth=100, max_features=1000, max_leaf_nodes=50,\n",
    "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "            min_samples_leaf=3, min_samples_split=10,\n",
    "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
    "            oob_score=False, random_state=None, verbose=0,\n",
    "            warm_start=False)\n",
    "\n",
    "print(rfc)\n",
    "print('=================================================================')\n",
    "\n",
    "# Train Model\n",
    "print('=========================== Training ============================')\n",
    "rfc.fit(train_features, y_train)\n",
    "\n",
    "# Evaluate Model\n",
    "print('========================== Evaluating ===========================')\n",
    "print('Accuracy:\\t{}'.format(round(100*rfc.score(train_features, y_train),4)))\n",
    "\n",
    "# K-Fold Cross Validation\n",
    "print('========================== Validating ===========================')\n",
    "(f1_rfc, acc_rfc) = test_single_model(rfc, X, y_train, [word_vec, char_vec], \n",
    "                                      n_splits=3, random_state=1)\n",
    "print('Mean F1 score (3-fold CV):\\t{}%'.format(round(100*f1_rfc,4)))\n",
    "print('Mean Accuracy (3-fold CV):\\t{}%'.format(round(100*acc_rfc,4)))\n",
    "\n",
    "# Predict on test set and write submission\n",
    "y_pred_proba_rfc = rfc.predict_proba(X_test)\n",
    "y_pred_rfc = np.argmax(y_pred_proba_rfc,1)\n",
    "make_submission(y_pred_rfc, {\"UNT\":0, \"TIN\":1}, df_test, \"submissions/taskb_rfc.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports & Manip for Deep Learning methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils import to_categorical\n",
    "from keras.preprocessing import sequence\n",
    "from utils.keras_utils import f1_loss\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Input, Embedding, Bidirectional, LSTM, GlobalMaxPool1D, Dropout, Dense\n",
    "from keras.preprocessing import text\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Conv1D, MaxPooling1D, Flatten, Embedding\n",
    "\n",
    "tokenizer = text.Tokenizer(num_words=15000)\n",
    "tokenizer.fit_on_texts(X)\n",
    "\n",
    "y_cat = to_categorical(y_train)\n",
    "\n",
    "list_tokenized_train = tokenizer.texts_to_sequences(X)\n",
    "X_test = df_test[\"clean_tweets\"]\n",
    "list_tokenized_test = tokenizer.texts_to_sequences(X_test)\n",
    "X_tr = sequence.pad_sequences(list_tokenized_train, maxlen=100)\n",
    "X_te = sequence.pad_sequences(list_tokenized_test, maxlen=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 100, 16)           240000    \n",
      "_________________________________________________________________\n",
      "conv1d_2 (Conv1D)            (None, 98, 128)           6272      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1 (None, 19, 128)           0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 2432)              0         \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 2432)              0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 64)                155712    \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 16)                1040      \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 2)                 34        \n",
      "=================================================================\n",
      "Total params: 403,058\n",
      "Trainable params: 403,058\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "=================================================================\n",
      "=========================== Training ============================\n",
      "========================== Evaluating ===========================\n",
      "13240/13240 [==============================] - 1s 81us/step\n",
      "loss:\t19.6541\n",
      "acc:\t83.8142%\n",
      "========================== Validating ===========================\n",
      "Mean F1 score (3-fold CV):\t63.9733%\n",
      "Mean Accuracy (3-fold CV):\t68.1425%\n"
     ]
    }
   ],
   "source": [
    "# Define Model\n",
    "cnn = Sequential()\n",
    "cnn.add(Embedding(15000, 16, input_length=100))\n",
    "cnn.add(Conv1D(128, 3, activation='relu'))\n",
    "cnn.add(MaxPooling1D(5))\n",
    "cnn.add(Flatten())\n",
    "cnn.add(Dropout(rate=0.25))\n",
    "cnn.add(Dense(units=64, activation='relu'))\n",
    "cnn.add(Dropout(rate=0.25))\n",
    "cnn.add(Dense(units=16, activation='relu'))\n",
    "cnn.add(Dense(units=2, activation='softmax'))\n",
    "cnn.compile(loss=f1_loss, optimizer='adam', metrics=['accuracy'])\n",
    "print(cnn.summary())\n",
    "print('=================================================================')\n",
    "\n",
    "# Train Model\n",
    "print('=========================== Training ============================')\n",
    "cnn.fit(X_tr, y_cat, epochs=10,  batch_size=64, verbose=False)\n",
    "\n",
    "# Evaluate Model\n",
    "print('========================== Evaluating ===========================')\n",
    "evaluation_cnn = cnn.evaluate(X_tr, y_cat, batch_size=64, verbose=1)\n",
    "print('{}:\\t{}'.format(cnn.metrics_names[0], round(100*evaluation_cnn[0],4)))\n",
    "print('{}:\\t{}%'.format(cnn.metrics_names[1], round(100*evaluation_cnn[1],4)))\n",
    "\n",
    "# K-Fold Cross Validation\n",
    "print('========================== Validating ===========================')\n",
    "(f1_cnn, acc_cnn) = test_single_model(cnn, X, y_train, tokenizer, n_splits=3, random_state=1, verbose=False)\n",
    "print('Mean F1 score (3-fold CV):\\t{}%'.format(round(100*f1_cnn,4)))\n",
    "print('Mean Accuracy (3-fold CV):\\t{}%'.format(round(100*acc_cnn,4)))\n",
    "\n",
    "# Predict on test set and write submission\n",
    "y_pred_proba_cnn = cnn.predict(X_te)\n",
    "y_pred_cnn = np.argmax(y_pred_proba_cnn, axis=1)\n",
    "make_submission(y_pred_cnn, {\"UNT\":0, \"TIN\":1}, df_test, \"submissions/taska_cnn.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_3 (Embedding)      (None, 100, 32)           480000    \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 100, 64)           16640     \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_1 (Glob (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 2)                 66        \n",
      "=================================================================\n",
      "Total params: 498,786\n",
      "Trainable params: 498,786\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "=================================================================\n",
      "=========================== Training ============================\n",
      "WARNING:tensorflow:From /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-c2f368dc9e80>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;31m# Train Model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'=========================== Training ============================'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0mlstm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_tr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_cat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m64\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;31m# Evaluate Model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1040\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2713\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2715\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2716\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2717\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2674\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2675\u001b[0;31m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2676\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1437\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1438\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1439\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1440\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#lstm = Sequential()\n",
    "#lstm.add(Embedding(15000, 32, input_length=100))\n",
    "#lstm.add(Bidirectional(LSTM(6, return_sequences=True)))\n",
    "#lstm.add(GlobalMaxPool1D())\n",
    "#lstm.add(Dropout(0.5))\n",
    "#lstm.add(Dense(6, activation=\"relu\"))\n",
    "#lstm.add(Dropout(0.5))\n",
    "#lstm.add(Dense(2, activation=\"softmax\"))\n",
    "\n",
    "# Define Model\n",
    "lstm = Sequential()\n",
    "lstm.add(Embedding(15000, 32, input_length=100))\n",
    "lstm.add(Bidirectional(LSTM(32, return_sequences=True)))\n",
    "lstm.add(GlobalMaxPool1D())\n",
    "lstm.add(Dropout(0.5))\n",
    "lstm.add(Dense(32, activation=\"relu\"))\n",
    "lstm.add(Dropout(0.5))\n",
    "lstm.add(Dense(2, activation=\"softmax\"))\n",
    "lstm.compile(loss=f1_loss, optimizer='adam', metrics=['accuracy'])\n",
    "print(lstm.summary())\n",
    "print('=================================================================')\n",
    "\n",
    "# Train Model\n",
    "print('=========================== Training ============================')\n",
    "lstm.fit(X_tr, y_cat, epochs=10,  batch_size=64, verbose=False)\n",
    "\n",
    "# Evaluate Model\n",
    "print('========================== Evaluating ===========================')\n",
    "evaluation_lstm = lstm.evaluate(X_tr, y_cat, batch_size=64, verbose=1)\n",
    "print('{}:\\t{}'.format(lstm.metrics_names[0], round(100*evaluation_lstm[0],4)))\n",
    "print('{}:\\t{}%'.format(lstm.metrics_names[1], round(100*evaluation_lstm[1],4)))\n",
    "\n",
    "# K-Fold Cross Validation\n",
    "print('========================== Validating ===========================')\n",
    "test_single_model(lstm, X, y_train, tokenizer, n_splits=3, random_state=1, verbose=False)\n",
    "(f1_lstm, acc_lstm) = test_single_model(lstm, X, y_train, tokenizer, n_splits=3, random_state=1, verbose=False)\n",
    "print('Mean F1 score (3-fold CV):\\t{}%'.format(round(100*f1_lstm,4)))\n",
    "print('Mean Accuracy (3-fold CV):\\t{}%'.format(round(100*acc_lstm,4)))\n",
    "\n",
    "# Predict on test set and write submission\n",
    "y_pred_proba_lstm = lstm.predict(X_te)\n",
    "y_pred_lstm = np.argmax(y_pred_proba_lstm, axis=1)\n",
    "make_submission(y_pred_lstm, {\"UNT\":0, \"TIN\":1}, df_test, \"submissions/taska_lstm.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bi-LSTM + ConvLayer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "blc = Sequential()\n",
    "blc.add(Embedding(15000, 32, input_length=100, trainable=True))\n",
    "blc.add(Bidirectional(LSTM(128, return_sequences=True, dropout=0.15, recurrent_dropout=0.15)))\n",
    "blc.add(Conv1D(128, kernel_size=3, padding='valid', kernel_initializer='glorot_uniform'))\n",
    "blc.add(GlobalMaxPool1D())\n",
    "blc.add(Dropout(0.2))\n",
    "blc.add(Dense(32, activation='relu'))\n",
    "blc.add(Dense(2, activation='softmax'))\n",
    "blc.compile(loss=f1_loss, optimizer='adam', metrics=['accuracy'])\n",
    "print(blc.summary())\n",
    "print('=================================================================')\n",
    "\n",
    "# Train Model\n",
    "print('=========================== Training ============================')\n",
    "blc.fit(X_tr, y_cat, epochs=10,  batch_size=64, verbose=False)\n",
    "\n",
    "# Evaluate Model\n",
    "print('========================== Evaluating ===========================')\n",
    "evaluation_blc = blc.evaluate(X_tr, y_cat, batch_size=64, verbose=1)\n",
    "print('{}:\\t{}'.format(blc.metrics_names[0], round(100*evaluation_blc[0],4)))\n",
    "print('{}:\\t{}%'.format(blc.metrics_names[1], round(100*evaluation_blc[1],4)))\n",
    "\n",
    "# K-Fold Cross Validation\n",
    "print('========================== Validating ===========================')\n",
    "test_single_model(blc, X, y_train, tokenizer, n_splits=3, random_state=1, verbose=False)\n",
    "(f1_blc, acc_blc) = test_single_model(blc, X, y_train, tokenizer, n_splits=3, random_state=1, verbose=False)\n",
    "print('Mean F1 score (3-fold CV):\\t{}%'.format(round(100*f1_blc,4)))\n",
    "print('Mean Accuracy (3-fold CV):\\t{}%'.format(round(100*acc_blc,4)))\n",
    "\n",
    "# Predict on test set and write submission\n",
    "y_pred_proba_blc = blc.predict(X_te)\n",
    "y_pred_blc = np.argmax(y_pred_proba_blc, axis=1)\n",
    "make_submission(y_pred_blc, {\"UNT\":0, \"TIN\":1}, df_test, \"submissions/taskba_blc.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ensemble RF + Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg = LogisticRegression(C=2, dual=True)\n",
    "logreg.fit(X_train, y_train)\n",
    "y_pred_proba_logreg = logreg.predict_proba(X_test)\n",
    "\n",
    "rf = RandomForestClassifier(n_estimators=100)\n",
    "rf.fit(X_train, y_train)\n",
    "y_pred_proba_rf = rf.predict_proba(X_test)\n",
    "\n",
    "y_pred_proba = y_pred_proba_logreg + y_pred_proba_rf\n",
    "y_pred = np.argmax(y_pred_proba, axis=1)\n",
    "#make_submission(y_pred, {0:\"OFF\", 1:\"NOT\"}, df_test, \"submissions/taska_logreg+rf.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}