Toxic Comment Classification Challenge.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Toxic Comment Classification Challenge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\compagnon\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from keras_tqdm import TQDMNotebookCallback\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "import codecs\n",
    "from tqdm import tqdm\n",
    "\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing import sequence\n",
    "\n",
    "import keras\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Conv1D, Conv2D, MaxPooling1D, GlobalMaxPool1D, Bidirectional, GlobalMaxPooling1D\n",
    "from keras.layers import LSTM, GRU, Dropout , BatchNormalization, Embedding, Flatten, GlobalAveragePooling1D, concatenate, Input\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "# Read train data\n",
    "train = pd.read_csv('train.csv')\n",
    "train.dropna(inplace=True)\n",
    "\n",
    "list_classes = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\n",
    "\n",
    "y_train = train[list_classes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "b9755707-f263-4074-bd4f-f2f0492d51cb",
    "_uuid": "68bf90a8cbce465d43c26c41ffb45ac76c69da9f"
   },
   "outputs": [],
   "source": [
    "# Read test data\n",
    "test = pd.read_csv('test.csv')\n",
    "test.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_cell_guid": "d6922dc3-b472-4546-9460-9d66aba67ea9",
    "_uuid": "52a7c76b92b01cf00da06169531754059d143a4d"
   },
   "outputs": [],
   "source": [
    "# Create tools to preprocess data\n",
    "# we will remove english stop words from text as well as punctuation\n",
    "porter = PorterStemmer()\n",
    "stop_words = set(stopwords.words('english'))\n",
    "stop_words.update(['.', ',', '\"', \"'\", ':', ';', '(', ')', '[', ']', '{', '}'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_cell_guid": "dc1e2746-877d-46b6-84d0-3e08aa9dd179",
    "_uuid": "da08cb620d34a7e65910c12178e606eb0f18925a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pre-processing train data...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████| 159571/159571 [00:05<00:00, 31835.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pre-processing test data...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████| 153164/153164 [00:04<00:00, 33923.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tokenizing input data...\n",
      "dictionary size:  348520\n",
      "Done !!\n"
     ]
    }
   ],
   "source": [
    "# the maximum number of words considered is 100000\n",
    "MAX_NB_WORDS = 100000\n",
    "# the size of the sentences will be 250\n",
    "max_seq_len = 250\n",
    "\n",
    "raw_docs_train = train['comment_text'].tolist()\n",
    "raw_docs_test = test['comment_text'].tolist()\n",
    "\n",
    "num_classes = len(list_classes)\n",
    "\n",
    "tokenizer = RegexpTokenizer(r'\\w+')\n",
    "\n",
    "print(\"pre-processing train data...\")\n",
    "processed_docs_train = []\n",
    "for doc in tqdm(raw_docs_train):\n",
    "    tokens = tokenizer.tokenize(doc)\n",
    "    filtered = [word for word in tokens if word not in stop_words]\n",
    "    processed_docs_train.append(\" \".join(filtered))\n",
    "\n",
    "print(\"pre-processing test data...\")\n",
    "processed_docs_test = []\n",
    "for doc in tqdm(raw_docs_test):\n",
    "    tokens = tokenizer.tokenize(doc)\n",
    "    filtered = [word for word in tokens if word not in stop_words]\n",
    "    processed_docs_test.append(\" \".join(filtered))\n",
    "\n",
    "\n",
    "print(\"tokenizing input data...\")\n",
    "tokenizer = Tokenizer(num_words=MAX_NB_WORDS, lower=True, char_level=False)\n",
    "tokenizer.fit_on_texts(processed_docs_train + processed_docs_test)  #leaky\n",
    "word_seq_train = tokenizer.texts_to_sequences(processed_docs_train)\n",
    "word_seq_test = tokenizer.texts_to_sequences(processed_docs_test)\n",
    "word_index = tokenizer.word_index\n",
    "print(\"dictionary size: \", len(word_index))\n",
    "\n",
    "#pad sequences\n",
    "word_seq_train = sequence.pad_sequences(word_seq_train, maxlen=max_seq_len)\n",
    "word_seq_test = sequence.pad_sequences(word_seq_test, maxlen=max_seq_len)\n",
    "\n",
    "print(\"Done !!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_cell_guid": "87e07100-0a09-421a-ac31-2b9ba1d4f8c2",
    "_uuid": "87fd8caea1e54d58fd2e0f25d5dd8e9719cda20d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading word embeddings...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2196018it [04:45, 7700.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found 2196016 word vectors\n",
      "preparing embedding matrix...\n",
      "number of null word embeddings: 23510\n"
     ]
    }
   ],
   "source": [
    "embed_dim = 300\n",
    "\n",
    "#load embeddings\n",
    "print('loading first word embeddings...')\n",
    "embeddings_index = {}\n",
    "f = codecs.open('glove.840B.300d.txt', encoding='utf-8')\n",
    "\n",
    "for line in tqdm(f):\n",
    "    values = line.rstrip().rsplit(' ')\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "print('found %s word vectors' % len(embeddings_index))\n",
    "\n",
    "#embedding matrix\n",
    "print('preparing embedding matrix...')\n",
    "words_not_found = []\n",
    "nb_words = min(MAX_NB_WORDS, len(word_index))\n",
    "embedding_matrix_glove = np.zeros((nb_words, embed_dim))\n",
    "for word, i in word_index.items():\n",
    "    if i >= nb_words:\n",
    "        continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if (embedding_vector is not None) and len(embedding_vector) > 0:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix_glove[i] = embedding_vector\n",
    "    else:\n",
    "        words_not_found.append(word)\n",
    "print('number of null word embeddings: %d' % np.sum(np.sum(embedding_matrix_glove, axis=1) == 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('loading second word embeddings...')\n",
    "EMBEDDING_FILE = 'crawl-300d-2M.vec'\n",
    "\n",
    "def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')\n",
    "embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE, encoding=\"utf8\"))\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "embedding_matrix_crawl = np.zeros((nb_words, embed_dim))\n",
    "for word, i in word_index.items():\n",
    "    if i >= MAX_NB_WORDS: continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None: embedding_matrix_crawl[i] = embedding_vector\n",
    "\n",
    "\n",
    "del embeddings_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_cell_guid": "b058c83a-7a35-461f-8507-4e9da8aa607b",
    "_uuid": "b168a84fff3286e2c13a298cc407e29ec78e0bea",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Creating model\n",
    "\n",
    "inp = Input(shape=(max_seq_len, ))\n",
    "emb_glove = Embedding(nb_words, 300, \n",
    "                weights=[embedding_matrix_glove], input_length = max_seq_len, trainable=False)(inp)\n",
    "\n",
    "emb_crawl = Embedding(nb_words, 300,\n",
    "                weights=[embedding_matrix_crawl], input_length = max_seq_len, trainable=False)(inp)\n",
    "\n",
    "conc1 = concatenate([emb_glove, emb_crawl])\n",
    "x = Bidirectional(LSTM(400, return_sequences=True))(conc1)\n",
    "x = Bidirectional(GRU (400, return_sequences=True))(x)\n",
    "avg_pool = GlobalAveragePooling1D()(x)\n",
    "max_pool = GlobalMaxPooling1D()(x)\n",
    "out = concatenate([avg_pool, max_pool])\n",
    "\n",
    "out = Dense(200, activation=\"relu\")(out)\n",
    "out = Dense(y_train.shape[1], activation=\"sigmoid\")(out)\n",
    "\n",
    "model = Model(inputs=inp, outputs=out)\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer = keras.optimizers.Adam(lr=0.001),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 250)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 250, 300)     30000000    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 250, 300)     30000000    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 250, 600)     0           embedding_1[0][0]                \n",
      "                                                                 embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 250, 800)     3203200     concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) (None, 250, 800)     2882400     bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling1d_1 (Glo (None, 800)          0           bidirectional_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalM (None, 800)          0           bidirectional_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 1600)         0           global_average_pooling1d_1[0][0] \n",
      "                                                                 global_max_pooling1d_1[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 200)          320200      concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 6)            1206        dense_1[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 66,407,006\n",
      "Trainable params: 6,407,006\n",
      "Non-trainable params: 60,000,000\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "_cell_guid": "04b08eee-44ae-4187-adc3-635885371d27",
    "_uuid": "5e420b32971c5fe1956b5881f2ff234cd54ec8ef"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 143613 samples, validate on 15958 samples\n",
      "Epoch 1/6\n",
      " - 2629s - loss: 0.0484 - acc: 0.9816 - val_loss: 0.0485 - val_acc: 0.9807\n",
      "Epoch 2/6\n",
      " - 2547s - loss: 0.0389 - acc: 0.9844 - val_loss: 0.0410 - val_acc: 0.9838\n",
      "Epoch 3/6\n",
      " - 2547s - loss: 0.0350 - acc: 0.9858 - val_loss: 0.0409 - val_acc: 0.9839\n",
      "Epoch 4/6\n",
      " - 2544s - loss: 0.0354 - acc: 0.9858 - val_loss: 0.0453 - val_acc: 0.9826\n",
      "Epoch 5/6\n",
      " - 2543s - loss: 0.0333 - acc: 0.9866 - val_loss: 0.0483 - val_acc: 0.9830\n",
      "Epoch 6/6\n",
      " - 2549s - loss: 0.0307 - acc: 0.9875 - val_loss: 0.0467 - val_acc: 0.9834\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2510b1d4048>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(word_seq_train, y_train, epochs=6, batch_size=80, shuffle=True, validation_split=0.1,verbose = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "_cell_guid": "ba358ae0-e491-494b-ba82-e5526efd1865",
    "_uuid": "61985840ff3f3bcac2d73b6c100693e1ee72a94e"
   },
   "outputs": [],
   "source": [
    "y_test = model.predict(word_seq_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "_cell_guid": "7f6af78a-94cd-473d-8c8a-6852103afec9",
    "_uuid": "ceac8763aa11e5030ae290db0678c7cedd92bae4"
   },
   "outputs": [],
   "source": [
    "sample_submission = pd.read_csv(\"sample_submission.csv\")\n",
    "\n",
    "sample_submission[list_classes] = y_test\n",
    "sample_submission.to_csv(\"results.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "a5611c5d-9184-49dc-a3d5-50d9bb48a9e5",
    "_kg_hide-input": false,
    "_kg_hide-output": false,
    "_uuid": "1f4f3161d77b08664c7110235639a0732a7940b3"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "50361861-b516-4f87-ab72-68542939edc8",
    "_uuid": "c65dedc9c53a25b7a80446dab85d17eb08133849"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "230569d7-eced-4d7e-894f-824508643907",
    "_uuid": "6534018c28edf5e4bf0eeac8a2b9019828e4c8ba"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}