examples/fake_news.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fake News Classification\n",
    "\n",
    "Solution: [Universal Sentence Encoder (USE) for English](https://www.aclweb.org/anthology/D18-2029)\n",
    "\n",
    "This [blog post](https://towardsdatascience.com/using-use-universal-sentence-encoder-to-detect-fake-news-dfc02dc32ae9) reaches ~ 90% accuracy with universal encoder from tf hub. But can we reach the same results with a simple encoder ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/talmago/git/simple-but-tough-to-beat-examples/.venv/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 216, got 192\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "import itertools\n",
    "import os\n",
    "\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "from encoder import build_from_fasttext_bin\n",
    "from nn import train_w2v, train_nn, load_model, fasttext\n",
    "from utils import preprocess_sentence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download the dataset (from github)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "CSV_FILE = 'fake_or_real_news.csv'\n",
    "\n",
    "! [[ ! -f { CSV_FILE } ]] && wget https://github.com/saadarshad102/Fake-News-Detection-Universal-Sentence-Encoder/raw/master/{ CSV_FILE }\n",
    "    \n",
    "def read_news(fname):\n",
    "    X = []\n",
    "    y = []\n",
    "    with open(fname) as f:\n",
    "        for row_num, row in enumerate(csv.reader(f)):\n",
    "            if row_num == 0:\n",
    "                continue\n",
    "            _, title, text, label = row\n",
    "            X.append(text)\n",
    "            y.append(label)\n",
    "    return X, y\n",
    "\n",
    "X, y = read_news(CSV_FILE)\n",
    "label_encoder = LabelEncoder()\n",
    "y = label_encoder.fit_transform(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build a corpus for word2vec training and pre-process with `textacy` lib:\n",
    "  - normalize unicode charset.\n",
    "  - deaccent (rèsume -> resume)\n",
    "  - unpack contractions (he's --> he is).\n",
    "  - remove emojis, hashtags, URLs, emails, etc\n",
    "  - remove punctuation marks\n",
    "  - strip whitespace\n",
    "  - lowercase\n",
    "  \n",
    "and train word2vec skipgram model as follows;\n",
    "  - dim = 200\n",
    "  - lr = relatively low.\n",
    "  - epochs = 15 (but should probably be ~ 25).\n",
    "  - ws = 5 (but should probably be ~ 7).\n",
    "  - sub-word information (minn = 3, maxn = 6).\n",
    "  \n",
    "alternatively, we can use a [pre-built model](https://fasttext.cc/docs/en/pretrained-vectors.html)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "W2V_PREBUILT_MODEL = 'cc.en.300.bin'\n",
    "W2V_MODEL = 'model.bin' # W2V_PREBUILT_MODEL\n",
    "\n",
    "if W2V_MODEL == W2V_PREBUILT_MODEL:\n",
    "    ! [[ ! -f {W2V_PREBUILT_MODEL} ]] && wget https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/{W2V_PREBUILT_MODEL}.gz\n",
    "    ! [[ ! -f {W2V_PREBUILT_MODEL} ]] && gzip -d {W2V_PREBUILT_MODEL}.gz\n",
    "    ! ls -lh {W2V_PREBUILT_MODEL}\n",
    "\n",
    "if not os.path.isfile(W2V_MODEL):\n",
    "    # build w2v corpus\n",
    "    corpus = []\n",
    "    for raw_sentence in tqdm(X):\n",
    "        sent = preprocess_sentence(raw_sentence)\n",
    "        corpus.append(sent)\n",
    "\n",
    "    # train word2vec\n",
    "    model = train_w2v(corpus,\n",
    "                      model='skipgram',\n",
    "                      dim=200,\n",
    "                      min_count=20,\n",
    "                      lr=0.015,\n",
    "                      epoch=20,\n",
    "                      ws=7,\n",
    "                      minn=3,\n",
    "                      maxn=6)\n",
    "    # save model\n",
    "    model.save_model(W2V_MODEL)\n",
    "\n",
    "else: # load prebuilt model\n",
    "    model = fasttext.load_model(W2V_MODEL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "word2vec ---> \"Simple But Tough to Beat ..\" encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_encoder = build_from_fasttext_bin(model, preprocessor=preprocess_sentence, weighted=True)\n",
    "\n",
    "del model # free some memory !"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split train/test and transform sentences to their embedding representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train.shape =  (5068, 200)\n",
      "X_test.shape =  (1267, 200)\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = \\\n",
    "    train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "X_train = sentence_encoder.fit_transform(X_train)\n",
    "X_test = sentence_encoder.transform(X_test)\n",
    "\n",
    "print('X_train.shape = ', X_train.shape)\n",
    "print('X_test.shape = ', X_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can train a binary classification net:\n",
    "  - 1 hidden layer (128).\n",
    "  - dropout ~ [0.2 - 0.5].\n",
    "  - binary logloss."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 5068 samples, validate on 1267 samples\n",
      "Epoch 1/25\n",
      "5068/5068 [==============================] - 1s 113us/sample - loss: 0.6582 - accuracy: 0.6845 - val_loss: 0.5938 - val_accuracy: 0.8240\n",
      "Epoch 2/25\n",
      "5068/5068 [==============================] - 0s 37us/sample - loss: 0.5274 - accuracy: 0.8094 - val_loss: 0.4482 - val_accuracy: 0.8564\n",
      "Epoch 3/25\n",
      "5068/5068 [==============================] - 0s 40us/sample - loss: 0.4261 - accuracy: 0.8348 - val_loss: 0.3838 - val_accuracy: 0.8611\n",
      "Epoch 4/25\n",
      "5068/5068 [==============================] - 0s 39us/sample - loss: 0.3768 - accuracy: 0.8500 - val_loss: 0.3473 - val_accuracy: 0.8737\n",
      "Epoch 5/25\n",
      "5068/5068 [==============================] - 0s 40us/sample - loss: 0.3421 - accuracy: 0.8696 - val_loss: 0.3059 - val_accuracy: 0.8848\n",
      "Epoch 6/25\n",
      "5068/5068 [==============================] - 0s 37us/sample - loss: 0.3190 - accuracy: 0.8757 - val_loss: 0.2913 - val_accuracy: 0.8863\n",
      "Epoch 7/25\n",
      "5068/5068 [==============================] - 0s 35us/sample - loss: 0.3018 - accuracy: 0.8820 - val_loss: 0.2944 - val_accuracy: 0.8934\n",
      "Epoch 8/25\n",
      "5068/5068 [==============================] - 0s 36us/sample - loss: 0.2894 - accuracy: 0.8828 - val_loss: 0.2677 - val_accuracy: 0.8895\n",
      "Epoch 9/25\n",
      "5068/5068 [==============================] - 0s 38us/sample - loss: 0.2792 - accuracy: 0.8897 - val_loss: 0.2596 - val_accuracy: 0.8982\n",
      "Epoch 10/25\n",
      "5068/5068 [==============================] - 0s 36us/sample - loss: 0.2721 - accuracy: 0.8923 - val_loss: 0.2532 - val_accuracy: 0.8990\n",
      "Epoch 11/25\n",
      "5068/5068 [==============================] - 0s 36us/sample - loss: 0.2653 - accuracy: 0.8984 - val_loss: 0.2505 - val_accuracy: 0.9029\n",
      "Epoch 12/25\n",
      "5068/5068 [==============================] - 0s 36us/sample - loss: 0.2594 - accuracy: 0.8968 - val_loss: 0.2459 - val_accuracy: 0.8974\n",
      "Epoch 13/25\n",
      "5068/5068 [==============================] - 0s 39us/sample - loss: 0.2541 - accuracy: 0.8990 - val_loss: 0.2431 - val_accuracy: 0.8982\n",
      "Epoch 14/25\n",
      "5068/5068 [==============================] - 0s 38us/sample - loss: 0.2474 - accuracy: 0.9021 - val_loss: 0.2402 - val_accuracy: 0.8958\n",
      "Epoch 15/25\n",
      "5068/5068 [==============================] - 0s 35us/sample - loss: 0.2455 - accuracy: 0.9011 - val_loss: 0.2410 - val_accuracy: 0.9100\n",
      "Epoch 16/25\n",
      "5068/5068 [==============================] - 0s 34us/sample - loss: 0.2415 - accuracy: 0.9051 - val_loss: 0.2456 - val_accuracy: 0.9084\n",
      "Epoch 17/25\n",
      "5068/5068 [==============================] - 0s 36us/sample - loss: 0.2410 - accuracy: 0.9021 - val_loss: 0.2413 - val_accuracy: 0.9108\n",
      "Epoch 18/25\n",
      "5068/5068 [==============================] - 0s 37us/sample - loss: 0.2356 - accuracy: 0.9067 - val_loss: 0.2361 - val_accuracy: 0.8974\n",
      "Epoch 19/25\n",
      "5068/5068 [==============================] - 0s 41us/sample - loss: 0.2324 - accuracy: 0.9071 - val_loss: 0.2348 - val_accuracy: 0.9116\n",
      "Epoch 20/25\n",
      "5068/5068 [==============================] - 0s 36us/sample - loss: 0.2326 - accuracy: 0.9043 - val_loss: 0.2324 - val_accuracy: 0.9084\n",
      "Epoch 21/25\n",
      "5068/5068 [==============================] - 0s 35us/sample - loss: 0.2307 - accuracy: 0.9045 - val_loss: 0.2333 - val_accuracy: 0.8958\n",
      "Epoch 22/25\n",
      "5068/5068 [==============================] - 0s 35us/sample - loss: 0.2283 - accuracy: 0.9102 - val_loss: 0.2408 - val_accuracy: 0.9077\n",
      "Epoch 23/25\n",
      "5068/5068 [==============================] - 0s 39us/sample - loss: 0.2252 - accuracy: 0.9122 - val_loss: 0.2291 - val_accuracy: 0.9108\n",
      "Epoch 24/25\n",
      "5068/5068 [==============================] - 0s 35us/sample - loss: 0.2239 - accuracy: 0.9086 - val_loss: 0.2299 - val_accuracy: 0.9116\n",
      "Epoch 25/25\n",
      "5068/5068 [==============================] - 0s 38us/sample - loss: 0.2239 - accuracy: 0.9106 - val_loss: 0.2284 - val_accuracy: 0.9061\n"
     ]
    }
   ],
   "source": [
    "MODEL_PT = 'model.h5'\n",
    "\n",
    "model = train_nn(\n",
    "    X_train,\n",
    "    y_train,\n",
    "    hidden_layers=(128,),\n",
    "    activation='relu',\n",
    "    dropout=0.4,\n",
    "    epochs=25,\n",
    "    batch_size=32,\n",
    "    # validation_split=0.1,\n",
    "    validation_data=(X_test, y_test),\n",
    "    patience=4,\n",
    "    shuffle=True,\n",
    "    optimizer='adam',\n",
    "    pt=MODEL_PT,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "91% accuracy with a pretty simple encoder ! that's nice !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "        FAKE       0.91      0.90      0.91       628\n",
      "        REAL       0.90      0.91      0.91       639\n",
      "\n",
      "    accuracy                           0.91      1267\n",
      "   macro avg       0.91      0.91      0.91      1267\n",
      "weighted avg       0.91      0.91      0.91      1267\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model = load_model(MODEL_PT)\n",
    "preds = model.predict_classes(X_test, batch_size=32)\n",
    "preds = preds.reshape(preds.shape[0])\n",
    "\n",
    "report = classification_report(y_test, preds, target_names=label_encoder.classes_)\n",
    "print(report)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}