session-01-naive-bayes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d50057d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import os\n",
    "import codecs\n",
    "import math\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from mllab.NaiveBayes import NaiveBayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "db46bca7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100 13.7M  100 13.7M    0     0  1699k      0  0:00:08  0:00:08 --:--:-- 2438k\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Downloading the datasets in specific directory\"\"\"\n",
    "!mkdir datasets/newsgroups\n",
    "!curl -o datasets/newsgroups/news.tar.gz \"http://qwone.com/~jason/20Newsgroups/20news-bydate.tar.gz\"\n",
    "\n",
    "\"\"\"Extract the files\"\"\"\n",
    "!gzip -d < datasets/newsgroups/news.tar.gz | tar xf - --directory datasets/newsgroups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "eb32099e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No. of categories :  20\n",
      "----------------------------------\n",
      "No. of samples:  480  in  alt.atheism\n",
      "No. of samples:  584  in  comp.graphics\n",
      "No. of samples:  591  in  comp.os.ms-windows.misc\n",
      "No. of samples:  590  in  comp.sys.ibm.pc.hardware\n",
      "No. of samples:  578  in  comp.sys.mac.hardware\n",
      "No. of samples:  593  in  comp.windows.x\n",
      "No. of samples:  585  in  misc.forsale\n",
      "No. of samples:  594  in  rec.autos\n",
      "No. of samples:  598  in  rec.motorcycles\n",
      "No. of samples:  597  in  rec.sport.baseball\n",
      "No. of samples:  600  in  rec.sport.hockey\n",
      "No. of samples:  595  in  sci.crypt\n",
      "No. of samples:  591  in  sci.electronics\n",
      "No. of samples:  594  in  sci.med\n",
      "No. of samples:  593  in  sci.space\n",
      "No. of samples:  599  in  soc.religion.christian\n",
      "No. of samples:  546  in  talk.politics.guns\n",
      "No. of samples:  564  in  talk.politics.mideast\n",
      "No. of samples:  465  in  talk.politics.misc\n",
      "No. of samples:  377  in  talk.religion.misc\n",
      "----------------------------------\n",
      "Total number of samples  11314\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Get all the directory and subdirectory in the folder\"\"\"\n",
    "\"\"\"Count documents in each category\"\"\"\n",
    "def showDirectory(path):\n",
    "    global subdirs\n",
    "    subdirs = [subdir for subdir in os.listdir(path)]\n",
    "    print(\"No. of categories : \", len(subdirs))\n",
    "    print(\"----------------------------------\")\n",
    "    sum = 0\n",
    "    for subdir in sorted(subdirs):\n",
    "        samples_per_category = len(os.listdir(os.path.join(path,subdir)))\n",
    "        sum = sum + samples_per_category\n",
    "        print(\"No. of samples: \", samples_per_category, \" in \", subdir)\n",
    "    print(\"----------------------------------\")    \n",
    "    print(\"Total number of samples \", sum)\n",
    "\n",
    "train_path = 'datasets/newsgroups/20news-bydate-train'\n",
    "test_path = 'datasets/newsgroups/20news-bydate-test'\n",
    "\n",
    "showDirectory(train_path)\n",
    "#showDirectory(test_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f20bc1a",
   "metadata": {},
   "source": [
    "# Extract and export datasets to pandas to csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8147b69d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def CreateDataFrame(path, writepath):\n",
    "    df = pd.DataFrame()\n",
    "    global subdirs\n",
    "    count = 0\n",
    "    subdirs = [subdir for subdir in os.listdir(path)]\n",
    "    for subdir in sorted(subdirs):\n",
    "        filepath = os.path.join(path,subdir)\n",
    "        uniquefile = [file for file in os.listdir(filepath)]\n",
    "        for unique in uniquefile:\n",
    "            finalpath = os.path.join(filepath, unique)\n",
    "            with codecs.open(finalpath, encoding='latin1') as doc:\n",
    "                doc = doc.read().lower()\n",
    "                _header, _blank_line, body = doc.partition('\\n\\n')\n",
    "                df = df.append({'body': body, 'news_category': subdir}, ignore_index=True)\n",
    "    df.to_csv(writepath, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8d9c4b65",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_path = 'datasets/newsgroups/20news-bydate-train'\n",
    "write_train = 'datasets/news_train.csv'\n",
    "test_path = 'datasets/newsgroups/20news-bydate-test'\n",
    "write_test = 'datasets/news_test.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "eb94e316",
   "metadata": {},
   "outputs": [],
   "source": [
    "CreateDataFrame(train_path, write_train)\n",
    "CreateDataFrame(test_path, write_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "725bf63a",
   "metadata": {},
   "outputs": [],
   "source": [
    "news_train = pd.read_csv('datasets/news_train.csv')\n",
    "news_test = pd.read_csv('datasets/news_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2691b396",
   "metadata": {},
   "source": [
    "__convert text category to numerical form__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fb3b4d0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "news_train['news_category'] = news_train['news_category'].astype('category')\n",
    "news_train['news_category'] = news_train['news_category'].cat.codes\n",
    "\n",
    "news_test['news_category'] = news_test['news_category'].astype('category')\n",
    "news_test['news_category'] = news_test['news_category'].cat.codes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8468b7f8",
   "metadata": {},
   "source": [
    "__cleaning data__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5e889625",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_data(df):\n",
    "    dfs = pd.DataFrame()\n",
    "    dfs['text'] = df\n",
    "    #convert text to lower case\n",
    "    dfs['text']  = dfs['text'].str.lower()\n",
    "    #remove punctuations\n",
    "    #remove all non alphabetic characters\n",
    "    regex_1 = re.compile(r'[^a-zA-Z]')\n",
    "    dfs['text'] = pd.Series(dfs['text']).str.replace(regex_1, ' ')\n",
    "    #reduce multi-space to single space\n",
    "    regex_2 = re.compile(r' +')\n",
    "    dfs['text'] = pd.Series(dfs['text']).str.replace(regex_2,' ')\n",
    "    return dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "602dfe8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "news_train['body'] = clean_data(news_train['body'])\n",
    "news_test['body'] = clean_data(news_test['body'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37719135",
   "metadata": {},
   "source": [
    "__Convert to unicode__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2a5a91a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainx = news_train['body'].astype('U')\n",
    "trainy = news_train['news_category']\n",
    "\n",
    "testx = news_test['body'].astype('U')\n",
    "testy = news_test['news_category']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9c2d425",
   "metadata": {},
   "source": [
    "__Convert to numpy array__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cb70ba88",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainx = np.array(trainx)\n",
    "trainy = np.array(trainy)\n",
    "\n",
    "testx = np.array(testx)\n",
    "testy = np.array(testy)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7bf9599",
   "metadata": {},
   "source": [
    "# Final magic : Naive Bayes :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c1ac112e",
   "metadata": {},
   "outputs": [],
   "source": [
    "nb = NaiveBayes()\n",
    "nb.train(trainx, trainy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "71ec7754",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = nb.infer(testx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8a9535b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 73.82 %\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "print('Accuracy: %2.2f %%' % (100. * metrics.accuracy_score(testy, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2504a07",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}