CMFS_20NG.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implementation of CMFS on 20NG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Authors: Abhirav Gholba\n",
    "         Bhargav Srinivasa\n",
    "         Devashish Deshpande\n",
    "         Gauri Kholkar\n",
    "\"\"\"\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from pprint import pprint\n",
    "from gensim.corpora import Dictionary\n",
    "import numpy as np\n",
    "import operator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "newsgroups_train = fetch_20newsgroups(subset='train', remove=('headers', 'footers', 'quotes'))\n",
    "vec = CountVectorizer(stop_words='english')\n",
    "matrix = vec.fit_transform(newsgroups_train.data).T\n",
    "documents = len(newsgroups_train.filenames)\n",
    "categories = len(newsgroups_train.target_names)\n",
    "terms = matrix.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No. of documents: 11314\n",
      "No. of categories: 20\n",
      "matrix.shape: (101322, 11314)\n",
      "8\n",
      "<class 'scipy.sparse.csc.csc_matrix'>\n"
     ]
    }
   ],
   "source": [
    "print \"No. of documents: %d\\nNo. of categories: %d\" % (documents, categories)\n",
    "print \"matrix.shape: {0}\".format(matrix.shape)\n",
    "print newsgroups_train.target[10]\n",
    "print type(matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Term-category feature-appearance matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "term_category_mat = np.zeros((terms, categories))\n",
    "for doc in range(documents):\n",
    "    cat = newsgroups_train.target[doc]\n",
    "    for row in matrix.getcol(doc).nonzero()[0]:\n",
    "        term_category_mat[row][cat] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Term-category matrix shape: (101322, 20)\n",
      "101322\n"
     ]
    }
   ],
   "source": [
    "print \"Term-category matrix shape: {0}\".format(term_category_mat.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Perform CMFS on term-category matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  2.92724117e-08   7.28602676e-06   2.78311080e-06 ...,   5.88311595e-07\n",
      "    2.72664657e-06   1.18473278e-07]\n",
      " [  1.33770769e-06   2.15001978e-06   3.43219337e-07 ...,   5.26946106e-05\n",
      "    1.01717453e-05   1.35351518e-06]\n",
      " [  2.85134973e-07   2.77231631e-07   2.24045956e-07 ...,   9.16894515e-07\n",
      "    2.65595573e-07   2.88504370e-07]\n",
      " ..., \n",
      " [  3.66602108e-07   3.56440669e-07   2.88059087e-07 ...,   2.94716094e-07\n",
      "    3.41480022e-07   3.70934190e-07]\n",
      " [  3.66602108e-07   3.56440669e-07   2.88059087e-07 ...,   2.94716094e-07\n",
      "    3.41480022e-07   3.70934190e-07]\n",
      " [  3.66602108e-07   3.56440669e-07   2.88059087e-07 ...,   2.94716094e-07\n",
      "    3.41480022e-07   3.70934190e-07]]\n"
     ]
    }
   ],
   "source": [
    "term_freq_per_cat = np.cumsum(term_category_mat, axis=0)[-1, :]\n",
    "for term in range(terms):\n",
    "    # Frequency of the term across all categories\n",
    "    total_term_freq = sum(term_category_mat[term, :])\n",
    "    for cat in range(categories):\n",
    "        numerator = float((term_category_mat[term][cat] + 1) ** 2)\n",
    "        denominator = (total_term_freq + categories) * (term_freq_per_cat[cat] + terms)\n",
    "        term_category_mat[term][cat] = numerator / denominator\n",
    "        \n",
    "# Final CMFS matrix\n",
    "print term_category_mat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create term-cmfs dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "term_cmfs_dict = {}\n",
    "cmfs_max = np.max(term_category_mat, axis=1)\n",
    "# Integer to term mapping dictionary\n",
    "dictionary = vec.get_feature_names()\n",
    "for i in range(terms):\n",
    "    term = dictionary[i]\n",
    "    term_cmfs_dict[term] = cmfs_max[i]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extract top 10 features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------Selected features are-------\n",
      "\n",
      "Term: bike \t CMFS: 0.00120489196406\n",
      "Term: sale \t CMFS: 0.00112834000168\n",
      "Term: car \t CMFS: 0.000970606167276\n",
      "Term: encryption \t CMFS: 0.000957596432953\n",
      "Term: god \t CMFS: 0.000932343449918\n",
      "Term: hockey \t CMFS: 0.00085429558318\n",
      "Term: clipper \t CMFS: 0.000844342753261\n",
      "Term: windows \t CMFS: 0.000837428432144\n",
      "Term: space \t CMFS: 0.000751994970671\n",
      "Term: israel \t CMFS: 0.000747811750338\n"
     ]
    }
   ],
   "source": [
    "sorted_feature_list = sorted(term_cmfs_dict.items(), key=operator.itemgetter(1), reverse=True)[:10]\n",
    "print \"-------Selected features are-------\\n\"\n",
    "for term, cmfs in sorted_feature_list:\n",
    "    print \"Term: {0} \\t CMFS: {1}\".format(term, cmfs)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}