Sentiment-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\daniela.miranda\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import re\n",
    "import nltk\n",
    "from sklearn.datasets import load_files\n",
    "nltk.download('stopwords')\n",
    "import pickle\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\daniela.miranda\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing the dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use the load_files function from the sklearn_datasets library to import the dataset into our application. The load_files function automatically divides the dataset into data and target sets. For instance, in our case, we will pass it the path to the \"txt_sentoken\" directory. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_data = load_files(r\"txt_sentoken_data\")\n",
    "X, y = movie_data.data, movie_data.target"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "documents = []\n",
    "\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "stemmer = WordNetLemmatizer()\n",
    "\n",
    "for sen in range(0, len(X)):\n",
    "    # Remove all the special characters\n",
    "    document = re.sub(r'\\W', ' ', str(X[sen]))\n",
    "    \n",
    "    # remove all single characters\n",
    "    document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)\n",
    "    \n",
    "    # Remove single characters from the start\n",
    "    document = re.sub(r'\\^[a-zA-Z]\\s+', ' ', document) \n",
    "    \n",
    "    # Substituting multiple spaces with single space\n",
    "    document = re.sub(r'\\s+', ' ', document, flags=re.I)\n",
    "    \n",
    "    # Removing prefixed 'b'\n",
    "    document = re.sub(r'^b\\s+', '', document)\n",
    "    \n",
    "    # Converting to Lowercase\n",
    "    document = document.lower()\n",
    "    \n",
    "    # Lemmatization\n",
    "    document = document.split()\n",
    "\n",
    "    document = [stemmer.lemmatize(word) for word in document]\n",
    "    document = ' '.join(document)\n",
    "    \n",
    "    documents.append(document)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Converting text to numbers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bag of Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer(max_features=1500, min_df=5, max_df=0.7, stop_words=stopwords.words('english'))\n",
    "X = vectorizer.fit_transform(documents).toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "tfidfconverter = TfidfTransformer()\n",
    "X = tfidfconverter.fit_transform(X).toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training-testing sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Text Classification Model and Predicting Sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "                       max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=1000,\n",
       "                       n_jobs=None, oob_score=False, random_state=0, verbose=0,\n",
       "                       warm_start=False)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier = RandomForestClassifier(n_estimators=1000, random_state=0)\n",
    "classifier.fit(X_train, y_train) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = classifier.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[180  28]\n",
      " [ 30 162]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.87      0.86       208\n",
      "           1       0.85      0.84      0.85       192\n",
      "\n",
      "    accuracy                           0.85       400\n",
      "   macro avg       0.85      0.85      0.85       400\n",
      "weighted avg       0.85      0.85      0.85       400\n",
      "\n",
      "0.855\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
    "\n",
    "print(confusion_matrix(y_test,y_pred))\n",
    "print(classification_report(y_test,y_pred))\n",
    "print(accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving and Loading the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('text_classifier', 'wb') as picklefile:\n",
    "    pickle.dump(classifier,picklefile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('text_classifier', 'rb') as training_model:\n",
    "    model = pickle.load(training_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[180  28]\n",
      " [ 30 162]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.87      0.86       208\n",
      "           1       0.85      0.84      0.85       192\n",
      "\n",
      "    accuracy                           0.85       400\n",
      "   macro avg       0.85      0.85      0.85       400\n",
      "weighted avg       0.85      0.85      0.85       400\n",
      "\n",
      "0.855\n"
     ]
    }
   ],
   "source": [
    "y_pred2 = model.predict(X_test)\n",
    "\n",
    "print(confusion_matrix(y_test, y_pred2))\n",
    "print(classification_report(y_test, y_pred2))\n",
    "print(accuracy_score(y_test, y_pred2)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp-workshop",
   "language": "python",
   "name": "nlp-workshop"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}