identify_intent_CV.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle as pk\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "import re\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from sklearn.model_selection import train_test_split\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset successfully loaded!\n"
     ]
    }
   ],
   "source": [
    "# Load the dataset and prepare it to the train the model\n",
    "\n",
    "# Importing dataset and splitting into words and labels\n",
    "dataset = pd.read_csv('intents.csv', names=[\"Query\", \"Intent\"])\n",
    "\n",
    "queries = dataset[\"Query\"]\n",
    "\n",
    "intent = list(dataset[\"Intent\"])\n",
    "unique_intent_list = list(set(intent))\n",
    "\n",
    "# print(queries)\n",
    "# print(unique_intent_list)\n",
    "print(\"Dataset successfully loaded!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus created\n"
     ]
    }
   ],
   "source": [
    "queryCorpus = []\n",
    "ps = PorterStemmer()\n",
    "\n",
    "for query in queries:\n",
    "    query = re.sub('[^a-zA-Z0-9]', ' ', query)\n",
    "\n",
    "    # Tokenize sentence\n",
    "    query = query.split(' ')\n",
    "\n",
    "    # Lemmatizing\n",
    "    tokenized_query = [ps.stem(word.lower()) for word in query]\n",
    "\n",
    "    # Recreate the sentence from tokens\n",
    "    tokenized_query = ' '.join(tokenized_query)\n",
    "\n",
    "    # Add to corpus\n",
    "    queryCorpus.append(tokenized_query)\n",
    "# print(queryCorpus)\n",
    "print(\"Corpus created\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " ...\n",
      " [0 0 0 ... 0 1 0]\n",
      " [0 0 0 ... 0 0 1]\n",
      " [0 0 0 ... 0 0 1]]\n",
      "Bag of words created!\n"
     ]
    }
   ],
   "source": [
    "intent_CV= CountVectorizer(max_features=1500)\n",
    "corpus = intent_CV.fit_transform(queryCorpus).toarray()\n",
    "print(corpus)\n",
    "# print(len(corpus))\n",
    "print(\"Bag of words created!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Intent vector saved!\n"
     ]
    }
   ],
   "source": [
    "# Save the CountVectorizer of Intent\n",
    "pk.dump(intent_CV, open('saved_state/intent_cv', 'wb'))\n",
    "print('Intent vector saved!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoded the classes!\n",
      "[ 2  2 10  2 25 25 25 15 20  7  2  9  1  2  2  7 23 23 23 23 23 23 23 23\n",
      " 23 23 23 23  3  3 19 20 20 10 21 21 21 21 20 13 13 13 17 17 17 12 11 16\n",
      "  7 15  4 15 15 15 15 12 12 12 12 15  3 16  3  1 12 23 21  8  8  8  8  8\n",
      "  8  8 22 22 22 22 22 24 24 24 24 24 24 24 24 24 24  5  5  5  0  0  0  0\n",
      "  0  0  0  0  0  0  0  0  0  0 14 14  6 14  6  6  6  6  6  6  6  6 14 14\n",
      " 14 18 18 18 18 18 18 18]\n"
     ]
    }
   ],
   "source": [
    "# Encode the intents\n",
    "labelencoder_intent = LabelEncoder()\n",
    "intent = labelencoder_intent.fit_transform(intent)\n",
    "print(\"Encoded the classes!\")\n",
    "print(intent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'AskingHelp': 0, 'BestPractice': 1, 'Cultivation': 2, 'CultivationSeason': 3, 'Diseases': 4, 'EnquireAboutDay': 5, 'Family': 6, 'Fertilizer': 7, 'Greeting': 8, 'HarvestTime': 9, 'Irrigation': 10, 'Location': 11, 'MarketPrice': 12, 'Maturity': 13, 'OutOfScope': 14, 'Pesticide': 15, 'Rainfall': 16, 'ReapingSeason': 17, 'Sarcasm': 18, 'SeedDensity': 19, 'Soil': 20, 'Varieties': 21, 'Wassup': 22, 'Weather': 23, 'Wellness': 24, 'Yield': 25}\n",
      "Label mapping obtained!\n"
     ]
    }
   ],
   "source": [
    "# Return a dict mapping labels to their integer values\n",
    "res = {}\n",
    "for cl in labelencoder_intent.classes_:\n",
    "    res.update({cl:labelencoder_intent.transform([cl])[0]})\n",
    "\n",
    "intent_label_map = res\n",
    "print(intent_label_map)\n",
    "print(\"Label mapping obtained!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset split into train and test set\n",
      "[[0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " ...\n",
      " [0 0 0 ... 0 1 0]\n",
      " [0 0 0 ... 0 0 1]\n",
      " [0 0 0 ... 0 0 0]]\n",
      "[11  2 21 12  6 22 18 21  8  0  6  2 22  3 23  8 18 18 17 22  3 24 25 23\n",
      " 16 10 20  6  0 17  0 18  2 24 15  7 23 13 14 12  0  2 23 23 17  6 24 24\n",
      " 14  1 18  2  5  3  9  0  5 15 23  5 23 12 24  1  0 15  8 15 20  8  3 12\n",
      " 12 21 15 14 24  0 24 25  8 22  7  8 14 24  0  7 13  6  4 24  2  0 25 21\n",
      " 23 12 21 20 22 15 23 13  0  6  6  0]\n"
     ]
    }
   ],
   "source": [
    "# Splitting the dataset into the Training set and Test set\n",
    "query_train, query_test, intent_train, intent_test = train_test_split(corpus, intent, test_size = 0.15, random_state = 19)\n",
    "\n",
    "print(\"Dataset split into train and test set\")\n",
    "print(query_train)\n",
    "print(intent_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model trained successfully!\n"
     ]
    }
   ],
   "source": [
    "# Fit the classifier to dataset\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "classifier = GaussianNB()\n",
    "classifier.fit(query_train, intent_train)\n",
    "print(\"Model trained successfully!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0 18 19 10 14 23  0  6 23 23 24  0 16  6 23 15  8 14 20 18]\n",
      "[ 0 24 12  7 18 23  0  6 23 23 24  0 12  6 23 15  8 18 20  6]\n"
     ]
    }
   ],
   "source": [
    "intent_pred = classifier.predict(query_test)\n",
    "print(intent_test)\n",
    "print(intent_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  1.0\n",
      "Precision:  0.6666666666666666\n",
      "Recall:  1.0\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "cm = confusion_matrix(intent_test, intent_pred)\n",
    "\n",
    "# Model Performace\n",
    "accuracy = (cm[1][1]+cm[0][0])/(cm[0][0]+cm[0][1]+cm[1][0]+cm[1][1])\n",
    "precision = cm[1][1]/(cm[0][0]+cm[0][1])\n",
    "recall = cm[1][1]/(cm[1][1]+cm[1][0])\n",
    "\n",
    "print(\"Accuracy: \", accuracy)\n",
    "print(\"Precision: \", precision)\n",
    "print(\"Recall: \", recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}