Twitter sentiment analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "318669ce",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\manis\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\manis\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn import model_selection, preprocessing, linear_model, metrics\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn import ensemble\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords\n",
    "from textblob import Word\n",
    "nltk.download('wordnet')\n",
    "\n",
    "from termcolor import colored\n",
    "from warnings import filterwarnings\n",
    "filterwarnings('ignore')\n",
    "\n",
    "from sklearn import set_config\n",
    "set_config(print_changed_only = False)\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "29f9d646",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\manis'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "13d906ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set = pd.read_csv(\"C:/Users/manis/Twitter sentiment analysis_train.csv\",\n",
    "                   encoding = \"utf-8\",\n",
    "                   engine = \"python\",\n",
    "                   header = 0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "546c0f15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>@user when a father is dysfunctional and is s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>@user @user thanks for #lyft credit i can't us...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>bihday your majesty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>#model   i love u take with u all the time in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>factsguide: society now    #motivation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>[2/2] huge fan fare and big talking before the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>@user camping tomorrow @user @user @user @use...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>the next school year is the year for exams.ð...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>we won!!! love the land!!! #allin #cavs #champ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>@user @user welcome here !  i'm   it's so #gr...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  label                                              tweet\n",
       "0   1      0   @user when a father is dysfunctional and is s...\n",
       "1   2      0  @user @user thanks for #lyft credit i can't us...\n",
       "2   3      0                                bihday your majesty\n",
       "3   4      0  #model   i love u take with u all the time in ...\n",
       "4   5      0             factsguide: society now    #motivation\n",
       "5   6      0  [2/2] huge fan fare and big talking before the...\n",
       "6   7      0   @user camping tomorrow @user @user @user @use...\n",
       "7   8      0  the next school year is the year for exams.ð...\n",
       "8   9      0  we won!!! love the land!!! #allin #cavs #champ...\n",
       "9  10      0   @user @user welcome here !  i'm   it's so #gr..."
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "12797309",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(31962, 3)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "07887a0e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 31962 entries, 0 to 31961\n",
      "Data columns (total 3 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   id      31962 non-null  int64 \n",
      " 1   label   31962 non-null  int64 \n",
      " 2   tweet   31962 non-null  object\n",
      "dtypes: int64(2), object(1)\n",
      "memory usage: 749.2+ KB\n"
     ]
    }
   ],
   "source": [
    "#get general information about train set\n",
    "\n",
    "train_set.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e6252e5d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check whether there are duplicated values\n",
    "\n",
    "train_set.duplicated().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3296f2a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>29720</td>\n",
       "      <td>29720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2242</td>\n",
       "      <td>2242</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  tweet\n",
       "label              \n",
       "0      29720  29720\n",
       "1       2242   2242"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#get the number of classes of the \"label\" variable of train set\n",
    "\n",
    "train_set.groupby(\"label\").count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36d52f8e",
   "metadata": {},
   "source": [
    "## Feature Engineering ( PROCESS DATA )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8d3cc6ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m\n",
      "CONVERTED SUCCESFULLY...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "#convert uppercase letters to lowercase letters in \"tweet\" columns\n",
    "\n",
    "train_set[\"tweet\"] = train_set[\"tweet\"].apply(lambda x: \" \".join(x.lower() for x in x.split()))\n",
    "\n",
    "print(colored(\"\\nCONVERTED SUCCESFULLY...\", \"green\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6b8aeb27",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m\n",
      "DELETED SUCCESFULLY...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "#delete punctuation marks from \"tweet\" columns\n",
    "\n",
    "train_set[\"tweet\"] = train_set[\"tweet\"].str.replace('[^\\w\\s]','')\n",
    "\n",
    "\n",
    "print(colored(\"\\nDELETED SUCCESFULLY...\", \"green\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "44f29e93",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m\n",
      " NUMBERS DELETED SUCCESFULLY...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "#delete numbers from \"tweet\" columns\n",
    "\n",
    "train_set['tweet'] = train_set['tweet'].str.replace('\\d','')\n",
    "\n",
    "\n",
    "print(colored(\"\\n NUMBERS DELETED SUCCESFULLY...\", \"green\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "81db50b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m\n",
      "STOPWORDS DELETED SUCCESFULLY...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "#delete stopwords from \"tweet\" columns\n",
    "\n",
    "sw = stopwords.words(\"english\")\n",
    "train_set['tweet'] = train_set['tweet'].apply(lambda x: \" \".join(x for x in x.split() if x not in sw))\n",
    "\n",
    "print(colored(\"\\nSTOPWORDS DELETED SUCCESFULLY...\", \"green\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6b4f5fef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m\n",
      "DONE SUCCESFULLY...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "#lemmatization. That is, we get the roots of the words in the \"tweet\" columns\n",
    "\n",
    "train_set['tweet'] = train_set['tweet'].apply(lambda x: \" \".join([Word(word).lemmatize() for word in x.split()]))\n",
    "\n",
    "print(colored(\"\\nDONE SUCCESFULLY...\", \"green\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a0c9a88b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m\n",
      "'ID' COLUMNS DROPPED SUCCESFULLY...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "#drop \"id\" column from datasets\n",
    "\n",
    "train_set = train_set.drop(\"id\", axis = 1)\n",
    "\n",
    "print(colored(\"\\n'ID' COLUMNS DROPPED SUCCESFULLY...\", \"green\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "fbb97457",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>user father dysfunctional selfish drag kid dys...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>user user thanks lyft credit cant use cause do...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>bihday majesty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>model love u take u time urð ðððð ððð</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>factsguide society motivation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>huge fan fare big talking leave chaos pay disp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>user camping tomorrow user user user user user...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>next school year year examsð cant think ð scho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>love land allin cavs champion cleveland clevel...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>user user welcome im gr</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label                                              tweet\n",
       "0      0  user father dysfunctional selfish drag kid dys...\n",
       "1      0  user user thanks lyft credit cant use cause do...\n",
       "2      0                                     bihday majesty\n",
       "3      0              model love u take u time urð ðððð ððð\n",
       "4      0                      factsguide society motivation\n",
       "5      0  huge fan fare big talking leave chaos pay disp...\n",
       "6      0  user camping tomorrow user user user user user...\n",
       "7      0  next school year year examsð cant think ð scho...\n",
       "8      0  love land allin cavs champion cleveland clevel...\n",
       "9      0                            user user welcome im gr"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#look at the latest condition of train set\n",
    "\n",
    "train_set.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2904a9e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# split dataset into train test\n",
    "\n",
    "\n",
    "x = train_set[\"tweet\"]\n",
    "y = train_set[\"label\"]\n",
    "\n",
    "train_x, test_x, train_y, test_y = model_selection.train_test_split(x, y,\n",
    "                                                                    test_size = 0.25,\n",
    "                                                                    shuffle = True,\n",
    "                                                                    random_state = 11)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e944765",
   "metadata": {},
   "source": [
    "## VECTORIZE DATA"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d5cae72",
   "metadata": {},
   "source": [
    "### \"Count Vectors\" method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7d06bf0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0]], dtype=int64)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = CountVectorizer()\n",
    "vectorizer.fit(train_x)\n",
    "\n",
    "x_train_count = vectorizer.transform(train_x)\n",
    "x_test_count = vectorizer.transform(test_x)\n",
    "\n",
    "x_train_count.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f230c79",
   "metadata": {},
   "source": [
    "### \"TF-IDF\" method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ac07c726",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_idf_word_vectorizer = TfidfVectorizer()\n",
    "tf_idf_word_vectorizer.fit(train_x)\n",
    "\n",
    "x_train_tf_idf_word = tf_idf_word_vectorizer.transform(train_x)\n",
    "x_test_tf_idf_word = tf_idf_word_vectorizer.transform(test_x)\n",
    "\n",
    "x_train_tf_idf_word.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30e8bdff",
   "metadata": {},
   "source": [
    "## BUILDING MACHINE LEARNING MODELS"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d00e6a3",
   "metadata": {},
   "source": [
    "### Logistic regression model with \"count-vectors\" method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "57feca4e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Logistic regression model with 'count-vectors' method\n",
      "Accuracy ratio:  0.9463148496240601\n"
     ]
    }
   ],
   "source": [
    "log = linear_model.LogisticRegression()\n",
    "log_model = log.fit(x_train_count, train_y)\n",
    "accuracy = model_selection.cross_val_score(log_model,\n",
    "                                           x_test_count,\n",
    "                                           test_y,\n",
    "                                           cv = 20).mean()\n",
    "\n",
    "print(\"\\nLogistic regression model with 'count-vectors' method\")\n",
    "print(\"Accuracy ratio: \", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "69cb4335",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred=log_model.predict(x_test_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "92b90bd5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[7352,   40],\n",
       "       [ 306,  293]], dtype=int64)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(test_y,pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "688ba076",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.99      0.98      7392\n",
      "           1       0.88      0.49      0.63       599\n",
      "\n",
      "    accuracy                           0.96      7991\n",
      "   macro avg       0.92      0.74      0.80      7991\n",
      "weighted avg       0.95      0.96      0.95      7991\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(test_y,pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "02b4a805",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqM0lEQVR4nO3dd5xU1f3/8deHhQWUDhYEEVQUISIKomIJdrAnYo8GjEEUTey9xJJi+flVgyUEDV+/Go0FBSuiBjEalSK9KEGQDRiKdClbPr8/zkU2yzI77O7MnfJ+Ph7z2Lkzd+589sLOZ84953yOuTsiIiLbUifuAEREJLMpUYiISEJKFCIikpAShYiIJKREISIiCSlRiIhIQkoUIiKSkBKFSAJmNt/M1pvZWjP71syGm1mjcs/3MrMPzGyNma0ys9fNrHOFYzQxs4fN7JvoOHOj7Vbp/41Etp8ShUjVTnX3RkA34EDgZgAzOwx4FxgJ7AZ0AKYAH5vZntE+hcD7QBegD9AE6AUsB3qm9bcQqSbTzGyRbTOz+cAl7v5etH0/0MXdTzazj4Bp7n55hde8DSx194vM7BLgt8Be7r42zeGL1Aq1KESSZGZtgb7AXDPbgdAyeKmSXV8Ejo/uHwe8oyQh2UyJQqRqr5nZGmAhsAS4E2hB+PtZXMn+i4HN/Q8tt7GPSNZQohCp2hnu3hjoDXQiJIEVQBnQupL9WwPLovvLt7GPSNZQohBJkrt/CAwHHnT3dcA/gbMq2fVsQgc2wHvAiWa2Y1qCFEkBJQqR7fMwcLyZdQNuAn5uZr8ys8Zm1tzM7gUOA+6K9v8/wiWrV8ysk5nVMbOWZnaLmZ0Uxy8gsr2UKES2g7svBZ4Bbnf3fwAnAj8l9EMsIAyfPcLdv4r230jo0J4NjAFWA58TLl99lvZfQKQaNDxWREQSUotCREQSUqIQEZGElChERCQhJQoREUmobtwBbK9WrVp5+/bt4w5DRCSrTJw4cZm771Sd12Zdomjfvj0TJkyIOwwRkaxiZguq+1pdehIRkYSUKEREJCElChERSUiJQkREElKiEBGRhJQoREQkoZQlCjN72syWmNn0bTxvZvaomc01s6lmdlCqYhERkepL5TyK4cAQQknmyvQFOka3Q4Anop8iOcPd2VhSFncYIjWSskTh7uPMrH2CXU4HnvFQ5/xTM2tmZq3dPaPWFy4rc4rLwh+6O0z6ZgXfrdvEhPkrKKxbhzpmAMz5djU71K+LJXHML75ZSatGhWDJ7C1p486UolU026EedevUzr/NsrWbauU4ItV1yDfT+MWEkTU6Rpwzs9sQVv7arCh6bKtEYWYDgYEA7dq1q9UgVn6/iW+++56/z17K8nUbaVQ/nJKV64t5c+piVq0vTvj6wrrh6l1xaRnusOdOVa94WVi3Dv9euZ4uuzWt+S8gtarXXi2pY8YeLXeotWOu21jCvrs2qbXjiSSjwcrlHPbkfXQaPYLVu7at0bHiTBSVfWWrdBUldx8KDAXo0aNHjVZaWrpmI6OmLGLsnCXMWLSa79Zt/Y2vXoFhGC12LGTXJg04aI9mtG0ePjhaNSqk4y6NKSyow947N6JBvYKahCMikhpn3gDvj4Kbb6bJbbfBjtVftj3ORFEE7F5uuy2wqDYOvHztRqYWreLDL5fy/aYSxn25jN1bNGTighWUVUgzfX+0K912b8bOTepzzL670HSHerURgohI+s2YAc2aQZs2cN99cPfd0KVLjQ8bZ6IYBVxhZi8QOrFX1bR/YsrCldzy6jRmLFq91XOrNxRzSIeWrFxfzCldW9Ove1t2adKgJm8nIpIZ1q2De+6B//f/4IILYPhw2HvvWjt8yhKFmT0P9AZamVkRcCdQD8DdnwTeAk4C5gLfAwOq+16Pvv8VD4358r8eG3B4e/p02ZUD2zX/oR9BRCTnvPkmDB4MCxbAxReHlkQtS+Wop/OqeN6BwTV9n7Oe/ITx81cA0LBeAXed3oV+B7WlTi2NWhERyViPPx6SROfOMG4cHHlkSt4m69ajKG/RyvU/JIkRl/fioHbNY45IRCTFSkpg6VJo3RrOPhvWr4crr4TCwpS9ZVZfk+n9wFgArj9xXyUJEcl9n38OBx8Mp50GpaXQqhVce21KkwRkcaI468lP2FQaJsINPrr2Om1ERDLOypVw+eVw6KGwZAnceCPUSd/Hd1ZeeiopLfvhktPY63rHG4yISCpNmwbHHx8uN/3qV2HIa5P0TuDMykTx7Kdh6dcLDmlH+1bVn0QiIpKxiouhXj3YZx84+mi4/no4KJ7aqVl56WnW4jUA3HFq55gjERGpZRs3bpkot3Yt1K8Pzz8fW5KALE0Uf5sQSkTVr6vyGSKSQz74ALp2hTvvDIlh48a4IwKyMFGUeajB0bShSm2ISI5Yvx4uvBCOPTYMf33nHXjhBWjZMu7IgCxMFBuKw0ingUftGXMkIiK1pEEDWLYMbrsNpk+HE0+MO6L/knWJYmNxKQCH7tki5khERGpg6tSQEIqKwto0b74Z6jU1bBh3ZFvJvkQRzZ1o31KjnUQkC61bt2UE06RJ8NVX4fE0zovYXpkb2TZsruDUYsfUzkQUEal1o0aFukwPPhgK+M2ZE4a+Zrism0fx/aZSdq5XgGkZURHJNq+9FibL/eMfcPjhcUeTtKxLFO5OSZkWqxeRLFBcDI8+GloNBx0EjzwSOq7rZdeozay79LSxpIzdW9TeesYiIinx6afQowdcdx28+GJ4rHHjrEsSkIWJwsxopjkUIpKpVqyAQYOgVy/47jt49VX4/e/jjqpGsi5RFJeW0WW3pnGHISJSuaFDYdgwuPpqmDkTzjgjDH/NYlnXRwGwZkNx3CGIiGwxZ06o7nrEEXDVVdC3byjFkSOyrkUB0E5zKEQkE2zYEOoyde0aliR1D0X8cihJQJYmisKC7G7GiUgOGDMG9t8/VHrt1w9Gj876S0zbkpWXnuoVZGV+E5FcMW4cnHACdOwYEsZxx8UdUUpl5SduQZ3czNoiksFKS8NqcwBHHglPPRXqNeV4koAsTRQbSzThTkTS6IsvwnDXww+H//wnXGK6+OIweS4PZGWiaN00P/5xRCRma9bANdeEiXPz58MTT8DOO8cdVdplZR+FVrYTkZRbtSp0Vi9cCJdeGibNNW8ed1SxyMpEsbGkNO4QRCRXrV4dCvc1bQoDB4ZV5w47LO6oYpWVl56aq8S4iNS24mK4/35o2zasEwFhxbk8TxKQpS2KOjk6VllEYvLxx6E+0/TpoeTGTjvFHVFGycoWhUbHikitufLKUHpj1SoYOTIU8dt997ijyihZmiiUKUSkBty33N9111AKfOZMOO20+GLKYFmZKJQnRKTaZs8OCwmNHBm2b70VHngAGjWKN64MlpWJQi0KEdlu69fD7beHgn1TpoRtSUpKE4WZ9TGzOWY218xuquT5pmb2uplNMbMZZjYgqePWfqgiksvefz/Mibj3Xjj33FAW/Nxz444qa6Rs1JOZFQCPAccDRcB4Mxvl7jPL7TYYmOnup5rZTsAcM3vO3TclOnYd9WaLyPYoKoK6dUPCOOaYuKPJOqlsUfQE5rr7vOiD/wXg9Ar7ONDYzAxoBHwHlFR1YOUJEUmotBQeewz+/OewfdFF4XKTkkS1pDJRtAEWltsuih4rbwiwH7AImAb82t23qvhnZgPNbIKZTYi2UxOxiGS/SZPg0EPhiivCGhEQRsDUrx9vXFkslYmisk9zr7B9IjAZ2A3oBgwxsyZbvch9qLv3cPceoM5sEanE6tXw61/DwQeH+kzPPw8vvRR3VDkhlYmiCCg/a6UtoeVQ3gBghAdzga+BTlUdWGlCRLYyZQoMGRJmWM+eHTqr9aWyVqQyUYwHOppZBzMrBM4FRlXY5xvgWAAz2wXYF5hX1YHVohARAL7+Gp5+Otw/8kiYOzf0TTRrFmtYuSZlicLdS4ArgNHALOBFd59hZoPMbFC02z1ALzObBrwP3Ojuy6o6dqlXvIIlInll06ZQ9rtzZ7j2WlixIjzeoUO8ceWolBYFdPe3gLcqPPZkufuLgBO297gN6mXlPEERqQ0ffRQuL82cCT/9KTzySN6uE5EuWVk9tq7Gx4rkp6VL4YQTYJdd4PXX4ZRT4o4oL2TpV3MlCpG84Q5jxoT7O+0Eb7wBM2YoSaRRViYKNShE8sSMGfDjH4dWxNix4bFjj4Udd4w1rHyTpYlCmUIkp33/PdxyC3TrFpLFsGFw1FFxR5W3srKPQolCJIe5hzLgn38OP/95KAGuFedilZWJQnlCJActXgw77wwFBaE10bQp9O4dd1RCll56UqIQySGlpfDoo7DvvvD44+Gx009XksggWZkodOlJJEdMmAA9e4YaTb16wUknxR2RVCLpRGFmGTPMQIlCJAfcf39IEosXw9/+Bm+/DXvtFXdUUokqE4WZ9TKzmYQyHJjZAWb2eMojS0DDY0WylDsUF4f7PXvC4MEwaxacfbauKWewZFoU/0MoB74cwN2nALGOU9N6FCJZ6F//gj594KZoVeTeveGPfwyd1pLRkrr05O4LKzxUmoJYkqY8IZJFNm4Ma1X/6Efwz3/q8lIWSmZ47EIz6wV4VC78V0SXoeKiPgqRLDFxIvzsZ2F9iLPOgocfht12izsq2U7JJIpBwCOEZUyLgHeBy1MZVFWUJkSyRKNG4RLAW29B375xRyPVlEyi2NfdLyj/gJkdDnycmpCqVq9uVo7qFcl9ZWXwl7+ES0zDhoW5EdOnQx39zWazZP71/pjkYyKSz6ZPD/WYLrkEvvoK1q0LjytJZL1ttijM7DCgF7CTmV1T7qkmQEGqAxORLLFuHdx9Nzz0UBjB9Je/hBpN6kvMGYkuPRUCjaJ9Gpd7fDXQL5VBiUgW2bAhJIeLLgqT6Fq2jDsiqWXbTBTu/iHwoZkNd/cFaYxJRDJdUVGoz/T734fEMHs2tGgRd1SSIsl0Zn9vZg8AXYAGmx9092NSFpWIZKaSkjBJ7o47QjG/c86B7t2VJHJcMr1MzwGzgQ7AXcB8YHwKY6qSrnyKxOCzz6BHD7jmmtBpPWNGSBKS85JJFC3d/Smg2N0/dPeLgUNTHJeIZJKyMhgwAJYuhZdfDutWd+gQd1SSJslceooqeLHYzE4GFgFtUxeSiGQE95AU+vSBxo1hxAho0ybcl7ySTIviXjNrClwLXAcMA65KZVAiErOvvoITTwxVXYcODY916qQkkaeqbFG4+xvR3VXA0fDDzGwRyTUbN8J998Hvfgf168OQITBoUNxRScwSTbgrAM4m1Hh6x92nm9kpwC1AQ+DA9IRYWWxxvbNIjhs8GJ56Cs49N0yga9067ogkA5i7V/6E2XBgd+Bz4BBgAXAYcJO7v5am+LZSv3VHX7FgFjsUJtO9IiJVWrIkdFbvumu45DRvXrjsJDnFzCa6e4/qvDbRp20PoKu7l5lZA2AZsLe7f1udNxKRDFNWFgr33XgjnHBCWI60Y8dwEyknUWf2JncvA3D3DcCXShIiOWLqVDjiCLj0UujWDe66K+6IJIMlalF0MrOp0X0D9oq2DXB375ry6ESk9r38cuiDaN4cnnkmLCykjj9JIFGi2C9tUWwn09xske23ejU0aRLWqh48GO68U6U3JCnb7MzOVPVbd/SVC2bTsFCVzkWS8s03cOWVsGgRfPopFOhvJx/VpDM7pSuKmFkfM5tjZnPN7KZt7NPbzCab2Qwz+zCV8YjkleJiePBB2G8/eO+9MHkuy74YSmZI2RjTaB7GY8DxhLW2x5vZKHefWW6fZsDjQB93/8bMdk5VPCJ5ZcECOO200Gl96qmh4usee8QdlWSppFoUZtbQzPbdzmP3BOa6+zx33wS8AJxeYZ/zgRHu/g2Auy/ZzvcQkfI2txh23RV22QVefRVGjlSSkBqpMlGY2anAZOCdaLubmY1K4thtgIXltouix8rbB2huZmPNbKKZXZRU1CLy39zh2Wfh4INh7dpQfuPdd+GMMzSiSWosmRbFbwitg5UA7j4ZaJ/E6yr731nxAmldoDtwMnAicLuZ7bPVgcwGmtkEM5sQtpN4d5F8MWcOHHssXHgh1K0Ly5fHHZHkmGQSRYm7r6rGsYsIJUA2a0soUV5xn3fcfZ27LwPGAQdUPJC7D3X3HtXtsRfJSSUlYYhr164waRI88QR88okuM0mtSyZRTDez84ECM+toZn8EPknideOBjmbWwcwKgXOBipesRgJHmlldM9uBUFNq1nbEL5K/Cgrgo4+gX7/Qqhg0COqkdCCj5Klk/lddSVgveyPwV0K58auqepG7lwBXAKMJH/4vuvsMMxtkZoOifWYR+j6mEooPDnP36dX4PUTyw7ffwsUXw8KF4RrsW2/Bc8+FjmuRFKlywp2ZHejuX6QpnirVb93RV30zmwb1NGlI8khpaVhA6OabYf360HF91llxRyVZJNUT7h4ys9lmdo+ZdanOm4hIDXzxBfTqBZdfDj16wLRpShKSVlUmCnc/GugNLAWGmtk0M7st1YGJSGTIEJg/P1xiGjMG9tlqYKBISm1XrScz2x+4ATjH3QtTFlUCuvQkOc8dXnsN2reHAw+EFSvC482bxxmVZLmUXnoys/3M7DdmNh0YQhjx1LY6byYiVZg/P5Te+OlP4eGHw2PNmytJSKySqfX0F+B54AR3rzgPQkRqQ3FxWKP6rrvCENcHH4Rf/zruqESAJBKFux+ajkBE8tqf/gQ33RRKbjzyCLRrF3dEIj/YZqIwsxfd/Wwzm8Z/l96IfYU7lfCQnLB8ebjU1L07/PKXsPfe0KdP3FGJbCVRi2Jzu/eUdAQikjfcwxKk110HjRvDl1+GIn5KEpKhttmZ7e6Lo7uXu/uC8jfg8vSEJ5JjZs2Co4+G/v2hY8cwuqluypaFEakVyUy4O76Sx/rWdiAiOW/KFDjggLCY0NCh8I9/hIJ+IhkuUR/FZYSWw55mNrXcU42Bj1MdmEjOKCqCtm1DUrjrLvjFL2BnLeYo2WObE+7MrCnQHPg9UH696zXu/l0aYqtU/dYdfc3CORTWVZVMyXCLFsHVV4fCfbNnQ5uK63aJpE+qJty5u88HBgNryt0wsxbVeTORvFBaGspu7LdfWIb0hhugVau4oxKptkS9aH8ljHiaSBgeW35QqgN7pjAukey0YQMcdRSMHw/HHw+PPx6GvYpksW0mCnc/JfrZIX3hiGSp4mKoVw8aNAijmq65Bs45R5N+JCckU+vpcDPbMbr/MzN7yMw0bVQEwpyIl18OrYZJk8Jj990H556rJCE5I5ke4SeA783sAELl2AXA/6U0KpFsMG8enHxyWBuiZUstQyo5K5n/2SUehkadDjzi7o8QhsjGRl/UJHYPPQRduoQ1qx9+GD7/HLp1izsqkZRIZkroGjO7GbgQONLMCoB6qQ1LJMOtXQsnnRQK+LVV1X3Jbcm0KM4BNgIXu/u3QBvggZRGJZJpli2DAQNg1Kiwfdtt8MorShKSF5JZCvVb4DmgqZmdAmxw92dSHplIJigrg6efhn33hWefhblzw+Pqj5A8ksyop7OBz4GzgLOBz8ysX6oDE4ndzJnQu3coudG5M0yeHIa9iuSZZPoobgUOdvclAGa2E/Ae8HIqA0tEfdmSFhMmwIwZ8NRTodqrWhGSp5JJFHU2J4nIcpLr2xDJPm+9FRYUuvDCcDvlFGihijWS35L5wH/HzEabWX8z6w+8CbyV2rBE0qyoCPr1C/MihgwJE+nMlCRESK4z+3rgT0BX4ABgqLvfmOrARNKipCQMcd1vP3jzTfjtb8PcCE3WEflBovUoOgIPAnsB04Dr3P3f6QpMJC0mToSrrgrLkD72GOypWpciFSVqUTwNvAGcSagg+8e0RJQE07c9qYlVq2DEiHD/kEPgs89C34SShEilEnVmN3b3P0f355jZpHQEJJIy7vDii6EFsXw5zJ8Pu+0GPXvGHZlIRkuUKBqY2YFsGY3asPy2uytxSPb4179g8GAYPRq6d4fXXw9JQkSqlChRLAYeKrf9bbltB45JVVAitWrNmpAcysrg0Ufh8suhoCDuqESyRqKFi45OZyAitW7qVOjaFRo3DpPmDj1U61aLVIMmzknuWboUfv5zOOCA0EkNcOaZShIi1ZTSRGFmfcxsjpnNNbObEux3sJmVJltDSmOepFJlZTBsWCjg9/zzcMstoVaTiNRIMiU8qiVat+Ix4HigCBhvZqPcfWYl+90HjE5VLJInzjwTXnsNjjoKnngiFPITkRpLpnqsRWtl3xFttzOzZMYT9gTmuvs8d98EvEBYJa+iK4FXgCWVPCeS2Lp1YXY1wHnnwfDhMHaskoRILUrm0tPjwGHAedH2GkJLoSptgIXltouix35gZm2AnwBPJjqQmQ00swlmNiGJ95V88frrISE8/njYPvvs0DehCZkitSqZRHGIuw8GNgC4+wqgMInXVfbX6hW2HwZudPfSRAdy96Hu3sPdeyTxvpLrFi6En/4UTjstjGjq3j3uiERyWjJ9FMVRP4LDD+tRlCXxuiJg93LbbYFFFfbpAbwQleRoBZxkZiXu/lqiA+sLYx579lkYNCh0XP/hD3D11VCYzPcWEamuZBLFo8CrwM5m9lugH3BbEq8bD3Q0sw7Av4FzgfPL7+DuHTbfN7PhwBtVJQnJU5vLfrdtG0Yy/fGP0KFDlS8TkZqrMlG4+3NmNhE4lnA56Qx3n5XE60rM7ArCaKYC4Gl3n2Fmg6LnE/ZLiACwciXcfDPsuCM8+GBIEhryKpJWVSYKM2sHfA+8Xv4xd/+mqte6+1tUWORoWwnC3ftXdTzJI+5hLsQ114QJdFdfvaVVISJplcylpzcJ/RMGNAA6AHOALimMS/LZ11/DwIHw3ntw8MHw9ttw4IFxRyWSt5K59LR/+W0zOwi4NGURiRQXhzpNjz0Gl16qAn4iMdvumdnuPsnMDk5FMMnSwkU56P33w1KkDz0E++wDCxZAgwZxRyUiJNdHcU25zTrAQcDSlEUk+eU//4Frr4XnnoO99oJbb4WWLZUkRDJIMhPuGpe71Sf0WVRWikMkeWVl8Kc/QadOYdW522+HadNCkhCRjJKwRRFNtGvk7tenKR7JF6tWwW23QbduoYBfp05xRyQi27DNFoWZ1Y1KaxyUxngkl61dG/ogSkuheXP47DP44AMlCZEMl6hF8TkhSUw2s1HAS8C6zU+6+4gUxya5ZORIuPLKUKepWzc45hjYc8+4oxKRJCTTR9ECWE5YI/sU4NTop0jVFiyA00+HM86AZs3g449DkhCRrJGoRbFzNOJpOlsm3G1WsQqsyNbcoV8/mDkT7r8frroK6tWLOyoR2U6JEkUB0IjkyoWLbPHpp9ClSygBPnQotGgBe+wRd1QiUk2JEsVid787bZFI9vvuu1DAb+hQuOMOuOsuld4QyQGJEoWmP0ty3MM6EddeG5LFtdfC9RpRLZIrEiWKY9MWhWS3W24JiwgdeiiMGQMHHBB3RCJSi7aZKNz9u3QGIllmw4YwL6JVKxgwIPRBDBwIdZIZSCci2UR/1bL9xoyB/feHX/4ybO+zT1ieVElCJCfpL1uS9+23cP75cMIJYQGhK66IOyIRSYPtLjMueervf4ef/ATWr4ff/AZuvFEVXkXyhBKFJFZcHCbJde0Kxx8Pv/1tuNQkInlDl56kcmvWhHWqjzwyFPFr2RJeeklJQiQPKVHIf3OHESNgv/3gkUfChLmNG+OOSkRipEQhWyxbBqeeCmeeGYa9fvJJWCtihx3ijkxEYqREIVs0bhyWJn3oIZgwIUygE5G8p0SR7/7xD+jbN0yeq18/LCZ09dVQV+McRCRQoshXy5fDJZeEzuqZM2HevPC4Js2JSAX6VMg37jB8OOy7b/h5/fUhUXTtGndkIpKhdH0hHz3zTEgUTz4ZSnGIiCSgFkU+WL8e7rwTiopC6Y1XXoGPPlKSEJGkKFHkutGj4Uc/grvvhpEjw2PNm6svQkSSlnWfFlpNKUmLFsE550CfPqEExwcfwODBcUclIlko6xKFJOnee0ML4u67YcoUOProuCMSkSxl7h53DNulQeuOvmHxV3GHkZkmTtxSwG/5clixAvbeO+6oRCQDmNlEd+9RndemtEVhZn3MbI6ZzTWzmyp5/gIzmxrdPjEzraFZHatXw69+BT17hmVJIRTxU5IQkVqQskRhZgXAY0BfoDNwnpl1rrDb18CP3b0rcA8wNFXx5CT3UNG1UycYMgQuuwyefTbuqEQkx6RyHkVPYK67zwMwsxeA04GZm3dw90/K7f8p0DaF8eSev/4VfvazUOF15Eg4+OC4IxKRHJTKRNEGWFhuuwg4JMH+vwDeruwJMxsIDAQo3DXPL6ds2hTKbXTqBP36hTkS/furNpOIpEwq+ygqG8laac+5mR1NSBQ3Vva8uw919x7u3iOvh8eOGwfduoU1qzdsCEX8LrlESUJEUiqViaII2L3cdltgUcWdzKwrMAw43d2XpzCe7LVsGQwYAD/+cWhBPPmk1qsWkbRJ5VfR8UBHM+sA/Bs4Fzi//A5m1g4YAVzo7l+mMJbsNW9e6HtYvRpuugluv10LCYlIWqUsUbh7iZldAYwGCoCn3X2GmQ2Knn8SuANoCTxuZgAl1R3nm3NWr4YmTaBDh9Ca6N8/lOIQEUkzTbjLNN9/D/fcA0OHhhnVbTUQTERqriYT7tQLmknefBOuuALmzw+tiIYN445IRESJIiOUlMB558HLL8N++8GHH8JRR8UdlYgIoKKA8dp82a9uXdhlF/jd72DyZCUJEckoShRxGT8eDjkEJk0K20OGwM03Q2FhvHGJiFSgRJFuq1aFfohDDgkrzi3X1BERyWxKFOm0uYDfE0+EZDF7Nhx/fNxRiYgkpM7sdJo1C9q0gddfhx6aLiIi2UHzKFJp40Z44AE44AA49VQoLg5rVRcUxB2ZiOSZjF24KK/9/e8hQdx+O7z/fnisXj0lCRHJOkoUtW3JEvj5z+GYY0IL4u234eGH445KRKTalChq27vvwvPPw623wvTp0KdP3BGJiNSIOrNrw7RpMGdOWEjoggugVy/Yc8+4oxIRqRVqUdTEunVwww1hKdIbbgiXmsyUJEQkpyhRVNfrr0PnzmFUU//+YaZ1vXpxRyUiUut06ak6pk+H006DLl3go4/giCPijkhEJGWyr0UR16LZJSUwdmy4/6MfwRtvwBdfKEmISM7LvkQRh88+CzOpjz0Wvoom+518si41iUheUKJIZMUKuOwyOOwwWLYs1Grae++4oxIRSSv1UWzLxo1hNNPChXDVVXDXXdC4cdxRiYiknRJFRf/+dyjcV78+/OY3oQzHgQfGHZWISGyy7tKTpao3e8OG0GrYc08YOTI81r+/koSI5D21KCAU7bvsstBRfd55YVEhEREBsrBFUeuuugqOOy6sX/3uu/DXv8Kuu8YdlYhIxsjPRFFWBqWl4X7PnnDHHaFek1abExHZSv4liilTQtG+xx4L2+efH/omGjSINy4RkQyVP4li7Vq49lro3h3mzdPlJRGRJOVHZ/Z778GAAVBUBAMHwh/+AM2bxx2ViEhWyI9EUVgILVrA3/4WLjuJiEjScjNRFBeH5UdXrYJ774WjjgoF/Orkz5U2EZHaknufnJ98EvohbrgBZs0KI5xASUJEpJpy59Pzu+9C/8Phh8PKlfDaa/DKK0oQIiI1lDufosuXh8ly110HM2fC6afHHZGISE7I7j6KOXNCB/Udd0DHjrBgAbRsGXdUIiI5JaUtCjPrY2ZzzGyumd1UyfNmZo9Gz081s4OSOvD69SE5dO0K//M/oRQ4KEmIiKRAyhKFmRUAjwF9gc7AeWbWucJufYGO0W0g8ERVx220cR3svz/ccw+cdRbMng27717L0YuIyGapbFH0BOa6+zx33wS8AFTsODgdeMaDT4FmZtY60UHbrPxP6KB+7z149lnYZZfURC8iIkBq+yjaAAvLbRcBFet3V7ZPG2Bx+Z3MbCChxQGw0b76ajrHHVe70WanVsCyuIPIEDoXW+hcbKFzscW+1X1hKhNFZSsMeTX2wd2HAkMBzGyCu/eoeXjZT+diC52LLXQuttC52MLMJlT3tam89FQElO88aAssqsY+IiISo1QmivFARzPrYGaFwLnAqAr7jAIuikY/HQqscvfFFQ8kIiLxSdmlJ3cvMbMrgNFAAfC0u88ws0HR808CbwEnAXOB74EBSRx6aIpCzkY6F1voXGyhc7GFzsUW1T4X5r5Vl4CIiMgPcqeEh4iIpIQShYiIJJSxiSJl5T+yUBLn4oLoHEw1s0/M7IA44kyHqs5Fuf0ONrNSM+uXzvjSKZlzYWa9zWyymc0wsw/THWO6JPE30tTMXjezKdG5SKY/NOuY2dNmtsTMpm/j+ep9brp7xt0Ind//AvYECoEpQOcK+5wEvE2Yi3Eo8Fncccd4LnoBzaP7ffP5XJTb7wPCYIl+cccd4/+LZsBMoF20vXPcccd4Lm4B7ovu7wR8BxTGHXsKzsVRwEHA9G08X63PzUxtUaSk/EeWqvJcuPsn7r4i2vyUMB8lFyXz/wLgSuAVYEk6g0uzZM7F+cAId/8GwN1z9Xwkcy4caGxmBjQiJIqS9IaZeu4+jvC7bUu1PjczNVFsq7TH9u6TC7b39/wF4RtDLqryXJhZG+AnwJNpjCsOyfy/2AdobmZjzWyimV2UtujSK5lzMQTYjzChdxrwa3cvS094GaVan5uZuh5FrZX/yAFJ/55mdjQhURyR0ojik8y5eBi40d1Lw5fHnJXMuagLdAeOBRoC/zSzT939y1QHl2bJnIsTgcnAMcBewBgz+8jdV6c4tkxTrc/NTE0UKv+xRVK/p5l1BYYBfd19eZpiS7dkzkUP4IUoSbQCTjKzEnd/LS0Rpk+yfyPL3H0dsM7MxgEHALmWKJI5FwOAP3i4UD/XzL4GOgGfpyfEjFGtz81MvfSk8h9bVHkuzKwdMAK4MAe/LZZX5blw9w7u3t7d2wMvA5fnYJKA5P5GRgJHmlldM9uBUL15VprjTIdkzsU3hJYVZrYLoZLqvLRGmRmq9bmZkS0KT135j6yT5Lm4A2gJPB59ky7xHKyYmeS5yAvJnAt3n2Vm7wBTgTJgmLtXOmwymyX5/+IeYLiZTSNcfrnR3XOu/LiZPQ/0BlqZWRFwJ1APava5qRIeIiKSUKZeehIRkQyhRCEiIgkpUYiISEJKFCIikpAShYiIJKREIRkpqvw6udytfYJ919bC+w03s6+j95pkZodV4xjDzKxzdP+WCs99UtMYo+NsPi/To2qozarYv5uZnVQb7y35S8NjJSOZ2Vp3b1Tb+yY4xnDgDXd/2cxOAB509641OF6NY6rquGb2v8CX7v7bBPv3B3q4+xW1HYvkD7UoJCuYWSMzez/6tj/NzLaqGmtmrc1sXLlv3EdGj59gZv+MXvuSmVX1AT4O2Dt67TXRsaab2VXRYzua2ZvR2gbTzeyc6PGxZtbDzP4ANIzieC56bm3082/lv+FHLZkzzazAzB4ws/EW1gm4NInT8k+igm5m1tPCWiRfRD/3jWYp3w2cE8VyThT709H7fFHZeRTZStz103XTrbIbUEoo4jYZeJVQRaBJ9FwrwszSzS3itdHPa4Fbo/sFQONo33HAjtHjNwJ3VPJ+w4nWrgDOAj4jFNSbBuxIKE09AzgQOBP4c7nXNo1+jiV8e/8hpnL7bI7xJ8D/RvcLCZU8GwIDgduix+sDE4AOlcS5ttzv9xLQJ9puAtSN7h8HvBLd7w8MKff63wE/i+43I9R92jHuf2/dMvuWkSU8RID17t5t84aZ1QN+Z2ZHEcpRtAF2Ab4t95rxwNPRvq+5+2Qz+zHQGfg4Km9SSPgmXpkHzOw2YCmhCu+xwKseiuphZiOAI4F3gAfN7D7C5aqPtuP3eht41MzqA32Ace6+Prrc1dW2rMjXFOgIfF3h9Q3NbDLQHpgIjCm3//+aWUdCNdB623j/E4DTzOy6aLsB0I7crAEltUSJQrLFBYSVybq7e7GZzSd8yP3A3cdFieRk4P/M7AFgBTDG3c9L4j2ud/eXN2+Y2XGV7eTuX5pZd0LNnN+b2bvufncyv4S7bzCzsYSy1+cAz29+O+BKdx9dxSHWu3s3M2sKvAEMBh4l1DL6u7v/JOr4H7uN1xtwprvPSSZeEVAfhWSPpsCSKEkcDexRcQcz2yPa58/AU4QlIT8FDjezzX0OO5jZPkm+5zjgjOg1OxIuG31kZrsB37v7s8CD0ftUVBy1bCrzAqEY25GEQnZEPy/b/Boz2yd6z0q5+yrgV8B10WuaAv+Onu5fbtc1hEtwm40GrrSoeWVmB27rPUQ2U6KQbPEc0MPMJhBaF7Mr2ac3MNnMviD0Izzi7ksJH5zPm9lUQuLolMwbuvskQt/F54Q+i2Hu/gWwP/B5dAnoVuDeSl4+FJi6uTO7gncJaxu/52HpTghricwEJpnZdOBPVNHij2KZQiirfT+hdfMxof9is78DnTd3ZhNaHvWi2KZH2yIJaXisiIgkpBaFiIgkpEQhIiIJKVGIiEhCShQiIpKQEoWIiCSkRCEiIgkpUYiISEL/HxKGTnLDcwivAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y = train_y\n",
    "X = x_train_count\n",
    "\n",
    "log_roc_auc = roc_auc_score(y, log_model.predict(X))\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y, log_model.predict_proba(X)[:,1])\n",
    "plt.figure()\n",
    "plt.plot(fpr, tpr, label='AUC (area = %0.2f)' % log_roc_auc)\n",
    "plt.plot([0, 1], [0, 1],'r--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba6fe798",
   "metadata": {},
   "source": [
    "### Logistic regression model with \"tf-idf\" method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "eded33ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Logistic regression model with 'tf-idf' method\n",
      "Accuracy ratio:  0.9353026315789474\n"
     ]
    }
   ],
   "source": [
    "log = linear_model.LogisticRegression()\n",
    "log_model = log.fit(x_train_tf_idf_word, train_y)\n",
    "accuracy = model_selection.cross_val_score(log_model,\n",
    "                                           x_test_tf_idf_word,\n",
    "                                           test_y,\n",
    "                                           cv = 20).mean()\n",
    "\n",
    "print(\"\\nLogistic regression model with 'tf-idf' method\")\n",
    "print(\"Accuracy ratio: \", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "433b58ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred=log_model.predict(x_test_tf_idf_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "87a0f0dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[7378,   14],\n",
       "       [ 429,  170]], dtype=int64)"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(test_y,pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "0cb9379c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      7392\n",
      "           1       0.92      0.28      0.43       599\n",
      "\n",
      "    accuracy                           0.94      7991\n",
      "   macro avg       0.93      0.64      0.70      7991\n",
      "weighted avg       0.94      0.94      0.93      7991\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(test_y,pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "9a242734",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAuT0lEQVR4nO3dd5xU1f3/8deH3aV3UEQQBQUBFVERFXsHe6KxRkXjF1HU2HuJJcXy86sGSwgav0ajsWPHFsUSC70JSlBgRaRKZ9ny+f1xZmGz7s5elp25M7Pv5+Mxj507c+fOh8vufOacc8/nmLsjIiJSnQZxByAiIplNiUJERJJSohARkaSUKEREJCklChERSUqJQkREklKiEBGRpJQoRJIws+/MbK2ZrTKzBWb2uJk1r/D8ADN738xWmtlyM3vVzHpXOkZLM7vPzOYmjjMrsd0+/f8ikU2nRCFSs2PdvTnQF9gNuA7AzPYB3gZGAVsDXYFJwCdm1i2xT0PgPWAnYCDQEhgALAH6p/VfIVJLppnZItUzs++A89z93cT2XcBO7n60mX0ETHH3Cyu95k1gkbufZWbnAb8Htnf3VWkOX6ROqEUhEpGZdQYGAbPMrCmhZfBcFbs+CxyeuH8Y8JaShGQzJQqRmr1sZiuBecBC4BagLeHv54cq9v8BKB9/aFfNPiJZQ4lCpGYnuHsL4CCgJyEJLAPKgI5V7N8RWJy4v6SafUSyhhKFSETu/iHwOHCPu68G/g38qopdTyYMYAO8CxxpZs3SEqRICihRiGya+4DDzawvcC1wtpldYmYtzKyNmd0B7APcmtj/74QuqxfMrKeZNTCzdmZ2vZkdFcc/QGRTKVGIbAJ3XwQ8Adzk7h8DRwK/JIxDzCFcPrufu3+T2L+IMKA9A3gHWAF8Qei++jzt/wCRWtDlsSIikpRaFCIikpQShYiIJKVEISIiSSlRiIhIUvlxB7Cp2rdv79ttt13cYYiIZJVx48YtdvctavParEsU2223HWPHjo07DBGRrGJmc2r7WnU9iYhIUkoUIiKSlBKFiIgkpUQhIiJJKVGIiEhSShQiIpJUyhKFmT1mZgvNbGo1z5uZPWBms8xsspntnqpYRESk9lI5j+JxYDihJHNVBgHdE7e9gIcTP0VEsoa7U1RSFncYKZWyROHuY8xsuyS7HA884aHO+Wdm1trMOrq71hdOMXdn+dpilqxeX+XzK9YWM27OMhoX5NXJ+33+7VIWryyiXfOGdXK8cquKSpi1cBXtmjUEszo9tgjAynXFzF60mtZNC8hvUPXv2OJVVf8dZYq95k7hN2NHbdYx4pyZ3Ymw8le5wsRjP0sUZjYEGALQpUuXtAQXt7Iyp7TSWiHL1xazcl0J839aS1niublL1zB3yRryEr/E60vKmDZ/BUtWF7FsTXGVx160sii1wVejZeN82rdoVGfHc4d1xWUUlzpbtKjbJCQC0LpJAa2aFLBVy8a0bVb979iqohJ6btUyjZHVrPFPS9jnkTvpOfpFVmzVebOOFWeiqCo9V7mKkruPAEYA9OvXL+NXWnJ3yhzWFZcybs4y8hoYhcvWUP65v2hlESvWFdMoP3xjHz93Gd//tJb2zRtRXFrG5MLlm/yeBXnhdBaXhjdp2jCPHh1a0Ktj1b+8K9cV06tjSzq3aVLl82ZGv23bkJ+3+d/UCxo0oE2SPzIRSYETr4b3XoHrrqPljTdCs9ov2x5noigEtqmw3RmYH1Msm8zdWbxqPe9M/5HZi1Yxd+kamjXK56UJ32/ScczYkEB+WlPMzp1asluX1hQ0aMCOW7WgQ8uN38CLSspo3bQhLRrl07F1Y5okuoY6tm5Cp9ZVf+CLSD0ybRq0bg2dOsGdd8Jtt8FOO232YeNMFK8AF5nZM4RB7OWZOj6xbPV6LnlmAh99s5gtEl0n1XXfdGjZiB9XFHFMn45037IFq9eXcEjPLTGgddOGtGwSTnmLxgU0b5R1NRlFJBOtXg233w7/7//BGWfA44/DDjvU2eFT9kllZk8DBwHtzawQuAUoAHD3R4A3gKOAWcAa4JxUxVJb0+ev4KgHPvqvxxatLOLE3TuztriEBmbs1a0dO2/dkt26tIkpShGp115/HYYNgzlz4NxzQ0uijqXyqqfTanjegWGpev/a+HHFOj6ZtZgGZvzpzRksWLFuw3PnH9CNK4/ckYI8zVEUkQzx0EMhSfTuDWPGwP77p+Rt6n3fx6yFq3jmi7m8PPH7Ki9zO3Pvbbn9hJ1jiExEpAolJbBoEXTsCCefDGvXwsUXQ8PUXTBSbxOFu/P3z+Zw86hp//X4GXt14bz9u5HfwOjcpgmm6/NFJFN88QWcfz7k58Nnn0H79nDFFSl/23qXKNaXlPHShEKueWHKhsd269Kafw7Zh4b56lYSkQz0009w/fXwyCOhJXH//dAgfZ9X9SpRzFywkiPvG/Nfj7192QH06NAipohERGowZQocfnjobrrkknDJa8v0Tu6rN4li1sKNSaJts4aMGrYv27RtGnNUIiLVKC6GggLo0QMOPhiuugp2j6d2ar3oa5mzZDWH3RuSxKE9t2T8TYcrSYhIZioq2jhRbtUqaNQInn46tiQB9SBRrCsu5cC7PwDgsF4deHTwnvEGJCJSnfffhz594JZbQmIoiqcuW2U53fU07B/jeX1ymOzdq2NLRp7dL+aIRESqsHYtDBkCTz4J3brBW2/BkUfGHdUGOduiuP216RuSxNG7dOSNS/aLOSIRkWo0bgyLF8ONN8LUqRmVJCBHWxSFy9bw6MffAjDmqoPp0k7jESKSYSZPDgPUjz4KnTuHUhxpvOR1U2RmVJvpvne/AUJLQklCRDLK6tUbr2AaPx6+CZ9XmZokIEcTxeipCwAYfvpuMUciIlLBK6+Eukz33BMK+M2cGS59zXA51/W0cOU6VhaVMGD7diq/ISKZ5eWXw2S5jz+GffeNO5rIci5RjPwojE2cufe2MUciIvVecTE88EBoNey+eyi90bhxmEiXRXKu6+lfMxYCMHDnrWKORETqtc8+g3794Mor4dlnw2MtWmRdkoAcTBT/WbSKVk0K1O0kIvFYtgyGDoUBA2DpUnjpJfjjH+OOarPkVKJYX1JGmcOA7dvFHYqI1FcjRsDIkXDZZTB9OpxwAmT5F9ecGqO46eWpAOy7Q/uYIxGRemXmzFDddb/94NJLYdCgUIojR+RUi2LmjyuBsPiQiEjKrVsX6jL16ROWJHUPRfxyKElAjiWKifN+okPLRhqfEJHUe+cd2GWXUOn1pJNg9Ois72KqTs50Pc1dsgaA3h3Tu6CHiNRDY8bAEUdA9+4hYRx2WNwRpVTOtCge+mAWAMf37RRzJCKSk0pLw2pzAPvvH2o0TZ6c80kCcihRjJ+7DIDj+24dcyQiknMmTAiXu+67L/z4Y+hiOvfcMHmuHsiZRPH1j6vovmVzjU+ISN1ZuRIuvzxMnPvuO3j4Ydhyy7ijSrucGKNYvrYYgF06t4o5EhHJGcuXh8HqefPg/PPDpLk2beKOKhY5kSjenf4jAHt1bRtzJCKS9VasCIX7WrUKq84deijss0/cUcUqJ7qeRk8LZcX7badEISK1VFwMd90VFhEaPz48duON9T5JQI60KD74ehEA22/RPOZIRCQrffJJqM80dWooubHFFnFHlFGyvkXh7qwvKaN984ZxhyIi2ejii0PpjeXLYdSoUMRvm23ijiqjZH2imDDvJwB+rfUnRCQq9433t9oqlAKfPh2OOy6+mDJY1ieKe9/+GoC9u6lirIhEMGNGWEho1KiwfcMNcPfd0Fxd19XJ+kSxtrgUUKIQkRqsXQs33RQK9k2aFLYlkpQmCjMbaGYzzWyWmV1bxfOtzOxVM5tkZtPM7JxNOX5ZmTNuzjJO7te57oIWkdzz3nthTsQdd8Cpp4ay4KeeGndUWSNlVz2ZWR7wIHA4UAh8aWavuPv0CrsNA6a7+7FmtgUw08yecvf1Ud5j8vfLAdi2XbO6DV5EckthIeTnh4RxyCFxR5N1Utmi6A/McvfZiQ/+Z4DjK+3jQAsLdTeaA0uBkqhv8O3iVQDstLUqxopIBaWl8OCD8Ne/hu2zzgrdTUoStZLKRNEJmFdhuzDxWEXDgV7AfGAK8Ft3L6t8IDMbYmZjzWzsokWLNjy+YHkRAN3aaxBKRBLGj4e994aLLgprREAo4teoUbxxZbFUJoqqqvN5pe0jgYnA1kBfYLiZ/ax54O4j3L2fu/fbosJEmB9XrAOgY+v6UcFRRJJYsQJ++1vYc89Qn+npp+G55+KOKiekMlEUAhVnrXQmtBwqOgd40YNZwLdAz6hvMObrRTRtmEdBXtZfvCUim2vSJBg+PMywnjEjDFarmnSdSOUn7JdAdzPramYNgVOBVyrtMxc4FMDMOgA7ArOjvkFeA2N9yc96qkSkvvj2W3jssXB///1h1qwwNtG6daxh5ZqUJQp3LwEuAkYDXwHPuvs0MxtqZkMTu90ODDCzKcB7wDXuvjjqeyxZvZ4Deqgmi0i9s359KPvduzdccQUsCwuX0bVrvHHlqJQWBXT3N4A3Kj32SIX784Ejanv8ktIySssqD3uISE776KPQvTR9Ovzyl3D//fV2nYh0ydrqse7OinUl9OigK55E6o1Fi+CII6BDB3j1VTjmmLgjqheydhR4xdow3aK4VC0KkZzmDu+8E+5vsQW89hpMm6YkkUZZmyiKSkKNpx22VItCJGdNmwYHHhhaER98EB479FBopmoM6ZS1iWLFurBOtq56EslBa9bA9ddD374hWYwcCQccEHdU9VbWjlHMWxoqP27brmnMkYhInXIPZcC/+ALOPjuUANeKc7HK2kRR+FNIFO2ba1q+SE744QfYckvIywutiVat4KCD4o5KyOKuJ0+sUKXyHSJZrrQUHngAdtwRHnooPHb88UoSGSRrE8WMBSsBaNm4IOZIRKTWxo6F/v1DjaYBA+Coo+KOSKoQOVGYWUZdZrBgeSgI2LggL+ZIRKRW7rorJIkffoB//hPefBO23z7uqKQKNSYKMxtgZtMJZTgws13N7KGUR1aD1UUltGqi1oRIVnGH4nDFIv37w7Bh8NVXcPLJKuCXwaK0KP6XUA58CYC7TwJiv05tcuFy2jVvGHcYIhLVf/4DAwfCtYlVkQ86CP785zBoLRktUteTu8+r9FBpCmLZJI0LGqhFIZINiorCWtU77wz//re6l7JQlMtj55nZAMAT5cIvIdENFad1xWXs0UWFwEQy2rhx8Otfh/UhfvUruO8+2HrruKOSTRQlUQwF7icsY1oIvA1cmMqgalJUUsra4lIK8rP2oi2R+qF58zD28MYbMGhQ3NFILUVJFDu6+xkVHzCzfYFPUhNSzZatDoNhzRrqiieRjFJWBn/7W+hiGjkyzI2YOhUa6EtdNovyv/fniI+lzdylawBop1nZIplj6tRQj+m88+Cbb2D16vC4kkTWq7ZFYWb7AAOALczs8gpPtQRi/SpfXjm2W/uMmtohUj+tXg233Qb33huuYPrb30KNJl3umjOSdT01BJon9mlR4fEVwEmpDKomi1cVAdCsUdaWqhLJHevWheRw1llhEl27dnFHJHWs2k9ad/8Q+NDMHnf3OWmMqUblixU10mC2SDwKC0N9pj/+MSSGGTOgbdu4o5IUifKVfI2Z3Q3sBGyowOfuh6Qsqhr8mCjf0bqpJtyJpFVJSZgkd/PNoZjfKafAHnsoSeS4KF/JnwJmAF2BW4HvgC9TGFONShOVY1s0VteTSNp8/jn06weXXx4GradNC0lCcl6URNHO3R8Fit39Q3c/F9g7xXElVVxaRl4DU9eTSLqUlcE558CiRfD882Hd6q5d445K0iTKV/JEBS9+MLOjgflA59SFVLPZi1aT38AwXVUhkjruISkMHAgtWsCLL0KnTuG+1CtRvpLfYWatgCuAK4GRwKWpDKomzRvlU6S1skVS55tv4MgjQ1XXESPCYz17KknUUzW2KNz9tcTd5cDBsGFmdmyWry2mq+ZQiNS9oiK48074wx+gUSMYPhyGDo07KolZsgl3ecDJhBpPb7n7VDM7BrgeaALslp4Qf27BinWUlnlcby+Su4YNg0cfhVNPDRPoOnaMOyLJAMlaFI8C2wBfAA+Y2RxgH+Bad385DbFVq3mjfNYVx17pXCQ3LFwYBqu32gquuSZUeT3yyLijkgySLFH0A/q4e5mZNQYWAzu4+4L0hFa9opIytmzRuOYdRaR6ZWWhcN8118ARR4TlSLt3DzeRCpINZq939zIAd18HfJ0JSQLg6wUrKcjTFU8itTZ5Muy3H5x/PvTtC7feGndEksGStSh6mtnkxH0Dtk9sG+Du3ifl0VWjZZMC1qxX15NIrTz/fBiDaNMGnngiLCykS80liWSJolfaothE3/+0lgN33CLuMESyy4oV0LJlWKt62DC45RaV3pBIkhUFzKhCgBU1MFhdVBJ3GCLZYe5cuPhimD8fPvsM2reH+++POyrJIimtgWFmA81sppnNMrNrq9nnIDObaGbTzOzDqMfu0rZp3QUqkouKi+Gee6BXL3j33TB5znVZuWy6lFXVS8zDeBA4nLDW9pdm9oq7T6+wT2vgIWCgu881sy2jHLvMoYH6VEWqN2cOHHdcGLQ+9thQ8XXbbeOOSrJUpBaFmTUxsx038dj9gVnuPtvd1wPPAMdX2ud04EV3nwvg7gtrOmj596H8BkoUIj9T3mLYaivo0AFeeglGjVKSkM1SY6Iws2OBicBbie2+ZvZKhGN3AuZV2C5MPFZRD6CNmX1gZuPM7Kwaj5r4O8jT5bEiG7nDk0/CnnvCqlWh/Mbbb8MJJ+iKJtlsUVoUvyO0Dn4CcPeJwHYRXlfVb2flDtJ8YA/gaOBI4CYz6/GzA5kNMbOxZjZ20eLF4YVqUYgEM2fCoYfCmWdCfj4sWRJ3RJJjoiSKEndfXotjFxJKgJTrTChRXnmft9x9tbsvBsYAu1Y+kLuPcPd+7t6vXWI9Xo1RSL1XUhIuce3TB8aPh4cfhk8/VTeT1LkoiWKqmZ0O5JlZdzP7M/BphNd9CXQ3s65m1hA4FajcZTUK2N/M8s2sKbAX8FWyg5Yl+mBX6fJYqe/y8uCjj+Ckk0KrYuhQaKDFvKTuRfmtupiwXnYR8A9CufFLa3qRu5cAFwGjCR/+z7r7NDMbamZDE/t8RRj7mEwoPjjS3acmP2742al1kwihi+SYBQvg3HNh3rww9vDGG/DUU2HgWiRFolweu6O73wDcsKkHd/c3gDcqPfZIpe27gbsjHzPxsyBP35ykHiktDQsIXXcdrF0LgwbBNttAYxXHlNSL8ml7r5nNMLPbzWynlEdUg/IWRb6uepL6YsIEGDAALrwQ+vWDKVNCKXCRNKkxUbj7wcBBwCJghJlNMbMbUx1YkngAyFdfrNQXw4fDd9+FLqZ33oEeP7swUCSlIn3auvsCd38AGEqYU3FzKoNKpnwwu7hUa2ZLjnIPE+UmTAjb99wDM2bA6adrToTEIsqEu15m9jszmwoMJ1zx1DnlkVWjfIyiffNGcYUgkjrffRdKb/zyl3DffeGxNm3CTSQmUQaz/wY8DRzh7pXnQaRd+RhFw3x1PUkOKS4Oa1Tfemu4xPWee+C3v407KhEgQqJw973TEUhUJWWhy6mREoXkkr/8Ba69NpTcuP9+6NIl7ohENqg2UZjZs+5+splN4b9Lb8S6wp0lKoNoZrZkvSVLQlfTHnvA//wP7LADDBwYd1QiP5OsRVHe7j0mHYFEF3JW04Z5McchUkvuYQnSK6+EFi3g669DET8lCclQ1fbfuPsPibsXuvucijfgwvSEV0VciZ95Kgoo2eirr+Dgg2HwYOjeHV5+ORTyE8lgUTr6D6/isUF1HUhUmnAnWWvSJNh117CY0IgR8PHHoaCfSIZLNkZxAaHl0M3MJld4qgXwSaoDq4laFJI1Cguhc+eQFG69FX7zG9gy0mKOIhkhWZv3H8CbwB+Biutdr3T3pSmNKokNLQrNzJZMN38+XHZZKNw3YwZ06hRqNYlkmWSftu7u3wHDgJUVbphZ29SHVrXyGdl5uupJMlVpaSi70atXWIb06quhffu4oxKptZpaFMcA4whjyBU/mR3olsK4qtUg0eXURFc9SSZatw4OOAC+/BIOPxweeihc9iqSxapNFO5+TOJn1/SFUzN3J6+BUaDBbMkkxcVQUBDKfh98MFx+OZxyimozSU6IUutpXzNrlrj/azO718ximzZaVFJGXgPD9AcomcAdnn8+tBrGjw+P3XknnHqqkoTkjCgjwg8Da8xsV+BqYA7w95RGlUReA2N9iSrHSgaYPRuOPjqsDdGunZYhlZwV5Te7xMMiEMcD97v7/YRLZGPhDlu11KpeErN774WddgprVt93H3zxBfTtG3dUIikRZUroSjO7DjgT2N/M8oCC1IZVPcc12U7it2oVHHVUKODXObaq+yJpEaVFcQpQBJzr7guATmzCGtd1zV3rZUsMFi+Gc86BV14J2zfeCC+8oCQh9UKUpVAXAE8BrczsGGCduz+R8siqsb6kjHzNypZ0KSuDxx6DHXeEJ5+EWbPC4xqPkHokylVPJwNfAL8CTgY+N7OTUh1YdfIbGEtXr4/r7aU+mT4dDjoolNzo3RsmTgyXvYrUM1HGKG4A9nT3hQBmtgXwLvB8KgOrTpnD9ls0j+Otpb4ZOxamTYNHHw3VXtWKkHoqSqJoUJ4kEpYQbWwjJdaXltGoQH+wkiJvvBEWFDrzzHA75hhoG1vFGpGMEOUT9y0zG21mg81sMPA68EZqw6qegbqepO4VFsJJJ4V5EcOHh6smzJQkRIg2mH0V8BegD7ArMMLdr0l1YMn06BDbNA7JNSUl4RLXXr3g9dfh978PcyM0q1pkg2TrUXQH7gG2B6YAV7r79+kKrDoOqvMkdWfcOLj00rAM6YMPQrdYal2KZLRkLYrHgNeAEwkVZP+clohqUFrmmkchm2f5cnjxxXB/r73g88/D2ISShEiVkg1mt3D3vybuzzSz8ekIqCZl7hSp1pPUhjs8+2xoQSxZAt99B1tvDf37xx2ZSEZLligam9lubFyHoknFbXePLXE01VoUsqn+8x8YNgxGj4Y99oBXXw1JQkRqlCxR/ADcW2F7QYVtBw5JVVA1ad20YVxvLdlo5cqQHMrK4IEH4MILIU9fNkSiSrZw0cHpDGRTNMrXGIVEMHky9OkDLVqESXN77x3WrRaRTZKVn7h5qvUkySxaBGefDbvuGgapAU48UUlCpJZSmijMbKCZzTSzWWZ2bZL99jSz0qg1pEpKNZgtVSgrg5EjQwG/p5+G668PtZpEZLNEKeFRK4l1Kx4EDgcKgS/N7BV3n17FfncCo6Meu2OrJnUZquSKE0+El1+GAw6Ahx8OhfxEZLNFqR5ribWyb05sdzGzKNcT9gdmuftsd18PPENYJa+yi4EXgIVVPFcl1XqSDVavDrOrAU47DR5/HD74QElCpA5F+cR9CNgHOC2xvZLQUqhJJ2Behe3CxGMbmFkn4BfAI8kOZGZDzGysmY0FtGa2BK++GhLCQw+F7ZNPDmMTKr8hUqeiJIq93H0YsA7A3ZcBUa5Preqv1Stt3wdc4+6lyQ7k7iPcvZ+79wNo3TS2lVglE8ybB7/8JRx3XLiiaY894o5IJKdFGaMoTowjOGxYjyLKV/pCYJsK252B+ZX26Qc8Y+EbYHvgKDMrcfeXkx3Y9I2x/nrySRg6NAxc/+lPcNll0FDzakRSKUqieAB4CdjSzH4PnATcGOF1XwLdzawr8D1wKnB6xR3cvWv5fTN7HHitpiQBVTdVJMeVl/3u3DlcyfTnP0PXrjW+TEQ2X42Jwt2fMrNxwKGEz+gT3P2rCK8rMbOLCFcz5QGPufs0MxuaeD7puEQyDdSiqD9++gmuuw6aNYN77glJQpe8iqRVjYnCzLoAa4BXKz7m7nNreq27v0GlRY6qSxDuPrim4218/6h7StZyD3MhLr88TKC77LKNrQoRSasoXU+vE8YnDGgMdAVmAjulMK6kTJ1Pue3bb2HIEHj3XdhzT3jzTdhtt7ijEqm3onQ97VJx28x2B85PWUQR6EtljisuDnWaHnwQzj9fBfxEYrbJM7PdfbyZ7ZmKYKJSnshB770XliK9917o0QPmzIHGjeOOSkSINkZxeYXNBsDuwKKURRSFMkXu+PFHuOIKeOop2H57uOEGaNdOSUIkg0SZcNeiwq0RYcyiqlIcaaMxihxQVgZ/+Qv07BlWnbvpJpgyJSQJEckoSVsUiYl2zd39qjTFE4nGKHLA8uVw443Qt28o4NezZ9wRiUg1qm1RmFl+orTG7mmMJxLNo8hSq1aFMYjSUmjTBj7/HN5/X0lCJMMla1F8QUgSE83sFeA5YHX5k+7+Yopjq5byRBYaNQouvjjUaerbFw45BLp1izsqEYkgyhhFW2AJYY3sY4BjEz9jozyRRebMgeOPhxNOgNat4ZNPQpIQkayRrEWxZeKKp6lsnHBXrnIV2LRSiyJLuMNJJ8H06XDXXXDppVCgyr8i2SZZosgDmhOtXHiaKVNktM8+g512CiXAR4yAtm1h223jjkpEailZovjB3W9LWySbQC2KDLV0aSjgN2IE3Hwz3HqrSm+I5IBkiSJjP44zNrD6yj2sE3HFFSFZXHEFXJVRV1SLyGZIligOTVsUm0gLF2WY668PiwjtvTe88w7sumvcEYlIHao2Ubj70nQGsimUJjLAunVhXkT79nDOOWEMYsgQaBDlQjoRySZZ+VetCXcxe+cd2GUX+J//Cds9eoTlSZUkRHJSVv5lK0/EZMECOP10OOKI8J9w0UVxRyQiabDJZcYzQUlZzFfn1kf/+hf84hewdi387ndwzTWq8CpST2RlomhckJUNoexUXBwmyfXpA4cfDr//fehqEpF6Iys/cfMbqO8p5VauDOtU779/KOLXrh0895yShEg9lJWJQtc9pZA7vPgi9OoF998fJswVFcUdlYjEKCsThRoUKbJ4MRx7LJx4Yrjs9dNPw1oRTZvGHZmIxChLE4UyRUq0aBGWJr33Xhg7NkygE5F6T4mivvv4Yxg0KEyea9QoLCZ02WWQn5XXOYhICmRlolCeqANLlsB554XB6unTYfbs8LgmzYlIJVn5qaBEsRnc4fHHYccdw8+rrgqJok+fuCMTkQyVlf0L6nraTE88ERLFI4+EUhwiIklkZYtCiWITrV0Lt9wChYWhOfbCC/DRR0oSIhJJViYK5YlNMHo07Lwz3HYbjBoVHmvTRmMRIhJZVn5aKFFEMH8+nHIKDBwYSnC8/z4MGxZ3VCKShbIyUajrKYI77ggtiNtug0mT4OCD445IRLKUuWdXJdZGHbv7D7Om0bZZw7hDyTzjxm0s4LdkCSxbBjvsEHdUIpIBzGycu/erzWtT2qIws4FmNtPMZpnZtVU8f4aZTU7cPjWzSGtoqj1RyYoVcMkl0L9/WJYUQhE/JQkRqQMpSxRmlgc8CAwCegOnmVnvSrt9Cxzo7n2A24ERUY6dl6dUAYQ5Ec89Bz17wvDhcMEF8OSTcUclIjkmlfMo+gOz3H02gJk9AxwPTC/fwd0/rbD/Z0DnKAfWGEXCP/4Bv/51qPA6ahTsuWfcEYlIDkplougEzKuwXQjslWT/3wBvVvWEmQ0BhgA03GqH+t31tH59KLfRsyecdFKYIzF4sGoziUjKpHKMoqrP8ypHzs3sYEKiuKaq5919hLv3q+1ATM4YMwb69g1rVq9bF4r4nXeekoSIpFQqE0UhsE2F7c7A/Mo7mVkfYCRwvLsviXLgetfztHgxnHMOHHhgaEE88ojWqxaRtEnlV9Evge5m1hX4HjgVOL3iDmbWBXgRONPdv456YKtPnU+zZ4exhxUr4Npr4aabtJCQiKRVyhKFu5eY2UXAaCAPeMzdp5nZ0MTzjwA3A+2Ahyw0E0rqffdSuRUroGVL6No1tCYGDw6lOERE0iwrJ9wtnzuDxgV5cYeSGmvWwO23w4gRYUZ150gXgomIJLU5E+40CppJXn8dLroIvvsutCKaNIk7IhERJYqMUFICp50Gzz8PvXrBhx/CAQfEHZWICJClRQFz5qqn8m6//Hzo0AH+8AeYOFFJQkQySlYmipzw5Zew114wfnzYHj4crrsOGqrYoYhklqxMFFl9eezy5WEcYq+9wopzSyJNHRERiU1WJoqsVV7A7+GHQ7KYMQMOPzzuqEREksrKweysHaP46ivo1AlefRX6abqIiGSHrJxHsbpwJvl5WdAYKiqCu++GXXeFY4+F4uKwVnVejs4BEZGMlbELF9Vr//pXSBA33QTvvRceKyhQkhCRrJOVicIyue9p4UI4+2w45JDQgnjzTbjvvrijEhGptexMFHEHkMzbb8PTT8MNN8DUqTBwYNwRiYhslqwczM44U6bAzJlhIaEzzoABA6Bbt7ijEhGpE9nZosiUJsXq1XD11WEp0quvDl1NZkoSIpJTsjJRZIRXX4XevcNVTYMHh5nWBQVxRyUiUueysusp9sHsqVPhuONgp53go49gv/3ijUdEJIXUooiqpAQ++CDc33lneO01mDBBSUJEcp4SRRSffx5mUh96KHzzTXjs6KPV1SQi9YISRTLLlsEFF8A++8DixaFW0w47xB2ViEhaZeUYRVoUFYWrmebNg0svhVtvhRYt4o5KRCTtlCgq+/77ULivUSP43e9CGY7ddos7KhGR2Kjrqdy6daHV0K0bjBoVHhs8WElCROo9tSggFO274IIwUH3aaWFRIRERAbKwRVHnMyguvRQOOyysX/322/CPf8BWW9X1u4iIZK2sSxR1oqwMSkvD/f794eabQ70mrTYnIvIz9S9RTJoUivY9+GDYPv30MDbRuHG8cYmIZKj6kyhWrYIrroA99oDZs9W9JCISUf0YzH73XTjnHCgshCFD4E9/gjZt4o5KRCQr1I9E0bAhtG0L//xn6HYSEZHIcjNRFBeH5UeXL4c77oADDggF/BrUn542EZG6knufnJ9+GsYhrr4avvoqXOEEShIiIrWUfZ+e1U2kWLo0jD/suy/89BO8/DK88IIShIjIZsqdT9ElS8JkuSuvhOnT4fjj445IRCQnZN0YhVVsUsycGQaob74ZuneHOXOgXbv4ghMRyUEpbVGY2UAzm2lms8zs2iqeNzN7IPH8ZDPbPdKB164NyaFPH/jf/w2lwEFJQkQkBVKWKMwsD3gQGAT0Bk4zs96VdhsEdE/chgAP13Tc5kWrYZdd4Pbb4Ve/ghkzYJtt6jh6EREpl8oWRX9glrvPdvf1wDNA5YGD44EnPPgMaG1mHZMdtNNPP4YB6nffhSefhA4dUhO9iIgAqR2j6ATMq7BdCFSu313VPp2AHyruZGZDCC0OgCL75pupHHZY3UabndoDi+MOIkPoXGykc7GRzsVGO9b2halMFFVdyOq12Ad3HwGMADCzse7eb/PDy346FxvpXGykc7GRzsVGZja2tq9NZddTIVBx8KAzML8W+4iISIxSmSi+BLqbWVczawicCrxSaZ9XgLMSVz/tDSx39x8qH0hEROKTsq4ndy8xs4uA0UAe8Ji7TzOzoYnnHwHeAI4CZgFrgHMiHHpEikLORjoXG+lcbKRzsZHOxUa1Phfm/rMhARERkQ1yp4SHiIikhBKFiIgklbGJImXlP7JQhHNxRuIcTDazT81s1zjiTIeazkWF/fY0s1IzOymd8aVTlHNhZgeZ2UQzm2ZmH6Y7xnSJ8DfSysxeNbNJiXMRZTw065jZY2a20MymVvN87T433T3jboTB7/8A3YCGwCSgd6V9jgLeJMzF2Bv4PO64YzwXA4A2ifuD6vO5qLDf+4SLJU6KO+4Yfy9aA9OBLontLeOOO8ZzcT1wZ+L+FsBSoGHcsafgXBwA7A5Mreb5Wn1uZmqLIiXlP7JUjefC3T9192WJzc8I81FyUZTfC4CLgReAhekMLs2inIvTgRfdfS6Au+fq+YhyLhxoYWYGNCckipL0hpl67j6G8G+rTq0+NzM1UVRX2mNT98kFm/rv/A3hG0MuqvFcmFkn4BfAI2mMKw5Rfi96AG3M7AMzG2dmZ6UtuvSKci6GA70IE3qnAL9197L0hJdRavW5manrUdRZ+Y8cEPnfaWYHExLFfimNKD5RzsV9wDXuXhq+POasKOciH9gDOBRoAvzbzD5z969THVyaRTkXRwITgUOA7YF3zOwjd1+R4tgyTa0+NzM1Uaj8x0aR/p1m1gcYCQxy9yVpii3dopyLfsAziSTRHjjKzErc/eW0RJg+Uf9GFrv7amC1mY0BdgVyLVFEORfnAH/y0FE/y8y+BXoCX6QnxIxRq8/NTO16UvmPjWo8F2bWBXgRODMHvy1WVOO5cPeu7r6du28HPA9cmINJAqL9jYwC9jezfDNrSqje/FWa40yHKOdiLqFlhZl1IFRSnZ3WKDNDrT43M7JF4akr/5F1Ip6Lm4F2wEOJb9IlnoMVMyOei3ohyrlw96/M7C1gMlAGjHT3Ki+bzGYRfy9uBx43symE7pdr3D3nyo+b2dPAQUB7MysEbgEKYPM+N1XCQ0REksrUricREckQShQiIpKUEoWIiCSlRCEiIkkpUYiISFJKFJKREpVfJ1a4bZdk31V18H6Pm9m3ifcab2b71OIYI82sd+L+9ZWe+3RzY0wcp/y8TE1UQ21dw/59zeyounhvqb90eaxkJDNb5e7N63rfJMd4HHjN3Z83syOAe9y9z2Ycb7Njqum4ZvZ/wNfu/vsk+w8G+rn7RXUdi9QfalFIVjCz5mb2XuLb/hQz+1nVWDPraGZjKnzj3j/x+BFm9u/Ea58zs5o+wMcAOyRee3niWFPN7NLEY83M7PXE2gZTzeyUxOMfmFk/M/sT0CQRx1OJ51Ylfv6z4jf8REvmRDPLM7O7zexLC+sEnB/htPybREE3M+tvYS2SCYmfOyZmKd8GnJKI5ZRE7I8l3mdCVedR5Gfirp+um25V3YBSQhG3icBLhCoCLRPPtSfMLC1vEa9K/LwCuCFxPw9okdh3DNAs8fg1wM1VvN/jJNauAH4FfE4oqDcFaEYoTT0N2A04Efhrhde2Svz8gPDtfUNMFfYpj/EXwP8l7jckVPJsAgwBbkw83ggYC3StIs5VFf59zwEDE9stgfzE/cOAFxL3BwPDK7z+D8CvE/dbE+o+NYv7/1u3zL5lZAkPEWCtu/ct3zCzAuAPZnYAoRxFJ6ADsKDCa74EHkvs+7K7TzSzA4HewCeJ8iYNCd/Eq3K3md0ILCJU4T0UeMlDUT3M7EVgf+At4B4zu5PQXfXRJvy73gQeMLNGwEBgjLuvTXR39bGNK/K1AroD31Z6fRMzmwhsB4wD3qmw//+ZWXdCNdCCat7/COA4M7sysd0Y6EJu1oCSOqJEIdniDMLKZHu4e7GZfUf4kNvA3cckEsnRwN/N7G5gGfCOu58W4T2ucvfnyzfM7LCqdnL3r81sD0LNnD+a2dvufluUf4S7rzOzDwhlr08Bni5/O+Bidx9dwyHWuntfM2sFvAYMAx4g1DL6l7v/IjHw/0E1rzfgRHefGSVeEdAYhWSPVsDCRJI4GNi28g5mtm1in78CjxKWhPwM2NfMysccmppZj4jvOQY4IfGaZoRuo4/MbGtgjbs/CdyTeJ/KihMtm6o8QyjGtj+hkB2JnxeUv8bMeiTes0ruvhy4BLgy8ZpWwPeJpwdX2HUloQuu3GjgYks0r8xst+reQ6ScEoVki6eAfmY2ltC6mFHFPgcBE81sAmEc4X53X0T44HzazCYTEkfPKG/o7uMJYxdfEMYsRrr7BGAX4ItEF9ANwB1VvHwEMLl8MLuStwlrG7/rYelOCGuJTAfGm9lU4C/U0OJPxDKJUFb7LkLr5hPC+EW5fwG9ywezCS2PgkRsUxPbIknp8lgREUlKLQoREUlKiUJERJJSohARkaSUKEREJCklChERSUqJQkREklKiEBGRpP4/3RCohiwYTXMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y = train_y\n",
    "X = x_train_tf_idf_word\n",
    "\n",
    "log_roc_auc = roc_auc_score(y, log_model.predict(X))\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y, log_model.predict_proba(X)[:,1])\n",
    "plt.figure()\n",
    "plt.plot(fpr, tpr, label='AUC (area = %0.2f)' % log_roc_auc)\n",
    "plt.plot([0, 1], [0, 1],'r--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee685f6f",
   "metadata": {},
   "source": [
    "### GradientBoostingClassifier model with \"count-vectors\" method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "8547e853",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "GradientBoosting Classifier model with 'count-vectors' method\n",
      "Accuracy ratio:  0.9419357769423558\n"
     ]
    }
   ],
   "source": [
    "gbc = GradientBoostingClassifier()\n",
    "gbc_model = gbc.fit(x_train_count,train_y)\n",
    "accuracy = model_selection.cross_val_score(gbc_model,\n",
    "                                           x_test_count,\n",
    "                                           test_y,\n",
    "                                           cv = 20).mean()\n",
    "\n",
    "print(\"\\nGradientBoosting Classifier model with 'count-vectors' method\")\n",
    "print(\"Accuracy ratio: \", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "83026290",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred=gbc_model.predict(x_test_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "5bab7b69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[7377,   15],\n",
       "       [ 449,  150]], dtype=int64)"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(test_y,pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "55e670f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97      7392\n",
      "           1       0.91      0.25      0.39       599\n",
      "\n",
      "    accuracy                           0.94      7991\n",
      "   macro avg       0.93      0.62      0.68      7991\n",
      "weighted avg       0.94      0.94      0.93      7991\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(test_y,pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "41fc4a2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0EklEQVR4nO3dd3yUZbbA8d9JgwRCSOgkQao0BQTEigsqCnbBta6u7q4sinp17b231euqi2W56nL36uruCoLYu2IHQm+CCCQBpBMghSRz7h/PBIaYTCZl5p1JzvfzySdT3pk5eSHvydPOI6qKMcYYU504rwMwxhgT3SxRGGOMCcoShTHGmKAsURhjjAnKEoUxxpigLFEYY4wJyhKFMcaYoCxRGBOEiKwRkSIR2S0iG0Vkioi0DHj+aBH5RER2ichOEZkpIv0qvUcrEXlSRNb532eV/37byP9ExtSeJQpjana6qrYEBgGHAbcCiMhRwAfADKAz0A1YAHwlIt39xyQBHwP9gdFAK+BoYCswLKI/hTF1JLYy25jqicga4A+q+pH//p+B/qp6qojMAhap6pWVXvMusFlVLxGRPwAPAj1UdXeEwzemQViLwpgQiUgWMAZYJSIpuJbBf6o49N/AKP/tE4H3LEmYWGaJwpiaTReRXUAusAm4G8jA/f5sqOL4DUDF+EObao4xJmZYojCmZmepaiowAuiDSwLbAR/QqYrjOwFb/Le3VnOMMTHDEoUxIVLVz4EpwOOqugf4Bvh1FYeeixvABvgIOFlEWkQkSGPCwBKFMbXzJDBKRAYBtwC/FZFrRCRVRNJF5AHgKOBe//H/h+uymioifUQkTkTaiMhtInKKFz+AMbVlicKYWlDVzcA/gDtV9UvgZGAsbhxiLW767LGqutJ/fAluQHs58CFQAHyP6776LuI/gDF1YNNjjTHGBGUtCmOMMUFZojDGGBOUJQpjjDFBWaIwxhgTVILXAdRW27ZttWvXrl6HYYwxMWXu3LlbVLVdXV4bc4mia9euzJkzx+swjDEmpojI2rq+1rqejDHGBGWJwhhjTFCWKIwxxgRlicIYY0xQliiMMcYEZYnCGGNMUGFLFCLykohsEpHF1TwvIvK0iKwSkYUiMjhcsRhjjKm7cLYopgCjgzw/Bujl/xoPPBfGWIwxpslRVcrKfRSXltfrfcK24E5VvxCRrkEOORP4h7o659+KSGsR6aSqtr+wMaZJUVW27tlL7rZCcrcXue/bCsndXsiu4jLKyhWfKmU+xedz38v9X2U+/3PlPnwKZT7fvud8CkesW8Tv58yoV3xerszOxO38VSHP/9gvEoWIjMe1OujSpUtEgjPGmIa0p6SM3O2F5G5ziWDdtkLyKu5vL6Rw74F/9bdtmURWegoZLZJIiBPi933FkRAnxIm473FywPMVj6UWbOdXLzxG/w/foKBjVr1i9zJRSBWPVbmLkqpOBiYDDB061HZaMsZEndJyH+t3FO278Fckg9ztReRtK2Trnr0HHN8iKZ7sjBS6tEnhmJ5tyc5IpktGCtkZKWSlJ5OSVM/L87hx8OlMuPVWWt1xB7So+7btXiaKPCA74H4WsN6jWIwxJihVZfPuEn+3UNG+rqF1/vsbdhbhC/gzNiFOyExPJjs9hZP6dyQ7w92uSAbpKYmIVPX3cj0sWQKtW0NmJjz6KNx3H/TvX++39TJRvAlcJSKvAUcAO218whjjpV3FpeRuKwroFnItgor7xaW+A45vn9qM7IwUDu+aTpeMTLIyUlwyaJNCx1bNiY9r4ERQnT174P774b//Gy66CKZMgZ49G+ztw5YoRORVYATQVkTygLuBRABVfR54BzgFWAUUApeFKxZjjAHYW+Yjf0dRQLdQIXn+xJC7vZAdhaUHHJ/aLIHsjBR6tGvBiIPb0aWNSwTZGclkpafQPDHeo58kwNtvw8SJsHYt/O53riXRwMI56+mCGp5XYGK4Pt8Y0/T4fMqmXSUHjhEEjBlsLChGA7qHkuLjyEpPJisjhQFZaW7MIGN/MkhLDkP3UEN69lmXJPr1gy++gOHDw/IxMbcfhTGmadtZWLrvwh84RpC7vZC87UXsLdvfPSQCHVKb0yUjhaN6tPEngIpxgmQ6pDYnLlLdQw2lrAw2b4ZOneDcc6GoCK6+GpKSwvaRliiMMVGluLScvO3+C3/FGMHWwn3JoaC47IDj05ITyc5Ipk/HVEb17eAfJ3AziDLTk2mWEAXdQw3l++/hj3+EhAT49lto2xauvz7sH2uJwhgTUeU+5eeCYn9LYP/00Ypxgp8LSg44vlmC6x7KzkhhyEHpB4wRZGekkJac6NFPEkE7dsBtt8Hzz7uWxFNPQVzkSvVZojDGNChVZYe/e6jyGEHutkLydxRRWr5/oEAEOqclk5WezPBe7fZ1C1V0E7Vr2Sz2uoca0qJFMGqU62665ho35bVVq4iGYInCGFNrRXvL3fTR7YX+bqGiA8pP7C45sHsoPSWRLhkp9M9MY/QhnQ5IBp1bJ5OUYIWsf6G0FBIT4eCDYeRIuPFGGOxN7VRLFMaYXygr97FhZ/Evpo+6mURFbNl9YPdQ88S4fYvJjuiWQVb6/lXG2RkptGxml5qQlZS4Ka4vvww5OdCyJbz6qqch2b+eMU1QsCJ0uduKWL+jiLKAZcbxcUKntOZkp6dwQp/2rjVQkQjSU2jbMim6p5HGik8+gSuugB9+gPPOc0mjZUuvo7JEYUxjVbkI3f5xguBF6AZmt+b0gZ0OmEraMa05ifHWPRQ2RUUwfrxrRXTvDu+9Byef7HVU+1iiMCZGlZb72LCjuFK3UM1F6LIzUji6Z5uAhWWuCF0L6x7yTvPmsGUL3HGHm92UnOx1RAew/xnGRKn9RegO7BoKrQhdB7ICCtBlpyeT0cK6h6LKwoVugPrFFyEry5XiiOCU19qwRGGMhyqK0AVOHw1WhK5dajO6+IvQZWdk7hsjyM5IpmOr5iRY91D027MH7rkH/vIXSE+HlStdoojSJAGWKIwJq8AidBWtgTx/Yli3reoidFkBRegCy01ETRE6U3dvvunKbaxbB5dfDo88AhkZXkdVI0sUxtRD5SJ0uQFTSfO2FbKhiiJ0melucdmph3ba1yKoSAZRX4TO1M/06W6x3JdfwjHHeB1NyCxRGFODnUWlv5g+ui8ZVCpCB9CxVXOyM5I5snubgCmkyXRpkxKbRehM3ZWWwtNPuwVzgwe70hvNm7uFdDHEEoVp8opLy8nf4R8XCFhXUFGLqLoidL07pHJi3w77EkF2RgqZrZOte8g4337rCvgtXAg33+wSRWqq11HViSUK0+hVFKGrPH20YpygchG6pIS4fRf+wV3S9+1l3KSK0Jm6274dbr0VJk92W5K+8QaceabXUdWLJQoT8wKL0OVWKjdRXRG6Tq2ak5WRwvBe7fbNGqqYStrki9CZ+pk8GV54Aa67zs1uitFWRCBLFCbqqColZT5KSn2UlJVTUuajuLSc4lIfm3cX708GNRShy85IoX9nV4QucGN7K0JnGtyKFa6667HHwrXXwpgxMGCA11E1GEsUJirMXbuNB95expL1Bb8YHK5KRRG67IwUhnVNP6DuUHZGMqnNrXvIREBxMTz8sJvm2qcPzJ8PzZo1qiQBliiMh1SV737axguzfuKjZT/ToVUzLj7yIFo2S6BZYhzNEuJpHvA9KT6ONi2bkZ2RTLuWzWwaqfHWhx/ClVfCqlVw4YXw3//t+jUbIUsUJuJKy328vXADL3y5msX5BWS0SOLaE3tx+fDuVm/IxIYvvoCTToJevVzCOPFEryMKK/utNBGjqrz87Vqe+fRHNhYU06NdCx4eeyhnH5ZpU0pN9Csvh6VL4dBDYfhwV6PpwgvduohGzhKFCbsNO4t4Y14+U+fm8ePmPQw9KJ2Hxx7Krw5uZ7OLTGyYNw8mTIBly1xtpg4d4He/8zqqiLFEYcKicG8Z7y3eyLScfL76cQuqMKxrBuOP6864wVlWvM7Ehl274O673Yrqtm3hueegfXuvo4o4SxSmwfh8yrc/bWXq3HzeXbyBwr3lZGckc83xvRg7OJOD2rTwOkRjQrdzp+tmys11K6wffthVe22CLFGYelu9eTfTcvJ5Y14++TuKSG2WwBkDOzN2cBaHd0232UkmthQUuMJ9aWlu17kTToCjjvI6Kk9ZojB1srOwlJkL1zM1J49563YQJzC8VztuHtOHk/p1sMFpE3tKS90eEQ88AJ995moz3XGH11FFBUsUJmSl5T4+X7GZafPy+GjpJvaW++jdIZXbTunDmYMy6dCq8c/+MI3UV1+5werFi+Gss6BdO68jiiqWKExQqsqS9QVMy8nnzQX5bNm9lzYtkrjoyC6MG5xF/86trGvJxLarr4ZJkyA7G2bMgDPO8DqiqGOJwlRpU0Ex0+fnMy0nn+Ubd5EUH8cJfdszbnAWv+rdjkSbtWRimer+VdQdO8INN7jZTS1behtXlLJEYfYpLi3ng6U/M3VuHrNWbsancFiX1tx/1iGcPqATrVOSvA7RmPpbvtx1M113nSv/ffvtXkcU9SxRNHGqypy125k6N4+3F25gV0kZndOac+WInpw9OJMe7ewvLNNIFBXBQw/Bo49CixbuvglJWBOFiIwGngLigRdU9ZFKz6cBLwNd/LE8rqp/D2dMxlm3tZBp8/KYlpPPum2FpCTFM+aQTowbnMmR3dvYimnTuHz8sVsL8eOPcPHF8PjjTXLhXF2FLVGISDzwDDAKyANmi8ibqro04LCJwFJVPV1E2gErROQVVd0brriasoLiUt5dtIGpc/P5fs02RODoHm249sRenNy/oxXkM41XXh4kJLiEcfzxXkcTc8J5ZRgGrFLV1QAi8hpwJhCYKBRIFTdtpiWwDSir/Eam7sp9yqyVm5mWk8/7SzZSUuaje7sW3Hhyb84+LJPOrZO9DtGYhldeDs8/D0lJcPnlcMklcP75bq8IU2vhTBSZQG7A/TzgiErHTALeBNYDqcB5qvqLXWtEZDwwHqBLly5hCbaxWbFxF1Nz8pg+L59Nu0pIS07k3KHZjBuSxcCsNJvSahqvnBzXzTRnDowb5xKFiCWJeghnoqjqSqSV7p8MzAeOB3oAH4rILFUtOOBFqpOByQBDhw6t/B7Gb+vuEmbMX8+0eXkszi8gIU4Y2ac94wZnMrJPe5ol2Gpp04gVFMCdd7o1Ee3awauvwnnneR1VoxDORJEHZAfcz8K1HAJdBjyiqgqsEpGfgD7A92GMq1EpKSvnk2WbmJqTx2crNlPmUw7NTOOe0/tx+sDOtGlpf0WZJmLBApckJkyABx+E1q29jqjRCGeimA30EpFuQD5wPnBhpWPWAScAs0SkA9AbWB3GmBoFVWV+7g6m5uQxc8EGdhaV0qFVM34/vBvjBmdxcIdUr0M0JjJ++gk+/dTtDTF8uNuWtFs3r6NqdMKWKFS1TESuAt7HTY99SVWXiMgE//PPA/cDU0RkEa6r6mZV3RKumGJd/o4ips/LZ2pOHqs376F5Yhwn9+/IuMFZHNOzLfE2pdU0FXv3uj2q77vP7TB39tmuBLglibAI63xIVX0HeKfSY88H3F4PnBTOGGLdnpIy3l28kWk5eXyzeqvbAKhbBhOO68GYQzuS2jzR6xCNiaxZs1z30tKlMHas21Soie4TESk2cT4K+XzKN6u3MjUnj/cWb6RwbzkHtUnh2hMOZuzgTLIzUrwO0RhvbN4MJ53ktiKdORNOO83riJoESxRR5MfNu5k6101pXb+zmNTmCZw5KJNxgzMZcpBtAGSaKFX46CMYNcrNZnrrLTjySFeGw0SEJQqP7Sjcy8wF63k9J58FuTuIjxOO69WWW0/pyyjbAMg0dUuWwBVXuO6mTz+FESPcjnMmoixReKC03MdnKzYzdW4eHy//mdJypU/HVO44tS9nDOpM+1TbAMg0cYWFbqe5xx5z25K+8AIcd5zXUTVZligiRFVZnF/A1Jw83lywnm179tK2ZRKXHNWVcYOz6Ne5ldchGhMdVGHkSPj+e/jtb12ysB3nPGWJIsx+LijmjXn5TMvJ44efd5MUH8eofh0YNyST4b1sAyBj9tmwwVV0jY+H226DtDTX1WQ8Z4kiDIr2lvPB0o1MzcnnS/8GQEMOSufBsw/htEM7k5ZiU1qN2ae8HJ55Bu64w62ovvpqt6GQiRqWKBqIz6fMXrONqTl5vLNoI7tLyshsnczEkT0ZOziLbm1thoYxvzBnjivgl5MDJ58Mp5zidUSmCiEnChFpoap7whlMLFq7dQ9Tc/J5Y14euduKaJEUzymHdmLs4CyO6JZhGwAZU50//xluucXtWf2vf8Gvf71/H2sTVWpMFCJyNPACbr+ILiIyEPijql4Z7uCi1c6iUt5ZtIGpc/OYs3Y7InBsz7b8adTBnNy/IylJ1lAzpkqqUFYGiYkwbBhMnOhmN6WleR2ZCSKUK9pfcOXA3wRQ1QUi0uTmqZWV+5i1cgtTc/L4YOnP7C3z0bN9S24e3YezDutMpzTbAMiYoH78Ea68Eg45xNVpGjHCBqtjREh/+qpqbqVVweXhCSf6LNtQwLScPKbPX8/mXSWkpyRyweHZjB2cxQDbAMiYmpWUuCmuDz7oWhI2UB1zQkkUuf7uJxWRJOAaYFl4w/LW5l0lzJifz7ScfJZuKCAxXhjZuz3jhmQxsnd7khJsSqsxIZk7F37zG1i+3I1BPPkkdO7sdVSmlkJJFBOAp3Bbm+YBHwCNbnyiuLScj/0bAH3+w2bKfcqArDTuPaM/pw/sTEaLJK9DNCb2tGzpBqjfeQfGjPE6GlNHoSSK3qp6UeADInIM8FV4QoocVSVnndsA6K0F6ykoLqNjq+ZcPrw74wZn0ss2ADKmdnw++Pvf4ZtvXNmN3r1h8WKIs1Z4LAslUfwVGBzCYzGjpKyc177PZcrXa/hpi9sAaMwhnRg7OJOje9gGQMbUyeLFbp+Ir75ydZn27HEVXi1JxLxqE4WIHAUcDbQTkT8FPNUKt2NdTFq/o4ijH/kEgMFdWnPFOQM45dBOtGxmU1qNqZM9e9xOc0884aa5/v3vrkaTTfRoNIJdHZNwaycSgMA+mALgnHAGFQ7lPuVfs3N57P3lNE+M48aT+/C7Y7rarCVj6qu42CWHSy5xi+jatPE6ItPAqk0Uqvo58LmITFHVtRGMqcF9/9M27p25hCXrCxjWNYOHxh5Cz/Y2/mBMneXlwdNPw8MPu8SwfDlkZHgdlQmTUPpbCkXkMaA/sG+jBFU9PmxRNZD1O4p4+N3lzFywns5pzfnrBYdx2oBO1oowpq7KyuCvf4W77nLF/M47D4YMsSTRyIWSKF4B/gWchpsq+1tgcziDqq9yn/KXD3/gxS9/wqfKNSf04opf9SA5KWaHVozx3nffuQJ+Cxa44n2TJkG3bl5HZSIglETRRlVfFJH/CuiO+jzcgdXHi1+uZtKnqziyewaPnTOQ7IwUr0MyJrb5fHDZZbBzJ7z+Oowda4PVTUgoiaLU/32DiJwKrAeywhdS/RSXlvPMpz+SlBDHK3840qa6GlNXqi4pjB4NqakwbRpkZrrbpkkJZYLzAyKSBlwP3ICrJHttOIOqjyc/WsnOolLuO6O/JQlj6mrlSrc/xLnnwuTJ7rE+fSxJNFE1tihU9S3/zZ3ASNi3MjvqFO0tZ1pOHkMOSue8w7O9DseY2FNSAo8+Cg89BM2auXGICRO8jsp4LNiCu3jgXFyNp/dUdbGInAbcBiQDh0UmxNBN+nQlm3eX8MS5g2xmkzF1MXEivPginH++W0DXqZPXEZkoEKxF8SKQDXwPPC0ia4GjgFtUdXoEYqsVn0/5+1drGN6rHcf2aut1OMbEjk2b3GB1x45w882uyuvJJ3sdlYkiwRLFUGCAqvpEpDmwBeipqhsjE1rtTPl6DYV7yzmim83nNiYkPp8r3HfzzXDSSW470l693JcxAYINZu9VVR+AqhYDP0RrkgBYsXEXAFeO6OFxJMbEgIUL4dhj3bqIQYPg3nu9jshEsWAtij4istB/W4Ae/vsCqKoOCHt0tTB7zTZO7NvBxiaMqcnrr7sxiPR0+Mc/3MZC9ntjggiWKPpGLIp62llYypqtezhtgA28GVOtggJo1crtUz1xItx9t5XeMCEJVhQwZgoBfrN6Cz6FY3u18zoUY6LPunVw9dWwfj18+y20bQtPPeV1VCaGhHVHEREZLSIrRGSViNxSzTEjRGS+iCypa2mQWSu30CIpnsO6tK5XvMY0KqWl8Pjj0LcvfPSRWzyn6nVUJgaFbbce/zqMZ4BRuL22Z4vIm6q6NOCY1sCzwGhVXSci7evyWV+u2sJRPdqQGG87aRkDwNq1cMYZbtD69NNdxdeDDvI6KhOjQrqyikiyiPSu5XsPA1ap6mpV3Qu8BpxZ6ZgLgWmqug5AVTfV8jPI3VbI2q2FHNvT1k4Ys6/F0LEjdOgAb7wBM2ZYkjD1UmOiEJHTgfnAe/77g0TkzRDeOxPIDbif538s0MFAuoh8JiJzReSSkKIOsHKTmxY7ILt1bV9qTOOhCi+/DIcfDrt3u/IbH3wAZ51lM5pMvYXSorgH1zrYAaCq84GuIbyuqv+dlTtIE4AhwKnAycCdInLwL95IZLyIzBGROZs3H7gVxo+b9gDQKa155ZcZ0zSsWAEnnAAXXwwJCbB1q9cRmUYmlERRpqo76/DeebgSIBWycCXKKx/znqruUdUtwBfAwMpvpKqTVXWoqg5t1+7AmU1vLVzPoZlpdEpLrkOIxsSwsjI3xXXAAMjJgeeeg6+/tm4m0+BCSRSLReRCIF5EeonIX4GvQ3jdbKCXiHQTkSTgfKByl9UMYLiIJIhICnAEsCzU4LfuLmFB3k5GH9Ix1JcY03jEx8OsWXDOOa5VMWECxNmEDtPwQvlfdTVuv+wS4J+4cuPX1vQiVS0DrgLex138/62qS0RkgohM8B+zDDf2sRBXfPAFVV0cavDT57sGyjE2kG2aio0b4Xe/g9xcN/bwzjvwyitu4NqYMAllemxvVb0duL22b66q7wDvVHrs+Ur3HwMeq+17AyzK2wHAwKy0urzcmNhRXu42ELr1VigqgjFjIDsbmtvYnAm/UFoUT4jIchG5X0T6hz2iWpi9Zjsje7ez+k6mcZs3D44+Gq68EoYOhUWLXClwYyKkxkShqiOBEcBmYLKILBKRO8IdWE2K9paTv6OIrPQUr0MxJrwmTYI1a1wX04cfwsG/mBhoTFiFNPKlqhtV9WlgAm5NxV3hDCoUGwuKAejatoXHkRjTwFTdQrl589z9xx+H5cvhwgttTYTxRCgL7vqKyD0ishiYhJvxlBX2yGrw/hK3NcaI3lYI0DQia9a40htjx8KTT7rH0tPdlzEeCWUw++/Aq8BJqlp5HYRn/j0nl8O7ptOjXUuvQzGm/kpL3R7V997rprg+/jj81395HZUxQAiJQlWPjEQgtVFW7mP15j2cMbCz16EY0zD+9je45RZXcuOpp6BLF68jMmafahOFiPxbVc8VkUUcWHrD8x3uCkvLAWjZLGzFb40Jv61bXVfTkCFw+eXQsyeMHu11VMb8QrArbUW797RIBFIbRXtdokhOivc4EmPqQNVtQXrDDZCaCj/84Ir4WZIwUarawWxV3eC/eaWqrg38Aq6MTHhVKy33AZBo5QpMrFm2DEaOhEsvhV69YPp0V8jPmCgWypV2VBWPjWnoQGrD5/IEcXE2VdDEkAULYOBAt5nQ5Mnw5ZeuoJ8xUS7YGMUVuJZDdxFZGPBUKvBVuAMLpty/OYvlCRMT8vIgK8slhXvvhd//HtrXaTNHYzwRrM37T+Bd4GEgcL/rXaq6LaxR1aDc5xJFvGUKE83Wr4frrnOF+5Yvh8xMV6vJmBgTrOtJVXUNMBHYFfCFiGSEP7Tq+fa1KCxRmChUXu7KbvTt67YhvekmaGsVjk3sqqlFcRowFzc9NvCqrED3MMYVlLUoTNQqLobjjoPZs2HUKHj2WTft1ZgYVm2iUNXT/N+7RS6c0FQkCmtRmKhRWgqJia7s98iR8Kc/wXnnWW0m0yiEUuvpGBFp4b/9GxF5QkQ8XTZa0fVkLQrjOVV4/XXXasjJcY89+iicf74lCdNohDI99jmgUEQGAjcBa4H/C2tUNahYcJcQb7+IxkOrV8Opp7q9Idq0sW1ITaMVyv/sMlVV4EzgKVV9CjdF1jOf/bCZ+DhhYFZrL8MwTdkTT0D//m7P6iefhO+/h0GDvI7KmLAIZUnoLhG5FbgYGC4i8UBieMMK7oMlGzm6RxsyWiR5GYZpynbvhlNOcQX8sjyvum9MWIXSojgPKAF+p6obgUzquMd1Q9m2Zy9d29iGRSaCtmyByy6DN9909++4A6ZOtSRhmoRQtkLdCLwCpInIaUCxqv4j7JEFsbfMR1KC9QebCPD54KWXoHdvePllWLXKPW7jEaYJCWXW07nA98CvgXOB70TknHAHFszecksUJgKWLoURI1zJjX79YP58N+3VmCYmlDGK24HDVXUTgIi0Az4CXg9nYMGUliuJ8ZYoTJjNmQNLlsCLL7pqr9aKME1UKIkiriJJ+G0ltLGNsPAvoaCZtShMOLzzjttQ6OKL3ddpp0GGpxVrjPFcKFfb90TkfRG5VEQuBd4G3glvWNVT/2Z7SdaiMA0pLw/OOceti5g0yf1FImJJwhhCG8y+EfgbMAAYCExW1ZvDHVh1/NU7SLTFdqYhlJW5Ka59+8Lbb8ODD7q1Ebaq2ph9gu1H0Qt4HOgBLAJuUNX8SAVWnYqup6QE2wbVNIC5c+Haa902pM88A909q3VpTNQK1qJ4CXgLGIerIPvXiERUA/VnCpv1ZOps506YNs3dPuII+O47NzZhScKYKgUbzE5V1f/x314hIjmRCKgm/gaFdT2Z2lOFf//btSC2boU1a6BzZxg2zOvIjIlqwRJFcxE5jP37UCQH3ldVTxJHRYvCZj2ZWvnxR5g4Ed5/H4YMgZkzXZIwxtQoWKLYADwRcH9jwH0Fjg9XUMH49o1RWKIwIdq1yyUHnw+efhquvBLibYzLmFAF27hoZCQDCVVpuQ+A9qnNPY7ERL2FC2HAAEhNdYvmjjzS7VttjKmVmPuzvKTMJYpuba0ooKnG5s3w29/CwIFukBpg3DhLEsbUUVgThYiMFpEVIrJKRG4JctzhIlIeSg0pnyoJcUKLZqEsKjdNis8HL7zgCvi9+ircdpur1WSMqZewXW39+1Y8A4wC8oDZIvKmqi6t4rhHgfdDeV9VG58w1Rg3DqZPh+OOg+eec4X8jDH1Fkr1WPHvlX2X/34XEQllPuEwYJWqrlbVvcBruF3yKrsamApsquK5X1BVSxRmvz173OpqgAsugClT4LPPLEkY04BCueI+CxwFXOC/vwvXUqhJJpAbcD/P/9g+IpIJnA08H+yNRGS8iMwRkTmFRcVWOdY4M2e6hPDss+7+uee6sQkrv2FMgwrlinuEqk4EigFUdTsQyh6kVf22aqX7TwI3q2p5sDdS1cmqOlRVhzZr3swKAjZ1ubkwdiyccYab0TRkiNcRGdOohTJGUeofR1DYtx+FL4TX5QHZAfezgPWVjhkKvCbuL8C2wCkiUqaq06t7U1VbbNekvfwyTJjgBq4feQSuuw6SbO90Y8IplETxNPAG0F5EHgTOAe4I4XWzgV4i0g3IB84HLgw8QFW7VdwWkSnAW8GShHuNDWY3SRVlv7Oy3Eymv/4VunWr8WXGmPqrMVGo6isiMhc4AdeddJaqLgvhdWUichVuNlM88JKqLhGRCf7ng45LBHlfSxRNyY4dcOut0KIFPP64SxI25dWYiKoxUYhIF6AQmBn4mKquq+m1qvoOlTY5qi5BqOqlNb0fuP4vG8xuAlTdWog//cktoLvuuv2tCmNMRIXS9fQ27vosQHOgG7AC6B/GuKqlqjaY3dj99BOMHw8ffQSHHw7vvguHHeZ1VMY0WaF0PR0aeF9EBgN/DFtENfDZGEXjV1rq6jQ98wz88Y9WwM8Yj9V6Zbaq5ojI4eEIJqTPxxJFo/Txx24r0ieegIMPhrVrobkVfjQmGoQyRvGngLtxwGBgc9giqoENZjcyP/8M118Pr7wCPXrA7bdDmzaWJIyJIqFccVMDvprhxiyqKsUREarYGEVj4PPB3/4Gffq4XefuvBMWLXJJwhgTVYK2KPwL7Vqq6o0RiqdGligaiZ074Y47YNAgV8CvTx+vIzLGVKPaK66IJPhLawyOYDw18mFdTzFr9243BlFeDunp8N138MknliSMiXLBWhTf45LEfBF5E/gPsKfiSVWdFubYqmQlPGLUjBlw9dWuTtOgQXD88dC9u9dRGWNCEMoVNwPYitsj+zTgdP93T/hUbdOiWLJ2LZx5Jpx1FrRuDV995ZKEMSZmBLvitvfPeFrM/gV3FSpXgY2owr1lXn68CZUqnHMOLF0Kf/4zXHstJCZ6HZUxppaCJYp4oCWhlQuPqC5tbL/sqPbtt9C/vysBPnkyZGTAQQd5HZUxpo6CJYoNqnpfxCKpheREW6kblbZtcwX8Jk+Gu+6Ce++10hvGNALBEkXUVl+z2bFRRtXtE3H99S5ZXH893Bg1M6qNMfUULFGcELEoaik+zjJFVLntNreJ0JFHwocfwsCBXkdkjGlA1SYKVd0WyUBqIyEuahs7TUdxsVsX0bYtXHaZG4MYPx4siRvT6MTkb3Wc7UngrQ8/hEMPhcsvd/cPPthtT2pJwphGKSZ/s8t8oWzZbRrcxo1w4YVw0kluA6GrrvI6ImNMBMTkyrW2LZt5HULT8+mncPbZUFQE99wDN99sFV6NaSJiMlFYracIKi11i+QGDIBRo+DBB11XkzGmyYjJK66NUETArl1un+rhw10RvzZt4D//sSRhTBMUk4nCaj2FkSpMmwZ9+8JTT7kFcyUlXkdljPFQTCYKa1GEyZYtcPrpMG6cm/b69ddur4iUFK8jM8Z4KDYThU2PDY/UVLc16RNPwJw5bgGdMabJi9FE4XUEjciXX8KYMW7xXLNmbjOh666DBOveM8Y4MZkobMFdA9i6Ff7wBzdYvXQprF7tHrdFc8aYSmLyqmBpoh5UYcoU6N3bfb/xRpcoBgzwOjJjTJSKyf4Fa1HU0z/+4RLF88+7UhzGGBNEbLYoLE/UTlER3H035OW5kzd1KsyaZUnCGBMSSxSN3fvvwyGHwH33wYwZ7rH0dBuLMMaELCavFjY9NgTr18N558Ho0a4ExyefwMSJXkdljIlBMZkobDuKEDzwgGtB3HcfLFgAI0d6HZExJkaJqnodQ60069RL1y5fRMc0q1z6C3Pn7i/gt3UrbN8OPXt6HZUxJgqIyFxVHVqX14a1RSEio0VkhYisEpFbqnj+IhFZ6P/6WkRC2kPTWhSVFBTANdfAsGFuW1JwRfwsSRhjGkDYEoWIxAPPAGOAfsAFItKv0mE/Ab9S1QHA/cDk0N68AQONZaquomufPjBpElxxBbz8stdRGWMamXCuoxgGrFLV1QAi8hpwJrC04gBV/Trg+G+BrFDe2NZR+P3zn/Cb37gKrzNmwOGHex2RMaYRCmeiyARyA+7nAUcEOf73wLtVPSEi44HxAEkdezbtBsXeva7cRp8+cM45bo3EpZdabSZjTNiEc4yiqut5lSPnIjISlyhurup5VZ2sqkMrBmKabIviiy9g0CC3Z3VxsSvi94c/WJIwxoRVOBNFHpAdcD8LWF/5IBEZALwAnKmqW0N544T4JpYotmyByy6DX/3KtSCef972qzbGREw4/xSdDfQSkW5APnA+cGHgASLSBZgGXKyqP4T6xonxMbn8o25Wr3ZjDwUFcMstcOedtpGQMSaiwpYoVLVMRK4C3gfigZdUdYmITPA//zxwF9AGeNa/2rqsrvN8G52CAmjVCrp1c62JSy91pTiMMSbCYnLB3c51y2meGO91KOFRWAj33w+TJ7sV1VkhTQQzxpig6rPgLiZHQRvtWPbbb8NVV8GaNa4VkZzsdUTGGBObiaLRKSuDCy6A11+Hvn3h88/huOO8jsoYY4AYLQoojWUlRUW3X0ICdOgADz0E8+dbkjDGRJWYTBSNwuzZcMQRkJPj7k+aBLfeCklJ3sZljDGVxGSiiOkxip073TjEEUe4Hee2hrR0xBhjPBObicLrAOqqooDfc8+5ZLF8OYwa5XVUxhgTlA1mR9KyZZCZCTNnwlBbLmKMiQ0xuY6iMP8H4mNhU4qSEnjsMRg4EE4/HUpL3V7V8Y10DYgxJmpF7cZFTdqnn7oEceed8PHH7rHEREsSxpiYE5OJIqrbEps2wW9/C8cf71oQ774LTz7pdVTGGFNnMZkootoHH8Crr8Ltt8PixTB6tNcRGWNMvcTkYHbUTY9dtAhWrHAbCV10ERx9NHTv7nVUxhjTIGKyRSHRkin27IGbbnJbkd50k+tqErEkYYxpVGIyUUSFmTOhXz83q+nSS91K68REr6MyxpgGF5NdT55bvBjOOAP694dZs+DYY72OyBhjwsZaFKEqK4PPPnO3DzkE3noL5s2zJGGMafQsUYTiu+/cSuoTToCVK91jp55qXU3GmCbBEkUw27fDFVfAUUfBli2uVlPPnl5HZYwxEWVjFNUpKXGzmXJz4dpr4d57ITXV66iMMSbiLFFUlp/vCvc1awb33OPKcBx2mNdRGWOMZ6zrqUJxsWs1dO8OM2a4xy691JKEMabJi7kWRViW2n38sRuLWLnS7V19xBHh+BRjjIlJ1qK49lo48US3f/UHH8A//wkdO3odlTHGRI2mmSh8Pigvd7eHDYO77nL1mmy3OWOM+YWmlygWLHBF+555xt2/8EI3NtG8ubdxGWNMlGo6iWL3brj+ehgyBFavtu4lY4wJUcwNZtfJRx/BZZdBXh6MHw+PPALp6V5HZYwxMaFpJIqkJMjIgH/9y3U7GWOMCVnjTBSlpW770Z074YEH4LjjXAG/uKbT02aMMQ2l8V05v/7ajUPcdBMsW+ZmOIElCWOMqaPYu3pWt+Ju2zY3/nDMMbBjB0yfDlOnWoIwxph6irmrqFSXKbZudYvlbrgBli6FM8+MbGDGGNNIxfYYxYoVboD6rrugVy9YuxbatPE6KmOMaVTC2qIQkdEiskJEVonILVU8LyLytP/5hSIyOKQ3LipyyWHAAPjLX1wpcLAkYYwxYRC2RCEi8cAzwBigH3CBiPSrdNgYoJf/azzwXE3v27JkDxx6KNx/P/z617B8OWRnN3D0xhhjKoSzRTEMWKWqq1V1L/AaUHng4EzgH+p8C7QWkU7B3jRzx89ugPqjj+Dll6FDh/BEb4wxBgjvGEUmkBtwPw+oXL+7qmMygQ2BB4nIeFyLA6BEVq5czIknNmy0saktsMXrIKKEnYv97FzsZ+div951fWE4E0VV05O0DsegqpOByQAiMkdVh9Y/vNhn52I/Oxf72bnYz87FfiIyp66vDWfXUx4QOHiQBayvwzHGGGM8FM5EMRvoJSLdRCQJOB94s9IxbwKX+Gc/HQnsVNUNld/IGGOMd8LW9aSqZSJyFfA+EA+8pKpLRGSC//nngXeAU4BVQCFwWQhvPTlMIcciOxf72bnYz87FfnYu9qvzuRDVXwwJGGOMMfvEXAkPY4wxkWWJwhhjTFBRmyjCVv4jBoVwLi7yn4OFIvK1iAz0Is5IqOlcBBx3uIiUi8g5kYwvkkI5FyIyQkTmi8gSEfk80jFGSgi/I2kiMlNEFvjPRSjjoTFHRF4SkU0isria5+t23VTVqPvCDX7/CHQHkoAFQL9Kx5wCvItbi3Ek8J3XcXt4Lo4G0v23xzTlcxFw3Ce4yRLneB23h/8vWgNLgS7+++29jtvDc3Eb8Kj/djtgG5DkdexhOBfHAYOBxdU8X6frZrS2KMJS/iNG1XguVPVrVd3uv/stbj1KYxTK/wuAq4GpwKZIBhdhoZyLC4FpqroOQFUb6/kI5VwokCoiArTEJYqyyIYZfqr6Be5nq06drpvRmiiqK+1R22Mag9r+nL/H/cXQGNV4LkQkEzgbeD6CcXkhlP8XBwPpIvKZiMwVkUsiFl1khXIuJgF9cQt6FwH/paq+yIQXVep03YzW/SgarPxHIxDyzykiI3GJ4tiwRuSdUM7Fk8DNqlru/nhstEI5FwnAEOAEIBn4RkS+VdUfwh1chIVyLk4G5gPHAz2AD0VklqoWhDm2aFOn62a0Jgor/7FfSD+niAwAXgDGqOrWCMUWaaGci6HAa/4k0RY4RUTKVHV6RCKMnFB/R7ao6h5gj4h8AQwEGluiCOVcXAY8oq6jfpWI/AT0Ab6PTIhRo07XzWjterLyH/vVeC5EpAswDbi4Ef61GKjGc6Gq3VS1q6p2BV4HrmyESQJC+x2ZAQwXkQQRScFVb14W4TgjIZRzsQ7XskJEOuAqqa6OaJTRoU7XzahsUWj4yn/EnBDPxV1AG+BZ/1/SZdoIK2aGeC6ahFDOhaouE5H3gIWAD3hBVaucNhnLQvx/cT8wRUQW4bpfblbVRld+XEReBUYAbUUkD7gbSIT6XTethIcxxpigorXryRhjTJSwRGGMMSYoSxTGGGOCskRhjDEmKEsUxhhjgrJEYaKSv/Lr/ICvrkGO3d0AnzdFRH7yf1aOiBxVh/d4QUT6+W/fVum5r+sbo/99Ks7LYn811NY1HD9IRE5piM82TZdNjzVRSUR2q2rLhj42yHtMAd5S1ddF5CTgcVUdUI/3q3dMNb2viPwv8IOqPhjk+EuBoap6VUPHYpoOa1GYmCAiLUXkY/9f+4tE5BdVY0Wkk4h8EfAX93D/4yeJyDf+1/5HRGq6gH8B9PS/9k/+91osItf6H2shIm/79zZYLCLn+R//TESGisgjQLI/jlf8z+32f/9X4F/4/pbMOBGJF5HHRGS2uH0C/hjCafkGf0E3ERkmbi+Sef7vvf2rlO8DzvPHcp4/9pf8nzOvqvNozC94XT/dvuyrqi+gHFfEbT7wBq6KQCv/c21xK0srWsS7/d+vB273344HUv3HfgG08D9+M3BXFZ83Bf/eFcCvge9wBfUWAS1wpamXAIcB44D/CXhtmv/7Z7i/3vfFFHBMRYxnA//rv52Eq+SZDIwH7vA/3gyYA3SrIs7dAT/ff4DR/vutgAT/7ROBqf7blwKTAl7/EPAb/+3WuLpPLbz+97av6P6KyhIexgBFqjqo4o6IJAIPichxuHIUmUAHYGPAa2YDL/mPna6q80XkV0A/4Ct/eZMk3F/iVXlMRO4ANuOq8J4AvKGuqB4iMg0YDrwHPC4ij+K6q2bV4ud6F3haRJoBo4EvVLXI3901QPbvyJcG9AJ+qvT6ZBGZD3QF5gIfBhz/vyLSC1cNNLGazz8JOENEbvDfbw50oXHWgDINxBKFiRUX4XYmG6KqpSKyBneR20dVv/AnklOB/xORx4DtwIeqekEIn3Gjqr5ecUdETqzqIFX9QUSG4GrmPCwiH6jqfaH8EKpaLCKf4cpenwe8WvFxwNWq+n4Nb1GkqoNEJA14C5gIPI2rZfSpqp7tH/j/rJrXCzBOVVeEEq8xYGMUJnakAZv8SWIkcFDlA0TkIP8x/wO8iNsS8lvgGBGpGHNIEZGDQ/zML4Cz/K9pges2miUinYFCVX0ZeNz/OZWV+ls2VXkNV4xtOK6QHf7vV1S8RkQO9n9mlVR1J3ANcIP/NWlAvv/pSwMO3YXrgqvwPnC1+JtXInJYdZ9hTAVLFCZWvAIMFZE5uNbF8iqOGQHMF5F5uHGEp1R1M+7C+aqILMQljj6hfKCq5uDGLr7HjVm8oKrzgEOB7/1dQLcDD1Tx8snAworB7Eo+wO1t/JG6rTvB7SWyFMgRkcXA36ihxe+PZQGurPafca2br3DjFxU+BfpVDGbjWh6J/tgW++8bE5RNjzXGGBOUtSiMMcYEZYnCGGNMUJYojDHGBGWJwhhjTFCWKIwxxgRlicIYY0xQliiMMcYE9f8veX5lj00OjgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y = train_y\n",
    "X = x_train_count.astype(\"float64\")\n",
    "\n",
    "gbc_roc_auc = roc_auc_score(y, gbc_model.predict(X))\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y, gbc_model.predict_proba(X)[:,1])\n",
    "plt.figure()\n",
    "plt.plot(fpr, tpr, label='AUC (area = %0.2f)' % gbc_roc_auc)\n",
    "plt.plot([0, 1], [0, 1],'r--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36ea89c0",
   "metadata": {},
   "source": [
    "### GradientBoostingClassifier model with \"tf-idf\" method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "f3f73194",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "GradientBoosting Classifier model with 'tf-idf' method\n",
      "Accuracy ratio:  0.9405592105263156\n"
     ]
    }
   ],
   "source": [
    "gbc = GradientBoostingClassifier()\n",
    "gbc_model = gbc.fit(x_train_tf_idf_word,train_y)\n",
    "accuracy = model_selection.cross_val_score(gbc_model, \n",
    "                                           x_test_tf_idf_word, \n",
    "                                           test_y, \n",
    "                                           cv = 20).mean()\n",
    "\n",
    "print(\"\\nGradientBoosting Classifier model with 'tf-idf' method\")\n",
    "print(\"Accuracy ratio: \", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "1c065f70",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred=gbc_model.predict(x_test_tf_idf_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "74cdc048",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[7373,   19],\n",
       "       [ 436,  163]], dtype=int64)"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(test_y,pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "eba6ef55",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97      7392\n",
      "           1       0.90      0.27      0.42       599\n",
      "\n",
      "    accuracy                           0.94      7991\n",
      "   macro avg       0.92      0.63      0.69      7991\n",
      "weighted avg       0.94      0.94      0.93      7991\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(test_y,pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "eed76dcf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAz8klEQVR4nO3deXiU5dX48e/JHiAkYYcEBGRHAQVxt6Ki4IatVhRrxdYiivZ1X3HXViu1al0or1p/fbXaVi3BXXHDXRHCDoIIkgRkkSWE7HN+f9wTEkIymSwzz8zkfK4rV+aZeWbm5CHMyb2dW1QVY4wxpj5xXgdgjDEmslmiMMYYE5AlCmOMMQFZojDGGBOQJQpjjDEBWaIwxhgTkCUKY4wxAVmiMCYAEVknIsUisltENonIsyLSrsbjR4nI+yJSKCI7ReRVERlS6zXai8jDIvKD/3XW+I87hf8nMqbxLFEY07AzVLUdMAI4BLgZQESOBN4BcoAeQB9gEfCpiPT1n5MEvAcMBcYB7YGjgG3A6LD+FMY0kdjKbGPqJyLrgEtUda7/+E/AUFU9TUQ+Bpao6uW1nvMmsEVVfy0ilwD3AQeq6u4wh29Mi7AWhTFBEpFsYDywRkTa4FoG/6nj1H8DY/23TwLesiRhopklCmMaNltECoENwGbgDqAD7v/PxjrO3whUjT90rOccY6KGJQpjGnaWqqYBxwODcElgO+ADutdxfndgq//2tnrOMSZqWKIwJkiq+hHwLDBDVYuAz4Ff1nHqubgBbIC5wCki0jYsQRoTApYojGmch4GxIjICuAm4SER+LyJpIpIpIvcCRwJ3+c//P1yX1csiMkhE4kSko4jcIiKnevEDGNNYliiMaQRV3QL8A7hNVT8BTgF+gRuHWI+bPnuMqq72n1+KG9BeCbwL7AK+wnVffRn2H8CYJrDpscYYYwKyFoUxxpiALFEYY4wJyBKFMcaYgCxRGGOMCSjB6wAaq1OnTtq7d2+vwzDGmKjyzTffbFXVzk15btQlit69ezN//nyvwzDGmKgiIuub+lzrejLGGBOQJQpjjDEBWaIwxhgTkCUKY4wxAVmiMMYYE5AlCmOMMQGFLFGIyDMisllEltbzuIjIoyKyRkQWi8ihoYrFGGNM04WyRfEsMC7A4+OB/v6vKcCTIYzFGGNaJZ9PKa2obNZrhGzBnarOE5HeAU6ZAPxDXZ3zL0QkQ0S6q6rtL2yMaXUqKn1s3FlC/o5i8rYXk7d9D/nbi9ldWkF5pY/ySqXC56O8Qin3+aioVP/9Pip8SnmFj3KfUuE/t+r+UesW89v5Oc2KzcuV2Vm4nb+q5Pnv2y9RiMgUXKuDXr16hSU4Y4xpSWUVPjbuLCZ/e3UiyPMnhfztxWzaVUKlr3p/IBHokpZMemoiCXFxJCbEkRgnJMQL7RITSIyPIyFOatwfR2J8HInxQkJcHO2LtnPS0zM4aO5sdnXLblbsXiYKqeO+OndRUtVZwCyAUaNG2U5LxpiIU1pRScGOEpcAtlclBP/tHS4R1NwnLk6gW/sUsjPbMLpPB7IzU8nOTCUrow3Zmal0z0ghOSG+6QGdfTZ8+BrcfDPtp0+Htk3ftt3LRJEH9KxxnA0UeBSLMcYEVFJeWd0ltLd7qJh8fzLYXFi6z/nxcUL39BSyM1M56sBOLgn4k0HPzDZ0S08hMb6Fh4mXLYOMDMjKggcegLvvhqFDm/2yXiaKOcAVIvIicDiw08YnjDFeKSqt8CeAPdXdQzuqk8HW3WX7nJ8YL/TISCUrI5XjB3YmO7MNWRkuEWR3aEPXtGQSWjoR1Bt8EdxzD/z5z3DBBfDss9CvX4u9fMgShYi8ABwPdBKRPOAOIBFAVWcCbwCnAmuAPcDFoYrFGGMKS8rr7BKqaiVs31O+z/lJCXF7P/iHDOnqv91mb8ugS1oK8XF19aCH2euvw7RpsH49/OY3riXRwkI56+n8Bh5XYFqo3t8Y03qoKruKK8jbsadGl9C+CWFn8b6JIDkhzj8u0IaDs9P33s7KSKVnZiqd2iUTFwmJIJAnnnBJYsgQmDcPjj02JG8TdftRGGNaH1Vlx57yOsYIqgeOC0sr9nlOm6T4vR/+Iw/IrE4E/nGCjm2TEInwRFCXigrYsgW6d4dzz4XiYrjySkhKCtlbWqIwxnhOVdlWVLbP+oHaSWFP2b6LxtolJ+ydKXRE347+GUPV3UMZbRKjMxEE8tVXcOmlkJAAX3wBnTrBtdeG/G0tURhjQs7nU7buLmXD3nGBfaeQ5u8opqTct89z0lMTycpIpXfHthzTr/PelkB2ZirZGW1on5oQe4mgPjt2wC23wMyZriXxyCMQF75SfZYojDHNVulTNheW1NkSqBojKKvYNxFktkkkO7MNA7qmMWZgl326hrIyU2mfkujRTxNhliyBsWNdd9Pvf++mvLZvH9YQLFEYYxpUUelj066aiaCY/B3VA8UFO4opr9x3LWyndklkZbZhSPf2nDyka411BG7AuG2yffwEVF4OiYkwYACMGQPXXw+HelM71f6ljDGUV/rYtLOEDft0CVW3DDbu3Le8BLjyElmZqQzLzuDUg7tXryHwJ4LUpGasKm7NSkvdFNfnnoMFC6BdO3jhBU9DskRhTCtQWlHJxh0l+7QEao4RbNpVQs08IP7yElkZqYw6IHOf2UJZGan0yEglJdESQYt7/3247DL49luYONEljXbtvI7KEoUxsaCkvJL8Hfu3BKpuby4s3a/OUPd01xV0xIEdya61mKx7eipJCbavWdgUF8OUKa4V0bcvvPUWnHKK11HtZYnCmCiwp6zCJYFaJairxgi21KozlBAndM9IITujDcf277xPl1B2Zmpo6gyZpktJga1bYfp0N7spNdXriPZhicKYCLC7tKKO0hLV4wXbivavM5SV4f76P8E/Y6hqoDg7M5Wu7SOkvISp3+LFboD66achO9uV4gjjlNfGsERhTBjsKikn76f9u4Sqbu+oo85Q1XjA0B7p1esH/GWou6RFQXkJU7eiIrjzTvjLXyAzE1avdokiQpMEWKIwptlUlZ3F1eUlaq4dqLqvsGTf8hKpifF7B4dH9MzYuwdBVcugU1tLBDFpzhxXbuOHH+B3v4P774cOHbyOqkGWKIxpgKryk7+8xP6rit1xUa3yEm2T4vd2Ax3WO3O/MYIO0VpnyDTP7Nlusdwnn8DRR3sdTdAsUZhWT1XZsru0zsVkVQmhuHzfRJCWnEB2hzb07NCGIw/sWKNryCWH9NQYrDNkGq+8HB591C2YO/RQV3ojJcUtpIsilihMzPP5lM2Fpft8+NccI8jfXkxprfISGW1cnaEDO7flZwM677uYzJ8IjAnoiy9cAb/Fi+HGG12iSEvzOqomsURhol6lT/lxV8n+lUd3uNsFO0ooq9w3EXRom0R2ZiqDuqVx4qAu+6whyMpIJc3qDJmm2r4dbr4ZZs1yW5L+978wYYLXUTWLJQoT8YrLKvn2x0L2lFXuP0awYw8bd5RQ4atdZyiZ7MxUDspK55SDurlEkFE9WNwmyX71TYjMmgVPPQVXX+1mN0VpK6Im+99iItL2ojKe/3I9767YzMqNu/brGuraPpnszDYc0jOTM4btuyFNlpWXMOG2apWr7nrMMXDVVTB+PAwb5nVULcYShYko67cV8fQn3/Of+XkUl1cy6oBMLjziAEb17kBaSgJZGal0z0ghOcESgYkAJSXwxz+6aa6DBkFuLiQnx1SSAEsUJox2FpdTXFZJaUUlJeW+fb7vLqlgzqIC3lq2iYQ4YcKILC45tg+DuoW37r4xQXv3Xbj8clizBiZNgj//2VVTjEGWKExI5W3fw5xFBczJLWDlpsKA57ZPSeCynx3IRUf1pmv7lDBFaEwTzJsHJ58M/fu7hHHSSV5HFFKWKEyL215UxutLNpKTm8/X67YDcGivDG4YN5CM1CRSEuNITognJTGOlMR4khPc976d29ogs4lclZWwfDkcfDAce6yr0TRpklsXEePsf6VpEXvKKnh3+Y/MyS3go2+3UOFT+nVpx3UnD+DM4Vn06tjG6xCNabqFC2HqVFixwtVm6toVfvMbr6MKG0sUpsnKK318smYrOQvzeWf5j+wpq6R7egq/PaYPZ47owZDu7W11soluhYVwxx1uRXWnTvDkk9Cli9dRhZ0lCtMoqsqCH7aTk1vA64s3sq2ojPYpCUwY0YMJI7IY3buDFbMzsWHnTtfNtGGDW2H9xz+6aq+tkCUKE5TVPxYyOzefnNwC8rYXk5wQx0lDujJheA9+NrCzTVc1sWPXLle4Lz3d7Tp34olw5JFeR+UpSxSmXgU7inl1UQGzcwtYsXEXcQLH9O/M1ScN4OShXa3MhYkt5eVuj4h774UPP3S1maZP9zqqiGCJwuxjx54y3liyiZzcfL5a9xOqMKJnBneeMYTThvWgc1qy1yEa0/I+/dQNVi9dCmedBZ07ex1RRLFEYSguq2Tuih/JyS3go283U16p9O3clqtPGsCEET04oGNbr0M0JnSuvBIeewx69oScHDjzTK8jijiWKFqpikofn363jZyF+by9bBNFZZV0bZ/M5KN6M2FEFkN72IwlE8NUq1dRd+sG113nZje1a+dtXBHKEkUroqos3LCDObkFvLa4gK27y0hLSeD0YT2YcEgPDu/TkXibsWRi3cqVrpvp6qtd+e9bb/U6oohniaIVWLN5Nzn+GUs//LSHpIQ4ThrchTOHZzFmkM1YMq1EcTH84Q/wwAPQtq07NkEJaaIQkXHAI0A88JSq3l/r8XTgOaCXP5YZqvr3UMbUWmzaWeKfsZTPsgI3Y+nofp248oR+nHJQN9rbjCXTmrz3nlsL8d13cOGFMGNGq1w411QhSxQiEg88DowF8oCvRWSOqi6vcdo0YLmqniEinYFVIvK8qpaFKq5YtnNPOW8u3cjs3Hy+/N7NWBqenc7tpw/h9GHd6WKF9kxrlZcHCQkuYZxwgtfRRJ1QtihGA2tUdS2AiLwITABqJgoF0sSNmrYDfgIqQhhTzCkpr+S9FZvJyc3nw1VbKKv00adTW/7nxP5MGJFFn042Y8m0QpWVMHMmJCXB734Hv/41nHee2yvCNFooE0UWsKHGcR5weK1zHgPmAAVAGjBRVX21zkFEpgBTAHr16hWSYKNJRaWPz9duY/bCAt5etondpRV0SUvmwiMPYMKIHhyclW4zlkzrtWCB62aaPx/OPtslChFLEs0QykRR1yeV1jo+BcgFTgAOBN4VkY9Vddc+T1KdBcwCGDVqVO3XaBVUlUV5O8nJzefVRRvZuruUtOQExh/UjbMOyeKIvjZjybRyu3bBbbe5NRGdO8MLL8DEiV5HFRNCmSjygJ41jrNxLYeaLgbuV1UF1ojI98Ag4KsQxhVVvtuym5zcAubk5rNu2x6S4uM4YVAXJozowZhBXWxvaGOqLFrkksTUqXDffZCR4XVEMSOUieJroL+I9AHygfOASbXO+QE4EfhYRLoCA4G1IYwpKvy4y81YysktYEn+TkTgqAM7cvnxbsZSeqrNWDIGgO+/hw8+cHtDHHus25a0Tx+vo4o5IUsUqlohIlcAb+Omxz6jqstEZKr/8ZnAPcCzIrIE11V1o6puDVVMkWxncTlvL93E7Nx8Pl+7DVU4OCud6acN5ozhPWxrUGNqKitze1TffbfbYe7nP3clwC1JhERI11Go6hvAG7Xum1njdgFwcihjiGQl5ZV8sHIzObkFvL9qM2UVPg7o2IYrT+jPhBE9OLCzlRMwZj8ff+y6l5Yvh1/8wm0q1Er3iQgXW5kdZpU+5Yu125i9MJ+3lm6isLSCTu2SueDwXkwYkcXwbJuxZEy9tmyBk092W5G++iqcfrrXEbUKlijCQFVZkr+TnNwCXl1UwObCUtolJ3DK0G6cdUgPjuzbkYT4OK/DNCYyqcLcuTB2rJvN9NprcMQRrgyHCQtLFCH0/dYicnLzmZNbwNqtRSTFx3H8wM6cdUgWJ9iMJWMatmwZXHaZ62764AM4/ni345wJK0sULWxzYQmvLtrInNx8FuW5GUtH9OnIlOP6Mv6g7qS3sRlLxjRozx6309yDD7ptSZ96Co47zuuoWi1LFC1gV4mbsZSTW8Bn323FpzC0R3tuPXUwpw/vTvf0VK9DNCZ6qMKYMfDVV3DRRS5Z2I5znrJE0USlFZV8sHILcxblM3eFm7HUq0Mbpo3px4QRPejXJc3rEI2JLhs3uoqu8fFwyy2Qnu66moznLFE0QqVP+fL7beQsLOCNpRspLKmgU7skJo3uxZkjenBIzwybsWRMY1VWwuOPw/TpbkX1lVe6DYVMxLBE0QBVZVnBLjcovaiAH3eV0jYpnlMO6saEEVkcfaDNWDKmyebPdwX8FiyAU06BU0/1OiJTh6AThYi0VdWiUAYTSdZvKyInt4Cc3Hy+21JEYrzwswFduO30Hpw4qCupSTZjyZhm+dOf4Kab3J7V//oX/PKX1ftYm4jSYKIQkaOAp3D7RfQSkeHApap6eaiDC7cthaW8ttjVWMrdsAOAw/t04LfH9OXUg7uR0SbJ2wCNiXaqUFEBiYkwejRMm+ZmN6Wnex2ZCSCYFsVfcOXA5wCo6iIRiZl5aoUl5byz7Edm5+bz6Ro3Y2lw9/bcPH4QZwzvQY8Mm7FkTIv47ju4/HI46CBXp+n4422wOkoE1fWkqhtqDdJWhiac8Nm8q4S7XlvO3OU/UlrhIzszlcuOP5AJI7IY0NVmLBnTYkpL3RTX++5zLQkbqI46wSSKDf7uJxWRJOD3wIrQhhVary/eyPTZS9i+p5wLjziAsw7pwaG9Mm3GkjEt7Ztv4Fe/gpUr3RjEww9Djx5eR2UaKZhEMRV4BLe1aR7wDhC14xNvLd3EtH8uQASevmgUJw7u6nVIxsSudu3cAPUbb8D48V5HY5oomEQxUFUvqHmHiBwNfBqakEJn/rqfuPHlxXRom8RnN51gtZaMaWk+H/z97/D5567sxsCBsHQpxNkU8mgWzL/eX4O8L2Kt21rEZc99wzkzPyclMY4/nT3MkoQxLW3pUleP6ZJLYPVqKPLPprckEfXqbVGIyJHAUUBnEbmmxkPtcTvWRbztRWU8+v5qnvtiPYnxcVwzdgCXHNuHNkm2ztCYFlNU5Haae+ghN8317393NZpszC9mBPrETMKtnUgAak4D2gWcE8qgWsLzX67n/jdXUlRawcTDenH12P50SbPtRI1pcSUlLjn8+tduEV3Hjl5HZFpYvYlCVT8CPhKRZ1V1fRhjaraPvt3Crf9dSnJCHG9ddZxNdzWmpeXlwaOPwh//6BLDypXQoYPXUZkQCaYPZo+IPAgMBfb+Sa6qJ4Qsqmaa+eF3ALx82VGWJIxpSRUV8Ne/wu23u2J+EyfCyJGWJGJcMKNMzwMrgT7AXcA64OsQxtQsu0sr+HztNiYf1ZuDsqwsgDEt5ssvYdQouOYaN2i9bJlLEibmBZMoOqrq00C5qn6kqr8BjghxXE3253dWAXDi4C4eR2JMDPH54OKLYcsWeOklt291nz5eR2XCJJiup3L/940ichpQAGSHLqSmy92wgxe++oHu6Skc06+T1+EYE91UXVIYNw7S0uCVVyAry902rUowLYp7RSQduBa4DldJ9qpQBtUUqsqNLy0mOSGeOVccY+U4jGmO1avd/hDnnguzZrn7Bg2yJNFKNdiiUNXX/Dd3AmNg78rsiPJTURmrfizk+lMG0jkt2etwjIlOpaXwwAPwhz9AcjI89hhMnep1VMZjgRbcxQPn4mo8vaWqS0XkdOAWIBU4JDwhBuf1JRsB6N2xrceRGBPFpk2Dp5+G885zC+i6d/c6IhMBArUongZ6Al8Bj4rIeuBI4CZVnR2G2IJWUl7J/32+nrSUBE4aYoPYxjTK5s1usLpbN7jxRlfl9ZRTvI7KRJBAiWIUMExVfSKSAmwF+qnqpvCEFrzXF29k9ebdzLpwJMkJUVFdxBjv+XyucN+NN8LJJ7vtSPv3d1/G1BBoMLtMVX0AqloCfBuJSQLg0fdX07FtEmMGWWvCmKAsXgzHHAOXXgojRsBdd3kdkYlggVoUg0Rksf+2AAf6jwVQVR0W8uiCUOlTthSWMqp3BxLjrUqlMQ166SU3BpGZCf/4h9tYyGYJmgACJYrBYYuiGQpLytlTVslx/W3dhDEB7doF7du7faqnTYM77rDSGyYogYoCRkUhwOUFuwDokZHqcSTGRKgffoArr4SCAvjiC+jUCR55xOuoTBQJaV+NiIwTkVUiskZEbqrnnONFJFdElonIR419j2c+XUf7lAROsPEJY/ZVXg4zZsDgwTB3rls8p+p1VCYKhWwHH/86jMeBsbi9tr8WkTmqurzGORnAE8A4Vf1BRBr1aa+qzF3xI0f362g71hlT0/r1cOaZbtD6jDNcxdcDDvA6KhOlgmpRiEiqiAxs5GuPBtao6lpVLQNeBCbUOmcS8Iqq/gCgqpsb8wYbd5YAMLJXZiNDMyZGVbUYunWDrl3hv/+FnBxLEqZZGkwUInIGkAu85T8eISJzgnjtLGBDjeM8/301DQAyReRDEflGRH4dVNR+by51s3XHHWSrR00rpwrPPQeHHQa7d7vyG++8A2edZTOaTLMF06K4E9c62AGgqrlA7yCeV9dvZ+0O0gRgJHAacApwm4gM2O+FRKaIyHwRmb9ly5a997/tTxSDulmhMtOKrVoFJ54IF14ICQmwbZvXEZkYE0yiqFDVnU147TxcCZAq2bgS5bXPeUtVi1R1KzAPGF77hVR1lqqOUtVRnTt3BqC80sf324oYeUAmcXH2F5NphSoq3BTXYcNgwQJ48kn47DPrZjItLphEsVREJgHxItJfRP4KfBbE874G+otIHxFJAs4DandZ5QDHikiCiLQBDgdWBBN47oYdbCks5aKjegdzujGxJz4ePv4YzjnHtSqmToU4W3RqWl4wv1VX4vbLLgX+iSs3flVDT1LVCuAK4G3ch/+/VXWZiEwVkan+c1bgxj4W44oPPqWqS4MJfJN/ILtb+5QGzjQmhmzaBL/5DWzY4MYe3ngDnn/eDVwbEyLBTI8dqKq3Arc29sVV9Q3gjVr3zax1/CDwYGNf2+ef3ZFq02JNa1BZ6TYQuvlmKC6G8eOhZ09IsT+UTOgF06J4SERWisg9IjI05BEFaWex26G1S3vbpMjEuIUL4aij4PLLYdQoWLLElQI3JkwaTBSqOgY4HtgCzBKRJSIyPdSBNeSDlZvpkZ5C53aWKEyMe+wxWLfOdTG9+y4M2G9ioDEhFdTIl6puUtVHgam4NRW3hzKohlRU+vh49VZOPbi7zXgysUfVLZRbuNAdz5gBK1fCpEm2JsJ4IpgFd4NF5E4RWQo8hpvxlB3yyALYUVxOhU/p1bGNl2EY0/LWrXOlN37xC3j4YXdfZqb7MsYjwQxm/x14AThZVWuvg/DE9qIyADLbJHkciTEtpLzc7VF9111uiuuMGfA//+N1VMYAQSQKVT0iHIE0xk+WKEys+dvf4KabXMmNRx6BXr28jsiYvepNFCLyb1U9V0SWsG/pDc93uCssqQAgPTXRqxCMab5t21xX08iR8LvfQb9+MG6c11EZs59ALYqqdu/p4QikMbbuLgUgLSVkVdKNCR1VtwXpdddBWhp8+60r4mdJwkSoegezVXWj/+blqrq+5hdweXjCq9u81VvokpZMrw42mG2izIoVMGYMTJ4M/fvD7NmukJ8xESyY6bFj67hvfEsHEiwFPly1hZOGdLWpsSa6LFoEw4e7zYRmzYJPPnEF/YyJcIHGKC7DtRz6isjiGg+lAZ+GOrD6VPqUPWWV9O/SzqsQjGmcvDzIznZJ4a674Le/hS62da+JHoHavP8E3gT+CNTc77pQVX8KaVRBiLfWhIl0BQVw9dWucN/KlZCV5Wo1GRNlAnU9qaquA6YBhTW+EJEOoQ+tvqjctzhboWoiVWWlK7sxeLDbhvSGG6BTJ6+jMqbJGmpRnA58g/t4rvnJrEDfEMZVL/VnCmtRmIhUUgLHHQdffw1jx8ITT7hpr8ZEsXoThaqe7v/eJ3zhNKxqQUe8tShMJCkvh8REV/Z7zBi45hqYONFqM5mYEEytp6NFpK3/9q9E5CER8W7ZaFXXk7UoTCRQhZdecq2GBQvcfQ88AOedZ0nCxIxgpsc+CewRkeHADcB64P9CGlUAlT6XKZISbMtH47G1a+G009zeEB072jakJmYF85tdoaoKTAAeUdVHcFNkPVHhTxRZGalehWCMK+A3dKjbs/rhh+Grr2DECK+jMiYkglkSWigiNwMXAseKSDzgYZEllygSrOvJeGn3bjj1VFfAL9vTqvvGhFwwLYqJQCnwG1XdBGTRhD2uW0rVYLZ1/5qw2roVLr4Y5sxxx9Onw8svW5IwrUIwW6FuAp4H0kXkdKBEVf8R8sjqDch9s3UUJix8PnjmGRg4EJ57DtascffbeIRpRYKZ9XQu8BXwS+Bc4EsROSfUgTXE8oQJueXL4fjjXcmNIUMgN9dNezWmlQlmjOJW4DBV3QwgIp2BucBLoQysPlWD2T6fF+9uWpX582HZMnj6aVft1VoRppUKJlHEVSUJv20EN7YREnEi+IDUpHivQjCx7I033IZCF17ovk4/HTp4V7HGmEgQzAf+WyLytohMFpHJwOvAG6ENKxAr4WFCIC8PzjnHrYt47DG3kE7EkoQxBDeYfT3wN2AYMByYpao3hjqweuPxf7cSHqZFVFS4Ka6DB8Prr8N997m1Efb7Zcxegfaj6A/MAA4ElgDXqWp+uAJriHUXmxbxzTdw1VVuG9LHH4e+ntS6NCaiBfq4fQZ4DTgbV0H2r2GJqAFq02NNc+3cCa+84m4ffjh8+aUbm7AkYUydAg1mp6nq//pvrxKRBeEIKFg2RmEaTRX+/W/Xgti2Ddatgx49YPRoryMzJqIFShQpInII1ftQpNY8VlVPEkfVGIW1KEyjfPcdTJsGb78NI0fCq6+6JGGMaVCgRLEReKjG8aYaxwqcEKqgAvJnCmtRmKAVFrrk4PPBo4/C5ZdDvE2vNiZYgTYuGhPOQIJVtcOd5QnToMWLYdgwSEtzi+aOOMLtW22MaZSomztk+1GYBm3ZAhddBMOHu0FqgLPPtiRhTBOF9NNWRMaJyCoRWSMiNwU47zARqQymhlRxeSWDuqXRJimYReWmVfH54KmnXAG/F16AW25xtZqMMc0Ssk9b/74VjwNjgTzgaxGZo6rL6zjvAeDtYF630qd0bJfU0uGaWHD22TB7Nhx3HDz5pCvkZ4xptmCqx4p/r+zb/ce9RCSY+YSjgTWqulZVy4AXcbvk1XYl8DKwuY7H9uNTSE20gUjjV1TkVlcDnH8+PPssfPihJQljWlAwXU9PAEcC5/uPC3EthYZkARtqHOf579tLRLKAnwMzA72QiEwRkfkiMr+8ooIUSxQG3BTXIUPgiSfc8bnnurEJmzptTIsKJlEcrqrTgBIAVd0OBNP3U9f/Vq11/DBwo6pWBnohVZ2lqqNUdVRcXLwlitZuwwb4xS/gzDPdjKaRI72OyJiYFswYRbl/HEFh734UwewGkQf0rHGcDRTUOmcU8KK4vwA7AaeKSIWqzq7vRX2q1vXUmj33HEyd6gau778frr4akmzMyphQCiZRPAr8F+giIvcB5wDTg3je10B/EekD5APnAZNqnqCqfapui8izwGuBkoR7DqQk2tTYVqeq7Hd2tpvJ9Ne/Qp8+DT7NGNN8DSYKVX1eRL4BTsR1J52lqiuCeF6FiFyBm80UDzyjqstEZKr/8YDjEvWxFkUrs2MH3HwztG0LM2a4JGFTXo0JqwYThYj0AvYAr9a8T1V/aOi5qvoGtTY5qi9BqOrkhl6vSrIlitin6tZCXHONW0B39dXVrQpjTFgF0/X0Om58QoAUoA+wChgawrgCssHsGPf99zBlCsydC4cdBm++CYcc4nVUxrRawXQ9HVzzWEQOBS4NWURBSIy3vypjWnm5q9P0+ONw6aVWwM8YjzV6ZbaqLhCRw0IRTLCsxHgMeu89txXpQw/BgAGwfj2kpHgdlTGG4MYorqlxGAccCmwJWURBsBLjMeTHH+Haa+H55+HAA+HWW6FjR0sSxkSQYOaZptX4SsaNWdRViiNs4q1FEf18Pvjb32DQILfr3G23wZIlLkkYYyJKwBaFf6FdO1W9PkzxBKWwtMLrEExz7dwJ06fDiBGugN+gQV5HZIypR70tChFJ8JfWODSM8QSlS1qy1yGYpti9241BVFZCZiZ8+SW8/74lCWMiXKAWxVe4JJErInOA/wBFVQ+q6ishjq1eNuspCuXkwJVXujpNI0bACSdA375eR2WMCUIwYxQdgG24PbJPB87wf/eMzXqKIuvXw4QJcNZZkJEBn37qkoQxJmoEalF08c94Wkr1grsqtavAhlWCtSiigyqccw4sXw5/+hNcdRUkJnodlTGmkQIlinigHcGVCw+r+DgrChjRvvgChg51JcBnzYIOHeCAA7yOyhjTRIESxUZVvTtskTRCgq2jiEw//eQK+M2aBbffDnfdZaU3jIkBgRKFfRqb4Ki6fSKuvdYli2uvhesjaka1MaYZAiWKE8MWRSN1aGsb1USUW25xmwgdcQS8+y4MH+51RMaYFlRvolDVn8IZSGPYrKcIUFLi1kV06gQXX+zGIKZMARs/MibmROX/6viojDqGvPsuHHww/O537njAALc9qSUJY2JSVP7PFmtReGPTJpg0CU4+2W0gdMUVXkdkjAmDRpcZjwTW9eSBDz6An/8ciovhzjvhxhutwqsxrUSUJgqvI2hFysvdIrlhw2DsWLjvPtfVZIxpNaKy68laFGFQWOj2qT72WFfEr2NH+M9/LEkY0wpFZ6KwJkXoqMIrr8DgwfDII27BXGmp11EZYzwUnYnC8kRobN0KZ5wBZ5/tpr1+9pnbK6JNG68jM8Z4KEoThWWKkEhLc1uTPvQQzJ/vFtAZY1q9qEwUlida0CefwPjxbvFccrLbTOjqqyEhKuc5GGNCICoTRYIt7Gq+bdvgkkvcYPXy5bB2rbvfrq0xppao/FRISojKsCODKjz7LAwc6L5ff71LFMOGeR2ZMSZCWf9Ca/SPf7hEMXOmK8VhjDEBROWf5jZE0UjFxXDHHZCX5wZ4Xn4ZPv7YkoQxJihRmShMI7z9Nhx0ENx9N+TkuPsyM20swhgTtKj8tLBZT0EoKICJE2HcOFeC4/33Ydo0r6MyxkShqEwUJgj33utaEHffDYsWwZgxXkdkjIlSoqpex9Aoyd376471K0lNivc6lMjzzTfVBfy2bYPt26FfP6+jMsZEABH5RlVHNeW5IW1RiMg4EVklImtE5KY6Hr9ARBb7vz4TEdtDsyl27YLf/x5Gj3bbkoIr4mdJwhjTAkKWKEQkHngcGA8MAc4XkSG1Tvse+JmqDgPuAWYF99otGWkUU3UVXQcNgsceg8sug+ee8zoqY0yMCeU6itHAGlVdCyAiLwITgOVVJ6jqZzXO/wLIDmE8seef/4Rf/cpVeM3JgcMO8zoiY0wMCmWiyAI21DjOAw4PcP5vgTfrekBEpgBTAJK6tfLulLIyV25j0CA45xy3RmLyZKvNZIwJmVCOUdTVQVTnyLmIjMElihvrelxVZ6nqqKYOxMSMefNgxAi3Z3VJiSvid8klliSMMSEVykSRB/SscZwNFNQ+SUSGAU8BE1R1WzAv3OrGKLZuhYsvhp/9zLUgZs60/aqNMWETyj9Fvwb6i0gfIB84D5hU8wQR6QW8Alyoqt+GMJbotXatG3vYtQtuugluu802EjLGhFXIEoWqVojIFcDbQDzwjKouE5Gp/sdnArcDHYEnxDUTKoLpXpLWUO1p1y5o3x769HGticmTXSkOY4wJs6hccFe4YVXslhrfswfuuQdmzXIrqrNtIpgxpvmas+AuKkdBY3aM4vXX4YorYN0614pITfU6ImOMic5EEXMqKuD88+Gll2DwYPjoIzjuOK+jMsYYIEqLAsZMg6Kq2y8hAbp2hT/8AXJzLUkYYyJKVCaKmPD113D44bBggTt+7DG4+WZISvI2LmOMqSUqE4VE8yDFzp1uHOLww92Oc9uCWjpijDGeicpEEbWqCvg9+aRLFitXwtixXkdljDEBReVgdtS2J1asgKwsePVVGNW6q5EYY6JHVK6jKM7/lri4KEgXpaXw4IMwfDiccQaUl7u9quNt0yVjTHhF7MZFoRIVQxQffOASxG23wXvvufsSEy1JGGOiTpQmigjOFJs3w0UXwQknuBbEm2/Cww97HZUxxjRZVCaKiPbOO/DCC3DrrbB0KYwb53VExhjTLFE5mB1xliyBVavcRkIXXABHHQV9+3odlTHGtAhrUTRHURHccIPbivSGG1xXk4glCWNMTLFE0VSvvgpDhrhZTZMnu5XWiYleR2WMMS3Oup6aYulSOPNMGDoUPv4YjjnG64iMMSZkrEURrIoK+PBDd/ugg+C112DhQksSxpiYZ4kiGF9+6VZSn3girF7t7jvtNOtqMsa0CpYoAtm+HS67DI48ErZudbWa+vXzOipjjAkrG6OoT2mpm820YQNcdRXcdRekpXkdlTHGhF3UJYqQr8nOz3eF+5KT4c47XRmOQw4J9bsaY0zEsq6nKiUlrtXQty/k5Lj7Jk+2JGGMafWirkUREu+958YiVq92e1cffrjXERljTMSwFsVVV8FJJ7n9q995B/75T+jWzeuojDEmYrTOROHzQWWluz16NNx+u6vXZLvNGWPMflpfoli0yBXte/xxdzxpkhubSEnxNi5jjIlQrSdR7N4N114LI0fC2rXWvWSMMUFqHYPZc+fCxRdDXh5MmQL33w+ZmV5HZYwxUaF1JIqkJOjQAf71L9ftZIwxJmixmSjKy932ozt3wr33wnHHuQJ+ca2np80YY1pK9H1yNrQ0+7PP3DjEDTfAihVuhhNYkjDGmCaKnU/Pn35y4w9HHw07dsDs2fDyy5YgjDGmmaLuU1Tqa1Js2+YWy113HSxfDhMmhDcwY4yJUdE9RrFqlRugvv126N8f1q+Hjh29jsoYY2JKSFsUIjJORFaJyBoRuamOx0VEHvU/vlhEDg3qhYuLXXIYNgz+8hdXChwsSRhjTAiELFGISDzwODAeGAKcLyJDap02Hujv/5oCPNnQ67YrLYKDD4Z77oFf/hJWroSePVs4emOMMVVC2aIYDaxR1bWqWga8CNQeOJgA/EOdL4AMEeke6EWzdvzoBqjnzoXnnoOuXUMTvTHGGCC0YxRZwIYax3lA7frddZ2TBWyseZKITMG1OABKZfXqpZx0UstGG506AVu9DiJC2LWoZteiml2LagOb+sRQJoq6pidpE85BVWcBswBEZL6qjmp+eNHPrkU1uxbV7FpUs2tRTUTmN/W5oex6ygNqDh5kAwVNOMcYY4yHQpkovgb6i0gfEUkCzgPm1DpnDvBr/+ynI4Cdqrqx9gsZY4zxTsi6nlS1QkSuAN4G4oFnVHWZiEz1Pz4TeAM4FVgD7AEuDuKlZ4Uo5Ghk16KaXYtqdi2q2bWo1uRrIar7DQkYY4wxe0VdCQ9jjDHhZYnCGGNMQBGbKEJW/iMKBXEtLvBfg8Ui8pmIDPciznBo6FrUOO8wEakUkXPCGV84BXMtROR4EckVkWUi8lG4YwyXIP6PpIvIqyKyyH8tghkPjToi8oyIbBaRpfU83rTPTVWNuC/c4Pd3QF8gCVgEDKl1zqnAm7i1GEcAX3odt4fX4igg0397fGu+FjXOex83WeIcr+P28PciA1gO9PIfd/E6bg+vxS3AA/7bnYGfgCSvYw/BtTgOOBRYWs/jTfrcjNQWRUjKf0SpBq+Fqn6mqtv9h1/g1qPEomB+LwCuBF4GNoczuDAL5lpMAl5R1R8AVDVWr0cw10KBNBERoB0uUVSEN8zQU9V5uJ+tPk363IzURFFfaY/GnhMLGvtz/hb3F0MsavBaiEgW8HNgZhjj8kIwvxcDgEwR+VBEvhGRX4ctuvAK5lo8BgzGLehdAvyPqvrCE15EadLnZqTuR9Fi5T9iQNA/p4iMwSWKY0IakXeCuRYPAzeqaqX74zFmBXMtEoCRwIlAKvC5iHyhqt+GOrgwC+ZanALkAicABwLvisjHqrorxLFFmiZ9bkZqorDyH9WC+jlFZBjwFDBeVbeFKbZwC+ZajAJe9CeJTsCpIlKhqrPDEmH4BPt/ZKuqFgFFIjIPGA7EWqII5lpcDNyvrqN+jYh8DwwCvgpPiBGjSZ+bkdr1ZOU/qjV4LUSkF/AKcGEM/rVYU4PXQlX7qGpvVe0NvARcHoNJAoL7P5IDHCsiCSLSBle9eUWY4wyHYK7FD7iWFSLSFVdJdW1Yo4wMTfrcjMgWhYau/EfUCfJa3A50BJ7w/yVdoTFYMTPIa9EqBHMtVHWFiLwFLAZ8wFOqWue0yWgW5O/FPcCzIrIE1/1yo6rGXPlxEXkBOB7oJCJ5wB1AIjTvc9NKeBhjjAkoUruejDHGRAhLFMYYYwKyRGGMMSYgSxTGGGMCskRhjDEmIEsUJiL5K7/m1vjqHeDc3S3wfs+KyPf+91ogIkc24TWeEpEh/tu31Hrss+bG6H+dquuy1F8NNaOB80eIyKkt8d6m9bLpsSYiichuVW3X0ucGeI1ngddU9SURORmYoarDmvF6zY6podcVkf8HfKuq9wU4fzIwSlWvaOlYTOthLQoTFUSknYi85/9rf4mI7Fc1VkS6i8i8Gn9xH+u//2QR+dz/3P+ISEMf4POAfv7nXuN/raUicpX/vrYi8rp/b4OlIjLRf/+HIjJKRO4HUv1xPO9/bLf/+79q/oXvb8mcLSLxIvKgiHwtbp+AS4O4LJ/jL+gmIqPF7UWy0P99oH+V8t3ARH8sE/2xP+N/n4V1XUdj9uN1/XT7sq+6voBKXBG3XOC/uCoC7f2PdcKtLK1qEe/2f78WuNV/Ox5I8587D2jrv/9G4PY63u9Z/HtXAL8EvsQV1FsCtMWVpl4GHAKcDfxvjeem+79/iPvrfW9MNc6pivHnwP/z307CVfJMBaYA0/33JwPzgT51xLm7xs/3H2Cc/7g9kOC/fRLwsv/2ZOCxGs//A/Ar/+0MXN2ntl7/e9tXZH9FZAkPY4BiVR1RdSAiicAfROQ4XDmKLKArsKnGc74GnvGfO1tVc0XkZ8AQ4FN/eZMk3F/idXlQRKYDW3BVeE8E/quuqB4i8gpwLPAWMENEHsB1V33ciJ/rTeBREUkGxgHzVLXY3901TKp35EsH+gPf13p+qojkAr2Bb4B3a5z//0SkP64aaGI9738ycKaIXOc/TgF6EZs1oEwLsURhosUFuJ3JRqpquYisw33I7aWq8/yJ5DTg/0TkQWA78K6qnh/Ee1yvqi9VHYjISXWdpKrfishIXM2cP4rIO6p6dzA/hKqWiMiHuLLXE4EXqt4OuFJV327gJYpVdYSIpAOvAdOAR3G1jD5Q1Z/7B/4/rOf5ApytqquCidcYsDEKEz3Sgc3+JDEGOKD2CSJygP+c/wWexm0J+QVwtIhUjTm0EZEBQb7nPOAs/3Pa4rqNPhaRHsAeVX0OmOF/n9rK/S2buryIK8Z2LK6QHf7vl1U9R0QG+N+zTqq6E/g9cJ3/OelAvv/hyTVOLcR1wVV5G7hS/M0rETmkvvcwpoolChMtngdGich8XOtiZR3nHA/kishC3DjCI6q6BffB+YKILMYljkHBvKGqLsCNXXyFG7N4SlUXAgcDX/m7gG4F7q3j6bOAxVWD2bW8g9vbeK66rTvB7SWyHFggIkuBv9FAi98fyyJcWe0/4Vo3n+LGL6p8AAypGszGtTwS/bEt9R8bE5BNjzXGGBOQtSiMMcYEZInCGGNMQJYojDHGBGSJwhhjTECWKIwxxgRkicIYY0xAliiMMcYE9P8BwxJIZMYk1+MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y = train_y\n",
    "X = x_train_tf_idf_word.astype(\"float64\")\n",
    "\n",
    "gbc_roc_auc = roc_auc_score(y, gbc_model.predict(X))\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y, gbc_model.predict_proba(X)[:,1])\n",
    "plt.figure()\n",
    "plt.plot(fpr, tpr, label='AUC (area = %0.2f)' % gbc_roc_auc)\n",
    "plt.plot([0, 1], [0, 1],'r--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37e70949",
   "metadata": {},
   "source": [
    "### AdaBoostClassifier model with \"count-vectors\" method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "655eafc4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "AdaBoostClassifier Classifier model with 'count-vectors' method\n",
      "Accuracy ratio:  0.9439404761904762\n"
     ]
    }
   ],
   "source": [
    "abc = AdaBoostClassifier()\n",
    "abc_model = abc.fit(x_train_count,train_y)\n",
    "accuracy = model_selection.cross_val_score(abc_model,\n",
    "                                           x_test_count,\n",
    "                                           test_y,\n",
    "                                           cv = 20).mean()\n",
    "\n",
    "print(\"\\nAdaBoostClassifier Classifier model with 'count-vectors' method\")\n",
    "print(\"Accuracy ratio: \", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "3d910c19",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred=abc_model.predict(x_test_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "255af9d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[7326,   66],\n",
       "       [ 384,  215]], dtype=int64)"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(test_y,pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "5a4267b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.99      0.97      7392\n",
      "           1       0.77      0.36      0.49       599\n",
      "\n",
      "    accuracy                           0.94      7991\n",
      "   macro avg       0.86      0.68      0.73      7991\n",
      "weighted avg       0.94      0.94      0.93      7991\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(test_y,pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "90f46e7c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA1vUlEQVR4nO3dd3hUZfbA8e9JBxJ6ld6bAgKiWBAEFGzYUWy4qyyK+rP33lZX11UXy7Lqsq5td23YFbCAlV4CodfQElpCQvqc3x/vAFkMk0nIlJucz/PkydyZO3dOrnjPvPd93/OKqmKMMcYcSkykAzDGGBPdLFEYY4wJyBKFMcaYgCxRGGOMCcgShTHGmIAsURhjjAnIEoUxxpiALFEYE4CIrBORPBHJEZGtIjJZRJJLvX68iHwjIntEJEtEPhGRHgcdo66IPCciG/zHWeXfbhz+v8iYirNEYUz5zlLVZKAPcDRwN4CIDAS+BqYARwDtgYXAjyLSwb9PAjAd6AmMAOoCxwM7gAFh/SuMqSSxmdnGHJqIrAOuVtVp/u0/AT1V9QwRmQksVtXrDnrPF0Cmql4hIlcDjwMdVTUnzOEbUyWsRWFMkESkFTASWCUitXEtg/+Wset/gOH+x8OALy1JGC+zRGFM+T4SkT3ARiADeBBoiPv/Z0sZ+28B9vU/NDrEPsZ4hiUKY8p3jqqmAIOBbrgksAvwAS3K2L8FsN3/eMch9jHGMyxRGBMkVf0emAw8o6q5wM/AhWXsehGuAxtgGnCaiNQJS5DGhIAlCmMq5jlguIj0Ae4CrhSRG0UkRUQaiMhjwEDgYf/+/8LdsnpfRLqJSIyINBKRe0Tk9Ej8AcZUlCUKYypAVTOBN4D7VfUH4DTgPFw/xHrc8NkTVXWlf/8CXIf2MmAqkA3Mwt2++jXsf4AxlWDDY40xxgRkLQpjjDEBWaIwxhgTkCUKY4wxAVmiMMYYE1BcpAOoqMaNG2u7du0iHYYxxnjK3Llzt6tqk8q813OJol27dsyZMyfSYRhjjKeIyPrKvtduPRljjAnIEoUxxpiALFEYY4wJyBKFMcaYgCxRGGOMCcgShTHGmIBClihE5HURyRCR1EO8LiLygoisEpFFItI3VLEYY4ypvFDOo5gMTMSVZC7LSKCz/+dY4GX/b2OMqTRVpahE8Vll7CoTskShqjNEpF2AXUYBb6irc/6LiNQXkRaqausLG1PNFJX42FtYQn5RCXmFJeQUFJOdV0R2fhHZecX+30Vk5RWRnf/b14pKfAGPX+JTCot9FJUoheXsW9Mcu2Exv58z5bCOEcmZ2S1xK3/tk+5/7jeJQkTGAeMA2rRpE5bgjDH/y+dT0rZm8/PqHWzNyievqIS8ogMXf7ftI3//45L9j4t95X+7F4G6SfHUrRXnfifF075xHVKS4kiIC3yXPEaEhLgYEuJiiI+NISFWiI2p2V2wSbt3MPCVp+j21QdkN291WMeKZKKQMp4r81+Tqk4CJgH079/f2pPGHAZVZf7G3WRkF7Anv4icgmJy8ovZU1DMnvzi/c/tyfc/n1/EnoJicgqK2Xc3p3ZCLLUTYkmKj6VWfCy1/I/r14qnVt2k/dvutRj/7zhqxceSFB9DcmIcdWu5ZFCvdjx1k+KokxBHTExZlwVTKeffAdM/hrvvpu5990Gdyi/bHslEkQ60LrXdCtgcoViMqXZKfMqazBw27c4jp6CYrVn5zF2/i1/X7mRnbuFv9k+KjyElKZ6UxDhSkuJIToqjSXIdkpPi3IU9KY62jepwfKdGtKhXKwJ/kSnXkiVQvz60bAlPPQWPPAI9ex72YSOZKD4GrheRd3Gd2FnWP2HM4Vm2NZv356YzZ/0u0rZkk1/0v/frWzWoxcldmtCndX36tW1AvVrxJCe6pBAfW7Nv1Xhabi48+ij8+c9w6aUweTJ06lRlhw9ZohCRd4DBQGMRSQceBOIBVPUV4HPgdGAVsBe4KlSxGFOdbd6dx/S0bUxLy+D7FZkADGjfkDED2tLziLq0a1yHuklxNKiTQOPkxAhHa6rcZ5/BhAmwfj387neuJVHFQjnq6ZJyXldgQqg+35jqSlVJ3ZTNtLRtTEvbxpLN2QC0b1yHq09szxm9WnB0mwYRjtKExUsvuSTRowfMmAEnnRSSj/HcehTG1ET5RSX8vHoH09K2MT0tg63Z+cQI9GvbgLtHdmNYj2Z0bJIc6TBNOBQXQ2YmtGgBF10EeXlwww2QkBCyj7REYUyU2p5TwDfLMpieto2ZK7ezt7CEOgmxDOrShGHdmzGkW1Ma1gndxcFEoVmz4A9/gLg4+OUXaNwYbr015B9ricKYKKGqrMrIYWraNqYt3cb8jbtRhRb1kji/byuG9WjGcR0akhgXG+lQTbjt3g333AOvvOJaEs8/D2GcJ2KJwpgIKirxMXvdTqYtzWD6sm2s37EXgKNa1uOmoV0Y1qMpPVrURcTmF9RYixfD8OHudtONN7ohr3XrhjUESxTGhFlWXhHfr8hk2tJtfLc8g+z8YhLiYjihYyPGDerA0G7NaF4vKdJhmkgrKoL4eOjSBYYMgdtvh76RqZ1qicKYMNiwY+/+UUqz1u6k2Kc0qpPAaT2bM6xHM07q3JjaCfa/owEKCtwQ1zffhHnzIDkZ3nknoiHZv0xjQsDnUxak72baUpccVmzLAaBz02SuGdSBYd2b0ad1fWKtZIUp7Ztv4NprYcUKGD3aJY3kyI9ms0RhTBXZW1jMDyu3My1tG98sy2B7TiGxMcKAdg25/8w2DOvelLaNKl9vx1RjeXkwbpxrRXToAF9+CaedFumo9rNEYcxh2Jadz/S0DKalbePHVdspKPaRkhTH4K5NGda9KYO7NKVe7fhIh2miXVISbN8O993nRjfViq5aWpYojKkAVWXpluz9yWFRehYArRvWYsyxbRjevRnHtG9odZNM+RYtch3Ur70GrVq5UhxRWhrdEoUx5SgoLuHXNTtdZ/TSbWzOykcEjm5dn9tP68rwHs3o3DTZhrCa4OTmwkMPwV/+Ag0awMqVLlFEaZIASxTGlGlXbiHfLnethu+XZ5JbWEKt+FhO6tyYm4Z1YUi3pjRJsQJ7poI+/tiV29iwAa65Bp58Eho2jHRU5bJEYYzf6swcpi11tZTmrN+JT6FpSiJn92nJ8B5NOb5jY5LibVa0OQwffeQmy/3wA5xwQqSjCZolClNjFZf4mLt+1/5Ce2u25wLQo0Vdrj+lM8O6N+XII+rZqmum8oqK4IUX3IS5vn1d6Y2kJDeRzkMsUZgaZU9+ETNWbGd62ja+WZ7B7r1FxMcKAzs2ZuwJ7RjavRkt60fXiBPjUb/84gr4LVoEd97pEkVKSqSjqhRLFKbaS9+1d/8opV/W7KCoRGlQO55TujVlWHc3KzolyVvf8EwU27UL7r4bJk1yS5J++CGMGhXpqA6LJQpT7fh8yuJNWf6SGRmkbXEL+3RoUoffndCeod2b0bdNfeJsCKsJhUmT4NVX4eab3egmj7YiSrNEYaqF/KISfly1fX9/Q8aeAmIE+rdryD2nd2Nod1vYx4TQ8uWuuuuJJ8JNN8HIkdCrV6SjqjKWKIxnZe4p4Jtl25i6NIMfVmWSX+QjOTGOk7s0YVgPNyu6gS3sY0IpPx/++Ec3zLVbN1iwABITq1WSAEsUxkNUlRXbcpiWto2pS7exMN0t7NOyfi1G92/NsB7NOLZ9IxLi7JaSCYOpU+G662DVKhgzBv78Z6imky4tUZioVljsFvaZ6q/Cmr4rD4Derepxy7AuDO3ejO4tUmxWtAmvGTPg1FOhc2eXMIYNi3REIWWJwkSdrL1FfLcig6lL3azoPQXFJMbFcGKnxkwY0olTujWlWV1b2MeEWUkJLF0KRx0FJ53kajSNGePmRVRzlihMVFi3PXf/wj6z1+2ixKc0Tk7g9KNaMKxHM07s1JhaCTYr2kTI/PkwfjykpbnaTM2awe9+F+mowsYShYmIEp8yf8MupvnnN6zKcAv7dG2WwviT3cI+vVvVt1nRJrL27IEHH3Qzqhs3hpdfhqZNIx1V2FmiMGGTW1DMzJWZTEvL4JtlGezMLSQuRji2Q0MuPbYNw7o3o3XD2pEO0xgnK8vdZtq40c2w/uMfXbXXGsgShQmpLVl5+2dF/7RqB4UlPuomxTHEPyt6UJcm1Ktls6JNFMnOdoX76tVzq84NHQoDB0Y6qoiyRGGqlKqyZHP2/v6G1E1uVnTbRrW5fGBbhnVvRv92DWxhHxN9iorcGhGPPQbffedqM913X6SjigqWKMxhyy8q4ec1O5i21K0VvcW/sE/fNg24c0Q3hvdoSscmtrCPiWI//ug6q1NT4ZxzoEmTSEcUVSxRmErZkVPAN8symJ6WwYyVmewtLKF2glvY55bhbmGfxsm2sI/xgBtugIkToXVrmDIFzj470hFFHUsUJiiqyurMHKYuzWB62jbmbtiFKjSvm8S5R7dkWI9mDOzQyBb2Md6gemAWdfPmcNttbnRTstUDK4slCnNIxSU+Zq/bt7DPNtbt2AtAzyPqcuMpnRneoxk9j6hrt5SMtyxb5m4z3XyzK/99772RjijqWaIwv7G3sJir/zmHJZuzycorIiE2hoEdG/H7kzowtFtTjrCFfYwX5eXBE0/AU09BnTpu2wQlpIlCREYAzwOxwKuq+uRBr9cD3gTa+GN5RlX/EcqYTGAbd+7l9Bdmsie/mN6t6nHt4I6c2LkJyYn2ncJ42PTpbi7E6tVw+eXwzDM1cuJcZYXs/34RiQVeBIYD6cBsEflYVZeW2m0CsFRVzxKRJsByEXlLVQtDFZcpW2Gxj1d/WMML01cSI8L5fVvxzIW97LaSqR7S0yEuziWMU06JdDSeE8qviQOAVaq6BkBE3gVGAaUThQIp4q5GycBOoDiEMZkyzFq7k3s/XMzKjBxG9GzOA2f1sNtLxttKSuCVVyAhAa65Bq64Ai6+2K0VYSoslLOeWgIbS22n+58rbSLQHdgMLAb+T1V9Bx9IRMaJyBwRmZOZmRmqeGucnbmF3P7fhVz0t5/ZW1jCa1f255XL+1mSMN42bx4cdxxcfz189ZV7TsSSxGEIZYuirHsWetD2acAC4BSgIzBVRGaqavb/vEl1EjAJoH///gcfw1SQz6e8Ny+dP36exp78Ysaf3JEbh3aidoL1QxgPy86G++93cyKaNIF33oHRoyMdVbUQyitDOtC61HYrXMuhtKuAJ1VVgVUishboBswKYVw12opte7jvw1RmrdtJ/7YNePzco+ja3PuLvxvDwoUuSYwfD48/DvXrRzqiaiOUiWI20FlE2gObgIuBMQftswEYCswUkWZAV2BNCGOqsfIKS3jhm5X8fcYakpPieOr8o7iwX2sr4228be1a+PZbtzbESSe5ZUnbt490VNVOyBKFqhaLyPXAV7jhsa+r6hIRGe9//RXgUWCyiCzG3aq6U1W3hyqmmuqbZdt4YMoS0nflcUG/Vtw9shuNrLyG8bLCQrdG9SOPuBXmzj3XlQC3JBESIb0praqfA58f9NwrpR5vBk4NZQw12ZasPB7+eClfLtlKp6bJvDvuOI7r0CjSYRlzeGbOdLeXli6F885ziwrV0HUiwsV6L6uh4hIf//x5Pc9+vZxin3L7aV255qQOJMRZaW/jcZmZcOqpbinSTz6BM8+MdEQ1giWKambBxt3c88Film7JZnDXJjxy9pG0aWSrxhkPU4Vp02D4cDea6dNP3fDXOnUiHVmNYYmimsjKK+Lpr5bx1q8baJqSyEuX9mXkkc1tZrXxtiVL4Npr3e2mb7+FwYPdinMmrCxReJyq8vHCzTz6aRo7cwsYe3w7bhnehZQkW17UeNjevW6luaefdsuSvvoqDBoU6ahqLEsUHrZ2ey73f5TKD6u206tVPSZfdQxHtqwX6bCMOTyqMGQIzJoFV17pkoWtOBdRlig8qKC4hJe/W81L360mMTaGR0b15NJj2xJrcyKMl23Z4iq6xsbCPfdAvXruVpOJOBsG4zE/rtrOyOdm8ty0lZzWsznTbz2ZKwa2syRhvKukBF54Abp2hZdecs+NGmVJIopYi8IjMvcU8NhnS5myYDNtG9Xmjd8NYFAXa44bj5szx60TMW8enHYanH56pCMyZQg6UYhIHVXNDWUw5rd8PuXtWRt46stl5BeVcOMpnbhuSCdbm9p435/+BHfd5das/ve/4cILD6xjbaJKuYlCRI4HXsWtF9FGRHoDf1DV60IdXE23ZHMW936YyoKNuxnYoRGPnnMknZra4u/Gw1ShuBji42HAAJgwwY1uqmeDMKJZMC2Kv+DKgX8MoKoLRcTGqYVQTkExf5m6gn/8uJYGtRP4y+jenNOnpc2JMN62ejVcdx0ceaSr0zR4sPVDeERQt55UdeNBF6mS0IRTs6kqXy3ZxsOfLGFLVj5jjm3Dnad1o15tmxNhPKygwA1xffxx15IYNSrSEZkKCiZRbPTfflIRSQBuBNJCG1bNs3HnXh76eAnTl2XQrXkKE8f0pV9bK3RmPG7uXLjsMli2zPVBPPccHHFEpKMyFRRMohgPPI9bxjQd+Bqw/okqUlTi49WZa3l++gpiRLj39O6MPaEd8bE2ctlUA8nJroP6889h5MhIR2MqKZhE0VVVLy39hIicAPwYmpBqjtnrdnLvh4tZsS2HU3s048Gze9LS1qs2XubzwT/+AT//7MpudO0KqakQY198vCyYRPFXoG8Qz5kg7cot5MkvlvHvORs5ol4Sf7+iP8N7NIt0WMYcntRUt07Ejz+6uky5ua7CqyUJzztkohCRgcDxQBMRuaXUS3VxK9aZClJV3pubzhOfp5GdX8wfBnXgxqGdqZNo8x6Nh+XmupXmnn3WDXP9xz9cjSYbpVdtBLpCJeDmTsQBKaWezwYuCGVQ1dGqjD3c82Eqs9bupF/bBjx+7pF0a1430mEZc/jy811yuOIKN4muka2iWN0cMlGo6vfA9yIyWVXXhzGmaiWvsISJ365k0ow11E6I48nzjuKi/q2JsdpMxsvS0119pj/+0SWGZcugYcNIR2VCJJh7HntF5GmgJ5C070lVPSVkUVUT3y7P4IEpqWzcmcd5fVtyz+ndaZycGOmwjKm84mL461/hgQdcMb/Ro6FfP0sS1VwwieIt4N/AmbihslcCmaEMyuu2ZuXzyKdL+HzxVjo0qcM71xzHwI7WHDce9+uvroDfwoWueN/EidC+faSjMmEQTKJopKqvicj/lbod9X2oA/Oi4hIf//plPX/+egVFJT5uO7UL1wzqQGKc9f0bj/P54KqrICsL3nsPzjvPOqtrkGASRZH/9xYROQPYDLQKXUjetHDjbu79aDGpm7IZ1KUJj47qSdtGtvi78TBVlxRGjICUFPjgA2jZ0j02NUowieIxEakH3IqbP1EXuCmUQXlJdn4Rz3y1nH/9sp4myYlMHHM0ZxzVwgr4GW9budJVdp06FZ55Bm69Fbp1i3RUJkLKTRSq+qn/YRYwBPbPzK7RVJVPFm3h0U+XsiOngCsHtuOWU7tQN8kK+BkPKyiAp56CJ56AxETXDzF+fKSjMhEWaMJdLHARrsbTl6qaKiJnAvcAtYCjwxNi9Fm3PZf7p6Qyc+V2jmpZj9eu7E+vVvUjHZYxh2/CBHjtNbj4YjeBrkWLSEdkokCgFsVrQGtgFvCCiKwHBgJ3qepHYYgt6hQUl/C379cw8dtVJMTG8PDZPbnsuLa2XrXxtowM11ndvDnceaer8nraaZGOykSRQImiP9BLVX0ikgRsBzqp6tbwhBZdflq1nfumpLImM5czerXggTN70KxuUvlvNCZa+XyucN+dd8Kpp7rlSDt3dj/GlBIoURSqqg9AVfNFZEVNTBLbcwp4/LM0Ppy/iTYNazP5qmMY3LVppMMy5vAsWuT6Hn7+2a0y9/DDkY7IRLFAiaKbiCzyPxago39bAFXVXiGPLoJ8PuXd2Rt58os08opKuOGUTkwY0omkeJsTYTzuvfdcH0SDBvDGG25hIRulZwIIlCi6hy2KKLN0czb3fbSYeRt2c1yHhjx2zpF0ampjx43HZWdD3bquBTFhAjz4oJXeMEEJVBSwxhUCzC0o5rlpK3j9x3XUqxXPny/szXl9W9qcCONtGzbADTfA5s3wyy/QuDE8/3ykozIeEtIVRURkhIgsF5FVInLXIfYZLCILRGRJJEuDfL1kK8Of/Z6/z1zLRf1b8c2tJ3N+v1aWJIx3FRW5yXLdu8O0aXDRRW62tTEVFLIVc/zzMF4EhuPW2p4tIh+r6tJS+9QHXgJGqOoGEQl7L/GqjBwemJLKT6t30K15Ci9ccjT921lz3Hjc+vVw9tmu0/qss1zF17ZtIx2V8aigEoWI1ALaqOryChx7ALBKVdf4j/EuMApYWmqfMcAHqroBQFUzKnD8w7Y9p4DRf/uZHbmF3H5aV8YN6kB8rC3baDxM1XVMN28OzZrBhx/CqFHWWW0OS7lXRRE5C1gAfOnf7iMiHwdx7JbAxlLb6f7nSusCNBCR70RkrohcEVTUVWBPfhFXvj6L3MJi3rnmOCYM6WRJwniXKrz5JhxzDOTkuPIbX38N55xjScIctmCujA/hWge7AVR1AdAuiPeV9a/z4BukcUA/4AzgNOB+EenymwOJjBOROSIyJzPz8JfCyC8q4ep/zmH51j28fFk/WyvCeNvy5TB0KFx+OcTFwY4dkY7IVDPBJIpiVc2qxLHTcSVA9mmFK1F+8D5fqmquqm4HZgC9Dz6Qqk5S1f6q2r9JkyaVCOWA4hIf1789n1nrdvLni3ozxCbPGa8qLnZDXHv1gnnz4OWX4aefrC/CVLlgEkWqiIwBYkWks4j8FfgpiPfNBjqLSHsRSQAuBg6+ZTUFOElE4kSkNnAskFaB+Cskr7CEq9+Yw7S0bTx0Vk9G9Tn4TpgxHhIbCzNnwgUXuFbF+PEQY7dPTdUL5l/VDbj1sguAt3Hlxm8q702qWgxcD3yFu/j/R1WXiMh4ERnv3ycN1/exCFd88FVVTa3E31Eun0+54/1FfLc8k0dH9eTK49uF4mOMCa2tW+F3v4ONG13fw+efw1tvuY5rY0IkmFFPXVX1XuDeih5cVT8HPj/ouVcO2n4aeLqix66oGSsz+WThZm4e1oXLB7YL9ccZU7VKSmDSJLj7bsjLg5EjoXVrSLLClCb0gmlRPCsiy0TkURHpGfKIQuQ/czbSoHY84wd3iHQoxlTM/Plw/PFw3XXQvz8sXuxKgRsTJuUmClUdAgwGMoFJIrJYRO4LdWBVacaKTL5I3cpF/VuTGGdF/YzHTJwI69a5W0xTp0KX3wwMNCakRCswpV9EjgLuAEarakLIogqgf//+OmfOnKD337w7jzNemEnTlCQ+nHA8tRNCNhndmKqhCh99BO3awdFHw65d7vkGDSIZlfE4EZmrqv0r895gJtx1F5GHRCQVmIgb8dSqMh8WboXFPia8PY+iEuWly/pakjDRb906V3rjvPPguefccw0aWJIwERXMlfMfwDvAqap68DyIqPbE52nM37CbF8f0pWOT5EiHY8yhFRW5NaofftgNcX3mGfi//4t0VMYAQSQKVT0uHIFUtU8XbWbyT+u46oR2nNHLFog3Ue5vf4O77nIlN55/Htq0iXRExux3yEQhIv9R1YtEZDH/W3oj6le4W52Zw53vLaJvm/rcPbLGrr9kot2OHe5WU79+cM010KkTjBgR6aiM+Y1ALYp97d4zwxFIVXr5u9XEiPDipX1JiLOZqibKqLolSG+7DVJSYMUKV8TPkoSJUoe8iqrqFv/D61R1fekf4LrwhFdxBcUl/Lx6BwM7NqJFvVqRDseY/5WWBkOGwNix0LmzG90UZ4MsTHQL5uv28DKeG1nVgVSVez5IZdPuPEYf07r8nY0Jp4ULoXdvt5jQpEnwww+uoJ8xUS5QH8W1uJZDBxFZVOqlFODHUAdWGTkFxbw/L53BXZswtLvVvjFRIj0dWrVySeHhh+H3v4emVrXYeEegNu/bwBfAH4HS613vUdWdIY2qklZn5ABw8TE2YsREgc2b4eabXeG+ZcugZUtXq8kYjwl060lVdR0wAdhT6gcRicpFpVf5E0XnZjZnwkRQSYkru9G9O0yZAnfcAY0bRzoqYyqtvBbFmcBc3PDY0ivWKRB11fVWbNtDfKzQtmHtSIdiaqr8fBg0CGbPhuHD4aWX3LBXYzzskIlCVc/0/24fvnAOz+JNWXRvUZc4W/vahFtREcTHu7LfQ4bALbfA6NG2XrWpFoKp9XSCiNTxP75MRJ4VkajrBPD5lMWbsjiqZb1Ih2JqElV47z3Xapg3zz331FNw8cWWJEy1EcxX75eBvSLSG1c5dj3wr5BGVQmTZq5hT34xAzs2inQopqZYswbOOMOtDdGokS1DaqqtYP5lF6urRT4KeF5Vn8cNkY0qs9buJDEuhtOPtLpOJgyefRZ69nRrVj/3HMyaBX36RDoqY0IimCmhe0TkbuBy4CQRiQXiQxtWxS3bks2pPZsTE2PNfRMGOTlw+umugF8rT1TdN6bSgmlRjAYKgN+p6lagJWFY47oitucUsDkrn17WP2FCZft2uOoq+Phjt33fffD++5YkTI0QzFKoW4G3gHoiciaQr6pvhDyyCli/Yy8AHZvWiXAkptrx+eD116FrV3jzTVi1yj1v/RGmBglm1NNFwCzgQuAi4FcRuSDUgVXE5t15ADSrmxThSEy1snQpDB7sSm706AELFrhhr8bUMMH0UdwLHKOqGQAi0gSYBrwXysAqYlraNhrUjqdTU5uRbarQnDmwZAm89pqr9mqtCFNDBZMoYvYlCb8dBNe3ETZrt+dyVKv6JMbFRjoU43Wff+4WFLr8cvdz5pnQMCor1hgTNsFc8L8Uka9EZKyIjAU+Az4PbVgVs31PAU2SEyMdhvGy9HS44AI3L2LiRDeRTsSShDEE15l9O/A3oBfQG5ikqneGOrBglfiUjD0FNKtricJUQnGxG+LavTt89hk8/ribG2Gzqo3ZL9B6FJ2BZ4COwGLgNlXdFK7AgpWxJ59in9Kyga1mZyph7ly46Sa3DOmLL0KHqKt1aUzEBWpRvA58CpyPqyD717BEVEGbdrkRT0fUt0RhgpSVBR984B4feyz8+qvrm7AkYUyZAnVmp6jq3/2Pl4vIvHAEVFGb/ENjW1miMOVRhf/8x7UgduyAdevgiCNgwIBIR2ZMVAuUKJJE5GgOrENRq/S2qkZF4tiXKKxFYQJavRomTICvvoJ+/eCTT1ySMMaUK1Ci2AI8W2p7a6ltBU4JVVAVsXl3HvVrx1MnMZiRvqZG2rPHJQefD154Aa67DmJtKLUxwQq0cNGQcAZSWZt25dHSWhOmLIsWQa9ekJLiJs0dd5xbt9oYUyFRNXGuMlZsy6F9Y6vxZErJzIQrr4TevV0nNcD551uSMKaSQpooRGSEiCwXkVUicleA/Y4RkZKK1pDKyM5n0+48jm7T4PCDNd7n88Grr7oCfu+8A/fc42o1GWMOS8hu7PvXrXgRGA6kA7NF5GNVXVrGfk8BX1X0M+Zv3A1An9b1DzNaUy2cfz589BEMGgQvv+wK+RljDlsw1WPFv1b2A/7tNiISzHjCAcAqVV2jqoXAu7hV8g52A/A+kFHGawHN37Cb+Fih5xF1K/pWU13k5rrZ1QCXXAKTJ8N331mSMKYKBXPr6SVgIHCJf3sPrqVQnpbAxlLb6f7n9hORlsC5wCuBDiQi40RkjojMyczM3P/8/A276HFEPZLibQRLjfTJJy4hvPSS277oItc3YeU3jKlSwSSKY1V1ApAPoKq7gIQg3lfW/6160PZzwJ2qWhLoQKo6SVX7q2r/Jk2a7H9+xbY91pqoiTZuhPPOg7PPdiOa+vWLdETGVGvB9FEU+fsRFPavR+EL4n3pQOtS262AzQft0x94V9w3wMbA6SJSrKofBXF8cgtLqJsUdct3m1B6800YP951XD/5JNx8MyQE873FGFNZwSSKF4APgaYi8jhwAXBfEO+bDXQWkfbAJuBiYEzpHVS1/b7HIjIZ+DTYJFHiUwqLfdSy2041w76y361auZFMf/0rtG9f7tuMMYev3EShqm+JyFxgKO520jmqmhbE+4pF5HrcaKZY4HVVXSIi4/2vB+yXKE9ekbtbVSvB81NBTCC7d8Pdd0OdOvDMMy5J2JBXY8Kq3EQhIm2AvcAnpZ9T1Q3lvVdVP+egRY4OlSBUdWx5xystJ9+NdKmVYKU7qiVVNxfillvcBLqbbz7QqjDGhFUwV9nPcP0TAiQB7YHlQM8QxlWuZVuzAWhl61BUP2vXwrhxMG0aHHMMfPEFHH10pKMypsYK5tbTUaW3RaQv8IeQRRSk1E1ZAHRtlhLhSEyVKypydZpefBH+8Acr4GdMhFX4vo2qzhORY0IRTEX8unYnnZsmW3nx6mL6dLcU6bPPQpcusH49JCVFOipjDMH1UdxSajMG6AtkHmL3sCko8tGwjg2L9Lxt2+DWW+Gtt6BjR7j3XmjUyJKEMVEkmCFDKaV+EnF9FmWV4girghIfCXE24smzfD7429+gWze36tz998PixS5JGGOiSsAWhX+iXbKq3h6meIJWWOwj0RKFd2VlwX33QZ8+roBft26RjsgYcwiHvNKKSJy/tEbfMMYTlBKfsmFHLi3qWf+Ep+TkuD6IkhJo0AB+/RW++caShDFRLlCLYhYuSSwQkY+B/wK5+15U1Q9CHNshrczYQ25hiZUX95IpU+CGG1ydpj594JRToEOHSEdljAlCMPduGgI7cGtknwmc5f8dMQs27Abg6Db1IxmGCcb69TBqFJxzDtSvDz/+6JKEMcYzArUomvpHPKVyYMLdPgdXgQ2r1Zk5JMbF2BKo0U4VLrgAli6FP/0JbroJ4q2IozFeEyhRxALJBFcuPKx8CvGxMYiVc4hOv/wCPXu6EuCTJkHDhtC2baSjMsZUUqBEsUVVHwlbJBWwt7DYRjxFo507XQG/SZPggQfg4Yet9IYx1UCgRBG1X9fX79hLm0a1Ix2G2UfVrRNx660uWdx6K9wedSOqjTGVFOhr+dCwRVFB63fspW1DSxRR45574Ior3MzquXNdOfDk5EhHZYypIodsUajqznAGEixV2JKVR5tGrSIdSs2Wn+/mRTRuDFdd5fogxo2DGLslaEx147n/q4tKfPgUWlt58ciZOhWOOgquucZtd+nilie1JGFMteS5/7P3FrqV7WxobARs3QpjxsCpp7oFhK6/PtIRGWPCwHPLwxX7fAB0tnUowuvbb+HccyEvDx56CO680yq8GlNDeC5RlPjUTfBI9Fzo3lRU5CbJ9eoFw4fD44+7W03GmBrDc7eeSlRJTowjNiZqR+9WD3v2uHWqTzrJFfFr1Aj++19LEsbUQJ5LFD4f1Em0pTFDRhU++AC6d4fnn3cT5goKIh2VMSaCPJcoFCU+1nNhe8P27XDWWXD++W7Y608/ubUiatucFWNqMs9dcdVf58mEQEqKW5r02Wdhzhw47rhIR2SMiQKeu+KqQpz1T1SdH36AkSPd5LnERLeY0M03Q5wNFjDGOJ5LFD5VaidYH8Vh27EDrr7adVYvXQpr1rjnbdKcMeYgnrsqlPiUerUTIh2Gd6nC5MnQtav7ffvtLlH06hXpyIwxUcpz9xd8qqQkeS7s6PLGGy5RvPKKK8VhjDEBeLJFkWKT7SomLw8efBDS013pjfffh5kzLUkYY4LiuUThU5uVXSFffQVHHgmPPAJTprjnGjSwvghjTNA8d7XwqZJst57Kt3kzjB4NI0a4EhzffAMTJkQ6KmOMB3kuUYC1KILy2GOuBfHII7BwIQwZEumIjDEe5ckrriWKQ5g790ABv0cfhVtugU6dIh2VMcbjQtqiEJERIrJcRFaJyF1lvH6piCzy//wkIr2DOa7dejpIdjbceCMMGOCWJQVXxM+ShDGmCoQsUYhILPAiMBLoAVwiIj0O2m0tcLKq9gIeBSYFc2xrUfipuoqu3brBxIlw7bXw5puRjsoYU82E8oo7AFilqmsARORdYBSwdN8OqvpTqf1/AYJaCDslKb4Kw/Swt9+Gyy5zFV6nTIFjjol0RMaYaiiUiaIlsLHUdjpwbID9fw98UdYLIjIOGAeQ0LwT9WvX4ERRWOjKbXTrBhdc4OZIjB1rtZmMMSETyj6Ksir3aZk7igzBJYo7y3pdVSepan9V7X+oA9cIM2ZAnz5uzer8fFfE7+qrLUkYY0IqlIkiHWhdarsVsPngnUSkF/AqMEpVdwRzYJEaliq2b4erroKTT3YtiFdesfWqjTFhE8qvorOBziLSHtgEXAyMKb2DiLQBPgAuV9UVwR64RlUZX7PG9T1kZ8Ndd8H999tCQsaYsApZolDVYhG5HvgKiAVeV9UlIjLe//orwANAI+AlfyuheN/tpUCkJtx8ys6GunWhfXvXmhg71pXiMMaYMBPVMrsNolZii866aslCWjespt+q9+51k+UmTXIzqlsFNRDMGGMCEpG5wXwRL4sne0GrbRfFZ5/B9dfDunWuFVGrVqQjMsYYryaKapYpiovhkkvgvfege3f4/nsYNCjSURljDODRooDVpjN7322/uDho1gyeeAIWLLAkYYyJKp5MFNWiM3v2bDj2WJg3z21PnAh33w0JtsyrMSa6eDNReDlPZGW5fohjj3Urzu0IauqIMcZEjCWKcNpXwO/ll12yWLYMhg+PdFTGGBOQNzuzvXrrKS0NWraETz6B/pUapWaMMWHnyXkUm1am0jg5MdKhlK+gAJ5+Gnr3hrPOgqIit1Z1bGykIzPG1DCHM4/Ck7eeYrxw7+nbb12CuP9+mD7dPRcfb0nCGOM5nkwUUZ0mMjLgyivhlFNcC+KLL+C55yIdlTHGVJonE0VUtyi+/hreeQfuvRdSU2HEiEhHZIwxh8WTndmxsVGWKBYvhuXL3UJCl14Kxx8PHTpEOipjjKkSnmxRRE2ayM2FO+5wS5HecYe71SRiScIYU614M1FEQ6b45BPo0cONaho71s20jq/BS7QaY6otT956ivg8itRUOPts6NkTZs6EE0+MbDzGGBNC1qIIVnExfPede3zkkfDppzB/viUJY0y1Z4kiGL/+6mZSDx0KK1e65844w241GWNqBG8minDdetq1C669FgYOhO3bXa2mTp3C89nGGBMlvNlHEY48UVDgRjNt3Ag33QQPPwwpKWH4YGOMiS7eTBShPPimTa5wX2IiPPSQK8Nx9NGh/ERjjIlq3rz1FIomRX6+azV06ABTprjnxo61JGGMqfGsRQGuaN+117qO6ksucYsKGWOMATzboqjCg910Ewwb5tav/vprePttaN68Cj/AGGO8zaOJ4jAzhc8HJSXu8YAB8MADrl6TrTZnjDG/4clEcVgWLnRF+1580W2PGeP6JpKSIhuXMcZEqZqTKHJy4NZboV8/WLPGbi8ZY0yQPNmZXWHTpsFVV0F6OowbB08+CQ0aRDoqY4zxhJqRKBISoGFD+Pe/3W0nY4wxQfNcogiqG7uoyC0/mpUFjz0Ggwa5An4xNedOmzHGVJXqd+X86SfXD3HHHZCW5kY4gSUJY4yppOpz9dy50/U/nHAC7N4NH30E779vCcIYYw5T9bmK7tjhJsvddhssXQqjRkU6ImOMqRY810fxP5Yvdx3UDzwAnTvD+vXQqFGkozLGmGolpC0KERkhIstFZJWI3FXG6yIiL/hfXyQifcs/KJCX55JDr17wl7+4UuBgScIYY0IgZIlCRGKBF4GRQA/gEhHpcdBuI4HO/p9xwMvlHTe5YC8cdRQ8+ihceCEsWwatW1dx9MYYY/YJZYtiALBKVdeoaiHwLnBwx8Eo4A11fgHqi0iLQAdttXub66CeNg3efBOaNQtN9MYYY4DQ9lG0BDaW2k4HDq7fXdY+LYEtpXcSkXG4FgdAgaxcmcqwYVUbrTc1BrZHOogoYefiADsXB9i5OKBrZd8YykRR1tw4rcQ+qOokYBKAiMxR1f6HH5732bk4wM7FAXYuDrBzcYCIzKnse0N56ykdKN150ArYXIl9jDHGRFAoE8VsoLOItBeRBOBi4OOD9vkYuMI/+uk4IEtVtxx8IGOMMZETsltPqlosItcDXwGxwOuqukRExvtffwX4HDgdWAXsBa4K4tCTQhSyF9m5OMDOxQF2Lg6wc3FApc+FqP6mS8AYY4zZr/qU8DDGGBMSliiMMcYEFLWJIiTlPzwqiHNxqf8cLBKRn0SkdyTiDIfyzkWp/Y4RkRIRuSCc8YVTMOdCRAaLyAIRWSIi34c7xnAJ4v+ReiLyiYgs9J+LYPpDPUdEXheRDBFJPcTrlbtuqmrU/eA6v1cDHYAEYCHQ46B9Tge+wM3FOA74NdJxR/BcHA808D8eWZPPRan9vsENlrgg0nFH8N9FfWAp0Ma/3TTScUfwXNwDPOV/3ATYCSREOvYQnItBQF8g9RCvV+q6Ga0tipCU//Cocs+Fqv6kqrv8m7/g5qNUR8H8uwC4AXgfyAhncGEWzLkYA3ygqhsAVLW6no9gzoUCKSIiQDIuURSHN8zQU9UZuL/tUCp13YzWRHGo0h4V3ac6qOjf+XvcN4bqqNxzISItgXOBV8IYVyQE8++iC9BARL4TkbkickXYoguvYM7FRKA7bkLvYuD/VNUXnvCiSqWum9G6HkWVlf+oBoL+O0VkCC5RnBjSiCInmHPxHHCnqpa4L4/VVjDnIg7oBwwFagE/i8gvqroi1MGFWTDn4jRgAXAK0BGYKiIzVTU7xLFFm0pdN6M1UVj5jwOC+jtFpBfwKjBSVXeEKbZwC+Zc9Afe9SeJxsDpIlKsqh+FJcLwCfb/ke2qmgvkisgMoDdQ3RJFMOfiKuBJdTfqV4nIWqAbMCs8IUaNSl03o/XWk5X/OKDccyEibYAPgMur4bfF0so9F6raXlXbqWo74D3gumqYJCC4/0emACeJSJyI1MZVb04Lc5zhEMy52IBrWSEizXCVVNeENcroUKnrZlS2KDR05T88J8hz8QDQCHjJ/026WKthxcwgz0WNEMy5UNU0EfkSWAT4gFdVtcxhk14W5L+LR4HJIrIYd/vlTlWtduXHReQdYDDQWETSgQeBeDi866aV8DDGGBNQtN56MsYYEyUsURhjjAnIEoUxxpiALFEYY4wJyBKFMcaYgCxRmKjkr/y6oNRPuwD75lTB500WkbX+z5onIgMrcYxXRaSH//E9B7320+HG6D/OvvOS6q+GWr+c/fuIyOlV8dmm5rLhsSYqiUiOqiZX9b4BjjEZ+FRV3xORU4FnVLXXYRzvsGMq77gi8k9ghao+HmD/sUB/Vb2+qmMxNYe1KIwniEiyiEz3f9tfLCK/qRorIi1EZEapb9wn+Z8/VUR+9r/3vyJS3gV8BtDJ/95b/MdKFZGb/M/VEZHP/GsbpIrIaP/z34lIfxF5Eqjlj+Mt/2s5/t//Lv0N39+SOV9EYkXkaRGZLW6dgD8EcVp+xl/QTUQGiFuLZL7/d1f/LOVHgNH+WEb7Y3/d/znzyzqPxvxGpOun24/9lPUDlOCKuC0APsRVEajrf60xbmbpvhZxjv/3rcC9/sexQIp/3xlAHf/zdwIPlPF5k/GvXQFcCPyKK6i3GKiDK029BDgaOB/4e6n31vP//g737X1/TKX22RfjucA//Y8TcJU8awHjgPv8zycCc4D2ZcSZU+rv+y8wwr9dF4jzPx4GvO9/PBaYWOr9TwCX+R/Xx9V9qhPp/972E90/UVnCwxggT1X77NsQkXjgCREZhCtH0RJoBmwt9Z7ZwOv+fT9S1QUicjLQA/jRX94kAfdNvCxPi8h9QCauCu9Q4EN1RfUQkQ+Ak4AvgWdE5Cnc7aqZFfi7vgBeEJFEYAQwQ1Xz/Le7esmBFfnqAZ2BtQe9v5aILADaAXOBqaX2/6eIdMZVA40/xOefCpwtIrf5t5OANlTPGlCmiliiMF5xKW5lsn6qWiQi63AXuf1UdYY/kZwB/EtEngZ2AVNV9ZIgPuN2VX1v34aIDCtrJ1VdISL9cDVz/igiX6vqI8H8EaqaLyLf4cpejwbe2fdxwA2q+lU5h8hT1T4iUg/4FJgAvICrZfStqp7r7/j/7hDvF+B8VV0eTLzGgPVRGO+oB2T4k8QQoO3BO4hIW/8+fwdewy0J+Qtwgojs63OoLSJdgvzMGcA5/vfUwd02mikiRwB7VfVN4Bn/5xysyN+yKcu7uGJsJ+EK2eH/fe2+94hIF/9nlklVs4Abgdv876kHbPK/PLbUrntwt+D2+Qq4QfzNKxE5+lCfYcw+liiMV7wF9BeRObjWxbIy9hkMLBCR+bh+hOdVNRN34XxHRBbhEke3YD5QVefh+i5m4fosXlXV+cBRwCz/LaB7gcfKePskYNG+zuyDfI1b23iauqU7wa0lshSYJyKpwN8op8Xvj2Uhrqz2n3Ctmx9x/Rf7fAv02NeZjWt5xPtjS/VvGxOQDY81xhgTkLUojDHGBGSJwhhjTECWKIwxxgRkicIYY0xAliiMMcYEZInCGGNMQJYojDHGBPT/cxohWjN6YIgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y = train_y\n",
    "X = x_train_count.astype(\"float64\")\n",
    "\n",
    "abc_roc_auc = roc_auc_score(y, abc_model.predict(X))\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y, abc_model.predict_proba(X)[:,1])\n",
    "plt.figure()\n",
    "plt.plot(fpr, tpr, label='AUC (area = %0.2f)' % abc_roc_auc)\n",
    "plt.plot([0, 1], [0, 1],'r--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "975c968f",
   "metadata": {},
   "source": [
    "### AdaBoostClassifier model with \"tf-idf\" method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "8c5d354e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "AdaBoostClassifier model with 'tf-idf' method\n",
      "Accuracy ratio:  0.9413126566416039\n"
     ]
    }
   ],
   "source": [
    "abc = AdaBoostClassifier()\n",
    "abc_model = abc.fit(x_train_tf_idf_word,train_y)\n",
    "accuracy = model_selection.cross_val_score(abc_model, \n",
    "                                           x_test_tf_idf_word, \n",
    "                                           test_y, \n",
    "                                           cv = 20).mean()\n",
    "\n",
    "print(\"\\nAdaBoostClassifier model with 'tf-idf' method\")\n",
    "print(\"Accuracy ratio: \", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "851c9155",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred=abc_model.predict(x_test_tf_idf_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "397ab4e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[7321,   71],\n",
       "       [ 395,  204]], dtype=int64)"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(test_y,pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "4fc0f917",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.99      0.97      7392\n",
      "           1       0.74      0.34      0.47       599\n",
      "\n",
      "    accuracy                           0.94      7991\n",
      "   macro avg       0.85      0.67      0.72      7991\n",
      "weighted avg       0.93      0.94      0.93      7991\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(test_y,pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "2402ebe7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA1GUlEQVR4nO3dd3xUZfb48c8hhdB7L9KrEFAEK4IKooLYsawurMqKyK69997borJ81XX3Z+8Q7CgKVkQgoReREkNvgfRyfn88E8myYTIJmblzJ+f9es0rc2fuzJxcwj3z3Od5ziOqijHGGHMgNbwOwBhjTHSzRGGMMSYoSxTGGGOCskRhjDEmKEsUxhhjgrJEYYwxJihLFMYYY4KyRGFMECKyVkRyRGSviGwSkVdEpG6p548Wka9EZI+I7BaRFBHptd971BeRp0VkfeB9Vge2m0b+NzKm4ixRGFO+UapaF+gH9AduARCRo4DPgWlAa6AjkAp8JyKdAvskAl8CvYERQH3gaGA7MDCiv4UxlSQ2M9uYAxORtcBlqjozsP0o0FtVTxOROcAiVb1yv9d8AmxV1UtE5DLgAaCzqu6NcPjGVAlrURgTIhFpC5wCrBaR2riWwTtl7Po2MCxw/yTgU0sSxs8sURhTvg9FZA+wAdgC3AU0xv3/2VjG/huBkv6HJgfYxxjfsERhTPnOUNV6wBCgBy4J7ASKgVZl7N8K2Ba4v/0A+xjjG5YojAmRqn4DvAI8rqpZwA/AuWXseh6uAxtgJnCyiNSJSJDGhIElCmMq5mlgmIj0A24G/iwifxOReiLSSETuB44C7gns//9wl6zeE5EeIlJDRJqIyK0icqoXv4AxFWWJwpgKUNWtwH+AO1T1W+Bk4CxcP8Q63PDZY1V1VWD/PFyH9nLgCyATmIu7fPVTxH8BYyrBhscaY4wJyloUxhhjgrJEYYwxJihLFMYYY4KyRGGMMSaoeK8DqKimTZtqhw4dvA7DGGN85Zdfftmmqs0q81rfJYoOHTowb948r8MwxhhfEZF1lX2tXXoyxhgTlCUKY4wxQVmiMMYYE5QlCmOMMUFZojDGGBOUJQpjjDFBhS1RiMjLIrJFRBYf4HkRkWdFZLWIpInIYeGKxRhjTOWFcx7FK8BkXEnmspwCdA3cBgEvBH4aYwwAqkphsVJUbFWuvRS2RKGqs0WkQ5BdRgP/UVfn/EcRaSgirVTV1hc2JsoVFhWTW1hMbkEROflF5BYUsTO7gJ3Z+ezMymdH4OfO7IL/2t6bV1jue6tCQVEx+UXF5BUWYyshHJxB6xdx6bxpB/UeXs7MboNb+atEeuCx/0kUIjIeGA/Qvn37iARnTKzLLyzmxzXbWbl5D7uyC9iRnc/unALyCorIKSgit8AlgtzA/bzCfY8VhvANPymhBo1rJ9KoTiKNaifStlFt6iXFIyHElhBXg5rxNUiMdz/jalh3akUl7drOUVMeocdn75PZsu1BvZeXiaKsv5cy//pUdSowFWDAgAH2/cKYMmTnF/Lrliw27MwmfWc227Pyyc4rIiu/kJz8IrLyi8jOKyQ7v4js/EK27MkjO78IgLgaQsNaCTSolUBSQhxJCTVISoijQa0EaiXEUTOwnRS/77mkhBqB5+KolRBHo9qJNKydQONAYqiVGOfxEanmzr4RvpwOt9xC/dtvhzqVX7bdy0SRDrQrtd0WyPAoFmN8RVVJ35nDik17WLYxk29Xb2P++p0UFO37HpUYX4M6iXHUToynTs04aiXGUycxjoa1E6lT053YB3drymHtG1E/KYEaNUL5rm+i2pIl0LAhtGkDjzwC994LvXsf9Nt6mSimA1eJyJu4Tuzd1j9hTNlUlSUZmazcvIdpCzNITd/FruwCAESgd+v6/OWYjvRv35B2jWvTrnFt6icleBy1iZisLLjvPnjiCbjoInjlFejSpcrePmyJQkTeAIYATUUkHbgLSABQ1SnAx8CpwGogGxgXrliM8aOS5JCSlsGM1I38visHcInh+G7NGNarBT1b1adT0zo0rJ3ocbTGMx99BBMnwrp18Je/uJZEFQvnqKcLynlegYnh+nxj/GrV5j2kpG1kRmoGa7ZlEV9DOLZrU64Z1o1+7RrStlEtkhLs+r8Bnn/eJYlevWD2bDjuuLB8jO/WozAmFq3bnsWMtI2kpGawfNMeROCoTk24fHAnRvRuSaM61mIwAYWFsHUrtGoF550HOTkwaRIkhu9vxBKFMR7J2JXDR2kbmZGWQWr6bgAOP6QRd4/qxal9WtG8fpLHEZqoM3cu/PWvEB8PP/4ITZvCddeF/WMtURgTQVv35PHJYtdy+HntTgD6tGnAraf24LS+rWnTsJbHEZqotGsX3HorTJniWhLPPAMRnFtiicKYMNuVnc+nizeRkpbBD79up1ihe4t6XDesGyOTW9OxaeXHt5tqYNEiGDbMXW7629/ckNf69SMagiUKY8JgT24BXyzdzIy0jcxeuZXCYqVDk9pMHNqFkX1b071lPa9DNNGuoAASEqBbNxg6FG64AQ7zpnaqJQpjqkhOfhFfLd9CSmoGX63YQn5hMa0bJHHpsR0Z2bc1h7apj4hNajPlyMtzQ1xffRXmz4e6deGNNzwNyRKFMQchr7CI2Su3kZKawcxlm8nOL6Jp3ZpcOLA9o5Jb0b9dI5vxbEL31VcwYQKsXAljxrikUbeu11FZojCmogqKivn+1+3MSM3g0yWb2JNbSMPaCYzu14ZRya0Y1LEJcZYcTEXk5MD48a4V0akTfPopnHyy11H9wRKFMSEoKlZ+XruDlNQMPlm8iR1Z+dSrGc+w3i0YldyaY7s0JSHOKpyaSkpKgm3b4Pbb3eimWtE1+s0ShTEHoKos3LCLlNSNfLQog82ZeSQl1OCkni45HN+tmc2QNpWXluY6qF96Cdq2daU4orScuiUKY0pRVZZuzCQl1U2ES9+ZQ2JcDYZ0b8bI5Nac2KM5dWrafxtzELKy4O674amnoFEjWLXKJYooTRJgicIYAFZv2cP0QHJYszWLuBrCsV2acvVJ3Rjeu4VVYjVVY/p0V25j/Xq4/HJ4+GFo3NjrqMplicJUW+u3Z5OSlvFf9ZWO7NiEy47txIhDW9LY6iuZqvbhh26y3LffwjHHeB1NyCxRmGpl425XXykldV99pcPaN+SuUb04zeormapWUADPPusmzB12mCu9kZTkJtL5iCUKE/NK6ivNSN3I3LU7ADi0TX1uOaUHp/VtRdtGtT2O0MSkH390BfzS0uCmm1yiqOfPGfmWKExM2pWdz2dLNpGSupHvf91GsULX5nWtvpIJv5074ZZbYOpUtyTpBx/A6NFeR3VQLFGYmLE3r5AvlrrkMGfVVgqKlEOa1ObKIV0YlWz1lUyETJ0KL74I11zjRjf5tBVRmiUK42s5+UXMWhGor7R8C3mB+krjjunIyL6t6NOmgdVXMuG3YoWr7nrssXD11XDKKdC3r9dRVRlLFMZ38gqLmLNyGylpGcxcupms/CKa1k3k/CPaMSq5NYe1t/pKJkJyc+Ghh9ww1x49YOFCqFkzppIEWKIwPlFYUl8pLYNPF28iM1Bf6fR+rRnVtzWDOll9JRNhX3wBV14Jq1fDhRfCE09AjLZeLVGYqFVcUl8pLYNPFm1ie1Y+dWvGM7yXK6FxTJemJMZH72xWE8Nmz4bhw6FrV5cwTjrJ64jCyhKFiSql6yt9vGgjmzJzSUqowYk9WzCqb2uGdLf6SsYjRUWwdCn06QPHHedqNF14oZsXEeMsURjPqSrLNu75Y5Z0SX2l47s349bknlZfyXhvwQK44gpYtszVZmrRAv7yF6+jihj732c8s3rLXlJSM0gpVV/pmC5N+fuJXRneuyUNavlr9qqJQXv2wF13uRnVTZvCCy9A8+ZeRxVxlihMRG3YUVJfaSPLNmYiAoM6NubSYzsyondLmtSt6XWIxji7d7vLTBs2uBnWDz3kqr1WQ5YoTNht2p3LjLQMUtI2krphFwD92zfkzpG9OK1vK1pYfSUTTTIzXeG+Bg3cqnMnnghHHeV1VJ6yRGHCYtvePD5ZtJGUtI38vHYHqtC7dX1uPqUHp/VpRbvGVl/JRJmCArdGxP33w9dfu9pMt9/udVRRwRKFqTK7swtcfaW0DL5b7eordWlel6tP7MbI5FZ0bub9IvHGlOm771xn9eLFcMYZ0KyZ1xFFFUsU5qDszStk5tLNpKRmMDtQX6l949pMGNLZ1VdqUc9KaJjoNmkSTJ4M7drBtGlw+uleRxR1LFGYCsstKGLW8i2kpGXw5TJXX6lVgyTGHt2BUcmtrb6SiX6q+2ZRt2wJ11/vRjfVtVZvWSxRmJDkFxYzZ9VWUlIz+KJUfaUxgfpKh1t9JeMXy5e7y0zXXOPKf992m9cRRT1LFOaACouK+WHNdmakbuSTxRvJzC2kQa0ERiW3ZlRyawZ1bEx8nJXQMD6RkwMPPgiPPAJ16rhtE5KwJgoRGQE8A8QBL6rqw/s93wB4FWgfiOVxVf1XOGMywRUXK/PW7SQlNYOPF21ke1Y+dRLjGN67JaOSW3Fsl2ZWX8n4z5dfurkQv/4KF18Mjz9eLSfOVVbYEoWIxAHPAcOAdOBnEZmuqktL7TYRWKqqo0SkGbBCRF5T1fxwxWX+l6qSmr6blNQMPkorVV+pRwtGJbdiSPfmVl/J+Ft6OsTHu4RxwgleR+M74WxRDARWq+oaABF5ExgNlE4UCtQT1/NZF9gBFIYxJhNQUl/JTYTLYMOOHBLihOO7NeeWU3twUs8WVl/J+FdREUyZAomJcPnlcMklcP75bq0IU2HhPBO0ATaU2k4HBu23z2RgOpAB1APGqGrx/m8kIuOB8QDt27cPS7DVSW5BEef98wfS0ncTV0M4unMTJp3QlZN7taRBbauvZHxu/nx3mWnePDj7bJcoRCxJHIRwJoqyhsDoftsnAwuBE4DOwBciMkdVM//rRapTgakAAwYM2P89TAUsydjNtW+lsmLzHk7r04p7R/e2+komNmRmwh13uDkRzZrBG2/AmDFeRxUTwpko0oF2pbbb4loOpY0DHlZVBVaLyG9AD2BuGOOqloqKlSnf/MrTM1fSsHYi/xp3BEO7W2eeiSGpqS5JXHEFPPAANGzodUQxI5yJ4megq4h0BH4Hzgcu3G+f9cCJwBwRaQF0B9aEMaZqae22LK57J5Vf1u3ktD6tuP+MQ2lUJ9HrsIw5eL/9BrNmubUhjjvOLUvasaPXUcWcsCUKVS0UkauAz3DDY19W1SUickXg+SnAfcArIrIId6nqJlXdFq6YqhtV5fW563ngo2XE1xCeOb8fpye3tlnTxv/y890a1ffe61aYO/NMVwLckkRYhHVYi6p+DHy832NTSt3PAIaHM4bqaktmLje+l8bXK7ZybJemPHZuX1o1qOV1WMYcvDlz3OWlpUvhrLPcokLVdJ2ISLHxjzHoo7SN3PbhInILirjn9N5cfOQhVl7DxIatW2H4cLcUaUoKjBzpdUTVgiWKGLI7u4C7pi/mw4UZJLdtwJNj+llpb+N/qjBzJgwb5kYzzZgBRx7pynCYiLBEESO+XbWN699JZevePK45qRsTh3a2OkzG/5YsgQkT3OWmWbNgyBC34pyJKEsUPpeTX8Qjny7nle/X0rlZHaZecjR92zb0OixjDk52tltp7rHH3LKkL74Igwd7HVW1ZYnCx1I37OKatxeyZmsWY4/uwM2n9LCaTMb/VGHoUJg7F/78Z5csbMU5T1mi8KGComImf7WaybNW07xeTV67bBDHdGnqdVjGHJyNG11F17g4uPVWaNDAXWoynrOL2D6zestezn7he575chWnJ7fm06sHW5Iw/lZUBM8+C927w/PPu8dGj7YkEUWsReETxcXKv39Yy8OfLKd2YhzPX3QYp/Zp5XVYxhycefNcAb/58+Hkk+HUU72OyJQh5EQhInVUNSucwZiyZezK4YZ3U/lu9XaGdm/GI2f3pXn9JK/DMubgPPoo3HyzW7P6rbfg3HP3rWNtokq5iUJEjgZexK0X0V5EkoG/quqV4Q6uulNVPlz4O3dOW0JRsfLQWX04/4h2VoLD+JcqFBZCQgIMHAgTJ7rRTQ0aeB2ZCSKUFsVTuHLg0wFUNVVEbJxamO3Iyuf2Dxfx8aJNDDikEU+cl8whTWyCkfGxX3+FK6+EQw91dZqGDLF+CJ8I6dKTqm7Y71tsUXjCMQCzlm/hxvfS2JWdz00jejB+cCfirASH8au8PDfE9YEHXEti9GivIzIVFEqi2BC4/KQikgj8DVgW3rCqp6y8Qu7/aBlvzF1Pj5b1+Pe4gfRqXd/rsIypvF9+gT/9CZYvd30QTz8NrVt7HZWpoFASxRXAM7ilTdOBzwHrn6hiv6zbwbVvp7J+RzZ/HdyJa4d3o2a8TZ4zPle3ruug/vhjOOUUr6MxlRRKouiuqheVfkBEjgG+C09I1Ut+YTFPzVzJP7/5ldYNa/HW+KMY2LGx12EZUznFxfCvf8EPP7iyG927w+LFUMOmbPlZKIniH8BhITxmKmj5pkyueSuVZRszGTOgHXeM6kXdmja1xfjU4sVunYjvvnN1mbKyXIVXSxK+d8CzkogcBRwNNBORa0s9VR+3Yp2ppKJi5cU5a3ji85XUrxXPi5cM4KReLbwOy5jKycpyK809+aQb5vqvf7kaTTaMO2YE+/qaiJs7EQ/UK/V4JnBOOIOKZRt2ZHPd26nMXbuDk3u34MEz+9Ckbk2vwzKm8nJzXXK45BI3ia5JE68jMlXsgIlCVb8BvhGRV1R1XQRjikmqytvzNnBvylJqiPDEucmcdVgbmzxn/Ck93dVneughlxiWL4fG1rcWq0K5IJ4tIo8BvYE/6kao6glhiyrGbN2Txy3vpzFz2RaO6tSEx87tS9tGtb0Oy5iKKyyEf/wD7rzTFfMbMwYOP9ySRIwLJVG8BrwFjMQNlf0zsDWcQcWSTxdv4tYPFrE3r5A7RvZi3NEdbP1q408//eQK+KWmuuJ9kydDx45eR2UiIJRE0URVXxKRv5e6HPVNuAPzu8zcAu6ZvpT35qdzaJv6PHVeP7q2qFf+C42JRsXFMG4c7N4N774LZ51lndXVSCiJoiDwc6OInAZkAG3DF5L//fDrdq5/J5WNu3OYdEIXJp3QlcR4GyJofEbVJYURI6BePXj/fWjTxt031UooZ6/7RaQBcB1wPa6S7NXhDMqvcguKuG/GUi74vx9JjK/BuxOO5rrh3S1JGP9ZtcqtD3HeeTB1qnusRw9LEtVUuS0KVZ0RuLsbGAp/zMw2pSz+fTfXvLWQVVv2cvGRh3DLqT2onWiT54zP5OXBI4/Agw9CzZquH+KKK7yOyngs2IS7OOA8XI2nT1V1sYiMBG4FagH9IxNidCssKuaFr3/lmS9X0aRuIv/+y0CO72YLwRufmjgRXnoJzj/fTaBrZasomuAtipeAdsBc4FkRWQccBdysqh9GILao99u2LK59eyEL1u9iVHJr7hvdm4a1E70Oy5iK2bLFdVa3bAk33eSqvJ58stdRmSgSLFEMAPqqarGIJAHbgC6quikyoUUvVeXVH9fx4MfLSYyvwbMX9Of0ZCudbHymuNgV7rvpJhg+3C1H2rWruxlTSrBEka+qxQCqmisiKy1JwKbdudz4XhqzV25lcLdmPHp2X1o2sPWrjc+kpbm+hx9+cKvM3XOP1xGZKBYsUfQQkbTAfQE6B7YFUFXtG/bookxKaga3f7iY/MJi7jvjUP40qL2V4DD+8+67rg+iUSP4z3/cwkL2d2yCCJYoekYsiii3KzufO6ctYXpqBv3aNeSpMf3o2NTWrzY+k5kJ9eu7FsTEiXDXXVZ6w4QkWFFAKwQIzF65lRveTWX73nyuG9aNCUM6Ex9n8yKMj6xfD5MmQUYG/PgjNG0KzzzjdVTGR8J6xhORESKyQkRWi8jNB9hniIgsFJEl0VQaJDu/kDunLeaSl+dSLymBD648hkkndrUkYfyjoAAefxx69oSZM93kOVWvozI+FLYZYYF5GM8Bw3Brbf8sItNVdWmpfRoCzwMjVHW9iDQPVzwVsWD9Tq59O5XftmVx6bEdueHk7iQl2FpNxkfWrYPTT3ed1qNGuYqvhxzidVTGp0JKFCJSC2ivqisq8N4DgdWquibwHm8Co4Glpfa5EHhfVdcDqOqWCrx/lSsoKubZL1fx3KzVtGpQi9cvH8TRnZt6GZIxFaPqOqZbtoQWLeCDD2D0aOusNgel3OsoIjIKWAh8GtjuJyLTQ3jvNsCGUtvpgcdK6wY0EpGvReQXEbkkpKjDYNXmPZz5/Hf846vVnNm/LZ9cfZwlCeMfqvDqq3DEEbB3ryu/8fnncMYZliTMQQulRXE3rnXwNYCqLhSRDiG8rqy/zv0vkMYDhwMn4sqC/CAiP6rqyv96I5HxwHiA9u3bh/DRoSsuVl7+7jce/WwFdWvGM+VPhzPi0JZV+hnGhNWKFTBhAsyaBYMGwfbtULeu11GZGBJKoihU1d2VmC+QjisBUqItrkT5/vtsU9UsIEtEZgPJwH8lClWdCkwFGDBgQJX1xv2+K4fr307lhzXbOalncx46qy/N6tn61cYnCgvhvvvg4YehVi144QUYPx5q2IALU7VCSRSLReRCIE5EugJ/A74P4XU/A11FpCPwO3A+rk+itGnAZBGJBxKBQcBToQZfWarK+/N/5+7pSyhW5dGz+3LugLY2ec74S1wczJkD55zjCvi1aOF1RCZGhZIoJgG3AXnA68BnwP3lvUhVC0XkqsD+ccDLqrpERK4IPD9FVZeJyKdAGlAMvKiqiyv3q4RmS2Yud05bwqdLNjGwQ2OeOC+Zdo1t/WrjE5s2wa23upIb7drBxx9DkpWQMeElWs64ahHpr6oLIhRPuQYMGKDz5s2r1Gt3ZuUz+rnvyNiVww0nd+ey4zoRZ+tXGz8oKnILCN1yC+TkuI7rc8/1OirjIyLyi6oOqMxrQ7mY+aSILBeR+0Skd2U+JBrk5Bdx6b9/ZtPuXJ6/6DD+enxnSxLGHxYsgKOPhiuvhAEDYNEiSxImospNFKo6FBgCbAWmisgiEbk93IFVpcKiYq56fT4LNuzi2Qv6Mby3jWoyPjJ5MqxdC6+9Bl98Ad26eR2RqWbKvfT0XzuL9AFuBMaoqicr9FT00pOqcvN7i3hr3gbuO+NQLj7SZqeaKKcKH34IHTpA//6wc6d7vFEjL6MyPhfWS08i0lNE7haRxcBk3IintpX5MC889cVK3pq3gUkndLEkYaLf2rWu9MZZZ8HTT7vHGjWyJGE8Fcqop38BbwDDVXX/eRBRbUdWPv+YtZoz+rXm2mHWXDdRrKDADXG95x43D+Lxx+Hvf/c6KmOAEBKFqh4ZiUDC4bHPlgNw+eBONkfCRLd//hNuvtmV3HjmGajiCgTGHIwDJgoReVtVzxORRfx36Q1frHA3Iy2DN+ZuYMKQzvRu3cDrcIz5X9u3u0tNhx8Ol18OXbrAiBFeR2XM/wjWoihp946MRCBVacOObG55bxH92ze0S04m+qi6JUivvx7q1YOVK10RP0sSJkodsDNbVTcG7l6pqutK34ArIxNexRUUFTPpjQUg8Oz5/UmwhYZMNFm2DIYOhbFjoWtXN7opPmzLwhhTJUI5iw4r47FTqjqQqvLE5ytZuGEXD5/V10pzmOiSmgrJyW4xoalT4dtvoW9UX8E1BgjeRzEB13LoJCJppZ6qB3wX7sAqY/76nUz55lcuHNSe0/q28jocY5z0dGjb1iWFe+6BSy+F5lGxmKMxIQnW5n0d+AR4CCi93vUeVd0R1qgq6ZXv1tKodgK3ndrT61CMgYwMuOYaV7hv+XJo08bVajLGZ4JdelJVXQtMBPaUuiEijcMfWsUt2LCTo7s0pU5Nu+ZrPFRU5Mpu9OwJ06bBjTdCU1st0fhXeS2KkcAvuOGxpSciKNApjHFVWG5BEek7czj7MN9MGjexKDcXBg+Gn3+GYcPg+efdsFdjfOyAiUJVRwZ+doxcOJW3ZmsWqtCluS0BaTxQUAAJCW5tiKFD4dprYcwYW6/axIRQaj0dIyJ1Avf/JCJPikjUTRtdvXUvYInCRJgqvPuuazXMn+8ee+QROP98SxImZoQyPPYFIFtEknGVY9cB/y+sUVXCkozdJMQJHZrU8ToUU12sWQOnnebWhmjSxNaqNjErlL/sQnW1yEcDz6jqM7ghslFl3tqd9GnTgKSEOK9DMdXBk09C795uzeqnn4a5c6FfP6+jMiYsQkkUe0TkFuBi4CMRiQMSwhtWxeQWFLEofTdHdIjKwVgmFu3dC6ee6mZa//3vNrvaxLRQEsUYIA/4i6puAtoAj4U1qgpakpFJflExhx1iNftNmGzbBuPGwfTpbvv22+G999xEOmNiXChLoW4CXgMaiMhIIFdV/xP2yCpg5eY9APRqVd/jSEzMKS6Gl1+G7t3h1Vdh9Wr3uPVHmGoklFFP5wFzgXOB84CfROSccAdWEau37CUpoQZtGtbyOhQTS5YuhSFDXMmNXr1g4UI37NWYaiaUC6u3AUeo6hYAEWkGzATeDWdgFbFqy146N6tLjRo2HNFUoXnzYMkSeOklV+3VWhGmmgolUdQoSRIB2wmtbyNi1m/PssWJTNX4+GO3oNDFF7vbyJHQ2AZJmOotlBP+pyLymYiMFZGxwEfAx+ENK3SqyqbMXFo1SPI6FONn6elwzjluXsTkyW4inYglCWMIrTP7BuCfQF8gGZiqqjeFO7BQZeYUkltQTEtLFKYyCgvdGtU9e8JHH8EDD7i5ETar2pg/BFuPoivwONAZWARcr6q/RyqwUG3MzAGwRGEq55df4Oqr3TKkzz0HnaKq1qUxUSFYi+JlYAZwNq6C7D8iElEFbdqdC0DL+pYoTIh274b333f3Bw2Cn35yfROWJIwpU7DO7Hqq+n+B+ytEZH4kAqqokkTRwhKFKY8qvP22a0Fs3w5r10Lr1jBwoNeRGRPVgiWKJBHpz751KGqV3lbVqEgcC9bvol7NeLv0ZIL79VeYOBE++wwOPxxSUlySMMaUK1ii2Ag8WWp7U6ltBU4IV1AV8du2LHq2rk9CXFSN2DXRZM8elxyKi+HZZ+HKKyHOikcaE6pgCxcNjWQglZW+M5sjOzfxOgwTjdLSoG9fqFfPTZo78ki3brUxpkJ8/TU8v7CYjZm5tG1U2+tQTDTZuhX+/GdITnad1ABnn21JwphKCmuiEJERIrJCRFaLyM1B9jtCRIoqWkMqY1cOqtC2kdV4MrhLSy++6Ar4vfEG3Hqrq9VkjDkoYSuiH1i34jlgGJAO/Cwi01V1aRn7PQJ8VtHPWLBhJwA9W1rVWINrNXz4IQweDC+84Ar5GWMOWijVYyWwVvadge32IhLKeMKBwGpVXaOq+cCbuFXy9jcJeA/YUsZzQc39bQf1k+Lp3doSRbWVleVmVwNccAG88gp8/bUlCWOqUCiXnp4HjgIuCGzvwbUUytMG2FBqOz3w2B9EpA1wJjAl2BuJyHgRmSci87Zu3frH40szMjm0TQOrGltdpaS4hPD88277vPNc34SV3zCmSoWSKAap6kQgF0BVdwKJIbyurP+tut/208BNqloU7I1UdaqqDlDVAc2aNQOgsKiY5Zv20NMWK6p+NmyAs86C0093I5oOP9zriIyJaaH0URQE+hEU/liPojiE16UD7UpttwUy9ttnAPCmuG+ATYFTRaRQVT8s781/35VDXmEx3VvUCyEUEzNefRWuuMJ1XD/8MFxzDSSG8r3FGFNZoSSKZ4EPgOYi8gBwDnB7CK/7GegqIh2B34HzgQtL76CqHUvui8grwIxQkgTAnlx3Xbph7YRQdjd+V1L2u21bN5LpH/+Ajh3LfZkx5uCVmyhU9TUR+QU4EXc56QxVXRbC6wpF5CrcaKY44GVVXSIiVwSeD9ovUZ7sfHe1qnZi2AZumWiwaxfccgvUqQOPP+6ShA15NSaiyj3Likh7IBtIKf2Yqq4v77Wq+jH7LXJ0oAShqmPLe7/SsvJdi6J2TSvFEJNU3VyIa691E+iuuWZfq8IYE1GhfB3/CNc/IUAS0BFYAfQOY1zlyvmjRWGJIub89huMHw8zZ8IRR8Ann0D//l5HZUy1Fcqlpz6lt0XkMOCvYYsoRFl5rkVRxy49xZ6CAlen6bnn4K9/tQJ+xniswmdZVZ0vIkeEI5iKyCmwFkVM+fJLtxTpk09Ct26wbh0kWel4Y6JBKH0U15barAEcBmw9wO4Rs31vPiJQN8laFL62eTNcdx289hp07gy33QZNmliSMCaKhDLhrl6pW01cn0VZpTgi6rdtWbSqn0TNeGtR+FJxMfzzn9Cjh1t17o47YNEilySMMVEl6NfxwES7uqp6Q4TiCdm67Vl0bl7X6zBMZe3eDbffDv36uQJ+PXp4HZEx5gAO2KIQkfhAaY3DIhhPyHIKiqwj22/27nV9EEVF0KgR/PQTfPWVJQljolywM+1cXJJYKCLTgXeArJInVfX9MMcWVE5BEbWsI9s/pk2DSZNcnaZ+/eCEE6BTJ6+jMsaEIJQ+isbAdtwa2SOBUYGfnlFVdmcXWKLwg3XrYPRoOOMMaNgQvvvOJQljjG8Ea1E0D4x4Wsy+CXcl9q8CG1Fb9+aRmVtIl2bWRxHVVOGcc2DpUnj0Ubj6akiw2lzG+E2wRBEH1CW0cuERtWrzXgC6t7TKsVHpxx+hd29XAnzqVGjcGA45xOuojDGVFCxRbFTVeyMWSQWs3LwHgK4trEURVXbscAX8pk6FO++Ee+6x0hvGxIBgiSJqq6+t3LyXhrUTaFa3ptehGHCXmF591U2c27HD/bwh6kZUG2MqKViiODFiUVTQqs176Na8HmKVRKPDrbe6RYSOPBK++AKSk72OyBhThQ6YKFR1RyQDqYiVm/cwKrm112FUb7m5bl5E06Ywbpzrgxg/HmqEMpDOGOMnvvtfXVCkZOYW0s2WQPXOF19Anz5w+eVuu1s3tzypJQljYpLv/mfnFbqqsV2tfEfkbdoEF14Iw4e7BYSuusrriIwxEeC7GhhFxW5kbuO6iR5HUs3MmgVnngk5OXD33XDTTVbh1ZhqwneJQgMzOBLifNcY8qeCAjdJrm9fGDYMHnjAXWoyxlQbvjvbFgcyRa0EK98RVnv2uHWqjzvOFfFr0gTeeceShDHVkG8TRT1bsCg8VOH996FnT3jmGTdhLi/P66iMMR7yYaJwPxPjfRd69Nu2DUaNgrPPdsNev//erRVRu7bXkRljPOS7s60GWhSJ1kdR9erVc0uTPvkkzJvnJtAZY6o9351ti4qV2olxNiu7qnz7LZxyips8V7OmW0zommsg3i7tGWMc/yUKVZrY0NiDt307XHaZ66xeuhTWrHGP26Q5Y8x+fHdWUIUEO5lVniq88gp07+5+3nCDSxR9+3odmTEmSvnu+oIqxMfZZaeD8p//uEQxZYorxWGMMUH47qt5UbESby2KisnJgbvugvR0V3rjvfdgzhxLEsaYkPjujJtXWEROQZHXYfjHZ5/BoYfCvffCtGnusUaNrC/CGBMy350tRIRDmti4/nJlZMCYMTBihCvB8dVXMHGi11EZY3zId4miuFjp0KSO12FEv/vvdy2Ie++F1FQYOtTriIwxPuW7zuxidfMoTBl++WVfAb/77oNrr4UuXbyOyhjjc2FtUYjICBFZISKrReTmMp6/SETSArfvRaTcNTQVqFPTd/ktvDIz4W9/g4ED3bKk4Ir4WZIwxlSBsCUKEYkDngNOAXoBF4hIr/12+w04XlX7AvcBU0N57zrWonBUXUXXHj1g8mSYMAFefdXrqIwxMSacX80HAqtVdQ2AiLwJjAaWluygqt+X2v9HoG0ob1zbWhTO66/Dn/7kKrxOmwZHHOF1RMaYGBTOM24bYEOp7XRgUJD9LwU+KesJERkPjAdIbNmFutU5UeTnu3IbPXrAOee4ORJjx1ptJmNM2ISzj6Ks6dNa5o4iQ3GJ4qaynlfVqao6QFUHANW3M3v2bOjXz61ZnZvrivhddpklCWNMWIUzUaQD7UpttwUy9t9JRPoCLwKjVXV7KG9c7Tqzt22DcePg+ONdC2LKFFuv2hgTMeE84/4MdBWRjsDvwPnAhaV3EJH2wPvAxaq6MtQ3rpNYjRLFmjWu7yEzE26+Ge64wxYSMsZEVNjOuKpaKCJXAZ8BccDLqrpERK4IPD8FuBNoAjwfWF+isOTyUjB1alaDS0+ZmVC/PnTs6FoTY8e6UhzGGBNhUrJinF/UbNVVf1+1mKZ1a3odSnhkZ7vJclOnuhnVbUMaCGaMMUGJyC+hfBEviy+v4dRKiNEWxUcfwVVXwdq1rhVRq5bXERljjD8TRVyNGFuPorAQLrgA3n0XevaEb76BwYO9jsoYYwAfFgWMKSWX/eLjoUULePBBWLjQkoQxJqpYovDKzz/DoEEwf77bnjwZbrkFEm09cGNMdPFlohA/X3navdv1Qwwa5Fac2x7S1BFjjPGMLxOFb5UU8HvhBZcsli+HYcO8jsoYY4LyZWe2lFkdxAeWLYM2bSAlBQZUapSaMcZEnC/nUexNX0FCnA8aQ3l58NhjkJwMo0ZBQYFbqzouRof3GmOi1sHMo/DB2danZs1yCeKOO+DLL91jCQmWJIwxvuPLRBHVF562bIE//xlOOMG1ID75BJ5+2uuojDGm0vyZKKJ52NPnn8Mbb8Btt8HixTBihNcRGWPMQfFlZ3bUWbQIVqxwCwlddBEcfTR06uR1VMYYUyX82aLwOoASWVlw441uKdIbb3SXmkQsSRhjYoo/E0U0ZIqUFOjVy41qGjvWzbROSPA6KmOMqXJ26akyFi+G00+H3r1hzhw49livIzLGmLDxaYvCgyZFYSF8/bW7f+ihMGMGLFhgScIYE/N8mSgi7qef3EzqE0+EVavcY6edZpeajDHVgiWKYHbuhAkT4KijYNs2V6upSxevozLGmIiyPooDyctzo5k2bICrr4Z77oF69byOyhhjIs4Sxf5+/90V7qtZE+6+25Xh6N/f66iMMcYzdumpRG6uazV06gTTprnHxo61JGGMqfasRQGuaN+ECa6j+oIL3KJCxhhjAGtRuP6Hk05y61d//jm8/jq0bOl1VMYYEzV8lyiqZAZFcTEUFbn7AwfCnXe6ek222pwxxvwP3yWKg5aa6or2Pfec277wQtc3kZTkbVzGGBOlqk+i2LsXrrsODj8c1qyxy0vGGBOi6tGZPXMmjBsH6ekwfjw8/DA0auR1VMYY4wvVI1EkJkLjxvDWW+6ykzHGmJDFZqIoKHDLj+7eDfffD4MHuwJ+NarPlTZjjKkqsXfm/P571w9x442wbJkb4QSWJIwxppL8d/Y80PjYHTtc/8Mxx8CuXfDhh/Dee5YgjDHmIMXOWXT7djdZ7vrrYelSGD3a64iMMSYm+K6PQko3KVascB3Ud94JXbvCunXQpIl3wRljTAwKa4tCREaIyAoRWS0iN5fxvIjIs4Hn00TksJDeOCfHJYe+feGpp1wpcLAkYYwxYRC2RCEiccBzwClAL+ACEem1326nAF0Dt/HAC+W9b928LOjTB+67D849F5Yvh3btqjh6Y4wxJcLZohgIrFbVNaqaD7wJ7N9xMBr4jzo/Ag1FpFWwN22za7ProJ45E159FVq0CE/0xhhjgPD2UbQBNpTaTgf2r99d1j5tgI2ldxKR8bgWB0CerFq1mJNOqtpo/akpsM3rIKKEHYt97FjsY8din+6VfWE4E0VZA1m1EvugqlOBqQAiMk9VBxx8eP5nx2IfOxb72LHYx47FPiIyr7KvDeelp3SgdOdBWyCjEvsYY4zxUDgTxc9AVxHpKCKJwPnA9P32mQ5cEhj9dCSwW1U37v9GxhhjvBO2S0+qWigiVwGfAXHAy6q6RESuCDw/BfgYOBVYDWQD40J466lhCtmP7FjsY8diHzsW+9ix2KfSx0JU/6dLwBhjjPlD7JTwMMYYExaWKIwxxgQVtYkibOU/fCiEY3FR4Bikicj3IpLsRZyRUN6xKLXfESJSJCLnRDK+SArlWIjIEBFZKCJLROSbSMcYKSH8H2kgIikikho4FqH0h/qOiLwsIltEZPEBnq/ceVNVo+6G6/z+FegEJAKpQK/99jkV+AQ3F+NI4Cev4/bwWBwNNArcP6U6H4tS+32FGyxxjtdxe/h30RBYCrQPbDf3Om4Pj8WtwCOB+82AHUCi17GH4VgMBg4DFh/g+UqdN6O1RRGW8h8+Ve6xUNXvVXVnYPNH3HyUWBTK3wXAJOA9YEskg4uwUI7FhcD7qroeQFVj9XiEciwUqCciAtTFJYrCyIYZfqo6G/e7HUilzpvRmigOVNqjovvEgor+npfivjHEonKPhYi0Ac4EpkQwLi+E8nfRDWgkIl+LyC8icknEoousUI7FZKAnbkLvIuDvqlocmfCiSqXOm9G6HkWVlf+IASH/niIyFJcojg1rRN4J5Vg8DdykqkXuy2PMCuVYxAOHAycCtYAfRORHVV0Z7uAiLJRjcTKwEDgB6Ax8ISJzVDUzzLFFm0qdN6M1UVj5j31C+j1FpC/wInCKqm6PUGyRFsqxGAC8GUgSTYFTRaRQVT+MSISRE+r/kW2qmgVkichsIBmItUQRyrEYBzys7kL9ahH5DegBzI1MiFGjUufNaL30ZOU/9in3WIhIe+B94OIY/LZYWrnHQlU7qmoHVe0AvAtcGYNJAkL7PzINOE5E4kWkNq5687IIxxkJoRyL9biWFSLSAldJdU1Eo4wOlTpvRmWLQsNX/sN3QjwWdwJNgOcD36QLNQYrZoZ4LKqFUI6Fqi4TkU+BNKAYeFFVyxw26Wch/l3cB7wiIotwl19uUtWYKz8uIm8AQ4CmIpIO3AUkwMGdN62EhzHGmKCi9dKTMcaYKGGJwhhjTFCWKIwxxgRlicIYY0xQliiMMcYEZYnCRKVA5deFpW4dguy7two+7xUR+S3wWfNF5KhKvMeLItIrcP/W/Z77/mBjDLxPyXFZHKiG2rCc/fuJyKlV8dmm+rLhsSYqicheVa1b1fsGeY9XgBmq+q6IDAceV9W+B/F+Bx1Tee8rIv8GVqrqA0H2HwsMUNWrqjoWU31Yi8L4gojUFZEvA9/2F4nI/1SNFZFWIjK71Dfu4wKPDxeRHwKvfUdEyjuBzwa6BF57beC9FovI1YHH6ojIR4G1DRaLyJjA41+LyAAReRioFYjjtcBzewM/3yr9DT/QkjlbROJE5DER+VncOgF/DeGw/ECgoJuIDBS3FsmCwM/ugVnK9wJjArGMCcT+cuBzFpR1HI35H17XT7eb3cq6AUW4Im4LgQ9wVQTqB55riptZWtIi3hv4eR1wW+B+HFAvsO9soE7g8ZuAO8v4vFcIrF0BnAv8hCuotwiogytNvQToD5wN/F+p1zYI/Pwa9+39j5hK7VMS45nAvwP3E3GVPGsB44HbA4/XBOYBHcuIc2+p3+8dYERguz4QH7h/EvBe4P5YYHKp1z8I/ClwvyGu7lMdr/+97Rbdt6gs4WEMkKOq/Uo2RCQBeFBEBuPKUbQBWgCbSr3mZ+DlwL4fqupCETke6AV8Fyhvkoj7Jl6Wx0TkdmArrgrvicAH6orqISLvA8cBnwKPi8gjuMtVcyrwe30CPCsiNYERwGxVzQlc7uor+1bkawB0BX7b7/W1RGQh0AH4Bfii1P7/FpGuuGqgCQf4/OHA6SJyfWA7CWhPbNaAMlXEEoXxi4twK5MdrqoFIrIWd5L7g6rODiSS04D/JyKPATuBL1T1ghA+4wZVfbdkQ0ROKmsnVV0pIofjauY8JCKfq+q9ofwSqporIl/jyl6PAd4o+Thgkqp+Vs5b5KhqPxFpAMwAJgLP4moZzVLVMwMd/18f4PUCnK2qK0KJ1xiwPgrjHw2ALYEkMRQ4ZP8dROSQwD7/B7yEWxLyR+AYESnpc6gtIt1C/MzZwBmB19TBXTaaIyKtgWxVfRV4PPA5+ysItGzK8iauGNtxuEJ2BH5OKHmNiHQLfGaZVHU38Dfg+sBrGgC/B54eW2rXPbhLcCU+AyZJoHklIv0P9BnGlLBEYfziNWCAiMzDtS6Wl7HPEGChiCzA9SM8o6pbcSfON0QkDZc4eoTygao6H9d3MRfXZ/Giqi4A+gBzA5eAbgPuL+PlU4G0ks7s/XyOW9t4prqlO8GtJbIUmC8ii4F/Uk6LPxBLKq6s9qO41s13uP6LErOAXiWd2biWR0IgtsWBbWOCsuGxxhhjgrIWhTHGmKAsURhjjAnKEoUxxpigLFEYY4wJyhKFMcaYoCxRGGOMCcoShTHGmKD+P8en/mjvyzKVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y = train_y\n",
    "X = x_train_tf_idf_word.astype(\"float64\")\n",
    "\n",
    "abc_roc_auc = roc_auc_score(y, abc_model.predict(X))\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y, abc_model.predict_proba(X)[:,1])\n",
    "plt.figure()\n",
    "plt.plot(fpr, tpr, label='AUC (area = %0.2f)' % abc_roc_auc)\n",
    "plt.plot([0, 1], [0, 1],'r--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "301ac14b",
   "metadata": {},
   "source": [
    "### RandomForestClassifier model with \"count-vectors\" method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "b7cc120a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "RandomForestClassifier model with 'count-vectors' method\n",
      "Accuracy ratio:  0.9488186090225563\n"
     ]
    }
   ],
   "source": [
    "rfc = RandomForestClassifier()\n",
    "rfc_model = rfc.fit(x_train_count,train_y)\n",
    "accuracy = model_selection.cross_val_score(rfc_model,\n",
    "                                           x_test_count,\n",
    "                                           test_y,\n",
    "                                           cv = 20).mean()\n",
    "\n",
    "print(\"\\nRandomForestClassifier model with 'count-vectors' method\")\n",
    "print(\"Accuracy ratio: \", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "74dc342a",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred=rfc_model.predict(x_test_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "0d540179",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[7358,   34],\n",
       "       [ 303,  296]], dtype=int64)"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(test_y,pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "fe9947dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      7392\n",
      "           1       0.90      0.49      0.64       599\n",
      "\n",
      "    accuracy                           0.96      7991\n",
      "   macro avg       0.93      0.74      0.81      7991\n",
      "weighted avg       0.96      0.96      0.95      7991\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(test_y,pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "27a8ef98",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAnqUlEQVR4nO3dd5xU1fnH8c8jVaWIYEHQgLpKiYBSVBRjQ8EexZqoYAyiqFGxYI81sbz8WbCEoOHnT6OxoIANUaNYolKkFyUIsqKhiDQBWXh+f5y7slmX2WHZO3fK9/16zWvn3rkz8+xlmWfOOfc8x9wdERGRTdkq6QBERCS7KVGIiEhKShQiIpKSEoWIiKSkRCEiIikpUYiISEpKFCIikpIShUgKZjbXzFab2Uoz+9bMhppZvTKPdzWzd8xshZktM7ORZtam3Gs0MLP7zeyr6HVmR9tNMv8biWw+JQqRyh3v7vWADsC+wLUAZnYg8CYwHNgFaAlMAj40s92jY2oDbwNtgR5AA6ArsAToktHfQqSKTDOzRTbNzOYC57v7W9H23UBbdz/WzN4Hprj7ReWe8zqwyN3PMbPzgTuAPdx9ZYbDF6kWalGIpMnMmgM9gdlmtg2hZfB8BYc+B3SP7h8JvKEkIblMiUKkci+b2QpgPrAQuBnYnvD/55sKjv8GKB1/aLyJY0RyhhKFSOVOcvf6wKFAK0ISWApsAJpWcHxTYHF0f8kmjhHJGUoUImly9/eAocC97r4K+BdwagWHnkYYwAZ4CzjazLbNSJAiMVCiENk89wPdzawDMBA418wuNbP6ZtbIzG4HDgRuiY7/P0KX1Ytm1srMtjKzxmZ2nZkdk8QvILK5lChENoO7LwKeBG509w+Ao4GTCeMQ8wiXzx7s7l9Ex68lDGjPBEYDy4FPCd1Xn2T8FxCpAl0eKyIiKalFISIiKSlRiIhISkoUIiKSkhKFiIikVDPpADZXkyZNvEWLFkmHISKSU8aPH7/Y3XeoynNzLlG0aNGCcePGJR2GiEhOMbN5VX2uup5ERCQlJQoREUlJiUJERFJSohARkZSUKEREJCUlChERSSm2RGFmT5jZQjObuonHzcweNLPZZjbZzPaLKxYREam6OOdRDAUGEUoyV6QnUBTd9gcejX6mpWT9Bko2qPKtiEjcYksU7j7GzFqkOORE4EkPdc4/NrPtzKypu6dcX/jfi1ZywqAPmPntCn4s2VCdIYuI5J39v5rC78YN36LXSHJmdjPCyl+liqN9P0sUZtYX6AtQe+c9aVC3FoftvQN779yArWvVyEiwIiK5pO73SzjwsbtoNWoYy3duvkWvlWSisAr2VdiX5O6DgcEAdZoW+WNnd6RenZyrPiIikjmnXA1vj4Brr6XBDTfAtlVftj3Jq56KgV3LbDcHFiQUi4hI7ps2Db7+Oty/6y6YOBHuvBO22WaLXjbJRDECOCe6+ukAYFll4xMiIlKBVatg4EDo0AGuvz7s23NPaNu2Wl4+tv4bM3sGOBRoYmbFwM1ALQB3fwx4DTgGmA38APSJKxYRkbz16qvQvz/MmwfnnRdaEtUszquezqzkcQf6x/X+IiJ575FHQpJo0wbGjIFu3WJ5G40Ii4jkkpISWLQImjaF006D1avhkkugdu3Y3lIlPEREcsWnn0LnznDCCbB+PTRpAgMGxJokQIlCRCT7ff89XHQRHHAALFwI11wDW2Xu4zsnu54qmoAhIpKXpkyB7t1Dd9Oll8Ktt0KDBhkNIScThYhI3lu3DmrVgr32gsMOg6uugv2SqZ2qricRkWyydm1oNbRtCytXQp068MwziSUJUKIQEcke77wD7drBzTeHxLB2bdIRAUoUIiLJW70azj4bjjgiXP76xhvw7LPQuHHSkQFKFCIiyatbFxYvhhtugKlT4eijk47ovyhRiIgkYfLkkBCKi8EslOK47TbYeuukI/uZnEwUputjRSRXrVq18QqmCRPgiy/C/gzOi9hc2RuZiEi+GTEi1GW6995QwG/WrHDpa5bTPAoRkUx5+eUwWe6DD+Cgg5KOJm1KFCIicVm3Dh58MLQa9tsPHnggDFzXqpV0ZJtFXU8iInH4+GPo1AmuvBKeey7sq18/55IEKFGIiFSvpUuhXz/o2hW++w5eegn+9Keko9oiShQiItVp8GAYMgQuvxymT4eTTsr5SzU1RiEisqVmzQrVXQ8+GC67DHr2DKU48kROtihMhcZFJBusWRPqMrVrF5YkdQ9F/PIoSUCOJgoRkcSNHg377BMqvfbqBaNG5XwX06ao60lEZHONGQNHHQVFRSFhHHlk0hHFSi0KEZF0rF8fVpsD6NYNHn881GvK8yQBShQiIpX77LNwuetBB8F//hO6mM47L0yeKwBKFCIim7JiBVxxRZg4N3cuPPoo7Lhj0lFlnMYoREQqsmxZGKyePx8uuCBMmmvUKOmoEqFEISJS1vLloXBfw4bQt29Yde7AA5OOKlE52fWUp1egiUiS1q2Du++G5s3DOhEQVpwr8CQBalGIiMCHH4b6TFOnhpIbO+yQdERZJSdbFCIi1eaSS0LpjWXLYPjwUMRv112TjiqrKFGISOFx33h/551DKfDp0+GEE5KLKYspUYhIYZk5MywkNHx42L7+erjnHqhXL9m4spgShYgUhtWr4cYbQ8G+SZPCtqQl1kRhZj3MbJaZzTazgRU83tDMRprZJDObZmZ94oxHRArU22+HORG33w5nnBHKgp9xRtJR5YzYrnoysxrAw0B3oBgYa2Yj3H16mcP6A9Pd/Xgz2wGYZWZPu/uPccUlIgWouBhq1gwJ4/DDk44m58TZougCzHb3OdEH/7PAieWOcaC+mRlQD/gOKIkxJhEpBOvXw8MPw1//GrbPOSd0NylJVEmciaIZML/MdnG0r6xBQGtgATAF+IO7byj/QmbW18zGmdm4uIIVkTwxYQIccABcfHFYIwLCLN06dZKNK4fFmSgqmj/t5baPBiYCuwAdgEFm1uBnT3If7O6d3L1TdQcpInli+XL4wx+gc+dQn+mZZ+D555OOKi/EmSiKgbKzVpoTWg5l9QGGeTAb+BJoFWNMIpKvJk2CQYPCDOuZM8Ngter9VIs4E8VYoMjMWppZbeAMYES5Y74CjgAws52AvYE5McYkIvnkyy/hiSfC/W7dYPbsMDax3XaJhpVvYksU7l4CXAyMAmYAz7n7NDPrZ2b9osNuA7qa2RTgbeAad18cV0wikid+/DGU/W7TBgYMgKVLw/6WLZONK0+Ze/lhg+xWp2mRL/tqJnVr1Ug6FBFJwvvvh+6l6dPh5JPhgQdCxVdJyczGV3WcNyerx6rbUaRALVoERx0FO+0EI0fCccclHVFBUAkPEclu7jB6dLi/ww7wyiswbZqSRAYpUYhI9po2DX71q9CKePfdsO+II2DbbRMNq9AoUYhI9vnhB7juOujQISSLIUPgkEOSjqpg5eQYhYjkMfdQBvzTT+Hcc0MJcK04lyglChHJDt98AzvuCDVqhNZEw4Zw6KFJRyWo60lEkrZ+PTz4IOy9NzzySNh34olKEllEiUJEkjNuHHTpEmo0de0KxxyTdERSgbQThZllzWUGVmG9QRHJKXffHZLEN9/AP/4Br78Oe+yRdFRSgUoThZl1NbPphDIcmFl7M3sk9shEJP+4w7p14X6XLtC/P8yYAaedppm0WSydFsX/EMqBLwFw90mArlMTkc3z739Djx4wMFoV+dBD4aGHwqC1ZLW0up7cfX65XetjiEVE8tHatWGt6l/+Ev71L3Uv5aB0Lo+db2ZdAY/KhV9K1A0lIpLS+PHw29+G9SFOPRXuvx922SXpqGQzpZMo+gEPEJYxLQbeBC6KMygRyRP16oWxh9deg549k45GqiidRLG3u/+m7A4zOwj4MJ6QRCRnbdgAf/tb6GIaMiTMjZg6FbbSlfi5LJ1/vYfS3JcxujhCJAtNnRrqMZ1/PnzxBaxaFfYrSeS8TbYozOxAoCuwg5ldUeahBoBWDRKRYNUquPVWuO++cAXT3/4WajTpG13eSNX1VBuoFx1Tv8z+5UCvOIMSkRyyZk1IDuecEybRNW6cdERSzTaZKNz9PeA9Mxvq7vMyGJOIZLvi4lCf6U9/Colh5kzYfvuko5KYpDOY/YOZ3QO0BeqW7nT3w2OLSkSyU0lJmCR3002hmN/pp0PHjkoSeS6dUaangZlAS+AWYC4wNsaYRCQbffIJdOoEV1wRBq2nTQtJQvJeOomisbs/Dqxz9/fc/TzggJjjEpFssmED9OkDixbBCy+Edatbtkw6KsmQdLqeogpefGNmxwILgObxhSQiWcE9JIUePaB+fRg2DJo1C/eloKTTorjdzBoCA4ArgSHAZXEGVRlddCcSsy++gKOPDlVdBw8O+1q1UpIoUJW2KNz9lejuMuAw+Glmtojkm7Vr4a674M47oU4dGDQI+vVLOipJWKoJdzWA0wg1nt5w96lmdhxwHbA1sG9mQhSRjOnfHx5/HM44I0yga9o06YgkC5i7V/yA2VBgV+BTYH9gHnAgMNDdX85QfD9Tp2mRryqeRc0aKgsgUi0WLgyD1TvvHLqc5swJ3U6SV8xsvLt3qspzU3U9dQLaufsGM6sLLAb2dPdvq/JGIpJlNmwIhfuuuQaOOiosR1pUFG4iZaT6Wv6ju28AcPc1wOdKEiJ5YvJkOPhguOAC6NABbrkl6Ygki6VqUbQys8nRfQP2iLYNcHdvF3t0IlL9XnghjEE0agRPPhkWFlIBP0khVaJonbEoRCR+y5dDgwZhrer+/eHmm1V6Q9KyycHsbFWnaZH/8PXn1NhK34BE0vLVV3DJJbBgAXz8MdTQKgGFaEsGs2O9dMjMepjZLDObbWYDN3HMoWY20cymmdl7ccYjUlDWrYN774XWreGtt8LkuRz7YijZIZ0SHlUSzcN4GOhOWGt7rJmNcPfpZY7ZDngE6OHuX5nZjnHFI1JQ5s2DE04Ig9bHHx8qvv7iF0lHJTkqrRaFmW1tZntv5mt3AWa7+xx3/xF4Fjix3DFnAcPc/SsAd1+4me8hImWVthh23hl22gleegmGD1eSkC1SaaIws+OBicAb0XYHMxuRxms3A+aX2S6O9pW1F9DIzN41s/Fmdk5aUYvIf3OHp56Czp1h5cpQfuPNN+Gkk3RFk2yxdFoUfyS0Dr4HcPeJQIs0nlfRX2f5DtKaQEfgWOBo4EYz2+tnL2TW18zGmdm4NN5XpLDMmgVHHAFnnw01a8KSJUlHJHkmnURR4u7LqvDaxYQSIKWaE0qUlz/mDXdf5e6LgTFA+/Iv5O6D3b1TVUfsRfJSSUm4xLVdO5gwAR59FD76SN1MUu3SSRRTzewsoIaZFZnZQ8BHaTxvLFBkZi3NrDZwBlC+y2o40M3MaprZNoSaUjMqe2E1pEUIl7m+/z706hVaFf36wVaqgSbVL52/qksI62WvBf5OKDd+WWVPcvcS4GJgFOHD/zl3n2Zm/cysX3TMDMLYx2RC8cEh7j61Cr+HSGH49ls47zyYPz+MPbz2Gjz9dBi4FolJpRPuzGxfd/8sQ/FUqk7TIl/99edspQl3UkjWrw8LCF17LaxeHQauTz016agkh8Q94e4+M5tpZreZWduqvImIbIHPPoOuXeGii6BTJ5gyRUlCMqrSROHuhwGHAouAwWY2xcxuiDswEYkMGgRz54YuptGjYa+fXRgoEqvNqvVkZvsAVwOnu3vt2KJKQV1Pkvfc4eWXoUUL2HdfWLo07G/UKMmoJMfF2vVkZq3N7I9mNhUYRLjiqXlV3kxEKjF3bii9cfLJcP/9YV+jRkoSkqh0aj39DXgGOMrdy8+DEJHqsG5dWKP6llvCJa733gt/+EPSUYkAaSQKdz8gE4FsDlUkkLzzl7/AwIGh5MYDD8BuuyUdkchPNpkozOw5dz/NzKbw36U3tMKdSHVYsiR0NXXsCL//Pey5J/TokXRUIj+TqkVR2u49LhOBiBQM97AE6ZVXQv368PnnoYifkoRkqU0OZrv7N9Hdi9x9XtkbcFFmwhPJMzNmwGGHQe/eUFQUrm6qGduyMCLVIp0Jd90r2NezugMRyXuTJkH79mExocGD4YMPQkE/kSyXaoziQkLLYXczm1zmofrAh3EHJpI3iouhefOQFG65BX73O9hRizlK7tjkhDszawg0Av4ElF3veoW7f5eB2CpUp2mRr1nwOaZLnyTbLVgAl18eCvfNnAnNyq/bJZI5cU24c3efC/QHVpS5YWbbV+XNRArC+vWh7Ebr1mEZ0quvhiZNko5KpMpSjaL9nXDF03jC5bFlv8I7sHuMcaWk1oRkrTVr4JBDYOxY6N4dHnkkXPYqksM2mSjc/bjoZ8vMhSOSo9atg1q1oG7dcFXTFVfA6adrdqjkhXRqPR1kZttG939rZveZmaaNikCYE/HCC6HVMGFC2HfXXXDGGUoSkjfSuTz2UeAHM2tPqBw7D/i/WKMSyQVz5sCxx4a1IRo31jKkkrfS+csu8XBp1InAA+7+AOESWZHCdd990LZtWLP6/vvh00+hQ4ekoxKJRTpTQleY2bXA2UA3M6sB1Io3LJEst3IlHHNMKODXXFX3Jb+l06I4HVgLnOfu3wLNgHtijUok2yxeDH36wIgRYfuGG+DFF5UkpCCksxTqt8DTQEMzOw5Y4+5Pxh6ZSDbYsAGeeAL23hueegpmzw77NR4hBSSdq55OAz4FTgVOAz4xs15xByaSuOnT4dBDQ8mNNm1g4sRw2atIgUlnjOJ6oLO7LwQwsx2At4AX4gxMJHHjxsG0afD446Haq1oRUqDSSRRblSaJyBLSG9sQyT2vvRYWFDr77HA77jjYXhVrpLCl84H/hpmNMrPeZtYbeBV4Ld6wRDKsuBh69QrzIgYNChPpzJQkREhvMPsq4C9AO6A9MNjdr4k7MJGMKCkJl7i2bg2vvgp33BHmRmhWtchPUq1HUQTcC+wBTAGudPevMxWYSEaMHw+XXRaWIX34Ydg9sVqXIlkrVYviCeAV4BRCBdmHMhKRSNyWLYNhw8L9/feHTz4JYxNKEiIVSjWYXd/d/xrdn2VmEzIRkEhs3OG550ILYskSmDsXdtkFunRJOjKRrJYqUdQ1s33ZuA7F1mW33V2JQ3LHv/8N/fvDqFHQsSOMHBmShIhUKlWi+Aa4r8z2t2W2HTg8rqBEqtWKFSE5bNgADz4IF10ENWokHZVIzki1cNFhmQxEpNpNngzt2kH9+mHS3AEHaN1qkSrQxDnJP4sWwbnnQvv2YZAa4JRTlCREqijWRGFmPcxslpnNNrOBKY7rbGbrVUNKtsiGDTBkSCjg98wzcN11oVaTiGyRdEp4VEm0bsXDQHegGBhrZiPcfXoFx90FjIorFikQp5wCL78MhxwCjz4aCvmJyBZLp3qsRWtl3xRt72Zm6VxP2AWY7e5z3P1H4FnCKnnlXQK8CCys4DGR1FatCrOrAc48E4YOhXffVZIQqUbpdD09AhwInBltryC0FCrTDJhfZrs42vcTM2sG/Bp4LNULmVlfMxtnZuPSeF8pFCNHhoTwyCNh+7TTwtiEym+IVKt0EsX+7t4fWAPg7kuB2mk8r6L/rV5u+37gGndfn+qF3H2wu3dy905pvK/ku/nz4eST4YQTwhVNHTsmHZFIXktnjGJdNI7g8NN6FBvSeF4xsGuZ7ebAgnLHdAKetfANsAlwjJmVuPvLaby+FKKnnoJ+/cLA9Z//DJdfDrXT+d4iIlWVTqJ4EHgJ2NHM7gB6ATek8byxQJGZtQS+Bs4Azip7gLu3LL1vZkOBV5QkpEKlZb+bNw9XMj30ELRsWenTRGTLVZoo3P1pMxsPHEHoTjrJ3Wek8bwSM7uYcDVTDeAJd59mZv2ix1OOS4gA8P33cO21sO22cO+9IUnokleRjKo0UZjZbsAPwMiy+9z9q8qe6+6vUW6Ro00lCHfvXdnrSQFxD3MhrrgiTKC7/PKNrQoRyah0up5eJYxPGFAXaAnMAtrGGJcUsi+/hL594a23oHNneP112HffpKMSKVjpdD3tU3bbzPYDLogtIpF160KdpocfhgsuUAE/kYRt9sxsd59gZp3jCEYK2Ntvh6VI77sP9toL5s2DunWTjkpESG+M4ooym1sB+wGLYotICst//gMDBsDTT8Mee8D110PjxkoSIlkknQl39cvc6hDGLCoqxSGSvg0b4C9/gVatwqpzN94IU6aEJCEiWSVliyKaaFfP3a/KUDxSKJYtgxtugA4dQgG/Vq2SjkhENmGTLQozqxmV1tgvg/FIPlu5MoxBrF8PjRrBJ5/AO+8oSYhkuVQtik8JSWKimY0AngdWlT7o7sNijk3yyfDhcMkloU5Thw5w+OGw++5JRyUiaUhnjGJ7YAlhjezjgOOjnyKVmzcPTjwRTjoJttsOPvwwJAkRyRmpWhQ7Rlc8TWXjhLtS5avAZozm5eYQd+jVC6ZPh7vvhssug1q1ko5KRDZTqkRRA6hHeuXCRTb6+GNo2zaUAB88GLbfHn7xi6SjEpEqSpUovnH3WzMWieS+774LBfwGD4abboJbblHpDZE8kCpRqJdH0uMe1okYMCAkiwED4CpdUS2SL1IliiMyFoXktuuuC4sIHXAAjB4N7dsnHZGIVKNNJgp3/y6TgUiOWbMmzIto0gT69AljEH37wlbpXEgnIrlE/6tl840eDfvsA7//fdjea6+wPKmShEhe0v9sSd+338JZZ8FRR4UFhC6+OOmIRCQDNrvMuBSof/4Tfv1rWL0a/vhHuOYaVXgVKRBKFJLaunVhkly7dtC9O9xxR+hqEpGCoa4nqdiKFWGd6m7dQhG/xo3h+eeVJEQKkBKF/Dd3GDYMWreGBx4IE+bWrk06KhFJkBKFbLR4MRx/PJxySrjs9aOPwloR22yTdGQikiAlCtmofv2wNOl998G4cWECnYgUPCWKQvfBB9CzZ5g8V6dOWEzo8suhpq5zEJFAiaJQLVkC558fBqunT4c5c8J+TZoTkXL0qVBo3GHoUNh77/DzqqtComjXLunIRCRLqX+hED35ZEgUjz0WSnGIiKSgFkUhWL0abr4ZiotD6Y0XX4T331eSEJG0KFHku1Gj4Je/hFtvheHDw75GjTQWISJp06dFvlqwAE4/HXr0CCU43nkH+vdPOioRyUFKFPnq9ttDC+LWW2HSJDjssKQjEpEcZe6edAybpW7TIl/zzRdJh5Gdxo/fWMBvyRJYuhT23DPpqEQkC5jZeHfvVJXnxtqiMLMeZjbLzGab2cAKHv+NmU2Obh+ZmdbQrIrly+HSS6FLl7AsKYQifkoSIlINYksUZlYDeBjoCbQBzjSzNuUO+xL4lbu3A24DBscVT15yDxVdW7WCQYPgwgvhqaeSjkpE8kyc8yi6ALPdfQ6AmT0LnAhMLz3A3T8qc/zHQPMY48k/f/87/Pa3ocLr8OHQuXPSEYlIHoozUTQD5pfZLgb2T3H874DXK3rAzPoCfQFq71zg3Sk//hjKbbRqBb16hTkSvXurNpOIxCbOMQqrYF+FI+dmdhghUVxT0ePuPtjdO7l7p4petGCMGQMdOoQ1q9esCUX8zj9fSUJEYhVnoigGdi2z3RxYUP4gM2sHDAFOdPclMcaTuxYvhj594Fe/Ci2Ixx7TetUikjFxfhUdCxSZWUvga+AM4KyyB5jZbsAw4Gx3/zzGWHLXnDlh7GH5chg4EG68UQsJiUhGxZYo3L3EzC4GRgE1gCfcfZqZ9Ysefwy4CWgMPGJmACVVvc437yxfDg0aQMuWoTXRu3coxSEikmGacJdtfvgBbrsNBg8OM6qb60IwEdlyWzLhTqOg2eTVV+Hii2Hu3NCK2HrrpCMSEVGiyAolJXDmmfDCC9C6Nbz3HhxySNJRiYgAKgqYrNJuv5o1Yaed4M47YeJEJQkRySpKFEkZOxb23x8mTAjbgwbBtddC7drJxiUiUo4SRaYtWxbGIfbfP6w4t0RTR0QkuylRZFJpAb9HHw3JYuZM6N496ahERFLKvcHsXK7hMWMGNGsGI0dCJ00XEZHckHvzKHYp8jULcmQexdq1cM890L49HH88rFsX1qquUSPpyESkwGTtwkUF7Z//DAnixhvh7bfDvlq1lCREJOcoUVS3hQvh3HPh8MNDC+L11+H++5OOSkSkypQoqtubb8Izz8D118PUqdCjR9IRiYhskdwbzM5GU6bArFlhIaHf/Aa6doXdd086KhGRaqEWxZZYtQquvjosRXr11aGryUxJQkTyihJFVY0cCW3ahKuaevcOM61r1Uo6KhGRapdzXU+WDRMppk6FE06Atm3h/ffh4IOTjkhEJDZqUaSrpATefTfc/+Uv4ZVX4LPPlCREJO8pUaTjk0/CTOojjoAvosl+xx6rriYRKQhKFKksXQoXXggHHgiLF4daTXvumXRUIiIZlXNjFBmzdm24mmn+fLjsMrjlFqhfP+moREQyTomivK+/DoX76tSBP/4xlOHYd9+koxIRSYy6nkqtWRNaDbvvDsOHh329eytJiEjBU4sCQtG+Cy8MA9VnnhkWFRIREUAtijD+cOSRYf3qN9+Ev/8ddt456ahERLJGYSaKDRtg/fpwv0sXuOmmUK9Jq82JiPxM4SWKSZNC0b6HHw7bZ50Vxibq1k02LhGRLFU4iWLlShgwADp2hDlz1L0kIpKmwhjMfust6NMHiouhb1/485+hUaOkoxIRyQmFkShq14btt4d//CN0O4mISNryM1GsWxeWH122DG6/HQ45JBTw26pwetpERKpL/n1yfvRRGIe4+mqYMSNc4QRKEiIiVZQ/n57ffRfGHw46CL7/Hl5+GV58UQlCRGQL5c+n6JIlYbLclVfC9Olw4olJRyQikhdye4xi1qwwQH3TTVBUBPPmQePGSUclIpJXYm1RmFkPM5tlZrPNbGAFj5uZPRg9PtnM9kvrhVevDsmhXTv4n/8JpcBBSUJEJAaxJQozqwE8DPQE2gBnmlmbcof1BIqiW1/g0cpet97aVbDPPnDbbXDqqTBzJuy6azVHLyIipeJsUXQBZrv7HHf/EXgWKD9wcCLwpAcfA9uZWdNUL9rs+/+EAeq33oKnnoKddoonehERAeIdo2gGzC+zXQyUr99d0THNgG/KHmRmfQktDoC19sUXUznyyOqNNjc1ARYnHUSW0LnYSOdiI52Ljfau6hPjTBRWwT6vwjG4+2BgMICZjXP3TlseXu7TudhI52IjnYuNdC42MrNxVX1unF1PxUDZwYPmwIIqHCMiIgmKM1GMBYrMrKWZ1QbOAEaUO2YEcE509dMBwDJ3/6b8C4mISHJi63py9xIzuxgYBdQAnnD3aWbWL3r8MeA14BhgNvAD0CeNlx4cU8i5SOdiI52LjXQuNtK52KjK58LcfzYkICIi8pP8KeEhIiKxUKIQEZGUsjZRxFb+IwelcS5+E52DyWb2kZm1TyLOTKjsXJQ5rrOZrTezXpmML5PSORdmdqiZTTSzaWb2XqZjzJQ0/o80NLORZjYpOhfpjIfmHDN7wswWmtnUTTxetc9Nd8+6G2Hw+9/A7kBtYBLQptwxxwCvE+ZiHAB8knTcCZ6LrkCj6H7PQj4XZY57h3CxRK+k407w72I7YDqwW7S9Y9JxJ3gurgPuiu7vAHwH1E469hjOxSHAfsDUTTxepc/NbG1RxFL+I0dVei7c/SN3XxptfkyYj5KP0vm7ALgEeBFYmMngMiydc3EWMMzdvwJw93w9H+mcCwfqm5kB9QiJoiSzYcbP3ccQfrdNqdLnZrYmik2V9tjcY/LB5v6evyN8Y8hHlZ4LM2sG/Bp4LINxJSGdv4u9gEZm9q6ZjTezczIWXWalcy4GAa0JE3qnAH9w9w2ZCS+rVOlzM1vXo6i28h95IO3f08wOIySKg2ONKDnpnIv7gWvcfX348pi30jkXNYGOwBHA1sC/zOxjd/887uAyLJ1zcTQwETgc2AMYbWbvu/vymGPLNlX63MzWRKHyHxul9XuaWTtgCNDT3ZdkKLZMS+dcdAKejZJEE+AYMytx95czEmHmpPt/ZLG7rwJWmdkYoD2Qb4kinXPRB/izh4762Wb2JdAK+DQzIWaNKn1uZmvXk8p/bFTpuTCz3YBhwNl5+G2xrErPhbu3dPcW7t4CeAG4KA+TBKT3f2Q40M3MaprZNoTqzTMyHGcmpHMuviK0rDCznQiVVOdkNMrsUKXPzaxsUXh85T9yTprn4iagMfBI9E26xPOwYmaa56IgpHMu3H2Gmb0BTAY2AEPcvcLLJnNZmn8XtwFDzWwKofvlGnfPu/LjZvYMcCjQxMyKgZuBWrBln5sq4SEiIilla9eTiIhkCSUKERFJSYlCRERSUqIQEZGUlChERCQlJQrJSlHl14llbi1SHLuyGt5vqJl9Gb3XBDM7sAqvMcTM2kT3ryv32EdbGmP0OqXnZWpUDXW7So7vYGbHVMd7S+HS5bGSlcxspbvXq+5jU7zGUOAVd3/BzI4C7nX3dlvwelscU2Wva2b/C3zu7nekOL430MndL67uWKRwqEUhOcHM6pnZ29G3/Slm9rOqsWbW1MzGlPnG3S3af5SZ/St67vNmVtkH+Bhgz+i5V0SvNdXMLov2bWtmr0ZrG0w1s9Oj/e+aWScz+zOwdRTH09FjK6Of/yj7DT9qyZxiZjXM7B4zG2thnYAL0jgt/yIq6GZmXSysRfJZ9HPvaJbyrcDpUSynR7E/Eb3PZxWdR5GfSbp+um66VXQD1hOKuE0EXiJUEWgQPdaEMLO0tEW8Mvo5ALg+ul8DqB8dOwbYNtp/DXBTBe83lGjtCuBU4BNCQb0pwLaE0tTTgH2BU4C/lnluw+jnu4Rv7z/FVOaY0hh/DfxvdL82oZLn1kBf4IZofx1gHNCygjhXlvn9ngd6RNsNgJrR/SOBF6P7vYFBZZ5/J/Db6P52hLpP2yb9761bdt+ysoSHCLDa3TuUbphZLeBOMzuEUI6iGbAT8G2Z54wFnoiOfdndJ5rZr4A2wIdReZPahG/iFbnHzG4AFhGq8B4BvOShqB5mNgzoBrwB3GtmdxG6q97fjN/rdeBBM6sD9ADGuPvqqLurnW1cka8hUAR8We75W5vZRKAFMB4YXeb4/zWzIkI10FqbeP+jgBPM7Mpouy6wG/lZA0qqiRKF5IrfEFYm6+ju68xsLuFD7ifuPiZKJMcC/2dm9wBLgdHufmYa73GVu79QumFmR1Z0kLt/bmYdCTVz/mRmb7r7ren8Eu6+xszeJZS9Ph14pvTtgEvcfVQlL7Ha3TuYWUPgFaA/8CChltE/3f3X0cD/u5t4vgGnuPusdOIVAY1RSO5oCCyMksRhwC/KH2Bmv4iO+SvwOGFJyI+Bg8ysdMxhGzPbK833HAOcFD1nW0K30ftmtgvwg7s/BdwbvU9566KWTUWeJRRj60YoZEf088LS55jZXtF7VsjdlwGXAldGz2kIfB093LvMoSsIXXClRgGXWNS8MrN9N/UeIqWUKCRXPA10MrNxhNbFzAqOORSYaGafEcYRHnD3RYQPzmfMbDIhcbRK5w3dfQJh7OJTwpjFEHf/DNgH+DTqAroeuL2Cpw8GJpcOZpfzJmFt47c8LN0JYS2R6cAEM5sK/IVKWvxRLJMIZbXvJrRuPiSMX5T6J9CmdDCb0PKoFcU2NdoWSUmXx4qISEpqUYiISEpKFCIikpIShYiIpKREISIiKSlRiIhISkoUIiKSkhKFiIik9P+INZUw6qKVJgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y = train_y\n",
    "X = x_train_count.astype(\"float64\")\n",
    "\n",
    "rfc_roc_auc = roc_auc_score(y, rfc_model.predict(X))\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y, rfc_model.predict_proba(X)[:,1])\n",
    "plt.figure()\n",
    "plt.plot(fpr, tpr, label='AUC (area = %0.2f)' % rfc_roc_auc)\n",
    "plt.plot([0, 1], [0, 1],'r--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e1e1550",
   "metadata": {},
   "source": [
    "### RandomForestClassifier model with \"tf-idf\" method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "b9103389",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "RandomForestClassifier model with 'tf-idf' method\n",
      "Accuracy ratio:  0.9466904761904761\n"
     ]
    }
   ],
   "source": [
    "rfc = RandomForestClassifier()\n",
    "rfc_model = rfc.fit(x_train_tf_idf_word,train_y)\n",
    "accuracy = model_selection.cross_val_score(rfc_model, \n",
    "                                           x_test_tf_idf_word, \n",
    "                                           test_y, \n",
    "                                           cv = 20).mean()\n",
    "\n",
    "print(\"\\nRandomForestClassifier model with 'tf-idf' method\")\n",
    "print(\"Accuracy ratio: \", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "8e37f636",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred=rfc_model.predict(x_test_tf_idf_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "136dacfb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[7370,   22],\n",
       "       [ 320,  279]], dtype=int64)"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(test_y,pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "9b977e33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      7392\n",
      "           1       0.93      0.47      0.62       599\n",
      "\n",
      "    accuracy                           0.96      7991\n",
      "   macro avg       0.94      0.73      0.80      7991\n",
      "weighted avg       0.96      0.96      0.95      7991\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(test_y,pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "53db3d5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAnTElEQVR4nO3dd5wV5fXH8c+RrhQRrKCCClIioBQVRUVFsWIUa6KCMYiiRsWCPdbE8jIWLCFoiD+NxoKCFVFjsESlSC9KEGRFQxFpArJwfn88s7JZl7uXZefOLd/363Vfe2fuzNyzw3LPfeaZ5zzm7oiIiGzKVkkHICIi2U2JQkREUlKiEBGRlJQoREQkJSUKERFJSYlCRERSUqIQEZGUlChEUjCzuWa22sxWmtm3ZjbMzOqWer2rmb1rZivMbJmZvWJmbcoco76Z3W9mX0XHmR0tN878bySy+ZQoRCp2grvXBToA+wLXApjZgcBbwAhgF6A5MAn40Mz2iLapCbwDtAV6AvWBrsASoEtGfwuRSjKNzBbZNDObC5zv7m9Hy3cDbd39ODN7H5ji7heV2ecNYJG7n2Nm5wN3AHu6+8oMhy9SJdSiEEmTmTUFjgFmm9nWhJbB8+Vs+hzQI3p+JPCmkoTkMiUKkYq9bGYrgPnAQuBmYDvC/59vytn+G6Ck/6HRJrYRyRlKFCIVO8nd6wGHAa0ISWApsAHYuZztdwYWR8+XbGIbkZyhRCGSJnf/FzAMuNfdVwH/Bk4tZ9PTCB3YAG8DR5vZNhkJUiQGShQim+d+oIeZdQAGAeea2aVmVs/MGprZ7cCBwC3R9v9HuGT1opm1MrOtzKyRmV1nZscm8QuIbC4lCpHN4O6LgCeBG939A+Bo4GRCP8Q8wu2zB7v7F9H2awkd2jOB0cBy4FPC5atPMv4LiFSCbo8VEZGU1KIQEZGUlChERCQlJQoREUlJiUJERFKqnnQAm6tx48berFmzpMMQEckp48ePX+zu21dm35xLFM2aNWPcuHFJhyEiklPMbF5l99WlJxERSUmJQkREUlKiEBGRlJQoREQkJSUKERFJSYlCRERSii1RmNkTZrbQzKZu4nUzswfNbLaZTTaz/eKKRUREKi/OcRTDgMGEkszlOQZoET32Bx6NfqaleP0Gijeo8q2ISNxiSxTuPsbMmqXYpBfwpIc65x+b2bZmtrO7p5xfePbClZw4+ANmfruCH4s3VGXIIiJ5Z/+vpvCbcSO26BhJjsxuQpj5q0RRtO5nicLM+gH9AGrutBcNt65J9723p9VO9aldo1pGghURySW1v1/CgY/dRatRw1m+U9MtOlaSicLKWVfutSR3HwIMAaizS0v/23ld4oxLRCT3nXI1vDMSrr2W+jfcANtUftr2JO96KgJ2LbXcFFiQUCwiIrlv2jT4+uvw/K67YOJEuPNO2HrrLTpskoliJHBOdPfTAcCyivonRESkHKtWwaBB0KEDXH99WLfXXtC2bZUcPrZLT2b2DHAY0NjMioCbgRoA7v4Y8DpwLDAb+AHoG1csIiJ567XXYMAAmDcPzjsvtCSqWJx3PZ1ZwesODIjr/UVE8t4jj4Qk0aYNjBkD3brF8jY5Nx+FiEhBKy6GRYtg553htNNg9Wq45BKoWTO2t1QJDxGRXPHpp9C5M5x4IqxfD40bw8CBsSYJUKIQEcl+338PF10EBxwACxfCNdfAVpn7+NalJxGRbDZlCvToES43XXop3Hor1K+f0RCUKEREstG6dVCjBrRsCd27w1VXwX7J1E7VpScRkWyydm1oNbRtCytXQq1a8MwziSUJUKIQEcke774L7drBzTeHxLB2bdIRAUoUIiLJW70azj4bjjgi3P765pvw7LPQqFHSkQFKFCIiyatdGxYvhhtugKlT4eijk47ofyhRiIgkYfLkkBCKisAslOK47TaoUyfpyH5GiUJEJJNWrdp4B9OECfDFF2F9BsdFbK7sjUxEJN+MHBnqMt17byjgN2tWuPU1y2kchYhIprz8chgs98EHcNBBSUeTNiUKEZG4rFsHDz4YWg377QcPPBA6rmvUSDqyzaJLTyIicfj4Y+jUCa68Ep57LqyrVy/nkgQoUYiIVK2lS6F/f+jaFb77Dl56Cf7wh6Sj2iJKFCIiVWnIEBg6FC6/HKZPh5NOCre/5jALE83ljjq7tPTVCz5POgwRkY1mzQrVXQ8+OJTdmDUrlOLIImY23t07VWZftShERCprzZpQl6lduzAlqXso4pdlSWJLKVGIiFTG6NGwzz6h0mvv3jBqVM5fYtoU3R4rIrK5xoyBo46CFi1CwjjyyKQjipVaFCIi6Vi/Psw2B9CtGzz+eKjXlOdJApQoREQq9tln4XbXgw6C//43XGI677wweK4AKFGIiGzKihVwxRVh4NzcufDoo7DDDklHlXHqoxARKc+yZaGzev58uOCCMGiuYcOko0qEEoWISGnLl4fCfQ0aQL9+Yda5Aw9MOqpE6dKTiAiEAn533w1Nm4Z5IiDMOFfgSQLUohARgQ8/DPWZpk4NJTe23z7piLKKWhQiUtguuSSU3li2DEaMCEX8dt016aiyihKFiBSe0jXudtoplAKfPh1OPDG5mLKYEoWIFJaZM8NEQiNGhOXrr4d77oG6dZONK4spUYhIYVi9Gm68MRTsmzQpLEtaYk0UZtbTzGaZ2WwzG1TO6w3M7BUzm2Rm08ysb5zxiEiBeuedMCbi9tvhjDNCGfAzzkg6qpwR211PZlYNeBjoARQBY81spLtPL7XZAGC6u59gZtsDs8zsaXf/Ma64RKQAFRVB9eohYRx+eNLR5Jw4WxRdgNnuPif64H8W6FVmGwfqmZkBdYHvgOIYYxKRQrB+PTz8MPzlL2H5nHPC5SYliUqJM1E0AeaXWi6K1pU2GGgNLACmAL9z9w1lD2Rm/cxsnJmNy7UZ+UQkwyZMgAMOgIsvDnNEQCjiV6tWsnHlsDgTRXkzeJT9lD8amAjsAnQABptZ/Z/t5D7E3Tu5eyfL04lBRGQLLV8Ov/sddO4c6jM98ww8/3zSUeWFOBNFEVB61EpTQsuhtL7AcA9mA18CrWKMSUTy1aRJMHhwGGE9c2borNYXyyoRZ6IYC7Qws+ZmVhM4AxhZZpuvgCMAzGxHYG9gTowxiUg++fJLeOKJ8LxbN5g9O/RNbLttomHlm9gShbsXAxcDo4AZwHPuPs3M+ptZ/2iz24CuZjYFeAe4xt0XxxWTiOSJH38MZb/btIGBA2Hp0rC+efNk48pTlmudw3V2aemrF3yedBgikpT33w+Xl6ZPh5NPhgceCBVfJSUzG+/unSqzr6rHikjuWLQIjjoKdtwRXnkFjj8+6YgKgkp4iEh2c4fRo8Pz7beHV1+FadOUJDJIiUJEste0aXDooaEV8d57Yd0RR8A22yQaVqFRohCR7PPDD3DdddChQ0gWQ4fCIYckHVXBUh+FiGQX91AG/NNP4dxzQwlwzTiXKCUKEckO33wDO+wA1aqF1kSDBnDYYUlHJejSk4gkbf16ePBB2HtveOSRsK5XLyWJLKJEISLJGTcOunQJNZq6doVjj006IilH2onCzHSbgYhUnbvvDknim2/gH/+AN96APfdMOiopR4WJwsy6mtl0QhkOzKy9mT0Se2Qikn/cYd268LxLFxgwAGbMgNNOUwG/LJZOi+JPhHLgSwDcfRKg+9REZPP85z/QsycMimZFPuwweOih0GktWS2tS0/uPr/MqvUxxCIi+Wjt2jBX9S9+Af/+ty4v5aB0bo+db2ZdAY/KhV9KdBlKRCSl8ePh178O80Oceircfz/sskvSUclmSidR9AceIExjWgS8BVwUZ1Aikifq1g19D6+/Dscck3Q0UknpJIq93f1XpVeY2UHAh/GEJCI5a8MG+OtfwyWmoUPD2IipU2Er3Ymfy9L513sozXUiUsimTg31mM4/H774AlatCuuVJHLeJlsUZnYg0BXY3syuKPVSfaBa3IGJSI5YtQpuvRXuuy/cwfTXv4YaTbrdNW+kuvRUE6gbbVOv1PrlQO84gxKRHLJmTUgO55wTBtE1apR0RFLFKpwK1cx2d/d5GYqnQpoKVSQLFBWF+kx/+EMo4vfdd7DddklHJSnEPRXqD2Z2D9AWqF2y0t0Pr8wbikgOKy4Og+RuuikU8zv9dOjYUUkiz6XTy/Q0MBNoDtwCzAXGxhiTiGSjTz6BTp3giitCp/W0aSFJSN5LJ1E0cvfHgXXu/i93Pw84IOa4RCSbbNgAffvCokXwwgth3urmzZOOSjIknUtPUQUvvjGz44AFQNP4QhKRrOAekkLPnlCvHgwfDk2ahOdSUNJpUdxuZg2AgcCVwFDgsjiDEpGEffEFHH10qOo6ZEhY16qVkkSBqrBF4e6vRk+XAd3hp5HZIpJv1q6Fu+6CO++EWrVg8GDo3z/pqCRhqQbcVQNOI9R4etPdp5rZ8cB1QB1g38yEKCIZM2AAPP44nHFGGEC3885JRyRZYJPjKMxsGLAr8CmwPzAPOBAY5O4vZyi+n9E4CpEqtnBh6KzeaadwyWnOnHDZSfJKXOMoOgHt3H2DmdUGFgN7ufu3lXkjEckyGzaEwn3XXANHHRWmI23RIjxESknVmf2ju28AcPc1wOdKEiJ5YvJkOPhguOAC6NABbrkl6Ygki6VqUbQys8nRcwP2jJYNcHdvF3t0IlL1Xngh9EE0bAhPPhkmFlIBP0khVaJonbEoRCR+y5dD/fphruoBA+Dmm1V6Q9JSYVHAbKPObJHN9NVXcMklsGABfPxxKOInBWdLOrNjnVHEzHqa2Swzm21mgzaxzWFmNtHMppnZv+KMR6SgrFsH994LrVvD22+HwXM59sVQskM6JTwqJRqH8TDQgzDX9lgzG+nu00ttsy3wCNDT3b8ysx3iikekoMybByeeGDqtTzghVHzdffeko5IclVaLwszqmNnem3nsLsBsd5/j7j8CzwK9ymxzFjDc3b8CcPeFm/keIlJaSYthp51gxx3hpZdgxAglCdkiFSYKMzsBmAi8GS13MLORaRy7CTC/1HJRtK60lkBDM3vPzMab2TlpRS0i/8sdnnoKOneGlStD+Y233oKTTtIdTbLF0mlR/J7QOvgewN0nAs3S2K+8v86yF0irAx2B44CjgRvNrOXPDmTWz8zGmdm4XOt8F4ndrFlwxBFw9tlQvTosWZJ0RJJn0kkUxe6+rBLHLiKUACnRlFCivOw2b7r7KndfDIwB2pc9kLsPcfdO7t7J9O1IJCguDre4tmsHEybAo4/CRx/pMpNUuXQSxVQzOwuoZmYtzOwh4KM09hsLtDCz5mZWEzgDKHvJagTQzcyqm9nWhJpSMzYjfpHCVa0avP8+9O4dWhX9+8NWsd7IKAUqnb+qSwjzZa8F/k4oN35ZRTu5ezFwMTCK8OH/nLtPM7P+ZtY/2mYGoe9jMqH44FB3n1qJ30OkMHz7LZx3HsyfH/oeXn8dnn46dFyLxKTCAXdmtq+7f5aheCqkAXdSkNavDxMIXXstrF4dOq5PPTXpqCSHxD3g7j4zm2lmt5lZ28q8iYhsgc8+g65d4aKLoFMnmDJFSUIyqsJE4e7dgcOARcAQM5tiZjfEHZiIRAYPhrlzwyWm0aOh5c9uDBSJ1WbVejKzfYCrgdPdvWZsUaWgS0+S99zh5ZehWTPYd19YujSsb9gwyagkx8V66cnMWpvZ781sKjCYcMdT08q8mYhUYO7cUHrj5JPh/vvDuoYNlSQkUenUevor8AxwlLuXHQchIlVh3bowR/Utt4RbXO+9F373u6SjEgHSSBTufkAmAhEpaH/+MwwaFEpuPPAA7LZb0hGJ/GSTicLMnnP308xsCv9bekMz3IlUhSVLwqWmjh3ht7+FvfaCnj2TjkrkZ1K1KEravcdnIhCRguEepiC98kqoVw8+/zwU8VOSkCy1yc5sd/8menqRu88r/QAuykx4Inlmxgzo3h369IEWLcLdTdVjmxZGpEqkM+CuRznrjqnqQETy3qRJ0L59mExoyBD44INQ0E8ky6Xqo7iQ0HLYw8wml3qpHvBh3IGJ5I2iImjaNCSFW26B3/wGdtBkjpI7NjngzswaAA2BPwCl57te4e7fZSC2cmnAneSMBQvg8stD4b6ZM6FJ2Xm7RDInrgF37u5zgQHAilIPzGy7yryZSEFYvz6U3WjdOkxDevXV0Lhx0lGJVFqqXrS/E+54Gk+4Pbb0jEEO7BFjXCK5ac0aOOQQGDsWevSARx4Jt72K5LBNJgp3Pz762Txz4YjkqHXroEYNqF073NV0xRVw+umar1ryQjq1ng4ys22i5782s/vMTMNGRSCMiXjhhdBqmDAhrLvrLjjjDCUJyRvp3B77KPCDmbUnVI6dB/xfrFGJ5II5c+C448LcEI0aaRpSyVvp/GUXe7g1qhfwgLs/QLhFVqRw3XcftG0b5qy+/3749FPo0CHpqERikc6Q0BVmdi1wNtDNzKoBNeINSyTLrVwJxx4bCvg1VdV9yW/ptChOB9YC57n7t0AT4J5YoxLJNosXQ9++MHJkWL7hBnjxRSUJKQjpTIX6LfA00MDMjgfWuPuTsUcmkg02bIAnnoC994annoLZs8N69UdIAUnnrqfTgE+BU4HTgE/MrHfcgYkkbvp0OOywUHKjTRuYODHc9ipSYNLpo7ge6OzuCwHMbHvgbeCFOAMTSdy4cTBtGjz+eKj2qlaEFKh0EsVWJUkisoT0+jZEcs/rr4cJhc4+OzyOPx62U8UaKWzpfOC/aWajzKyPmfUBXgNejzcskQwrKoLevcO4iMGDw0A6MyUJEdLrzL4K+DPQDmgPDHH3a+IOTCQjiovDLa6tW8Nrr8Edd4SxERpVLfKTVPNRtADuBfYEpgBXuvvXmQpMJCPGj4fLLgvTkD78MOyhWpciZaVqUTwBvAqcQqgg+1BGIhKJ27JlMHx4eL7//vDJJ6FvQklCpFypOrPruftfouezzGxCJgISiY07PPdcaEEsWQJz58Iuu0CXLklHJpLVUiWK2ma2LxvnoahTetndlTgkd/znPzBgAIwaBR07wiuvhCQhIhVKlSi+Ae4rtfxtqWUHDo8rKJEqtWJFSA4bNsCDD8JFF0G1aklHJZIzUk1c1D2TgYhUucmToV07qFcvDJo74ADNWy1SCRo4J/ln0SI491xo3z50UgOccoqShEglxZoozKynmc0ys9lmNijFdp3NbL1qSMkW2bABhg4NBfyeeQauuy7UahKRLZJOCY9KieateBjoARQBY81spLtPL2e7u4BRccUiBeKUU+Dll+GQQ+DRR0MhPxHZYulUj7VoruybouXdzCyd+wm7ALPdfY67/wg8S5glr6xLgBeBheW8JpLaqlVhdDXAmWfCsGHw3ntKEiJVKJ1LT48ABwJnRssrCC2FijQB5pdaLorW/cTMmgC/BB5LdSAz62dm48xsXJiVVYRwi2ubNvDII2H5tNNC34TKb4hUqXQSxf7uPgBYA+DuS4GaaexX3v/Wsp/y9wPXuPv6VAdy9yHu3sndO5k+BGT+fDj5ZDjxxHBHU8eOSUckktfS6aNYF/UjOPw0H8WGNPYrAnYttdwUWFBmm07As9GHf2PgWDMrdveX0zi+FKKnnoL+/UPH9R//CJdfDjXT+d4iIpWVTqJ4EHgJ2MHM7gB6Azeksd9YoIWZNQe+Bs4Aziq9gbs3L3luZsOAV5UkpFwlZb+bNg13Mj30EDRvXuFuIrLlKkwU7v60mY0HjiBcTjrJ3WeksV+xmV1MuJupGvCEu08zs/7R6yn7JUQA+P57uPZa2GYbuPfekCR0y6tIRlWYKMxsN+AH4JXS69z9q4r2dffXKTPJ0aYShLv3qeh4UkDcw1iIK64IA+guv3xjq0JEMiqdS0+vEfonDKgNNAdmAW1jjEsK2ZdfQr9+8Pbb0LkzvPEG7Ltv0lGJFKx0Lj3tU3rZzPYDLogtIpF160KdpocfhgsuUAE/kYRt9shsd59gZp3jCEYK2DvvhKlI77sPWraEefOgdu2koxIR0uujuKLU4lbAfsCi2CKSwvLf/8LAgfD007DnnnD99dCokZKESBZJZ8BdvVKPWoQ+i/JKcYikb8MG+POfoVWrMOvcjTfClCkhSYhIVknZoogG2tV196syFI8UimXL4IYboEOHUMCvVaukIxKRTdhki8LMqkelNfbLYDySz1auDH0Q69dDw4bwySfw7rtKEiJZLlWL4lNCkphoZiOB54FVJS+6+/CYY5N8MmIEXHJJqNPUoQMcfjjssUfSUYlIGtLpo9gOWEKYI/t44ITop0jF5s2DXr3gpJNg223hww9DkhCRnJGqRbFDdMfTVDYOuCuhWt9SMXfo3RumT4e774bLLoMaNZKOSkQ2U6pEUQ2oS3rlwkU2+vhjaNs2lAAfMgS22w523z3pqESkklIlim/c/daMRSK577vvQgG/IUPgppvglltUekMkD6RKFKq+JulxD/NEDBwYksXAgXCV7qgWyRepEsURGYtCctt114VJhA44AEaPhvbtk45IRKrQJhOFu3+XyUAkx6xZE8ZFNG4MffuGPoh+/WCrdG6kE5Fcov/VsvlGj4Z99oHf/jYst2wZpidVkhDJS/qfLen79ls46yw46qgwgdDFFycdkYhkwGaXGZcC9c9/wi9/CatXw+9/D9dcowqvIgVCiUJSW7cuDJJr1w569IA77giXmkSkYOjSk5RvxYowT3W3bqGIX6NG8PzzShIiBUiJQv6XOwwfDq1bwwMPhAFza9cmHZWIJEiJQjZavBhOOAFOOSXc9vrRR2GuiK23TjoyEUmQEoVsVK9emJr0vvtg3LgwgE5ECp4SRaH74AM45pgweK5WrTCZ0OWXQ3Xd5yAigRJFoVqyBM4/P3RWT58Oc+aE9Ro0JyJl6FOh0LjDsGGw997h51VXhUTRrl3SkYlIltL1hUL05JMhUTz2WCjFISKSgloUhWD1arj5ZigqCqU3XnwR3n9fSUJE0qJEke9GjYJf/AJuvRVGjAjrGjZUX4SIpE2fFvlqwQI4/XTo2TOU4Hj3XRgwIOmoRCQHKVHkq9tvDy2IW2+FSZOge/ekIxKRHGXunnQMm6XOLi199YLPkw4jO40fv7GA35IlsHQp7LVX0lGJSBYws/Hu3qky+8baojCznmY2y8xmm9mgcl7/lZlNjh4fmZnm0KyM5cvh0kuhS5cwLSmEIn5KEiJSBWJLFGZWDXgYOAZoA5xpZm3KbPYlcKi7twNuA4bEFU9ecg8VXVu1gsGD4cIL4amnko5KRPJMnOMougCz3X0OgJk9C/QCppds4O4fldr+Y6BpjPHkn7//HX7961DhdcQI6Nw56YhEJA/FmSiaAPNLLRcB+6fY/jfAG+W9YGb9gH4AtXYq8MspP/4Yym20agW9e4cxEn36qDaTiMQmzj4KK2dduT3nZtadkCiuKe91dx/i7p3cvZNZeYctEGPGQIcOYc7qNWtCEb/zz1eSEJFYxZkoioBdSy03BRaU3cjM2gFDgV7uviTGeHLX4sXQty8cemhoQTz2mOarFpGMifOr6FighZk1B74GzgDOKr2Bme0GDAfOdnfd81qeOXNC38Py5TBoENx4oyYSEpGMii1RuHuxmV0MjAKqAU+4+zQz6x+9/hhwE9AIeCS6pFRc2ft8887y5VC/PjRvHloTffqEUhwiIhmmAXfZ5ocf4LbbYMiQMKK6qW4EE5EttyUD7tQLmk1eew0uvhjmzg2tiDp1ko5IRESJIisUF8OZZ8ILL0Dr1vCvf8EhhyQdlYgIoKKAySq57Fe9Ouy4I9x5J0ycqCQhIllFiSIpY8fC/vvDhAlhefBguPZaqFkz2bhERMpQosi0ZctCP8T++4cZ55Zo6IiIZDclikwqKeD36KMhWcycCT16JB2ViEhK6szOpBkzoEkTeOUV6KThIiKSGzSOIk5r18I990D79nDCCbBuXZirulq1pCMTkQKTtRMXFbR//jMkiBtvhHfeCetq1FCSEJGco0RR1RYuhHPPhcMPDy2IN96A++9POioRkUpToqhqb70FzzwD118PU6dCz55JRyQiskXUmV0VpkyBWbPCREK/+hV07Qp77JF0VCIiVUItii2xahVcfXWYivTqq8OlJjMlCRHJK0oUlfXKK9CmTbirqU+fMNK6Ro2koxIRqXK69FQZU6fCiSdC27bw/vtw8MFJRyQiEhu1KNJVXAzvvRee/+IX8Oqr8NlnShIikveUKNLxySdhJPURR8AXX4R1xx2nS00iUhCUKFJZuhQuvBAOPBAWLw61mvbaK+moREQySn0Um7J2bbibaf58uOwyuOUWqFcv6ahERDJOiaKsr78Ohftq1YLf/z6U4dh336SjEhFJjC49lVizJrQa9tgDRowI6/r0UZIQkYKnFgWEon0XXhg6qs88M0wqJCIigFoUof/hyCPD/NVvvQV//zvstFPSUYmIZI3CTBQbNsD69eF5ly5w002hXpNmmxMR+ZnCSxSTJoWifQ8/HJbPOiv0TdSunWxcIiJZqnASxcqVMHAgdOwIc+bo8pKISJoKozP77behb18oKoJ+/eCPf4SGDZOOSkQkJxRGoqhZE7bbDv7xj3DZSURE0pafiWLdujD96LJlcPvtcMghoYDfVoVzpU1EpKrk3yfnRx+Ffoirr4YZM8IdTqAkISJSSfnz6fndd6H/4aCD4Pvv4eWX4cUXlSBERLZQ/nyKLlkSBstdeSVMnw69eiUdkYhIXsjtPopZs0IH9U03QYsWMG8eNGqUdFQiInkl1haFmfU0s1lmNtvMBpXzupnZg9Hrk81sv7QOvHp1SA7t2sGf/hRKgYOShIhIDGJLFGZWDXgYOAZoA5xpZm3KbHYM0CJ69AMerei4ddeugn32gdtug1NPhZkzYdddqzh6EREpEWeLogsw293nuPuPwLNA2Y6DXsCTHnwMbGtmO6c6aJPv/xs6qN9+G556CnbcMZ7oRUQEiLePogkwv9RyEVC2fnd52zQBvim9kZn1I7Q4ANbaF19M5cgjqzba3NQYWJx0EFlC52IjnYuNdC422ruyO8aZKKycdV6JbXD3IcAQADMb5+6dtjy83KdzsZHOxUY6FxvpXGxkZuMqu2+cl56KgNKdB02BBZXYRkREEhRnohgLtDCz5mZWEzgDGFlmm5HAOdHdTwcAy9z9m7IHEhGR5MR26cndi83sYmAUUA14wt2nmVn/6PXHgNeBY4HZwA9A3zQOPSSmkHORzsVGOhcb6VxspHOxUaXPhbn/rEtARETkJ/lTwkNERGKhRCEiIillbaKIrfxHDkrjXPwqOgeTzewjM2ufRJyZUNG5KLVdZzNbb2a9MxlfJqVzLszsMDObaGbTzOxfmY4xU9L4P9LAzF4xs0nRuUinPzTnmNkTZrbQzKZu4vXKfW66e9Y9CJ3f/wH2AGoCk4A2ZbY5FniDMBbjAOCTpONO8Fx0BRpGz48p5HNRart3CTdL9E467gT/LrYFpgO7Rcs7JB13gufiOuCu6Pn2wHdAzaRjj+FcHALsB0zdxOuV+tzM1hZFLOU/clSF58LdP3L3pdHix4TxKPkonb8LgEuAF4GFmQwuw9I5F2cBw939KwB3z9fzkc65cKCemRlQl5AoijMbZvzcfQzhd9uUSn1uZmui2FRpj83dJh9s7u/5G8I3hnxU4bkwsybAL4HHMhhXEtL5u2gJNDSz98xsvJmdk7HoMiudczEYaE0Y0DsF+J27b8hMeFmlUp+b2TofRZWV/8gDaf+eZtadkCgOjjWi5KRzLu4HrnH39eHLY95K51xUBzoCRwB1gH+b2cfu/nncwWVYOufiaGAicDiwJzDazN539+Uxx5ZtKvW5ma2JQuU/Nkrr9zSzdsBQ4Bh3X5Kh2DItnXPRCXg2ShKNgWPNrNjdX85IhJmT7v+Rxe6+ClhlZmOA9kC+JYp0zkVf4I8eLtTPNrMvgVbAp5kJMWtU6nMzWy89qfzHRhWeCzPbDRgOnJ2H3xZLq/BcuHtzd2/m7s2AF4CL8jBJQHr/R0YA3cysupltTajePCPDcWZCOufiK0LLCjPbkVBJdU5Go8wOlfrczMoWhcdX/iPnpHkubgIaAY9E36SLPQ8rZqZ5LgpCOufC3WeY2ZvAZGADMNTdy71tMpel+XdxGzDMzKYQLr9c4+55V37czJ4BDgMam1kRcDNQA7bsc1MlPEREJKVsvfQkIiJZQolCRERSUqIQEZGUlChERCQlJQoREUlJiUKyUlT5dWKpR7MU266sgvcbZmZfRu81wcwOrMQxhppZm+j5dWVe+2hLY4yOU3JepkbVULetYPsOZnZsVby3FC7dHitZycxWunvdqt42xTGGAa+6+wtmdhRwr7u324LjbXFMFR3XzP4GfO7ud6TYvg/Qyd0vrupYpHCoRSE5wczqmtk70bf9KWb2s6qxZrazmY0p9Y27W7T+KDP7d7Tv82ZW0Qf4GGCvaN8romNNNbPLonXbmNlr0dwGU83s9Gj9e2bWycz+CNSJ4ng6em1l9PMfpb/hRy2ZU8ysmpndY2ZjLcwTcEEap+XfRAXdzKyLhblIPot+7h2NUr4VOD2K5fQo9iei9/msvPMo8jNJ10/XQ4/yHsB6QhG3icBLhCoC9aPXGhNGlpa0iFdGPwcC10fPqwH1om3HANtE668Bbirn/YYRzV0BnAp8QiioNwXYhlCaehqwL3AK8JdS+zaIfr5H+Pb+U0yltimJ8ZfA36LnNQmVPOsA/YAbovW1gHFA83LiXFnq93se6Bkt1weqR8+PBF6MnvcBBpfa/07g19HzbQl1n7ZJ+t9bj+x+ZGUJDxFgtbt3KFkwsxrAnWZ2CKEcRRNgR+DbUvuMBZ6Itn3Z3Sea2aFAG+DDqLxJTcI38fLcY2Y3AIsIVXiPAF7yUFQPMxsOdAPeBO41s7sIl6ve34zf6w3gQTOrBfQExrj76uhyVzvbOCNfA6AF8GWZ/euY2USgGTAeGF1q+7+ZWQtCNdAam3j/o4ATzezKaLk2sBv5WQNKqogSheSKXxFmJuvo7uvMbC7hQ+4n7j4mSiTHAf9nZvcAS4HR7n5mGu9xlbu/ULJgZkeWt5G7f25mHQk1c/5gZm+5+63p/BLuvsbM3iOUvT4deKbk7YBL3H1UBYdY7e4dzKwB8CowAHiQUMvon+7+y6jj/71N7G/AKe4+K514RUB9FJI7GgALoyTRHdi97AZmtnu0zV+AxwlTQn4MHGRmJX0OW5tZyzTfcwxwUrTPNoTLRu+b2S7AD+7+FHBv9D5lrYtaNuV5llCMrRuhkB3RzwtL9jGzltF7lsvdlwGXAldG+zQAvo5e7lNq0xWES3AlRgGXWNS8MrN9N/UeIiWUKCRXPA10MrNxhNbFzHK2OQyYaGafEfoRHnD3RYQPzmfMbDIhcbRK5w3dfQKh7+JTQp/FUHf/DNgH+DS6BHQ9cHs5uw8BJpd0ZpfxFmFu47c9TN0JYS6R6cAEM5sK/JkKWvxRLJMIZbXvJrRuPiT0X5T4J9CmpDOb0PKoEcU2NVoWSUm3x4qISEpqUYiISEpKFCIikpIShYiIpKREISIiKSlRiIhISkoUIiKSkhKFiIik9P/p07FPnonnSAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y = train_y\n",
    "X = x_train_tf_idf_word\n",
    "\n",
    "rfc_roc_auc = roc_auc_score(y, rfc_model.predict(X))\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y, rfc_model.predict_proba(X)[:,1])\n",
    "plt.figure()\n",
    "plt.plot(fpr, tpr, label='AUC (area = %0.2f)' % rfc_roc_auc)\n",
    "plt.plot([0, 1], [0, 1],'r--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14488941",
   "metadata": {},
   "source": [
    "### ExtraTreesClassifier model with \"count-vectors\" method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "e3b05c0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "ExtraTreesClassifier model with 'count-vectors' method\n",
      "Accuracy ratio:  0.9542011278195488\n"
     ]
    }
   ],
   "source": [
    "etc = ExtraTreesClassifier()\n",
    "etc_model = etc.fit(x_train_count,train_y)\n",
    "accuracy = model_selection.cross_val_score(etc_model,\n",
    "                                           x_test_count,\n",
    "                                           test_y,\n",
    "                                           cv = 20).mean()\n",
    "\n",
    "print(\"\\nExtraTreesClassifier model with 'count-vectors' method\")\n",
    "print(\"Accuracy ratio: \", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "4fffd8fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred=etc_model.predict(x_test_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "fa5be8c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[7334,   58],\n",
       "       [ 254,  345]], dtype=int64)"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(test_y,pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "6833b3d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.99      0.98      7392\n",
      "           1       0.86      0.58      0.69       599\n",
      "\n",
      "    accuracy                           0.96      7991\n",
      "   macro avg       0.91      0.78      0.83      7991\n",
      "weighted avg       0.96      0.96      0.96      7991\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(test_y,pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "465554ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAnPElEQVR4nO3deZgU1dXH8e+RXUFEcAUVVJAlAsoiohgUUdyN4pqoYAyiiFFRwT2uicvj64JoCBpeX43GBQVXRI3BJcom+6IEQUY0LCIIArKc949bI5Nx6GmGqa5efp/n6We6qquqzxRDn751655r7o6IiMiWbJd0ACIikt2UKEREJCUlChERSUmJQkREUlKiEBGRlJQoREQkJSUKERFJSYlCJAUzm29ma8xslZl9Y2bDzax2idc7m9m7Zva9ma0ws1fMrGWpY+xoZg+Y2ZfRceZGyw0y/xuJbD0lCpHyneTutYG2wEHAdQBmdijwFjAS2BNoAkwBPjSzfaNtqgPvAK2AHsCOQGdgGdAxo7+FSAWZRmaLbJmZzQcucve3o+V7gFbufoKZvQ9Mc/dLS+3zBrDE3c83s4uAO4H93H1VhsMXqRRqUYikycwaAccBc81se0LL4PkyNn0O6B49Pxp4U0lCcpkShUj5Xjaz74GFwGLgFmBnwv+fr8vY/muguP+h/ha2EckZShQi5TvV3esAXYHmhCSwHNgE7FHG9nsAS6Pny7awjUjOUKIQSZO7/xMYDtzn7quBfwFnlLHpmYQObIC3gWPNbIeMBCkSAyUKka3zANDdzNoCg4ALzOxyM6tjZvXM7A7gUODWaPv/I1yyetHMmpvZdmZW38yuN7Pjk/gFRLaWEoXIVnD3JcCTwE3u/gFwLHAaoR9iAeH22cPd/fNo+3WEDu3ZwBhgJTCOcPnqk4z/AiIVoNtjRUQkJbUoREQkJSUKERFJSYlCRERSUqIQEZGUqiYdwNZq0KCBN27cOOkwRERyysSJE5e6+y4V2TfnEkXjxo2ZMGFC0mGIiOQUM1tQ0X116UlERFJSohARkZSUKEREJCUlChERSUmJQkREUlKiEBGRlGJLFGb2hJktNrPpW3jdzOwhM5trZlPN7OC4YhERkYqLcxzFcGAwoSRzWY4DmkaPQ4BHo58pucPa9RsrKUQRESlPbInC3ceaWeMUm5wCPOmhzvnHZraTme3h7innF56xaAXNb3qzMkMVEclbh3w5jd9OGLlNx0hyZHZDwsxfxYqidT9LFGbWB+gDUH33/bm2xwEYlpEgRURyUc3vlnHoY3fTfPQIVu7eaJuOlWSiKOuTvsxZlNx9KDAUoNaezfzSrvvHGZeISO47/Vp4ZxRcdx073ngj7FDxaduTvOupCNirxHIjYFFCsYiI5L4ZM+Crr8Lzu++GyZPhrrtg++236bBJJopRwPnR3U+dgBXl9U+IiEgZVq+GQYOgbVu44Yawbv/9oVWrSjl8bJeezOwZoCvQwMyKgFuAagDu/hjwOnA8MBf4AegdVywiInnrtdegXz9YsAAuvDC0JCpZnHc9nVPO6w70i+v9RUTy3pAhIUm0bAljx0KXLrG8Tc7NRyEiUtA2bIAlS2CPPeDMM2HNGujfH6pXj+0tVcJDRCRXjBsHHTrAySfDxo3QoAEMGBBrkgAlChGR7Pfdd3DppdCpEyxeDAMHwnaZ+/jWpScRkWw2bRp07x4uN11+Odx2G+y4Y0ZDUKIQEclG69dDtWrQrBkceSRccw0cnEztVF16EhHJJuvWhVZDq1awahXUqAHPPJNYkgAlChGR7PHuu9C6NdxyS0gM69YlHRGgRCEikrw1a+C886Bbt3D765tvwrPPQv36SUcGKFGIiCSvZk1YuhRuvBGmT4djj006ov+iRCEikoSpU0NCKCoCs1CK4/bboVatpCP7GSUKEZFMWr168x1MkybB55+H9RkcF7G1sjcyEZF8M2pUqMt0332hgN+cOeHW1yyncRQiIpny8sthsNwHH8BhhyUdTdqUKERE4rJ+PTz0UGg1HHwwPPhg6LiuVi3pyLaKLj2JiMTh44+hfXu4+mp47rmwrk6dnEsSoEQhIlK5li+Hvn2hc2f49lt46SX44x+TjmqbKFGIiFSmoUNh2DC48kqYORNOPTXc/prDLEw0lztq7dnM1yz6LOkwREQ2mzMnVHc9/PBQdmPOnFCKI4uY2UR3b1+RfdWiEBGpqLVrQ12m1q3DlKTuoYhfliWJbaVEISJSEWPGwIEHhkqvPXvC6NE5f4lpS3R7rIjI1ho7Fo45Bpo2DQnj6KOTjihWalGIiKRj48Yw2xxAly7w+OOhXlOeJwlQohARKd+nn4bbXQ87DP7zn3CJ6cILw+C5AqBEISKyJd9/D1ddFQbOzZ8Pjz4Ku+6adFQZpz4KEZGyrFgROqsXLoSLLw6D5urVSzqqRChRiIiUtHJlKNxXty706RNmnTv00KSjSpQuPYmIQCjgd8890KhRmCcCwoxzBZ4kQC0KERH48MNQn2n69FByY5ddko4oq6hFISKFrX//UHpjxQoYOTIU8dtrr6SjyipKFCJSeErWuNt991AKfOZMOPnk5GLKYkoUIlJYZs8OEwmNHBmWb7gB7r0XatdONq4spkQhIoVhzRq46aZQsG/KlLAsaYk1UZhZDzObY2ZzzWxQGa/XNbNXzGyKmc0ws95xxiMiBeqdd8KYiDvugLPPDmXAzz476ahyRmx3PZlZFeARoDtQBIw3s1HuPrPEZv2Ame5+kpntAswxs6fd/ce44hKRAlRUBFWrhoRx1FFJR5Nz4mxRdATmuvu86IP/WeCUUts4UMfMDKgNfAtsiDEmESkEGzfCI4/AX/4Sls8/P1xuUpKokDgTRUNgYYnlomhdSYOBFsAiYBrwe3ffVPpAZtbHzCaY2YRcm5FPRDJs0iTo1AkuuyzMEQGhiF+NGsnGlcPiTBRlzeBR+lP+WGAysCfQFhhsZjv+bCf3oe7e3t3bW55ODCIi22jlSvj976FDh1Cf6Zln4Pnnk44qL8SZKIqAkqNWGhFaDiX1BkZ4MBf4AmgeY0wikq+mTIHBg8MI69mzQ2e1vlhWijgTxXigqZk1MbPqwNnAqFLbfAl0AzCz3YADgHkxxiQi+eSLL+CJJ8LzLl1g7tzQN7HTTomGlW9iSxTuvgG4DBgNzAKec/cZZtbXzPpGm90OdDazacA7wEB3XxpXTCKSJ378MZT9btkSBgyA5cvD+iZNko0rT1mudQ7X2rOZr1n0WdJhiEhS3n8/XF6aORNOOw0efDBUfJWUzGyiu7evyL6qHisiuWPJEjjmGNhtN3jlFTjxxKQjKggq4SEi2c0dxowJz3fZBV59FWbMUJLIICUKEcleM2bAL38ZWhHvvRfWdesGO+yQaFiFRolCRLLPDz/A9ddD27YhWQwbBkcckXRUBUt9FCKSXdxDGfBx4+CCC0IJcM04lyglChHJDl9/DbvuClWqhNZE3brQtWvSUQm69CQiSdu4ER56CA44AIYMCetOOUVJIosoUYhIciZMgI4dQ42mzp3h+OOTjkjKkHaiMDPdZiAileeee0KS+Ppr+Pvf4Y03YL/9ko5KylBuojCzzmY2k1CGAzNrY2ZDYo9MRPKPO6xfH5537Aj9+sGsWXDmmSrgl8XSaVH8D6Ec+DIAd58C6D41Edk6//439OgBg6JZkbt2hYcfDp3WktXSuvTk7gtLrdoYQywiko/WrQtzVf/iF/Cvf+nyUg5K5/bYhWbWGfCoXPjlRJehRERSmjgRfvObMD/EGWfAAw/AnnsmHZVspXQSRV/gQcI0pkXAW8ClcQYlInmidu3Q9/D663DccUlHIxWUTqI4wN1/XXKFmR0GfBhPSCKSszZtgr/+NVxiGjYsjI2YPh220534uSydf72H01wnIoVs+vRQj+mii+Dzz2H16rBeSSLnbbFFYWaHAp2BXczsqhIv7QhUiTswEckRq1fDbbfB/feHO5j++tdQo0m3u+aNVJeeqgO1o23qlFi/EugZZ1AikkPWrg3J4fzzwyC6+vWTjkgqWblToZrZPu6+IEPxlEtToYpkgaKiUJ/pj38MRfy+/RZ23jnpqCSFuKdC/cHM7gVaATWLV7r7URV5QxHJYRs2hEFyN98civmddRa0a6ckkefS6WV6GpgNNAFuBeYD42OMSUSy0SefQPv2cNVVodN6xoyQJCTvpZMo6rv748B6d/+nu18IdIo5LhHJJps2Qe/esGQJvPBCmLe6SZOko5IMSefSU1TBi6/N7ARgEdAovpBEJCu4h6TQowfUqQMjRkDDhuG5FJR0WhR3mFldYABwNTAMuCLOoEQkYZ9/DsceG6q6Dh0a1jVvriRRoMptUbj7q9HTFcCR8NPIbBHJN+vWwd13w113QY0aMHgw9O2bdFSSsFQD7qoAZxJqPL3p7tPN7ETgeqAWcFBmQhSRjOnXDx5/HM4+Owyg22OPpCOSLLDFcRRmNhzYCxgHHAIsAA4FBrn7yxmK72c0jkKkki1eHDqrd989XHKaNy9cdpK8Etc4ivZAa3ffZGY1gaXA/u7+TUXeSESyzKZNoXDfwIFwzDFhOtKmTcNDpIRUndk/uvsmAHdfC3ymJCGSJ6ZOhcMPh4svhrZt4dZbk45IsliqFkVzM5saPTdgv2jZAHf31rFHJyKV74UXQh9EvXrw5JNhYiEV8JMUUiWKFhmLQkTit3Il7LhjmKu6Xz+45RaV3pC0lFsUMNuoM1tkK335JfTvD4sWwccfhyJ+UnC2pTM71hlFzKyHmc0xs7lmNmgL23Q1s8lmNsPM/hlnPCIFZf16uO8+aNEC3n47DJ7LsS+Gkh3SKeFRIdE4jEeA7oS5tseb2Sh3n1lim52AIUAPd//SzHaNKx6RgrJgAZx8cui0PumkUPF1n32SjkpyVFotCjOrZWYHbOWxOwJz3X2eu/8IPAucUmqbc4ER7v4lgLsv3sr3EJGSilsMu+8Ou+0GL70EI0cqScg2KTdRmNlJwGTgzWi5rZmNSuPYDYGFJZaLonUlNQPqmdl7ZjbRzM5PK2oR+W/u8NRT0KEDrFoVym+89RaceqruaJJtlk6L4g+E1sF3AO4+GWicxn5l/XWWvkBaFWgHnAAcC9xkZs1+diCzPmY2wcwm5Frnu0js5syBbt3gvPOgalVYtizpiCTPpJMoNrj7igocu4hQAqRYI0KJ8tLbvOnuq919KTAWaFP6QO4+1N3bu3t707cjkWDDhnCLa+vWMGkSPPoofPSRLjNJpUsnUUw3s3OBKmbW1MweBj5KY7/xQFMza2Jm1YGzgdKXrEYCXcysqpltT6gpNWsr4hcpXFWqwPvvQ8+eoVXRty9sF+uNjFKg0vmr6k+YL3sd8DdCufErytvJ3TcAlwGjCR/+z7n7DDPra2Z9o21mEfo+phKKDw5z9+kV+D1ECsM338CFF8LChaHv4fXX4emnQ8e1SEzKHXBnZge5+6cZiqdcGnAnBWnjxjCB0HXXwZo1oeP6jDOSjkpySNwD7u43s9lmdruZtarIm4jINvj0U+jcGS69FNq3h2nTlCQko8pNFO5+JNAVWAIMNbNpZnZj3IGJSGTwYJg/P1xiGjMGmv3sxkCRWG1VrSczOxC4FjjL3avHFlUKuvQkec8dXn4ZGjeGgw6C5cvD+nr1koxKclysl57MrIWZ/cHMpgODCXc8NarIm4lIOebPD6U3TjsNHnggrKtXT0lCEpVOrae/As8Ax7h76XEQIlIZ1q8Pc1Tfemu4xfW+++D3v086KhEgjUTh7p0yEYhIQfvzn2HQoFBy48EHYe+9k45I5CdbTBRm9py7n2lm0/jv0hua4U6kMixbFi41tWsHv/sd7L8/9OiRdFQiP5OqRVHc7j0xE4GIFAz3MAXp1VdDnTrw2WehiJ+ShGSpLXZmu/vX0dNL3X1ByQdwaWbCE8kzs2bBkUdCr17QtGm4u6lqbNPCiFSKdAbcdS9j3XGVHYhI3psyBdq0CZMJDR0KH3wQCvqJZLlUfRSXEFoO+5rZ1BIv1QE+jDswkbxRVASNGoWkcOut8Nvfwq6azFFyxxYH3JlZXaAe8Eeg5HzX37v7txmIrUwacCc5Y9EiuPLKULhv9mxoWHreLpHMiWvAnbv7fKAf8H2JB2a2c0XeTKQgbNwYym60aBGmIb32WmjQIOmoRCosVS/a3wh3PE0k3B5bcsYgB/aNMS6R3LR2LRxxBIwfD927w5Ah4bZXkRy2xUTh7idGP5tkLhyRHLV+PVSrBjVrhruarroKzjpL81VLXkin1tNhZrZD9Pw3Zna/mWnYqAiEMREvvBBaDZMmhXV33w1nn60kIXkjndtjHwV+MLM2hMqxC4D/izUqkVwwbx6ccEKYG6J+fU1DKnkrnb/sDR5ujToFeNDdHyTcIitSuO6/H1q1CnNWP/AAjBsHbdsmHZVILNIZEvq9mV0HnAd0MbMqQLV4wxLJcqtWwfHHhwJ+jVR1X/JbOi2Ks4B1wIXu/g3QELg31qhEss3SpdC7N4waFZZvvBFefFFJQgpCOlOhfgM8DdQ1sxOBte7+ZOyRiWSDTZvgiSfggAPgqadg7tywXv0RUkDSuevpTGAccAZwJvCJmfWMOzCRxM2cCV27hpIbLVvC5MnhtleRApNOH8UNQAd3XwxgZrsAbwMvxBmYSOImTIAZM+Dxx0O1V7UipEClkyi2K04SkWWk17chkntefz1MKHTeeeFx4omwsyrWSGFL5wP/TTMbbWa9zKwX8BrwerxhiWRYURH07BnGRQweHAbSmSlJiJBeZ/Y1wJ+B1kAbYKi7D4w7MJGM2LAh3OLaogW89hrceWcYG6FR1SI/STUfRVPgPmA/YBpwtbt/lanARDJi4kS44oowDekjj8C+qnUpUlqqFsUTwKvA6YQKsg9nJCKRuK1YASNGhOeHHAKffBL6JpQkRMqUqjO7jrv/JXo+x8wmZSIgkdi4w3PPhRbEsmUwfz7suSd07Jh0ZCJZLVWiqGlmB7F5HopaJZfdXYlDcse//w39+sHo0dCuHbzySkgSIlKuVInia+D+EsvflFh24Ki4ghKpVN9/H5LDpk3w0ENw6aVQpUrSUYnkjFQTFx2ZyUBEKt3UqdC6NdSpEwbNdeqkeatFKkAD5yT/LFkCF1wAbdqETmqA009XkhCpoFgThZn1MLM5ZjbXzAal2K6DmW1UDSnZJps2wbBhoYDfM8/A9deHWk0isk3SKeFRIdG8FY8A3YEiYLyZjXL3mWVsdzcwOq5YpECcfjq8/DIccQQ8+mgo5Cci2yyd6rEWzZV9c7S8t5mlcz9hR2Cuu89z9x+BZwmz5JXWH3gRWFzGayKprV4dRlcDnHMODB8O772nJCFSidK59DQEOBQ4J1r+ntBSKE9DYGGJ5aJo3U/MrCHwK+CxVAcysz5mNsHMJoRZWUUIt7i2bAlDhoTlM88MfRMqvyFSqdJJFIe4ez9gLYC7Lweqp7FfWf9bS3/KPwAMdPeNqQ7k7kPdvb27tzd9CMjChXDaaXDyyeGOpnbtko5IJK+l00exPupHcPhpPopNaexXBOxVYrkRsKjUNu2BZ6MP/wbA8Wa2wd1fTuP4Uoieegr69g0d13/6E1x5JVRP53uLiFRUOoniIeAlYFczuxPoCdyYxn7jgaZm1gT4CjgbOLfkBu7epPi5mQ0HXlWSkDIVl/1u1CjcyfTww9CkSbm7ici2KzdRuPvTZjYR6Ea4nHSqu89KY78NZnYZ4W6mKsAT7j7DzPpGr6fslxAB4Lvv4LrrYIcd4L77QpLQLa8iGVVuojCzvYEfgFdKrnP3L8vb191fp9QkR1tKEO7eq7zjSQFxD2MhrroqDKC78srNrQoRyah0Lj29RuifMKAm0ASYA7SKMS4pZF98AX36wNtvQ4cO8MYbcNBBSUclUrDSufR0YMllMzsYuDi2iETWrw91mh55BC6+WAX8RBK21SOz3X2SmXWIIxgpYO+8E6Yivf9+aNYMFiyAmjWTjkpESK+P4qoSi9sBBwNLYotICst//gMDBsDTT8N++8ENN0D9+koSIlkknQF3dUo8ahD6LMoqxSGSvk2b4M9/hubNw6xzN90E06aFJCEiWSVliyIaaFfb3a/JUDxSKFasgBtvhLZtQwG/5s2TjkhEtmCLLQozqxqV1jg4g/FIPlu1KvRBbNwI9erBJ5/Au+8qSYhkuVQtinGEJDHZzEYBzwOri1909xExxyb5ZORI6N8/1Glq2xaOOgr23TfpqEQkDen0UewMLCPMkX0icFL0U6R8CxbAKafAqafCTjvBhx+GJCEiOSNVi2LX6I6n6WwecFdMtb6lfO7QsyfMnAn33ANXXAHVqiUdlYhspVSJogpQm/TKhYts9vHH0KpVKAE+dCjsvDPss0/SUYlIBaVKFF+7+20Zi0Ry37ffhgJ+Q4fCzTfDrbeq9IZIHkiVKFR9TdLjHuaJGDAgJIsBA+Aa3VEtki9SJYpuGYtCctv114dJhDp1gjFjoE2bpCMSkUq0xUTh7t9mMhDJMWvXhnERDRpA796hD6JPH9gunRvpRCSX6H+1bL0xY+DAA+F3vwvLzZqF6UmVJETykv5nS/q++QbOPReOOSZMIHTZZUlHJCIZsNVlxqVA/eMf8KtfwZo18Ic/wMCBqvAqUiCUKCS19evDILnWraF7d7jzznCpSUQKhi49Sdm+/z7MU92lSyjiV78+PP+8koRIAVKikP/mDiNGQIsW8OCDYcDcunVJRyUiCVKikM2WLoWTToLTTw+3vX70UZgrYvvtk45MRBKkRCGb1akTpia9/36YMCEMoBORgqdEUeg++ACOOy4MnqtRI0wmdOWVUFX3OYhIoERRqJYtg4suCp3VM2fCvHlhvQbNiUgp+lQoNO4wfDgccED4ec01IVG0bp10ZCKSpXR9oRA9+WRIFI89FkpxiIikoBZFIVizBm65BYqKQumNF1+E999XkhCRtChR5LvRo+EXv4DbboORI8O6evXUFyEiadOnRb5atAjOOgt69AglON59F/r1SzoqEclBShT56o47QgvitttgyhQ48sikIxKRHGXunnQMW6XWns18zaLPkg4jO02cuLmA37JlsHw57L9/0lGJSBYws4nu3r4i+8baojCzHmY2x8zmmtmgMl7/tZlNjR4fmZnm0KyIlSvh8suhY8cwLSmEIn5KEiJSCWJLFGZWBXgEOA5oCZxjZi1LbfYF8Et3bw3cDgyNK5685B4qujZvDoMHwyWXwFNPJR2ViOSZOMdRdATmuvs8ADN7FjgFmFm8gbt/VGL7j4FGMcaTf/72N/jNb0KF15EjoUOHpCMSkTwUZ6JoCCwssVwEHJJi+98Cb5T1gpn1AfoA1Ni9wC+n/PhjKLfRvDn07BnGSPTqpdpMIhKbOPsorIx1Zfacm9mRhEQxsKzX3X2ou7d39/ZmZR22QIwdC23bhjmr164NRfwuukhJQkRiFWeiKAL2KrHcCFhUeiMzaw0MA05x92UxxpO7li6F3r3hl78MLYjHHtN81SKSMXF+FR0PNDWzJsBXwNnAuSU3MLO9gRHAee6ue17LMm9e6HtYuRIGDYKbbtJEQiKSUbElCnffYGaXAaOBKsAT7j7DzPpGrz8G3AzUB4ZEl5Q2VPQ+37yzciXsuCM0aRJaE716hVIcIiIZpgF32eaHH+D222Ho0DCiupFuBBORbbctA+7UC5pNXnsNLrsM5s8PrYhatZKOSEREiSIrbNgA55wDL7wALVrAP/8JRxyRdFQiIoCKAiar+LJf1aqw225w110webKShIhkFSWKpIwfD4ccApMmheXBg+G666B69WTjEhEpRYki01asCP0QhxwSZpxbpqEjIpLdlCgyqbiA36OPhmQxezZ07550VCIiKakzO5NmzYKGDeGVV6C9houISG7QOIo4rVsH994LbdrASSfB+vVhruoqVZKOTEQKTNZOXFTQ/vGPkCBuugneeSesq1ZNSUJEco4SRWVbvBguuACOOiq0IN54Ax54IOmoREQqTImisr31FjzzDNxwA0yfDj16JB2RiMg2UWd2ZZg2DebMCRMJ/frX0Lkz7Ltv0lGJiFQKtSi2xerVcO21YSrSa68Nl5rMlCREJK8oUVTUK69Ay5bhrqZevcJI62rVko5KRKTS6dJTRUyfDiefDK1awfvvw+GHJx2RiEhs1KJI14YN8N574fkvfgGvvgqffqokISJ5T4kiHZ98EkZSd+sGn38e1p1wgi41iUhBUKJIZflyuOQSOPRQWLo01Graf/+koxIRySj1UWzJunXhbqaFC+GKK+DWW6FOnaSjEhHJOCWK0r76KhTuq1ED/vCHUIbjoIOSjkpEJDG69FRs7drQath3Xxg5Mqzr1UtJQkQKnloUEIr2XXJJ6Kg+55wwqZCIiABqUYT+h6OPDvNXv/UW/O1vsPvuSUclIpI1CjNRbNoEGzeG5x07ws03h3pNmm1ORORnCi9RTJkSivY98khYPvfc0DdRs2aycYmIZKnCSRSrVsGAAdCuHcybp8tLIiJpKozO7Lffht69oagI+vSBP/0J6tVLOioRkZxQGImienXYeWf4+9/DZScREUlbfiaK9evD9KMrVsAdd8ARR4QCftsVzpU2EZHKkn+fnB99FPohrr0WZs0KdziBkoSISAXlz6fnt9+G/ofDDoPvvoOXX4YXX1SCEBHZRvnzKbpsWRgsd/XVMHMmnHJK0hGJiOSF3O6jmDMndFDffDM0bQoLFkD9+klHJSKSV2JtUZhZDzObY2ZzzWxQGa+bmT0UvT7VzA5O68Br1oTk0Lo1/M//hFLgoCQhIhKD2BKFmVUBHgGOA1oC55hZy1KbHQc0jR59gEfLO27tdavhwAPh9tvhjDNg9mzYa69Kjl5ERIrF2aLoCMx193nu/iPwLFC64+AU4EkPPgZ2MrM9Uh204Xf/CR3Ub78NTz0Fu+0WT/QiIgLE20fREFhYYrkIKF2/u6xtGgJfl9zIzPoQWhwA6+zzz6dz9NGVG21uagAsTTqILKFzsZnOxWY6F5sdUNEd40wUVsY6r8A2uPtQYCiAmU1w9/bbHl7u07nYTOdiM52LzXQuNjOzCRXdN85LT0VAyc6DRsCiCmwjIiIJijNRjAeamlkTM6sOnA2MKrXNKOD86O6nTsAKd/+69IFERCQ5sV16cvcNZnYZMBqoAjzh7jPMrG/0+mPA68DxwFzgB6B3GoceGlPIuUjnYjOdi810LjbTudiswufC3H/WJSAiIvKT/CnhISIisVCiEBGRlLI2UcRW/iMHpXEufh2dg6lm9pGZtUkizkwo71yU2K6DmW00s56ZjC+T0jkXZtbVzCab2Qwz+2emY8yUNP6P1DWzV8xsSnQu0ukPzTlm9oSZLTaz6Vt4vWKfm+6edQ9C5/e/gX2B6sAUoGWpbY4H3iCMxegEfJJ03Amei85Avej5cYV8Lkps9y7hZomeSced4N/FTsBMYO9oedek407wXFwP3B093wX4FqiedOwxnIsjgIOB6Vt4vUKfm9naooil/EeOKvdcuPtH7r48WvyYMB4lH6XzdwHQH3gRWJzJ4DIsnXNxLjDC3b8EcPd8PR/pnAsH6piZAbUJiWJDZsOMn7uPJfxuW1Khz81sTRRbKu2xtdvkg639PX9L+MaQj8o9F2bWEPgV8FgG40pCOn8XzYB6ZvaemU00s/MzFl1mpXMuBgMtCAN6pwG/d/dNmQkvq1ToczNb56OotPIfeSDt39PMjiQkisNjjSg56ZyLB4CB7r4xfHnMW+mci6pAO6AbUAv4l5l97O6fxR1chqVzLo4FJgNHAfsBY8zsfXdfGXNs2aZCn5vZmihU/mOztH5PM2sNDAOOc/dlGYot09I5F+2BZ6Mk0QA43sw2uPvLGYkwc9L9P7LU3VcDq81sLNAGyLdEkc656A38ycOF+rlm9gXQHBiXmRCzRoU+N7P10pPKf2xW7rkws72BEcB5efhtsaRyz4W7N3H3xu7eGHgBuDQPkwSk939kJNDFzKqa2faE6s2zMhxnJqRzLr4ktKwws90IlVTnZTTK7FChz82sbFF4fOU/ck6a5+JmoD4wJPomvcHzsGJmmueiIKRzLtx9lpm9CUwFNgHD3L3M2yZzWZp/F7cDw81sGuHyy0B3z7vy42b2DNAVaGBmRcAtQDXYts9NlfAQEZGUsvXSk4iIZAklChERSUmJQkREUlKiEBGRlJQoREQkJSUKyUpR5dfJJR6NU2y7qhLeb7iZfRG91yQzO7QCxxhmZi2j59eXeu2jbY0xOk7xeZkeVUPdqZzt25rZ8ZXx3lK4dHusZCUzW+XutSt72xTHGA686u4vmNkxwH3u3nobjrfNMZV3XDP7X+Azd78zxfa9gPbuflllxyKFQy0KyQlmVtvM3om+7U8zs59VjTWzPcxsbIlv3F2i9ceY2b+ifZ83s/I+wMcC+0f7XhUda7qZXRGt28HMXovmNphuZmdF698zs/Zm9iegVhTH09Frq6Kffy/5DT9qyZxuZlXM7F4zG29hnoCL0zgt/yIq6GZmHS3MRfJp9POAaJTybcBZUSxnRbE/Eb3Pp2WdR5GfSbp+uh56lPUANhKKuE0GXiJUEdgxeq0BYWRpcYt4VfRzAHBD9LwKUCfadiywQ7R+IHBzGe83nGjuCuAM4BNCQb1pwA6E0tQzgIOA04G/lNi3bvTzPcK3959iKrFNcYy/Av43el6dUMmzFtAHuDFaXwOYADQpI85VJX6/54Ee0fKOQNXo+dHAi9HzXsDgEvvfBfwmer4Toe7TDkn/e+uR3Y+sLOEhAqxx97bFC2ZWDbjLzI4glKNoCOwGfFNin/HAE9G2L7v7ZDP7JdAS+DAqb1Kd8E28LPea2Y3AEkIV3m7ASx6K6mFmI4AuwJvAfWZ2N+Fy1ftb8Xu9ATxkZjWAHsBYd18TXe5qbZtn5KsLNAW+KLV/LTObDDQGJgJjSmz/v2bWlFANtNoW3v8Y4GQzuzpargnsTX7WgJJKokQhueLXhJnJ2rn7ejObT/iQ+4m7j40SyQnA/5nZvcByYIy7n5PGe1zj7i8UL5jZ0WVt5O6fmVk7Qs2cP5rZW+5+Wzq/hLuvNbP3CGWvzwKeKX47oL+7jy7nEGvcva2Z1QVeBfoBDxFqGf3D3X8Vdfy/t4X9DTjd3eekE68IqI9CckddYHGUJI4E9im9gZntE23zF+BxwpSQHwOHmVlxn8P2ZtYszfccC5wa7bMD4bLR+2a2J/CDuz8F3Be9T2nro5ZNWZ4lFGPrQihkR/TzkuJ9zKxZ9J5lcvcVwOXA1dE+dYGvopd7ldj0e8IluGKjgf4WNa/M7KAtvYdIMSUKyRVPA+3NbAKhdTG7jG26ApPN7FNCP8KD7r6E8MH5jJlNJSSO5um8obtPIvRdjCP0WQxz90+BA4Fx0SWgG4A7yth9KDC1uDO7lLcIcxu/7WHqTghzicwEJpnZdODPlNPij2KZQiirfQ+hdfMhof+i2D+AlsWd2YSWR7UotunRskhKuj1WRERSUotCRERSUqIQEZGUlChERCQlJQoREUlJiUJERFJSohARkZSUKEREJKX/B+4Xqc+xrSi2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y = train_y\n",
    "X = x_train_count.astype(\"float64\")\n",
    "\n",
    "etc_roc_auc = roc_auc_score(y, etc_model.predict(X))\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y, etc_model.predict_proba(X)[:,1])\n",
    "plt.figure()\n",
    "plt.plot(fpr, tpr, label='AUC (area = %0.2f)' % etc_roc_auc)\n",
    "plt.plot([0, 1], [0, 1],'r--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb7605df",
   "metadata": {},
   "source": [
    "### ExtraTreesClassifier model with \"tf-idf\" method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "bf5544e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "ExtraTreesClassifier model with 'tf-idf' method\n",
      "Accuracy ratio:  0.9521963391739675\n"
     ]
    }
   ],
   "source": [
    "etc = ExtraTreesClassifier()\n",
    "etc_model = etc.fit(x_train_tf_idf_word,train_y)\n",
    "accuracy = model_selection.cross_val_score(etc_model, \n",
    "                                           x_test_tf_idf_word, \n",
    "                                           test_y, \n",
    "                                           cv = 10).mean()\n",
    "\n",
    "print(\"\\nExtraTreesClassifier model with 'tf-idf' method\")\n",
    "print(\"Accuracy ratio: \", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "8ab64031",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred=etc_model.predict(x_test_tf_idf_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "1fb2bf63",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[7352,   40],\n",
       "       [ 250,  349]], dtype=int64)"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(test_y,pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "69dddbf6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.99      0.98      7392\n",
      "           1       0.90      0.58      0.71       599\n",
      "\n",
      "    accuracy                           0.96      7991\n",
      "   macro avg       0.93      0.79      0.84      7991\n",
      "weighted avg       0.96      0.96      0.96      7991\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(test_y,pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "c7e067a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAnPElEQVR4nO3deZgU1dXH8e+RXUFEcAUVVJAlAsoiohgUUdyN4pqoYAyiiFFRwT2uicvj64JoCBpeX43GBQVXRI3BJcom+6IEQUY0LCIIArKc949bI5Nx6GmGqa5efp/n6We6qquqzxRDn751655r7o6IiMiWbJd0ACIikt2UKEREJCUlChERSUmJQkREUlKiEBGRlJQoREQkJSUKERFJSYlCJAUzm29ma8xslZl9Y2bDzax2idc7m9m7Zva9ma0ws1fMrGWpY+xoZg+Y2ZfRceZGyw0y/xuJbD0lCpHyneTutYG2wEHAdQBmdijwFjAS2BNoAkwBPjSzfaNtqgPvAK2AHsCOQGdgGdAxo7+FSAWZRmaLbJmZzQcucve3o+V7gFbufoKZvQ9Mc/dLS+3zBrDE3c83s4uAO4H93H1VhsMXqRRqUYikycwaAccBc81se0LL4PkyNn0O6B49Pxp4U0lCcpkShUj5Xjaz74GFwGLgFmBnwv+fr8vY/muguP+h/ha2EckZShQi5TvV3esAXYHmhCSwHNgE7FHG9nsAS6Pny7awjUjOUKIQSZO7/xMYDtzn7quBfwFnlLHpmYQObIC3gWPNbIeMBCkSAyUKka3zANDdzNoCg4ALzOxyM6tjZvXM7A7gUODWaPv/I1yyetHMmpvZdmZW38yuN7Pjk/gFRLaWEoXIVnD3JcCTwE3u/gFwLHAaoR9iAeH22cPd/fNo+3WEDu3ZwBhgJTCOcPnqk4z/AiIVoNtjRUQkJbUoREQkJSUKERFJSYlCRERSUqIQEZGUqiYdwNZq0KCBN27cOOkwRERyysSJE5e6+y4V2TfnEkXjxo2ZMGFC0mGIiOQUM1tQ0X116UlERFJSohARkZSUKEREJCUlChERSUmJQkREUlKiEBGRlGJLFGb2hJktNrPpW3jdzOwhM5trZlPN7OC4YhERkYqLcxzFcGAwoSRzWY4DmkaPQ4BHo58pucPa9RsrKUQRESlPbInC3ceaWeMUm5wCPOmhzvnHZraTme3h7innF56xaAXNb3qzMkMVEclbh3w5jd9OGLlNx0hyZHZDwsxfxYqidT9LFGbWB+gDUH33/bm2xwEYlpEgRURyUc3vlnHoY3fTfPQIVu7eaJuOlWSiKOuTvsxZlNx9KDAUoNaezfzSrvvHGZeISO47/Vp4ZxRcdx073ngj7FDxaduTvOupCNirxHIjYFFCsYiI5L4ZM+Crr8Lzu++GyZPhrrtg++236bBJJopRwPnR3U+dgBXl9U+IiEgZVq+GQYOgbVu44Yawbv/9oVWrSjl8bJeezOwZoCvQwMyKgFuAagDu/hjwOnA8MBf4AegdVywiInnrtdegXz9YsAAuvDC0JCpZnHc9nVPO6w70i+v9RUTy3pAhIUm0bAljx0KXLrG8Tc7NRyEiUtA2bIAlS2CPPeDMM2HNGujfH6pXj+0tVcJDRCRXjBsHHTrAySfDxo3QoAEMGBBrkgAlChGR7Pfdd3DppdCpEyxeDAMHwnaZ+/jWpScRkWw2bRp07x4uN11+Odx2G+y4Y0ZDUKIQEclG69dDtWrQrBkceSRccw0cnEztVF16EhHJJuvWhVZDq1awahXUqAHPPJNYkgAlChGR7PHuu9C6NdxyS0gM69YlHRGgRCEikrw1a+C886Bbt3D765tvwrPPQv36SUcGKFGIiCSvZk1YuhRuvBGmT4djj006ov+iRCEikoSpU0NCKCoCs1CK4/bboVatpCP7GSUKEZFMWr168x1MkybB55+H9RkcF7G1sjcyEZF8M2pUqMt0332hgN+cOeHW1yyncRQiIpny8sthsNwHH8BhhyUdTdqUKERE4rJ+PTz0UGg1HHwwPPhg6LiuVi3pyLaKLj2JiMTh44+hfXu4+mp47rmwrk6dnEsSoEQhIlK5li+Hvn2hc2f49lt46SX44x+TjmqbKFGIiFSmoUNh2DC48kqYORNOPTXc/prDLEw0lztq7dnM1yz6LOkwREQ2mzMnVHc9/PBQdmPOnFCKI4uY2UR3b1+RfdWiEBGpqLVrQ12m1q3DlKTuoYhfliWJbaVEISJSEWPGwIEHhkqvPXvC6NE5f4lpS3R7rIjI1ho7Fo45Bpo2DQnj6KOTjihWalGIiKRj48Yw2xxAly7w+OOhXlOeJwlQohARKd+nn4bbXQ87DP7zn3CJ6cILw+C5AqBEISKyJd9/D1ddFQbOzZ8Pjz4Ku+6adFQZpz4KEZGyrFgROqsXLoSLLw6D5urVSzqqRChRiIiUtHJlKNxXty706RNmnTv00KSjSpQuPYmIQCjgd8890KhRmCcCwoxzBZ4kQC0KERH48MNQn2n69FByY5ddko4oq6hFISKFrX//UHpjxQoYOTIU8dtrr6SjyipKFCJSeErWuNt991AKfOZMOPnk5GLKYkoUIlJYZs8OEwmNHBmWb7gB7r0XatdONq4spkQhIoVhzRq46aZQsG/KlLAsaYk1UZhZDzObY2ZzzWxQGa/XNbNXzGyKmc0ws95xxiMiBeqdd8KYiDvugLPPDmXAzz476ahyRmx3PZlZFeARoDtQBIw3s1HuPrPEZv2Ame5+kpntAswxs6fd/ce44hKRAlRUBFWrhoRx1FFJR5Nz4mxRdATmuvu86IP/WeCUUts4UMfMDKgNfAtsiDEmESkEGzfCI4/AX/4Sls8/P1xuUpKokDgTRUNgYYnlomhdSYOBFsAiYBrwe3ffVPpAZtbHzCaY2YRcm5FPRDJs0iTo1AkuuyzMEQGhiF+NGsnGlcPiTBRlzeBR+lP+WGAysCfQFhhsZjv+bCf3oe7e3t3bW55ODCIi22jlSvj976FDh1Cf6Zln4Pnnk44qL8SZKIqAkqNWGhFaDiX1BkZ4MBf4AmgeY0wikq+mTIHBg8MI69mzQ2e1vlhWijgTxXigqZk1MbPqwNnAqFLbfAl0AzCz3YADgHkxxiQi+eSLL+CJJ8LzLl1g7tzQN7HTTomGlW9iSxTuvgG4DBgNzAKec/cZZtbXzPpGm90OdDazacA7wEB3XxpXTCKSJ378MZT9btkSBgyA5cvD+iZNko0rT1mudQ7X2rOZr1n0WdJhiEhS3n8/XF6aORNOOw0efDBUfJWUzGyiu7evyL6qHisiuWPJEjjmGNhtN3jlFTjxxKQjKggq4SEi2c0dxowJz3fZBV59FWbMUJLIICUKEcleM2bAL38ZWhHvvRfWdesGO+yQaFiFRolCRLLPDz/A9ddD27YhWQwbBkcckXRUBUt9FCKSXdxDGfBx4+CCC0IJcM04lyglChHJDl9/DbvuClWqhNZE3brQtWvSUQm69CQiSdu4ER56CA44AIYMCetOOUVJIosoUYhIciZMgI4dQ42mzp3h+OOTjkjKkHaiMDPdZiAileeee0KS+Ppr+Pvf4Y03YL/9ko5KylBuojCzzmY2k1CGAzNrY2ZDYo9MRPKPO6xfH5537Aj9+sGsWXDmmSrgl8XSaVH8D6Ec+DIAd58C6D41Edk6//439OgBg6JZkbt2hYcfDp3WktXSuvTk7gtLrdoYQywiko/WrQtzVf/iF/Cvf+nyUg5K5/bYhWbWGfCoXPjlRJehRERSmjgRfvObMD/EGWfAAw/AnnsmHZVspXQSRV/gQcI0pkXAW8ClcQYlInmidu3Q9/D663DccUlHIxWUTqI4wN1/XXKFmR0GfBhPSCKSszZtgr/+NVxiGjYsjI2YPh220534uSydf72H01wnIoVs+vRQj+mii+Dzz2H16rBeSSLnbbFFYWaHAp2BXczsqhIv7QhUiTswEckRq1fDbbfB/feHO5j++tdQo0m3u+aNVJeeqgO1o23qlFi/EugZZ1AikkPWrg3J4fzzwyC6+vWTjkgqWblToZrZPu6+IEPxlEtToYpkgaKiUJ/pj38MRfy+/RZ23jnpqCSFuKdC/cHM7gVaATWLV7r7URV5QxHJYRs2hEFyN98civmddRa0a6ckkefS6WV6GpgNNAFuBeYD42OMSUSy0SefQPv2cNVVodN6xoyQJCTvpZMo6rv748B6d/+nu18IdIo5LhHJJps2Qe/esGQJvPBCmLe6SZOko5IMSefSU1TBi6/N7ARgEdAovpBEJCu4h6TQowfUqQMjRkDDhuG5FJR0WhR3mFldYABwNTAMuCLOoEQkYZ9/DsceG6q6Dh0a1jVvriRRoMptUbj7q9HTFcCR8NPIbBHJN+vWwd13w113QY0aMHgw9O2bdFSSsFQD7qoAZxJqPL3p7tPN7ETgeqAWcFBmQhSRjOnXDx5/HM4+Owyg22OPpCOSLLDFcRRmNhzYCxgHHAIsAA4FBrn7yxmK72c0jkKkki1eHDqrd989XHKaNy9cdpK8Etc4ivZAa3ffZGY1gaXA/u7+TUXeSESyzKZNoXDfwIFwzDFhOtKmTcNDpIRUndk/uvsmAHdfC3ymJCGSJ6ZOhcMPh4svhrZt4dZbk45IsliqFkVzM5saPTdgv2jZAHf31rFHJyKV74UXQh9EvXrw5JNhYiEV8JMUUiWKFhmLQkTit3Il7LhjmKu6Xz+45RaV3pC0lFsUMNuoM1tkK335JfTvD4sWwccfhyJ+UnC2pTM71hlFzKyHmc0xs7lmNmgL23Q1s8lmNsPM/hlnPCIFZf16uO8+aNEC3n47DJ7LsS+Gkh3SKeFRIdE4jEeA7oS5tseb2Sh3n1lim52AIUAPd//SzHaNKx6RgrJgAZx8cui0PumkUPF1n32SjkpyVFotCjOrZWYHbOWxOwJz3X2eu/8IPAucUmqbc4ER7v4lgLsv3sr3EJGSilsMu+8Ou+0GL70EI0cqScg2KTdRmNlJwGTgzWi5rZmNSuPYDYGFJZaLonUlNQPqmdl7ZjbRzM5PK2oR+W/u8NRT0KEDrFoVym+89RaceqruaJJtlk6L4g+E1sF3AO4+GWicxn5l/XWWvkBaFWgHnAAcC9xkZs1+diCzPmY2wcwm5Frnu0js5syBbt3gvPOgalVYtizpiCTPpJMoNrj7igocu4hQAqRYI0KJ8tLbvOnuq919KTAWaFP6QO4+1N3bu3t707cjkWDDhnCLa+vWMGkSPPoofPSRLjNJpUsnUUw3s3OBKmbW1MweBj5KY7/xQFMza2Jm1YGzgdKXrEYCXcysqpltT6gpNWsr4hcpXFWqwPvvQ8+eoVXRty9sF+uNjFKg0vmr6k+YL3sd8DdCufErytvJ3TcAlwGjCR/+z7n7DDPra2Z9o21mEfo+phKKDw5z9+kV+D1ECsM338CFF8LChaHv4fXX4emnQ8e1SEzKHXBnZge5+6cZiqdcGnAnBWnjxjCB0HXXwZo1oeP6jDOSjkpySNwD7u43s9lmdruZtarIm4jINvj0U+jcGS69FNq3h2nTlCQko8pNFO5+JNAVWAIMNbNpZnZj3IGJSGTwYJg/P1xiGjMGmv3sxkCRWG1VrSczOxC4FjjL3avHFlUKuvQkec8dXn4ZGjeGgw6C5cvD+nr1koxKclysl57MrIWZ/cHMpgODCXc8NarIm4lIOebPD6U3TjsNHnggrKtXT0lCEpVOrae/As8Ax7h76XEQIlIZ1q8Pc1Tfemu4xfW+++D3v086KhEgjUTh7p0yEYhIQfvzn2HQoFBy48EHYe+9k45I5CdbTBRm9py7n2lm0/jv0hua4U6kMixbFi41tWsHv/sd7L8/9OiRdFQiP5OqRVHc7j0xE4GIFAz3MAXp1VdDnTrw2WehiJ+ShGSpLXZmu/vX0dNL3X1ByQdwaWbCE8kzs2bBkUdCr17QtGm4u6lqbNPCiFSKdAbcdS9j3XGVHYhI3psyBdq0CZMJDR0KH3wQCvqJZLlUfRSXEFoO+5rZ1BIv1QE+jDswkbxRVASNGoWkcOut8Nvfwq6azFFyxxYH3JlZXaAe8Eeg5HzX37v7txmIrUwacCc5Y9EiuPLKULhv9mxoWHreLpHMiWvAnbv7fKAf8H2JB2a2c0XeTKQgbNwYym60aBGmIb32WmjQIOmoRCosVS/a3wh3PE0k3B5bcsYgB/aNMS6R3LR2LRxxBIwfD927w5Ah4bZXkRy2xUTh7idGP5tkLhyRHLV+PVSrBjVrhruarroKzjpL81VLXkin1tNhZrZD9Pw3Zna/mWnYqAiEMREvvBBaDZMmhXV33w1nn60kIXkjndtjHwV+MLM2hMqxC4D/izUqkVwwbx6ccEKYG6J+fU1DKnkrnb/sDR5ujToFeNDdHyTcIitSuO6/H1q1CnNWP/AAjBsHbdsmHZVILNIZEvq9mV0HnAd0MbMqQLV4wxLJcqtWwfHHhwJ+jVR1X/JbOi2Ks4B1wIXu/g3QELg31qhEss3SpdC7N4waFZZvvBFefFFJQgpCOlOhfgM8DdQ1sxOBte7+ZOyRiWSDTZvgiSfggAPgqadg7tywXv0RUkDSuevpTGAccAZwJvCJmfWMOzCRxM2cCV27hpIbLVvC5MnhtleRApNOH8UNQAd3XwxgZrsAbwMvxBmYSOImTIAZM+Dxx0O1V7UipEClkyi2K04SkWWk17chkntefz1MKHTeeeFx4omwsyrWSGFL5wP/TTMbbWa9zKwX8BrwerxhiWRYURH07BnGRQweHAbSmSlJiJBeZ/Y1wJ+B1kAbYKi7D4w7MJGM2LAh3OLaogW89hrceWcYG6FR1SI/STUfRVPgPmA/YBpwtbt/lanARDJi4kS44oowDekjj8C+qnUpUlqqFsUTwKvA6YQKsg9nJCKRuK1YASNGhOeHHAKffBL6JpQkRMqUqjO7jrv/JXo+x8wmZSIgkdi4w3PPhRbEsmUwfz7suSd07Jh0ZCJZLVWiqGlmB7F5HopaJZfdXYlDcse//w39+sHo0dCuHbzySkgSIlKuVInia+D+EsvflFh24Ki4ghKpVN9/H5LDpk3w0ENw6aVQpUrSUYnkjFQTFx2ZyUBEKt3UqdC6NdSpEwbNdeqkeatFKkAD5yT/LFkCF1wAbdqETmqA009XkhCpoFgThZn1MLM5ZjbXzAal2K6DmW1UDSnZJps2wbBhoYDfM8/A9deHWk0isk3SKeFRIdG8FY8A3YEiYLyZjXL3mWVsdzcwOq5YpECcfjq8/DIccQQ8+mgo5Cci2yyd6rEWzZV9c7S8t5mlcz9hR2Cuu89z9x+BZwmz5JXWH3gRWFzGayKprV4dRlcDnHMODB8O772nJCFSidK59DQEOBQ4J1r+ntBSKE9DYGGJ5aJo3U/MrCHwK+CxVAcysz5mNsHMJoRZWUUIt7i2bAlDhoTlM88MfRMqvyFSqdJJFIe4ez9gLYC7Lweqp7FfWf9bS3/KPwAMdPeNqQ7k7kPdvb27tzd9CMjChXDaaXDyyeGOpnbtko5IJK+l00exPupHcPhpPopNaexXBOxVYrkRsKjUNu2BZ6MP/wbA8Wa2wd1fTuP4Uoieegr69g0d13/6E1x5JVRP53uLiFRUOoniIeAlYFczuxPoCdyYxn7jgaZm1gT4CjgbOLfkBu7epPi5mQ0HXlWSkDIVl/1u1CjcyfTww9CkSbm7ici2KzdRuPvTZjYR6Ea4nHSqu89KY78NZnYZ4W6mKsAT7j7DzPpGr6fslxAB4Lvv4LrrYIcd4L77QpLQLa8iGVVuojCzvYEfgFdKrnP3L8vb191fp9QkR1tKEO7eq7zjSQFxD2MhrroqDKC78srNrQoRyah0Lj29RuifMKAm0ASYA7SKMS4pZF98AX36wNtvQ4cO8MYbcNBBSUclUrDSufR0YMllMzsYuDi2iETWrw91mh55BC6+WAX8RBK21SOz3X2SmXWIIxgpYO+8E6Yivf9+aNYMFiyAmjWTjkpESK+P4qoSi9sBBwNLYotICst//gMDBsDTT8N++8ENN0D9+koSIlkknQF3dUo8ahD6LMoqxSGSvk2b4M9/hubNw6xzN90E06aFJCEiWSVliyIaaFfb3a/JUDxSKFasgBtvhLZtQwG/5s2TjkhEtmCLLQozqxqV1jg4g/FIPlu1KvRBbNwI9erBJ5/Au+8qSYhkuVQtinGEJDHZzEYBzwOri1909xExxyb5ZORI6N8/1Glq2xaOOgr23TfpqEQkDen0UewMLCPMkX0icFL0U6R8CxbAKafAqafCTjvBhx+GJCEiOSNVi2LX6I6n6WwecFdMtb6lfO7QsyfMnAn33ANXXAHVqiUdlYhspVSJogpQm/TKhYts9vHH0KpVKAE+dCjsvDPss0/SUYlIBaVKFF+7+20Zi0Ry37ffhgJ+Q4fCzTfDrbeq9IZIHkiVKFR9TdLjHuaJGDAgJIsBA+Aa3VEtki9SJYpuGYtCctv114dJhDp1gjFjoE2bpCMSkUq0xUTh7t9mMhDJMWvXhnERDRpA796hD6JPH9gunRvpRCSX6H+1bL0xY+DAA+F3vwvLzZqF6UmVJETykv5nS/q++QbOPReOOSZMIHTZZUlHJCIZsNVlxqVA/eMf8KtfwZo18Ic/wMCBqvAqUiCUKCS19evDILnWraF7d7jzznCpSUQKhi49Sdm+/z7MU92lSyjiV78+PP+8koRIAVKikP/mDiNGQIsW8OCDYcDcunVJRyUiCVKikM2WLoWTToLTTw+3vX70UZgrYvvtk45MRBKkRCGb1akTpia9/36YMCEMoBORgqdEUeg++ACOOy4MnqtRI0wmdOWVUFX3OYhIoERRqJYtg4suCp3VM2fCvHlhvQbNiUgp+lQoNO4wfDgccED4ec01IVG0bp10ZCKSpXR9oRA9+WRIFI89FkpxiIikoBZFIVizBm65BYqKQumNF1+E999XkhCRtChR5LvRo+EXv4DbboORI8O6evXUFyEiadOnRb5atAjOOgt69AglON59F/r1SzoqEclBShT56o47QgvitttgyhQ48sikIxKRHGXunnQMW6XWns18zaLPkg4jO02cuLmA37JlsHw57L9/0lGJSBYws4nu3r4i+8baojCzHmY2x8zmmtmgMl7/tZlNjR4fmZnm0KyIlSvh8suhY8cwLSmEIn5KEiJSCWJLFGZWBXgEOA5oCZxjZi1LbfYF8Et3bw3cDgyNK5685B4qujZvDoMHwyWXwFNPJR2ViOSZOMdRdATmuvs8ADN7FjgFmFm8gbt/VGL7j4FGMcaTf/72N/jNb0KF15EjoUOHpCMSkTwUZ6JoCCwssVwEHJJi+98Cb5T1gpn1AfoA1Ni9wC+n/PhjKLfRvDn07BnGSPTqpdpMIhKbOPsorIx1Zfacm9mRhEQxsKzX3X2ou7d39/ZmZR22QIwdC23bhjmr164NRfwuukhJQkRiFWeiKAL2KrHcCFhUeiMzaw0MA05x92UxxpO7li6F3r3hl78MLYjHHtN81SKSMXF+FR0PNDWzJsBXwNnAuSU3MLO9gRHAee6ue17LMm9e6HtYuRIGDYKbbtJEQiKSUbElCnffYGaXAaOBKsAT7j7DzPpGrz8G3AzUB4ZEl5Q2VPQ+37yzciXsuCM0aRJaE716hVIcIiIZpgF32eaHH+D222Ho0DCiupFuBBORbbctA+7UC5pNXnsNLrsM5s8PrYhatZKOSEREiSIrbNgA55wDL7wALVrAP/8JRxyRdFQiIoCKAiar+LJf1aqw225w110webKShIhkFSWKpIwfD4ccApMmheXBg+G666B69WTjEhEpRYki01asCP0QhxwSZpxbpqEjIpLdlCgyqbiA36OPhmQxezZ07550VCIiKakzO5NmzYKGDeGVV6C9houISG7QOIo4rVsH994LbdrASSfB+vVhruoqVZKOTEQKTNZOXFTQ/vGPkCBuugneeSesq1ZNSUJEco4SRWVbvBguuACOOiq0IN54Ax54IOmoREQqTImisr31FjzzDNxwA0yfDj16JB2RiMg2UWd2ZZg2DebMCRMJ/frX0Lkz7Ltv0lGJiFQKtSi2xerVcO21YSrSa68Nl5rMlCREJK8oUVTUK69Ay5bhrqZevcJI62rVko5KRKTS6dJTRUyfDiefDK1awfvvw+GHJx2RiEhs1KJI14YN8N574fkvfgGvvgqffqokISJ5T4kiHZ98EkZSd+sGn38e1p1wgi41iUhBUKJIZflyuOQSOPRQWLo01Graf/+koxIRySj1UWzJunXhbqaFC+GKK+DWW6FOnaSjEhHJOCWK0r76KhTuq1ED/vCHUIbjoIOSjkpEJDG69FRs7drQath3Xxg5Mqzr1UtJQkQKnloUEIr2XXJJ6Kg+55wwqZCIiABqUYT+h6OPDvNXv/UW/O1vsPvuSUclIpI1CjNRbNoEGzeG5x07ws03h3pNmm1ORORnCi9RTJkSivY98khYPvfc0DdRs2aycYmIZKnCSRSrVsGAAdCuHcybp8tLIiJpKozO7Lffht69oagI+vSBP/0J6tVLOioRkZxQGImienXYeWf4+9/DZScREUlbfiaK9evD9KMrVsAdd8ARR4QCftsVzpU2EZHKkn+fnB99FPohrr0WZs0KdziBkoSISAXlz6fnt9+G/ofDDoPvvoOXX4YXX1SCEBHZRvnzKbpsWRgsd/XVMHMmnHJK0hGJiOSF3O6jmDMndFDffDM0bQoLFkD9+klHJSKSV2JtUZhZDzObY2ZzzWxQGa+bmT0UvT7VzA5O68Br1oTk0Lo1/M//hFLgoCQhIhKD2BKFmVUBHgGOA1oC55hZy1KbHQc0jR59gEfLO27tdavhwAPh9tvhjDNg9mzYa69Kjl5ERIrF2aLoCMx193nu/iPwLFC64+AU4EkPPgZ2MrM9Uh204Xf/CR3Ub78NTz0Fu+0WT/QiIgLE20fREFhYYrkIKF2/u6xtGgJfl9zIzPoQWhwA6+zzz6dz9NGVG21uagAsTTqILKFzsZnOxWY6F5sdUNEd40wUVsY6r8A2uPtQYCiAmU1w9/bbHl7u07nYTOdiM52LzXQuNjOzCRXdN85LT0VAyc6DRsCiCmwjIiIJijNRjAeamlkTM6sOnA2MKrXNKOD86O6nTsAKd/+69IFERCQ5sV16cvcNZnYZMBqoAjzh7jPMrG/0+mPA68DxwFzgB6B3GoceGlPIuUjnYjOdi810LjbTudiswufC3H/WJSAiIvKT/CnhISIisVCiEBGRlLI2UcRW/iMHpXEufh2dg6lm9pGZtUkizkwo71yU2K6DmW00s56ZjC+T0jkXZtbVzCab2Qwz+2emY8yUNP6P1DWzV8xsSnQu0ukPzTlm9oSZLTaz6Vt4vWKfm+6edQ9C5/e/gX2B6sAUoGWpbY4H3iCMxegEfJJ03Amei85Avej5cYV8Lkps9y7hZomeSced4N/FTsBMYO9oedek407wXFwP3B093wX4FqiedOwxnIsjgIOB6Vt4vUKfm9naooil/EeOKvdcuPtH7r48WvyYMB4lH6XzdwHQH3gRWJzJ4DIsnXNxLjDC3b8EcPd8PR/pnAsH6piZAbUJiWJDZsOMn7uPJfxuW1Khz81sTRRbKu2xtdvkg639PX9L+MaQj8o9F2bWEPgV8FgG40pCOn8XzYB6ZvaemU00s/MzFl1mpXMuBgMtCAN6pwG/d/dNmQkvq1ToczNb56OotPIfeSDt39PMjiQkisNjjSg56ZyLB4CB7r4xfHnMW+mci6pAO6AbUAv4l5l97O6fxR1chqVzLo4FJgNHAfsBY8zsfXdfGXNs2aZCn5vZmihU/mOztH5PM2sNDAOOc/dlGYot09I5F+2BZ6Mk0QA43sw2uPvLGYkwc9L9P7LU3VcDq81sLNAGyLdEkc656A38ycOF+rlm9gXQHBiXmRCzRoU+N7P10pPKf2xW7rkws72BEcB5efhtsaRyz4W7N3H3xu7eGHgBuDQPkwSk939kJNDFzKqa2faE6s2zMhxnJqRzLr4ktKwws90IlVTnZTTK7FChz82sbFF4fOU/ck6a5+JmoD4wJPomvcHzsGJmmueiIKRzLtx9lpm9CUwFNgHD3L3M2yZzWZp/F7cDw81sGuHyy0B3z7vy42b2DNAVaGBmRcAtQDXYts9NlfAQEZGUsvXSk4iIZAklChERSUmJQkREUlKiEBGRlJQoREQkJSUKyUpR5dfJJR6NU2y7qhLeb7iZfRG91yQzO7QCxxhmZi2j59eXeu2jbY0xOk7xeZkeVUPdqZzt25rZ8ZXx3lK4dHusZCUzW+XutSt72xTHGA686u4vmNkxwH3u3nobjrfNMZV3XDP7X+Azd78zxfa9gPbuflllxyKFQy0KyQlmVtvM3om+7U8zs59VjTWzPcxsbIlv3F2i9ceY2b+ifZ83s/I+wMcC+0f7XhUda7qZXRGt28HMXovmNphuZmdF698zs/Zm9iegVhTH09Frq6Kffy/5DT9qyZxuZlXM7F4zG29hnoCL0zgt/yIq6GZmHS3MRfJp9POAaJTybcBZUSxnRbE/Eb3Pp2WdR5GfSbp+uh56lPUANhKKuE0GXiJUEdgxeq0BYWRpcYt4VfRzAHBD9LwKUCfadiywQ7R+IHBzGe83nGjuCuAM4BNCQb1pwA6E0tQzgIOA04G/lNi3bvTzPcK3959iKrFNcYy/Av43el6dUMmzFtAHuDFaXwOYADQpI85VJX6/54Ee0fKOQNXo+dHAi9HzXsDgEvvfBfwmer4Toe7TDkn/e+uR3Y+sLOEhAqxx97bFC2ZWDbjLzI4glKNoCOwGfFNin/HAE9G2L7v7ZDP7JdAS+DAqb1Kd8E28LPea2Y3AEkIV3m7ASx6K6mFmI4AuwJvAfWZ2N+Fy1ftb8Xu9ATxkZjWAHsBYd18TXe5qbZtn5KsLNAW+KLV/LTObDDQGJgJjSmz/v2bWlFANtNoW3v8Y4GQzuzpargnsTX7WgJJKokQhueLXhJnJ2rn7ejObT/iQ+4m7j40SyQnA/5nZvcByYIy7n5PGe1zj7i8UL5jZ0WVt5O6fmVk7Qs2cP5rZW+5+Wzq/hLuvNbP3CGWvzwKeKX47oL+7jy7nEGvcva2Z1QVeBfoBDxFqGf3D3X8Vdfy/t4X9DTjd3eekE68IqI9CckddYHGUJI4E9im9gZntE23zF+BxwpSQHwOHmVlxn8P2ZtYszfccC5wa7bMD4bLR+2a2J/CDuz8F3Be9T2nro5ZNWZ4lFGPrQihkR/TzkuJ9zKxZ9J5lcvcVwOXA1dE+dYGvopd7ldj0e8IluGKjgf4WNa/M7KAtvYdIMSUKyRVPA+3NbAKhdTG7jG26ApPN7FNCP8KD7r6E8MH5jJlNJSSO5um8obtPIvRdjCP0WQxz90+BA4Fx0SWgG4A7yth9KDC1uDO7lLcIcxu/7WHqTghzicwEJpnZdODPlNPij2KZQiirfQ+hdfMhof+i2D+AlsWd2YSWR7UotunRskhKuj1WRERSUotCRERSUqIQEZGUlChERCQlJQoREUlJiUJERFJSohARkZSUKEREJKX/B+4Xqc+xrSi2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y = train_y\n",
    "X = x_train_tf_idf_word.astype(\"float64\")\n",
    "\n",
    "etc_roc_auc = roc_auc_score(y, etc_model.predict(X))\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y, etc_model.predict_proba(X)[:,1])\n",
    "plt.figure()\n",
    "plt.plot(fpr, tpr, label='AUC (area = %0.2f)' % etc_roc_auc)\n",
    "plt.plot([0, 1], [0, 1],'r--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd5d7fe9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}