he_amazon.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading important libraries for reading, analyzing and visualizing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import re\n",
    "import random\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from sklearn.utils import compute_class_weight, compute_sample_weight\n",
    "from sklearn.preprocessing import LabelEncoder, MultiLabelBinarizer\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.model_selection import RandomizedSearchCV, KFold\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "from xgboost import XGBClassifier, XGBRFClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "from keras.preprocessing import text, sequence\n",
    "from keras.models import Model\n",
    "from keras.layers import Input, LSTM, Bidirectional, Dense, Dropout, Conv1D, GlobalAveragePooling1D, Embedding, BatchNormalization, MaxPool1D, Activation\n",
    "from keras.optimizers import Adam\n",
    "from keras.metrics import top_k_categorical_accuracy\n",
    "from keras.callbacks import ReduceLROnPlateau, EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Review Text</th>\n",
       "      <th>Review Title</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Did nothing for me, didn't help lost even with...</td>\n",
       "      <td>Useless</td>\n",
       "      <td>Shipment and delivery</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Did nothing for me, didn't help lost even with...</td>\n",
       "      <td>Useless</td>\n",
       "      <td>Not Effective</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I have bought these bags and  immediately open...</td>\n",
       "      <td>TRASH!!! Do not buy these bags it’s a waist of...</td>\n",
       "      <td>Customer Service</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Gave me an allergic reaction on my face :(</td>\n",
       "      <td>Do not recommend</td>\n",
       "      <td>Allergic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>These don't compare to the name brand wipes. F...</td>\n",
       "      <td>Can't tackle big messes</td>\n",
       "      <td>Texture</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         Review Text  \\\n",
       "0  Did nothing for me, didn't help lost even with...   \n",
       "1  Did nothing for me, didn't help lost even with...   \n",
       "2  I have bought these bags and  immediately open...   \n",
       "3         Gave me an allergic reaction on my face :(   \n",
       "4  These don't compare to the name brand wipes. F...   \n",
       "\n",
       "                                        Review Title                  topic  \n",
       "0                                            Useless  Shipment and delivery  \n",
       "1                                            Useless          Not Effective  \n",
       "2  TRASH!!! Do not buy these bags it’s a waist of...       Customer Service  \n",
       "3                                   Do not recommend               Allergic  \n",
       "4                            Can't tackle big messes                Texture  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reading training dataframe and displaying top 10 rows\n",
    "train_df = pd.read_csv('train.csv')\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Review Text</th>\n",
       "      <th>Review Title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I use chia seed in my protein shakes. These ta...</td>\n",
       "      <td>Bad tast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I use chia seed in my protein shakes. These ta...</td>\n",
       "      <td>Bad tast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Don’t waste your money.</td>\n",
       "      <td>No change. No results.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I use the book 'Fortify Your Life' by Tieraona...</td>\n",
       "      <td>Good Vegan Choice, Poor Non Vegan Choice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I use the book 'Fortify Your Life' by Tieraona...</td>\n",
       "      <td>Good Vegan Choice, Poor Non Vegan Choice</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         Review Text  \\\n",
       "0  I use chia seed in my protein shakes. These ta...   \n",
       "1  I use chia seed in my protein shakes. These ta...   \n",
       "2                            Don’t waste your money.   \n",
       "3  I use the book 'Fortify Your Life' by Tieraona...   \n",
       "4  I use the book 'Fortify Your Life' by Tieraona...   \n",
       "\n",
       "                               Review Title  \n",
       "0                                  Bad tast  \n",
       "1                                  Bad tast  \n",
       "2                    No change. No results.  \n",
       "3  Good Vegan Choice, Poor Non Vegan Choice  \n",
       "4  Good Vegan Choice, Poor Non Vegan Choice  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reading test dataframe and displaying top 10 rows\n",
    "test_df = pd.read_csv('test.csv')\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAF4CAYAAABDzx54AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsnXe4XFX1v99P6C20BERaQFBEBIVQBFQECyC9SBVEFFFAEAsI+kPAgh1FREBAmhQJXaRI75BI6KBIjaCEIvAVBAPr98fakzt3Mndm76k3uet9nnnunDNnnb1n7jln7b1Xk5kRBEEQBLmM6ncHgiAIgpmLUBxBEARBEaE4giAIgiJCcQRBEARFhOIIgiAIigjFEQRBEBQRiiMIgiAoIhRHEARBUEQojiAIgqCI2fvdgW4wZswYGzduXL+7EQRBMFMxadKk58xsbLPjZknFMW7cOCZOnNjvbgRBEMxUSHoi57hYqgqCIAiKCMURBEEQFBGKIwiCICgiFEcQBEFQRNcUh6STJT0r6b6qfT+W9JCkeyRdIGmhqs++KekRSQ9L+kTV/o3TvkckHdyt/gZBEAR5dHPG8Ttg45p9VwGrmNmqwF+BbwJIWhnYEXhPkvm1pNkkzQYcC2wCrAzslI4NgiAI+kTXFIeZ3QC8ULPvSjObljZvA5ZK77cEzjaz183sMeARYK30esTMHjWzN4Cz07FBEARBn+injeOzwJ/S+yWBp6o+m5L2DbV/BiTtJWmipIlTp07tQneDIAgC6FMAoKRDgWnAmZVddQ4z6iu2ukXSzewE4ASA8ePHRyH1WYA9Lqhd6WzMKVtf3qWeBEFQTc8Vh6Tdgc2Ajcys8oCfAixdddhSwNPp/VD7gyAIgj7Q06UqSRsDBwFbmNmrVR9dDOwoaS5JywErAncAdwIrSlpO0py4Af3iXvY5CIIgGEzXZhySzgI2AMZImgIchntRzQVcJQngNjPb28zul3Qu8AC+hLWPmb2ZzrMvcAUwG3Cymd3frT4HQRAEzema4jCznersPqnB8d8Dvldn/2XAZR3sWhAEQdAGETkeBEEQFBGKIwiCICgiFEcQBEFQRCiOIAiCoIhQHEEQBEERoTiCIAiCIkJxBEEQBEWE4giCIAiKCMURBEEQFBGKIwiCICgiFEcQBEFQRCiOIAiCoIhQHEEQBEERoTiCIAiCIkJxBEEQBEWE4giCIAiKCMURBEEQFBGKIwiCICgiFEcQBEFQRCiOIAiCoIhQHEEQBEERoTiCIAiCIkJxBEEQBEWE4giCIAiKCMURBEEQFBGKIwiCICiia4pD0smSnpV0X9W+RSRdJelv6e/Cab8k/VLSI5LukbR6lczu6fi/Sdq9W/0NgiAI8ujmjON3wMY1+w4GrjazFYGr0zbAJsCK6bUXcBy4ogEOA9YG1gIOqyibIAiCoD90TXGY2Q3ACzW7twROTe9PBbaq2n+aObcBC0laAvgEcJWZvWBmLwJXMaMyCoIgCHpIr20ci5vZMwDp72Jp/5LAU1XHTUn7hto/A5L2kjRR0sSpU6d2vONBEASBM1yM46qzzxrsn3Gn2QlmNt7Mxo8dO7ajnQuCIAgG6LXi+FdagiL9fTbtnwIsXXXcUsDTDfYHQRAEfaLXiuNioOIZtTtwUdX+3ZJ31TrAS2kp6wrg45IWTkbxj6d9QRAEQZ+YvVsnlnQWsAEwRtIU3DvqKOBcSXsCTwLbp8MvAzYFHgFeBfYAMLMXJB0J3JmOO8LMag3uQRAEQQ/pmuIws52G+GijOscasM8Q5zkZOLmDXQuCIAjaYLgYx4MgCIKZhFAcQRAEQRGhOIIgCIIiQnEEQRAERYTiCIIgCIoIxREEQRAUEYojCIIgKCIURxAEQVBEKI4gCIKgiFAcQRAEQRGhOIIgCIIiQnEEQRAERYTiCIIgCIoIxREEQRAUEYojCIIgKCIURxAEQVBEKI4gCIKgiFAcQRAEQRGhOIIgCIIiQnEEQRAERYTiCIIgCIoIxREEQRAUEYojCIIgKCIURxAEQVBEKI4gCIKgiL4oDklfkXS/pPsknSVpbknLSbpd0t8knSNpznTsXGn7kfT5uH70OQiCIHB6rjgkLQl8GRhvZqsAswE7Aj8Efm5mKwIvAnsmkT2BF81sBeDn6bggCIKgT/RrqWp2YB5JswPzAs8AGwLnpc9PBbZK77dM26TPN5KkHvY1CIIgqKLnisPM/gH8BHgSVxgvAZOAf5vZtHTYFGDJ9H5J4KkkOy0dv2gv+xwEQRAM0I+lqoXxWcRywNuB+YBN6hxqFZEGn1Wfdy9JEyVNnDp1aqe6GwRBENTQj6WqjwKPmdlUM/sfcD6wLrBQWroCWAp4Or2fAiwNkD5fEHih9qRmdoKZjTez8WPHju32dwiCIBix9ENxPAmsI2neZKvYCHgAuBbYLh2zO3BRen9x2iZ9fo2ZzTDjCIIgCHpDP2wct+NG7r8A96Y+nAAcBBwo6RHchnFSEjkJWDTtPxA4uNd9DoIgCAaYvfkhncfMDgMOq9n9KLBWnWP/C2zfi34FQRAEzYnI8SAIgqCIUBxBEARBEaE4giAIgiJCcQRBEARFhOIIgiAIigjFEQRBEBQRiiMIgiAoIhRHEARBUEQojiAIgqCIvkSOBzMHJ5328WKZPXe7sgs9CYJgOBEzjiAIgqCIUBxBEARBEaE4giAIgiKyFIekq3P2BUEQBLM+DY3jkuYG5gXGpJKvlTKuo/Gyr0EQBMEIo5lX1ReAA3AlMYkBxfEycGwX+xUEQRAMUxoqDjP7BfALSfuZ2TE96lMQBEEwjMmK4zCzYyStC4yrljGz07rUryAIgmCYkqU4JJ0OvAOYDLyZdhsQiiMIgmCEkRs5Ph5Y2cysm50JgiAIhj+5cRz3AW/rZkeCIAiCmYPcGccY4AFJdwCvV3aa2RZd6VUQBEEwbMlVHN/pZieCIAiCmYdcr6rru92RIAiCYOYg16vqFdyLCmBOYA7gP2Y2ulsdC4IgCIYnuTOOBaq3JW0FrNWVHgVBEATDmpay45rZhcCGrTYqaSFJ50l6SNKDkj4gaRFJV0n6W/q7cDpWkn4p6RFJ90havdV2gyAIgvbJXarapmpzFB7X0U5Mxy+Ay81sO0lz4okUDwGuNrOjJB0MHAwcBGwCrJheawPHpb9d51/Hfb9YZvEvHtKFngRBEAwfcr2qNq96Pw14HNiylQYljQY+BHwGwMzeAN6QtCWwQTrsVOA6XHFsCZyWgg9vS7OVJczsmVbaD4IgCNoj18axRwfbXB6YCpwiaTU86+7+wOIVZWBmz0haLB2/JPBUlfyUtG+Q4pC0F7AXwDLLLNPB7gZBEATV5BZyWkrSBZKelfQvSRMkLdVim7MDqwPHmdn7gf/gy1JDNl9n3wzLZGZ2gpmNN7PxY8eObbFrQRAEQTNyjeOnABfjdTmWBC5J+1phCjDFzG5P2+fhiuRfkpYASH+frTp+6Sr5pYCnW2w7CIIgaJNcxTHWzE4xs2np9TugpWG9mf0TeErSu9KujYAHcMW0e9q3O3BRen8xsFvyrloHeCnsG0EQBP0j1zj+nKRdgbPS9k7A8220ux9wZvKoehTYA1di50raE3gS2D4dexmwKfAI8Go6NgiCIOgTuYrjs8CvgJ/j9oVbaOMBbmaTcZfeWjaqc6wB+7TaVhAEQdBZchXHkcDuZvYigKRFgJ/gCiXoEjeeuFmxzAc/f2kXehIEQTBAro1j1YrSADCzF4D3d6dLQRAEwXAmV3GMqqQAgekzjtzZShAEQTALkfvw/ylwi6TzcBvHp4Dvda1XQRAEwbAlN3L8NEkT8cSGArYxswe62rMgCIJgWJK93JQURSiLIAiCEU5LadWDIAiCkUsojiAIgqCIUBxBEARBEaE4giAIgiJCcQRBEARFhOIIgiAIigjFEQRBEBQRiiMIgiAoIhRHEARBUEQojiAIgqCIUBxBEARBEZEaPQiCGdh2wp3FMhO2XbMLPQmGIzHjCIIgCIoIxREEQRAUEYojCIIgKCIURxAEQVBEKI4gCIKgiFAcQRAEQRGhOIIgCIIiQnEEQRAERfRNcUiaTdJdki5N28tJul3S3ySdI2nOtH+utP1I+nxcv/ocBEEQ9DdyfH/gQWB02v4h8HMzO1vSb4A9gePS3xfNbAVJO6bjduhHh2c2Ljx5k2KZrT77py70JAiCWYm+zDgkLQV8Evht2hawIXBeOuRUYKv0fsu0Tfp8o3R8EARB0Af6tVR1NPAN4K20vSjwbzOblranAEum90sCTwGkz19Kxw9C0l6SJkqaOHXq1G72PQiCYETTc8UhaTPgWTObVL27zqGW8dnADrMTzGy8mY0fO3ZsB3oaBEEQ1KMfNo71gC0kbQrMjds4jgYWkjR7mlUsBTydjp8CLA1MkTQ7sCDwQk5DU487o7hzY7+4a7FMEATBSKLnMw4z+6aZLWVm44AdgWvMbBfgWmC7dNjuwEXp/cVpm/T5NWY2w4wjCIIg6A3DKY7jIOBASY/gNoyT0v6TgEXT/gOBg/vUvyAIgoA+F3Iys+uA69L7R4G16hzzX2D7nnYsCIIgGJLhNOMIgiAIZgJCcQRBEARFhOIIgiAIigjFEQRBEBQRiiMIgiAoIhRHEARBUEQojiAIgqCIUBxBEARBEaE4giAIgiJCcQRBEARFhOIIgiAIigjFEQRBEBQRiiMIgiAoIhRHEARBUEQojiAIgqCIUBxBEARBEaE4giAIgiJCcQRBEARFhOIIgiAIigjFEQRBEBQRiiMIgiAoIhRHEARBUMTs/e7ArMzffrVlscyK+17UhZ4EQRB0jlAcQdf44dmfKDr+oB2v6FJPgiDoJKE4glmWTS88uOj4y7Y6qks9CYJZi57bOCQtLelaSQ9Kul/S/mn/IpKukvS39HfhtF+SfinpEUn3SFq9130OgiAIBujHjGMa8FUz+4ukBYBJkq4CPgNcbWZHSToYOBg4CNgEWDG91gaOS3+DYJZls/P+UCxz6Xbbd6EnQTAjPZ9xmNkzZvaX9P4V4EFgSWBL4NR02KnAVun9lsBp5twGLCRpiR53OwiCIEj01R1X0jjg/cDtwOJm9gy4cgEWS4ctCTxVJTYl7QuCIAj6QN8Uh6T5gQnAAWb2cqND6+yzOufbS9JESROnTp3aqW4GQRAENfRFcUiaA1caZ5rZ+Wn3vypLUOnvs2n/FGDpKvGlgKdrz2lmJ5jZeDMbP3bs2O51PgiCYITTD68qAScBD5rZz6o+uhjYPb3fHbioav9uybtqHeClypJWEARB0Hv64VW1HvBp4F5Jk9O+Q4CjgHMl7Qk8CVRcRC4DNgUeAV4F9uhtd4MgCIJqeq44zOwm6tstADaqc7wB+3S1U0EQBEE2keQwCIIgKCIURxAEQVBEKI4gCIKgiFAcQRAEQRGhOIIgCIIiQnEEQRAERUQ9jiCYBdnyvPKiWBdtV1Z4Kxi5xIwjCIIgKCJmHEEwBJ88/+ii4/+4zQFd6kkQDC9ixhEEQRAUEYojCIIgKCKWqoIgGHacen55TZ3dt4lyCr0iFEcQBB3nyxc81fygGn659dLNDwqGBaE4gqALbDbhlGKZS7eNigHBzEHYOIIgCIIiQnEEQRAERYTiCIIgCIoIxREEQRAUEYojCIIgKCIURxAEQVBEKI4gCIKgiFAcQRAEQRERABgEQVDDfcf/q1hmlS8s3oWeDE9CcQRBMMtx9e/Lc11ttHPkusolFEcQBEGH+eePnyiWedvXl+1CT7pD2DiCIAiCImYaxSFpY0kPS3pE0sH97k8QBMFIZaZQHJJmA44FNgFWBnaStHJ/exUEQTAymVlsHGsBj5jZowCSzga2BB7oa6+CIAi6wL+OvrNYZvED1pz+/tlfXVEsv9i+n8g+VmZW3ECvkbQdsLGZfS5tfxpY28z2rTpmL2CvtPku4OEGpxwDPNdGl0I+5EN+ZMrPzH3PkV/WzJq6l80sMw7V2TdI45nZCcAJWSeTJprZ+JY7E/IhH/IjUn5m7nsn5CvMFDYOYApQXVdyKeDpPvUlCIJgRDOzKI47gRUlLSdpTmBH4OI+9ykIgmBEMlMsVZnZNEn7AlcAswEnm9n9bZwya0kr5EM+5EN+GLU9HOSBmcQ4HgRBEAwfZpalqiAIgmCYEIojCIIgKCIURxAEQVDEiFAckiZI+qSkvnxfSfNK+rakE9P2ipI260dfZjYkzSbp7jbPcYqkPSSt0Kl+Fba/Tc6+oDtI2j9nX8Z5Fpa0amd61RskbShp3o6fdyQYxyV9FNgDWAf4A/A7M3uoQH5x4PvA281sk5Qn6wNmdlKm/DnAJGA3M1tF0jzArWb2vtLv0grpgXks8DYzWy1d/J80sx/0qP12f7+zgK+Z2T9abP/jwPrAB/F4oEnADWZ2bBO5Y6gJNK3GzL6c2f5fzGz1mn2TzGyNHPk651sYWNrM7imQmQicAvzezF5spd1SJB3Y6HMz+1nBuVavs/sl4Akzm9ZEtt7vf5eZvT+j3euALXAP1MnAVOB6M2v43ark3wkcByye7v1VgS3M7LuZ8j8Cvgu8BlwOrAYcYGZnZMqfhj/3ngduTK+b2r0GRoTiqCBpQWAn4FDgKeBE4Awz+18TuT/hN92h6cE7O3CXmb03s92JZja++mKVdLeZrZYpfxWwvZn9O20vDJxtZlnJZdLFfwhwrJm9X5KA+8zsPRmy99L44dl0BNaB3+8qYG3gVuA/VW1nj9rTbHN1YCNgH+ANM2s4A5G0e3q7Hp5c85y0vT0wycy+0kT+E8DGwM7AmVUfjQZWM7M16wrWP9d1tPcAWwEfPO0AVJTIlZb5AJD0yzq7XwImmtlFQ8gclt6+C1iTgdirzXHF/bmcttO5bsP/f/fgmSRWSe8XBfY2syvryOyE//br4w/MCgsAb5rZRzPavSvdM5/DlfVhku7Jue6T/PXA14Hjq+79+8xslUz5yWb2PklbA1sBXwGuzX12VJ3n7cB2wNfwAVxboRgzRRxHJ5C0KLAr8GngLvxGXh/YHdigifgYMztX0jdhelzJmwXNv5FmGZb68g7g9QL5MRWlkdp/UdJiBfLzmdktri/AzExSQ2VZRWVJbZ/09/T0dxfg1cxztPv7HVVw7AxIugJYEA8kvRFYx8yaZh4ws1OT/GeAj1QGGJJ+A8zwoKrDs8B9wH+B6rijV4DS0gALmtnL6QF2SuUBlitsZo8Ah0r6Nv4/PRl4S9LJwC/M7IUmp5gbWAmfsQNsi3+nPSV9xMwOqNPm4QCSrgRWN7NX0vZ3qs6Ty+PAnpX4rTRr/TpwJHA+9f8ftwDP4PmZflq1/xVc6eQwu6QlgE/hA85S5jWzOyr3XqLhDKmGOdLfTYGzzOyFmnM1RNKu+Ez7vXiOql8xWIm2xIhQHJLOxy/604HNzeyZ9NE5aQrfjP8kxVN58K+Dj7ZyOQyfZi4t6Ux8BPuZAvm3JC1jZk+m9pelwSygDs9LWo6B/m8F/DNH0MyeSDLrmdl6VR8dLOlm4IiM07T1+5nZ1ZLGAJUcOxPNrCTR21+B9wMrAv8C/inpeTPLVd5vx0eplYfr/Glfs37fBdyV/udvAcukB3grtPsAIy2T7IE/hCYwMHi6Bmi2bLoCsGFlWUjScfjD+mPAvU1klwHeqNp+AxhX2P2VqoN+zewBSe83s0eHepCma/cJ4AOFbVVzOB54fJOZ3SlpeeBvBfLPpYFi5drfDldmuVwi6SF8qepLksbiA5Fcjgb+DvwGn6k8XiA7NGY2S79wB4D/1+Y51gBuxh92N+MPolULz7Eo8El8tDemUHZj4Elc8Z2O3wyfKJBfAX84vJpkbwPGFfZhMrB+1fa6wORM2dXb+f3w0e2T+IPu9/joc+sW/o8LAvul3+C1Ark9kszv0usxYPcC+U/i2ZofS9vvAy4o7Pt2+Cj512l7eWBCgfwk4Gp86Waums/Oz5B/GJ/1VP+WD6X3dzWRPRS4G/gOPoiaDBxS+P3PwW0FH06vXwPnAnMBdzaR3QZ/2L8EvIzPOF7OaHM24Cul11nNOZYH/pzuvX8AN7Vw7y0MzJbez4fbKkvk3wN8Md0/dwCnt/OdzGzWVxzph7u1A+eYPf0DVgHmKJTduuamWwjYqvAcY3ClszkFiidd/Num9wsCC7X4/ddIN//j6cE5GV9+aCY3Clcy7fx+d+PGxcr24sDdBfJ7p5vmr8D1+PLGxwv78Da8BsyWLdy4k9L//K6qffcWnmO9nH0N/gdFD+o659gz/d9PwZXno8Dn0oPsxxnyqwP7p9f7W2h/HuCrwAXAhfha/bzpu83fRPYR4N0tfu9r2/zd5kp/5wMWSO8XKZCfF/gWcELaXhHYrEB+NF4A7yh80PYwcGo738nMRoZxXNLh+GjtfGvhCyd30HOAc8zs7y3IT7YaD6ocrw5JK5nZQ0N4lGBmf8ls/0Yz+2B+jxueazTuVJG91CTpVjNreblA0r1WZUhPhu67Ld+4/k3gBnxk+kaz44c4x8L4TTt3ZZ+Z3ZApe5uZrVPjHJFtYE3H1/MMmmFfA/kbzOxDue0NcY4l8KJqAu6wDDtRlez6wIpmdkpabpnfzB5rpz8Fbd9sg5dZS2S/hw+4zmGwY0buvfdHYEsbWOJ7G/BHy/SoU5semckOdlN63WBmU3LkmjEibBzAgbjGf1PSa/iFb2Y2OlN+C9wb5VxJb+EX0bmWbA4Z1IsfyfntD8SLU/20zmcGbJjZ/hWSDmDGi//lTPl2XWqvlLQtLSpu4CpJl+HLVODZkbNLnJnZDyStAnw2rYffaAVJMpNBen88nf9k3L3xVvJ//wclfQoYlWxN++PLhTltfwCfsY3VYPfW0fhsMperJH2NGa+BZkbxakbh3lyzAytIWiFHeSbvqvG4d9UpuMH3DNzWl4Wk9fClrmWpunfMbPkM8YnpAXwhVU4pZnZ+huy66W+1La/k3rsQOC9d/0vjnmVfy5QFeIeZ7ZA8xDCz11RgHa8MTiTNZ2b/aXZ8LiNixtFJJK0IfBvYxcyybtzkufJvPJbC8HX2hc3sM93qZ037T9XZbWa2TME5WnaplfQKrrin4Ya9IsWdbpTtcUOu8NnDeblKSNI+uFfYhWnXlrhr8q8z5e/F3UlvM3eNXAk43Mx2yJSfD/h/wMfTriuAI8ysqVeapA/jXn974wbOCq8Al5hZlqFWUr3RvWU+eJH0Q3zwdD9u6K/Ib5EhOxl3TvhLGzOuh3BX1EnAdI88M3s+Q/aUOrvNzD6b2347pOtvY9wh4AtmdkuB7C24C/nNZrZ6MrSfZWZrZcp/ADgJn+EtI2m11IcvlX6PQecdCYojPXh2AZYzsyMlLQ0sYWZ3FJxjHO7RsgN+4Z5jZvVmAvVk58OVzUfxB9+VwHdzRwCqH2X8Er5O/mzOOdpF0p1mtmbNcssMS3Bdans/4A9mluUJVkf+HmBdM/u/tD0/cEvug6vqu0/GSxa/3sp3lzSX5Xty1coua8nDrR9Iehh3aCjuv6Q7zGytytJauh9uLVQct5vZ2qVtt4s89uswoLLMdz2u9Bsu1dbMDoWHAdyLhwJgmcGPkj6G2zhWxp8b6wGfMbPrMuVvxx0rLrYW4kiGYqQsVf0aHyVtiBtG/w8f/WcFYKUffw7c93x7M3u0pPGkIEr99qvZE3cpvDZtb4AvdbxT0hFmdvpQggCSdh6iX7+vt38IWnaplVR3bT3XRgAsBlwn6RngbNybqMQdV0B13Mr/0r5cpkhaCJ+xXCXpRQoqUEpaG/gtvlZeGfV9zsz2K+jDXJJOwEet1Us1DZdMJG1oZtcMMfjIXa4BN4bPQVn8UYVzJR0PLCTp88Bn8eDbEq6V9GM8ZqN6uamprUHtRW+fjMfifCptfxqfeTcLPl2gZvuCIfY3xMyukvQXfHlUwP6F1z5m9lTN6lZJDFVdRsqMozLSaTVyeyUrSFFSJXe0mR0g6RLqxF3kTPPTeS7BHzT/StuL4zfC53CDV8PRg9znvsLcuAKdZGWR16sDx+BeUfcBY3El2jSPVOp/dftrpfZz14mr+7AD7qX2qJltnCn3DTxjwIS0a2t8uv+TkvbTuT6MK4DLcw3t8qjnHYALWx31yR00fsOMSzWTmsgdbh4s2NZyjaQJeLqLqxn84M5Nu/IxfKlOwBVmdlWOXJX8tXV2W841pDait+vNLHs1005ttTXoknQe8DM88G8d4MvAeDPbsZ1+jZQZx/8kzcbAaHksA+u0QyJpV/OcMJtK2rT284zpZmUmUPyAqmFcRWkkngXeaR5F2jQC3My+WL0t9xD6XWEf7sf959+F3/wPk5kk08w2r2l/aeBHhe2Dp4l5HA+gyrbPmNmP0oPng3jf9zazO0sblyeL+w/wQK7SSIwysyfaHPVNM7Pjmh82GDM7LP3do1S2hotpsVyzpK/gS41FyqIaM/tIq7K0F739mqT1zewmmG6kf62ZUKcGjbjCqzB90EW+cX5v4BfAksAUfLlrn4YSGYwUxfFLfKq4mNy9bjt83bAZ86W/9aaXTadqVaPBRYDLWl3fBm6UdCmD0z3ckNaK/z202JC8AryzUOZWc9fP6d5IaQqd5Q5awxR85pJFWt7YAfdqmgDsZxkJ/uSuwxUeTq/pn1kTrzJJW+DXzgv49XIsHnk+TtJBllKSZPCUpLUASwOY/fCYkhIukfQl/DquHvFneUWlZcbDcAcDw90zj8gxLqd2cr9rPUbjnn0v4EuN59UMhIakMnjTEAkTM20F7URv7w2clmwdAC/iaYqa0ZFBY7uDrrSstUs7fajHiFiqAl9uwr0TBFxtZg8WyK5nZjc329dA/hR8hHADfuNcYU0yetbIC19TrXgV3YSv8+d6FV3AgKIbhQfiXWRmTd0C5X7nS+LukzszYBsYDfzGzFbKOEd1ltlReOT042a2a2b/f4IndcxJD1Mt91Rqt9LnSh8qXl0NZy1peWh7fGnqWtw4/Kg8T9jVlh9HshiugCpJ9f4M7FuyVq32vaKuwq+/SlbVXYANrEmiP0nnmtmnNESyy0ID96r4AGBbYEqztpPMF8zseA0kTKxt//CMcyyP19peF3/wPwbsahnpNyQtZ2aPVQYh5vnClrM2YlAknWOZHnl1ZAXmSw0sAAAgAElEQVTcU3DtjQU+z4y2sbY8ykaE4pD0C9wLKtsNrka+reCrdPwceATnDrgCuMoysoOmEeoVOTdZg3NsVLU5DU9F/Xim7O54Xq3xeFbVCq/g6embGlc1kGW20v7juUq35jyLMDgAL9tA3Qo1NrHaIMSstNzp2DkLl7Y6juqkcVfK2txEbgkze0aeH20GrMDTKw1CtsfjcBYoUTqdIM3QR1lKtpgp09GU+En+yWaDlqpj2x103YInNay1jU0YUiiDkbJU9RfgW8m74gJciTQdvapzwVeY2f/ksRCGp0/YEjduN5N7U9Krkha0gmjtGjYys0Oqd0j6fu2+Ido/FThV0rZtXGwLmdkvatrfv3bfUCT70tH4UtXzeILBv+GJKxvJNXwwZSx3jUr2oFF4osmFGZi9lBQFeyjNfm7ER/23WHINziXZVw7EEyXuJY8nepeZXZp5imsl7YjndwJfrv1jMyFLCUFLFEQtkr6ID5jGAucBnzezBwrPcSruUVRdWuCnOSNnuUfcbqRRtwayRA9p2E8rFO8BFtRgj7TRVA1eekD1c2oa7tRRMuia18wO6nCfRsaMo0IasW6Lj3iWMbMVmxzfqeCrjVObHwGuw6N3r8xdrpJ0Lu4RcRWDo37bKSSU7VVWJfNJ/GaqHvU3zY47RPslI/bJeBbWK81rI3wMz7+1dxO5RumjzZqk4JD0OO5EUc91N3uZKJ1redw4vx7uXfRcs9F+jXy7qScqQZiVUedsDFxLZk2CMZN87cPiJfzB9lVr4KIu6Sh8qXFyTl+HOMcM10vuNZRG3bfhcRTTnWIa2W0kbYnXv9iCwU4Br+DfpeHqhYZIE4RfS5ea2RLN+l3nnK0U8PouPlC5rLS9RoyUGUeFFfBR6jig6YjHzK4Hrpf0u3ZGXLgx7Rw8YrMVA/kfyRgd1iLpC7jSe1cyZFdYAH8IlZzrN3jCtY/gMQnb4Zk2G8lUCuksJ6n65lsAnznkMs3MpkoaJUnmvu3fayZkbebnMrNx7chXSEs0a+BxQ+8BHsITzpXQbuqJoviBOvwMj135Pf7w2xFP/PgwHuuwQa2ABhwQfpS2F6npU1G6E0kLW6pcl86V+/ya2zILXlX17SLgIkkfMLNbS2QTjYKDS6qPXkdNAS9JTQt4VSl6AYdIep2B+KWmA4WmWJtZEmeGF/BDfGnjcjz4qChDLD7SX6hqe2Hc7pAjOxvw5w58h3nwpYkSmYVxZfkH4B1Vr8VaaP+emr/z4zOARjLL4g+UWxlIh/1h3BNr9oK2r8ZHy8fi3io/xdN/lPR/JdzBYOfKq4fX31vA7fjy5KgWz3FLugb+krbfgScazJXfs851eViB/O119t2W/tbNVIyPrMGN0Y+mv5XXo4XffzfgQTyA90j84fvpTNmv4AbiJXAPx0VokqEW+Eb6ewzu2DDo1cNr567093N4mpvp92A/XyNlxvEYnpCvKOKyipYr8FkHbBSSNsfd+ubER+/vw10pG/qCm4/OXpR0rNVk9ZW0i5mdOYRoPSq+66/Ky1A+DyzXpP0ngCck7Wk1a9qSNsCX7XLYCndBPQB/gCyIp5fPQtK38OWhlfA8UZ/APdNKIufbYU3cIeLTwEHyvEvXW5mL62G0VwxsI3mivT3xFP0n4+kzcnlLnqjxvLS9XdVndde7zWyz9LfhdZKDmZ0mL7q2IT5q3qb2mmrAG8CP8boglb4aXitjKCpel0WefF2gpQJe8rLFC5jZeTX7dwamWhsxNcCsPePAq4aBj3BneBWcZxJuE6lsL0sa+WXKn4sXIjqJFkYtqf0FabGeA74scgw+Yh1LqmlQ+Ft+G68psS1ePfAZ4MhM2fuAb+A3/DypL23XSCno+734CPvutL0Enrunl9fiPLg77hF4UagpLZyj5WJgSX4HvHzok2TW8qiSXR64JMlPTe9XSN9r/Saywss2fzttLwOsldnu6PR3kXqvzHP8vZXfK8lun7Ovi9fN9rRQwAu36Yyts/9tnbj3ZmnjuKQTzezzaiNdQTrPxrgfeGWE9iFgLzPLSu1d445a3YGsEadSgje1WM9BXr/iG/gIdTZ8ttIwv1WT882Frxvn5qqaD18uXAO3b5wJ/NDMGkbvy3NC1btAK+u0i9T5rN55Kkn2JuFLZ/+HK96sIER58NgU8+SGGwCrAqdZ1Sy0ifxtuDfOrQzURciq66LO1WRZETgVV6Lvxm18B1pGht52kae8eQsvPfvuZOS90sya5oqTdKmZbSaPY6m+FirXQFMHhWRf27GV7zqEY0eRK34/aPR8KHl2DMUsvVRlZp9Pf9tJV4CZXZ5u3Eqisa9YwbKXmZ2avGCWMbOHmwrMyH1pijlbegB8GV/zzmU0nmdoCj7aXjwZmYtGDZLWpSqQSBJmdlqG6P/wpa55cI+sx5opjcSYkv414K7kknkyvvTwMu6incsEYLykFfBZ48X4MtcMaWiGYGsbqHNfylfx9fl2a7JcAuxjXr9duGvvnbixvimS5saXuWq96nICyda2lCsuybwoac6cdpPSEPBhy69/U8ubwOQ0gMzKsyVpE/z/u6SkX1Z9NJr8dCVIutrMNmq2r4H8j4Dv4vfP5fh9fIB5KqRGzC1pdqvx3JTHk82T2/+hmKUVh4bICFrBMjODpgt3Y2B5MztC0jKS1rLMtOyt2iiq2A9f33wdOAtfpz8yUxbc++knZnZCGv3/GI8pWD/3BJJOxw2ykxlw6TQgR3HcCVyEr/UvChwvaTsz266xGJuZe7dUe+gUY2ZfSG+PlXQFvvxRojjeMrNpkrYGjjazYyoPwUbI40/uqzzwJB2CL/U9gQ8+mnrqdWrwgy8NvZzOZcBPazzdmnE6bpD+BL7ctgsDdoBmtJQrroKZmTz7QatBdxcyUIsll6fxQcYWDPZAfAU3tjckKdp5gTEaHP8zGo9DyuXjZvaNdO1NwZeurmUgA8BQnA+cKGlfS+Ub0r3/y/RZe3R6TW44vfD0x0O9Ti44z3G4R8+DaXthvAxprnxbNooO/A7L1dm3YeE5HiTF/bTQ/vg6+5p6xFBlR6LAplTnPCcDewArtCh/O55d977Kb4krhGZydwPzpfefxGtfr427SF+e2fY2jV4Z8t+oer99zWffL/gNKt49Fa+6OYBrMmV3wWdpU4Dv4S68RXaCdP+t2cY1UOyVWPU9W/Fo3B93ynmdwR5ld+PpZnLPc3/6eyKwceW6ypCbHa8z/lx6/kzCbVNHAXO0+jtOP3+7JxgJLwZcIKsf/E3/eVXH3l5HvqlLHb68cPFQr8LvsCNevQ+8hOUahfJ/wItftfobrg/skd6PoY4yqyNzV733LbT9cXyUfG16eJ+DL9vkyq+Mj9R2StvLAQdnyE2uen9StUzu92FgoPNHPM/ShPR6AS/Fm3Xt1r6vt93kPHekvzfgCSrHUOBSi3u07QPsC7y7hf/hA/hM9++4sfjenHsoyW6OK6vH0vb7cu+fdmTT8fu1et0m+aPwmd5duBIbSx3X6Aby8wDvTa952ulL9WuWXqqqoPbqZUObU21at1G0m44dAEm/wi+6D+Ejvv/gkfBZhawSY4AHJN3B4HXinNKhhzG45vSc5NWcnkfSe/H0HnOn99OD3iwzgtbMrpT0Z9ybbiP8AbYGPorNkX8A/59Vth/Db+hmjJKnCnkttXt81WdzZba9B7iRGFjZkq0kuWjm9F9DvK+33YgT0pLLt/CBy/x4OdzmHfCiX/eb2bFpewFJa5vZ7QXtb1JwbC3fwdORXwdgZpPltd9blR3XTEgDdTSK0/dXY2YHy8v2vmzu2v8fPB4oV/41XMl2lBGhOPDaE6cw4Af9V3zUmas4Wk3LXqElG4V55HonWNcGGydfyDVOVvGdNtrfmlRzOrX/tKScSObn8OqNte/BlXjDlCEVkl1jQfwmvhFYxzISJGqIjLDTO9DcM+UYfKT4EvA3SzYxeQXA0jK442ywgf1f5KXGtyHe19tuxOnmWQ9uIMU/qCYSvAHHMTj9/n/q7KtLshXsjbv+3gucZAWZpRPTzOwlDQ60z/3u9WRzqNQ/+TfuGltEPftsTR/at1O0wUhRHGPM7FxJ3wQwN3Q2LaSjlD7ZzM5MrpyVtOxbWUFadnM3wEOBQ9Oo7d+W5pFN2h/qwSXcYJuba+p/ySW3MmNalLIZU7tK7A0zM0mV9udrJpDYxjpTU/2vuOJaEX/g/lPS89Y8/ctm7TRqZicmpbU4g724nsMzGJRwXTrXWfj/cUcGSgk3YjVJL5NiaNJ70nZJsr7zJW1lZv+D6WlU/kiewXqQB5+ZvSUp99lzKu6VdyM+61gZtx+U0I5XYkuy1n7hrEYBrkYojp7Qar3s84A1qtznisrHSvp/wLnmfvhzAX/C3enelLSzmf25ySnqPbiEZ4ltmtm2yh3vWHxdfKykw/Eo1KZ1DNI5bjKz9TVjkruSnDf1ak7/NkPuDEnzA9fgroi3WJ4b7yAs1faWF+PZDfcQWowmbonWXn6yyjmexAPuqvf9o4Xz7Js8ayqzrBPM7IJGMkmuKItzAy4E/iCPPl8aX65qWs8l8aikL+OzDIAv4QbjHFa2lM5e0kk0yY82BO14Jbbl0Zju+22ZsR5Gw+SgHVA81X3YgoHr5nozu6TR8VnnzBj4zvSofr3s7ZqtkaelnQvxPDE/r/3cmlQfk3Q/sEoabe+F50jaCF9iONXM1ir4Du9L8p/CvTMmmNmvmshMD1SS9B48cll47qz7ctvuBGqx5nSyEWyIjzbXwR84f8K9krLqcUjaG89MuyYe8X4DcKOZXdlErlpZVheD6kyiuELkNTFWNLM/p99lNiuoLdGB9vfB3dLH4Qk7s0btGihktSH++12NxyI0nU2qJtiudruUZKuczwpdu+WFnKz095Z0OT5Ira2H0SgJYseQ9APcRlNJL7QTMNHMvtnWeUeC4gAffVNVL7sy5W4i8y48T9IBDE6rDjSvPqbBkd4T8GjZ49N20xtAXj9kR/yf/Txul/mamS3brO+17XeCtMy2NINHTk3jIST90GpqAtTbl9mHd+IPr43xlBPrZMh8E1cWd1qfCyq1Spqp7YV/53ekZZPfWGYgWRvtVmdhFZ5v617cdtN08NSB9t9kIP17JWXNqxQob0m/x+0kbzLgGv8zM/txhuyauDt3xSb3EvBZGygL3Uz+PsvMUNANJN0DvK8yU0+K864M+1xDZumlqnoGpsQ75VHPzdYJNzazH0qaq9nUcghel7QKvq7+EQZP7efNkH8IX9vd3MweAZDUNPioitoCVIMoueklHYmnLHmUAftIbuTyx4BaJbFJnX2N2l8KH21fK+lJ/GZuqvwTZwFPm9kbktbHU4acUTLqTHIrmtkpksbgCeQeK5BfhYGAyxvN7P5Gx9dhH3zkeDuAmf1NmYk226TWieGCIfYPSVL2xwGLm9cSWRXYwsy+20y2Q0ttK5uXfN0FuAy/7ibhgbDNOAn4kpndCNOvg1PwayiHWyS918yKPZuSXXKd3JldAxbC3bfBlWbbzNKKgwED02J4Jb9r0naloFIzxbEH8At81tGK4tgft5OMBX5eedDII4qbRh4zUHTq2jTlPZsyF8rZcLfJYpeQOnwKrwmRPWKXV377ErB8GvlUWICCehSSPov7/y+IR68vgyd9yy2neyGwpjzn1Gm4Uff3ZBq/1bo7cUV+X/x3qEQvnyvPWPzrBmK1vJ4UX+Wcs1PmFdUSzWbVmZwIfJ3kjmxm96RZQFPF0SHmkKfa2Ar4lXk1ztzf7pWK0gAws5vSEmYu6wOfkefaep2BmVJTxZOcCH4KfKCgvVp+gKfcuTa1/SEy7KNNsQ4FhAznF3ApVcFreL6mnOCps4DH8anyPVWv7OCjDvV/Pjz69lJ8mn4cnoqgmVzL0dZ1zjWBwjoe+IN+XPodl616ZWU1rTrPZPxh3Wp24EoA59eBL6f32QGFqX1RGMBZfSwwf9X2/KXXD14M6RB8FvoxfOT/vR5eg+3UpLmz9jenKjiyB33/MvAPfLahdA3emCn7c1zhbYDXkvk1HguVlWG75rqf/iro++H4ALKlrA3pHEvgqVO2BN7Wid90Vp9xVBhnLfjAm9lOye3wCvyH7wvmuWbOBM6U+85vDxwMNDTu0pmZRoXKyOU+MgMAzbPnvoTbaNrhvzZ4tF26fDFN0vb4+vxWad8cBfKtuhNXEIOX1SqV2Eo4GE8yeC/wBfwhmOOZ1inGWos1aYDn0myv8vtthzsp9AQzq5QyqPCEpNzcX5XSvIfV7F+XBku1Gohxadd54UBSyV9Jr1HomFHlEXpxnX0tM1IUR0s+8PLEev/EXWhrP1um473MwLzc5vEMjkIeik4aTk/FU6MPqtvcI26W9A08evwj+Hr/pQXyn8WXin5kZo/Ko4bPKpCv5058YoH86cBtyUFCuPLKLuKUFOWpZrZrYbud5E1Jy9hAwsZlyV8q2wcvS7CSpH/gXoG7dKebA0ja1czOaGDna2rjs9aTS05iwANvGTxdjHB7w5PQuAhaVfstlfxV55Is1j9/msrM8tT4wN9gGT7wNe6sg7R0rltgBw1cfUVe5/jDfWp7NtyjaLo7L+5R1LOLt1V34ir5NXGXYOHXX1EqijTw2dz65BWmFmvSpOt/O/MA3Pnw0rk9cSGW9AUzOz7ZqGoxy3B4kafj340Z4zCGTMleI/8bPLfVZWl7E+CjZvbVTHnhSnY5MztS0tL4snvDeBZJ++PeoG/Hl+kqiuNl4ERr4srftF8jRXG0Qo077SDX1hJXV0m3mlk7Bq6+I+ln+BLVxQxeqipJT95q2/vWXuj19vWgH6MZ/PB4ocHhtbKr4IrDaMGrKs14Vsd//4p7KtZld9iaPoxhoCbNrZZZk0bSDWaWlR6mG0haz8xubrZvCNlb8JQhg2ball+EbZKZrVGzb6KZjc+Ub7kIVpLfz8yOyTm2hJGyVNUqncrzc6U84vb8Xo6SO0xFSVbHTTR0x9WM0eaDyF2nxZeGapXEnnX2dQVJX8C96l7Db2LRvGZ1tXzFq+qCJNuKV9XT6TWKAlfYDvMm8CyeqmTl5NJ+Q4bcVZK+hschVSu9bMXbJscwY16sevvqMbeZDenSnsFz8pr3Z+DXzK54TFYuLRfBSsd3XGlAKI5mLJbWR1X1nrQ9tuA8bRm4hgOtrPVW1mclHYEn9Tsd/+67kPHwk7QDbo9aTlK16/RoPHlcEfIqjJhnDC3ha8B7ckfYddgLL6T0f6kf38fzHWUrDuuMW2zLSPoc7l6+FO5ltg5eCjcnjqeSl2ufqn3ZirdVJH0AN2LXxjONxl3Vczg92bUuZfBMO1fp7YQb1itL4zdQ5izSbmburhCKozEnMvCAq34PBR4trRq4hhuSPsmMpUNz4ls+YWZrV20fJ+l23MW0EXfgo7OlGJxC/BXy4mAAKulWfoe7JY6S9BReG+SBzFP8HXeDbpW2vaokXcKMs7eX8Cp1x5vZf9voXw774ylbbjOzj0haicx8Z2aWm8K808yJuz7PzuB792U8w3UOb+CBgocy8PtnK72kYEqTMlbTbmburjBLKw61mRa7U6O8Vg1cw4lk5JsXD578LX4B5/b/TXnU7tn4/2MnqvL2DIV5wORjaZ35teQS+w48EK9kye944JCKQVvSR3FDb27p3G/iEcC3k1mzOrVTSTJZ7VUFnmY+26sq8Sg+y614g+3AgFv5ibircTf5r5n9VxLyTAoPyVPyDFvM7HpJNwHvbeNePhCvHFk025R0tJkdMITCb+jGXnNcS5m55fn5Gp23LdvkLG0cTy6DMDBFPj393QV4NXO03Il+tGXgGg5IusfMVq36Oz9us/l4huw4PAJ/PfwmuhlPcvd4ZtsTcS+eSk2Nu4AXzWy3TPm7rSYFfb19DeTvAG6i0EBa45XXrlfVDAbmyj5J95vZe0rOV4q85vceuKfOhrh76Rxmtmk32+0Ekq4xs5wltXqyFwM7mpdGKJFbw8wmSarriWgFZQrUQo44eaQ4+OrAeLxkrfBUKbebWe6gqS6z9IzDUlrs5EFRnR7iYEk301oakVZoy8A1TKjYBV6V9HZ8CSnXF/1xCqqW1WGUmb0qTz3yKzM7StLkAvnH5YkOKwOHXYGSlOnTWjSQVlcrvJP2qsGN1eA4imXwqozgyyldxcy2Tm+/kx5KC+Kp7mcG7koK4A8MNs7n1LR4E5icvnP2bNNSEsQ06xmb3k8t7bgGcsT9ncFLZQ0VYcUmKels3G363rS9Cvnp8IdkllYcVcwnaX0zuwlA0rq4sToLpYJOzfY1YFgauAq5VO7T/mO8KJGRaedJ3/fzzOgLn1vMaFQase+MG5oh37gJbpw9Eo+2BjdQfqZA/lp5WvxLKDOQdizJJPBV4CZJf8cV0nLAl+SxEaXLXtmofgW+oqJeqhOpXG9fF1kEH+hUP2xziyFdyECOsWzS8vRheI414dfwNOCYwpWO4hxxNaxkVQkWzew+eYmGthgpimNP4GR5IR9wj5ySCmwTmNF17zzyqp/BMDVwFfIj84p5E+T1r+cGcg2yF+FZfv9Mhm2jDgfihtg/pgt/+XS+XD5sZl+q3iHPnJxbRW3n9Le6hkGOgbRjSSbN7DJ5KvWV0vkeqjKIH93u+RvQcgU+dTl6ORdroyiSmZ2aVgcqKYqySjLgS3rrAWvaQHLT5XHHkK+Y2Qz1fYbgPjzavNVKmA9J+i2D3YGzq5cOxSxt46hFHsAl8xxKOcevhHsR/QhPkFdhNPD1knXldK6KgevqHAPXcEJ1IuXr7RtCdrKZtT3KaZUh+j5DYFYv2m3zfOsy46zttE6df4g277WBCnyzA3fkfid1OXo5F7WR1l3SBrjyfBzv/9LA7s3iV9Ky9Mdqjepp9n2l5QcPj8cHXtk54mrk5wa+SFXWDOC4dr3wRsSMQzXlG5WS5WVMGd+Fp95eiME1gF/Bl15K+Bc+apsdr/28erueDb1AnuRxSbzP72fwqDGnpgj4MtemltIutNCHMfhSTa0rcEPDvKRP4AWflpRHvlcYTcZSYVoee8o8XxmSdsOvoyeA72QsVXUsyaSk0/GU8pMZmLUZnia+m0wfXZvZtMq9k4OZ/QL4hboUvVxAO2ndf4pnon4Ypiuhs2i+2jBHPU8sM5sqT/GeS8s54tLy+G/Nc5zlznCyGBGKA9fYlfKNrzc5djpmdhFwkaQPmNmtrTbeqoFrmPAJvO9LMTgp3Cvk5/XfHzhE0usMxDCY5QdAnoEv9W2Ne8jtjgcUNuNZfKT2X6A6xccreLbZZhyPl9tF0oeAo/Aa1O/D3XmbxQJ0cg1/PF6QqNdLBKtJqhS8Ej6AeJmy/+FbkhaylF03LVvtZGWR8+0wr5ndUaP0pmXKzlFRGgBm9tfMB38jm0SJveI58+y+xZjZm5LGSpqzDRtJXUbEUpXaLN8orz53DAPupDcB+5vZlEz5h3Ff8pmybCmApG3NbELzI7vS9iQzW6PKFVjAtWa2Qab83K1MzatddiUdC0w1s++k7Z4uv0n6A15LpGfpyDtFvd9KHS5r3KT9P+FG6j8k78btgD3NbJMM2ZPxe77alX/2ZnYTDS55O+gjPI1J1qxDbeaIU5dynI2UGUfL5RsTp+AV47ZP27umfR/LlG/XwDUcuFTSzsy4xp7lIZJGmSsyeKkpJ88RDCyX/DMtPz2NrzVn0cZ67mwaCOLbiAGPLuj9vTMGeEAeU1K81t1nRklSZbaUllB66Y7eTlr3Lyb5L5NicMhIFWOdKXkLLeSIq6ErOc5GyozjAdydsLh8Y5KvF0CWPeJs18A1HJCXrq0s9033jDKzn2bI1s1zZJlBWZK2wNN5L4unHhkNHJ7ph98ykg4FNgWew2sqrG5mJmkFvD5GVunYDvWl7UCyfiHpx/iA4zf4Q29v3HaUlVq8A+3PlpZtitO6J5n/mtmblXMBc1lhQOCsxkhRHMvW228pQDBD/s94rqNKuoed8FxHWWvYku7H18trI4+H/U1foZ3lPnnql0qeo/clD7PDzWyHJnI9T51epw/r4DmurjSvxFgxkM4/Mzg3DAfkNTm+wIBX4ZW40bYV1+xW2n8SD1Y8B7imxE4k6Ta8fkYlQeX8+LWwblc6O2P7C+LxIBWvqOuBIwo8Q8cC32BGx5K27KsjQnFUkJe6rP7xnsyUWwZP4f0BfMR0C27jyFU8fSuC1CkknYAHLxUv90m608zWlEd7r21mr+fM2DrlzipPlzFUgsATh7PtSUOnpp/pMiz3C3lW5M3xTMur45luz7YUENxEtp59pmf2LXl+s/sYCPL8NLCamW2TKX8lrjC/hs/0dsdtdQe11a+RoDjSUsdPcX/yZ/Eljwety/l9qtrvWxGkTtHOcp9azHPUQcXxS+BtDE4Q+A88OG9uM9u93TaCoZH0GPUT/XU1rfoQfVkYz5u2S44dQp6aaL/KvSppDTztTU8Ks7WruGodS9K+tgeyI8U4fiS+rv5nM3u/vG5105z4kv5fg4/NzI7MbL9dA9dwoKkHylBY63mOVq1yBa2mdLS9WvWNIulC4HrzBIG5qdWD1qmudjc37mSySC87kGxEO+DX8Z14Ko8cDgD+IOnptL1EOk+veE2D0yWtx0DeuBwqjiXPyMsiPI3bGttipMw4JprZeEl3A+83s7ck3WFmazWRq2e8mw9PYbKomc3fjf4OVySthmd4BS9/eneX2+uIy6akh/B16ilpe0l8EPHuXrqFBgNIusnazNBa0NZjuFPGuXj973puso3k58CDgSupXnJSjnSEdM+dhg+2wGfru5vZPZnym+GBx0vjIQUVx5KL2+nXSJlx/DsZtW4AzpT0LBkBQNUeQ5IWwD2D9sDrSjT1JqqSbcvANRyQp4/4PAP5nc6QdIL1NyI4l28AtyYFIjzv0L7JY+bMvvZsBKDBtSFG4TOQXhY3W83M6s1cs0iK4r4O9qeEl81sNXm6JMzsZViPz6cAABPiSURBVEnZhbHM7NL09iW8lk5HGCkzjvnw6d0o3H97QeBMM2ta+1fSIniSvV1wA9UvzOzFwvbbMnANByTdA3ygyrNoPtylNsulucU2DzGz73foXPPgCfoE3G/l5WODFtFAbQjwAdvjwE+sKiK7S+0eQ+NCbg1Tow8H6tn5lJFnrdvffUTMOKqmpm9RkII6+Z9vgwcPvbfiktcC7zCzbau2D1dZPYnhgBic2fbNtK+xkPu9X2FmHy1tsF2lIenD5vUQauNl3i6JdqfrQR7WQr36DjEx/V0PHzSck7a3x+ORhi0aSLC6oDyTc4XRVHmGNmBi1fvD8RWPjjEiFEcbfBX3IPoWcKgGct2UGmfbNXANB04Bbk8eUgBbASc1E0qBV69KWrAPS3Mfw5cFt6/zmeFebkGXUINaJNB+2otmWKrQKOkzwEcqtgl5GeQrc8+TBh7Tl5nN7JIOd7UebSVYtarqlJIOsCbVKksZEUtV/aZdA9dwIa1Vrw/Ty5/elSl3Lu5RdhWD8+UM+6WCoHUkVUa578IDQCuKenP8+vlcj/rxML7M+kLaXhgPRm1aM13SD4C1GLCF7QRMNLNvDi3VOdRmgtV0jo6m94cRMuOQtL95iueG+7pIWwaufiJPLT7GzP6UfNkr/uxbSBplqURmE/6YXqVtd2TEmh4U38aVXiVJ5XdLbVVBGWZ2OEwPQlvdUqoPSd/By7j2iqPw8rEVW8uH8eWbHD4JvM/M3gKQdCpe874nigPYWp554jXchX014AAzO6NH7ddlVD8b7yH1Arw+08P2J4ArjCrvjvN62H47/Jj6FcMeSJ81JU2Tz8LXlScBv8+cOi+QXuPxZHNLptfe+Jp1LmfjU/xd8ASVLzOw3h10n2UYnEr8DTx3VU8ws1OAtfHU/Bfgs4/fFZxioar3Cw55VHf4eHpmbAZMwT0Cv95YxDMOSHo5xUGtWnlf2d9up2bpGYeknfCyn8vJi9VXWACvQdzt9ts1cA0HFjWzx2t3mtkjkhbNOYHqVFGTtLs1yY7bwRHrGDOrNg4eLmlYG0dnMU4H7qixj3WtTno9zItxXSTpHcDnJO1oebnXfsDAbEW4rSO3Dk0nqKRf3xQ4y8xeUEYxLTPrqrvzLK048JxSz+ApqavjLl4BemFf6GQFwX4xT4PP5ss8R6tV1Cq0O2K9XtJ2ZnZean8b4E8F8kEbmNn35DUxPogvFe6Rax/rBJIq0d47A6viyqBp5ggAMztL0nW4jUbAQUkJ9YpLUvzRa8CX5EkL2yr72gnCON4DOmHg6hfJA+V54FtWdbFIOhxYwsz2GlJ44NjpeXIa7WsgfyieIqKSrHBr4Nxm7rqSXkzHC19i+F/anhP4t5n1NO3FSCY5iHwI//27nnUgtfl5XEEshUeNnwtcZGbZ9kVJV1tNFux6+7pJstG9nDwU5wVG91h5zcCsPuMApo8wfwgshj9Eep1ZdFgauDL5Ku52+0hV7MlquJ94rlfMREknMbiKWvZSURqxXo4btyF/xDomt42ge1RlHZiA33u9yjpwLHArsLOZTUx9yRopS5obmBcYkx7clfWh0Xiy1J4gr3NfeV/9UbdrzTdkRMw4JD0CbG5m9Yy8vWh/snkdiq3x9d2v4KVPV2si2neUKuBJWh6314BHXj9acI658Cpq0115gV+bWXb993SeltLiB/2lH1kHUjtj8BienYDF8RnHZ8ysafXIpOwOwJXEPxhQHC/jqfh7UicmRYBXmBuvafIXM2tW776rjBTFcbP1sFpbnfbvN7P3SDoRmGBml6tOVcHhiKSJuDfH5cDl9QzlPehDbVr8ZfBkcz1Jix+0h1IhL0slfNNo/k4ze28P+7AUXo9jJ3wmcYGZNTVyS9qvBzOjbOR57063PlcPHRFLVfhSyTnAhQyuh9HV0qNVDEsDVw7mWYWXxdNRHy3PLHsTbly+vtGsIT0wGuXLyR1xtpQWPxg2tJR1oJOYZ0b+CfATSe/ClUiO3LBRGolXgRX73YmRMuM4pc5uM7PP9rAPw87A1QryFNMfBDYGNsCriX1yiGPrluytYPkVFFtKi19zjlUYsJHcaGb358oG7dNq1oGRjqRLGBh8jcLjl841s4P716sRojj6TbWBqxoz66uBqwR5Xv/LKhG0VfuXNLN/ZMgvjrs0AtxhZs8WtP1nfJT6A9zg/Sy+9JFV91nSvsCX8BknwJbAsWb269w+BK0hrzd+T2bMxLBCbo1eysye6mMfqiv1TQOeSLOnvjIiFEeKGzgOWNzMVpG0KrCFmX23R+0PSwNXCZLOwGuuTwBOKXE0kPQpPMr8OnzE+UHg65W4igz5ltPiJ/l7gHUtZTeW12a5pdvG2cCRdCbwzZnRmUEZKcy71O4K+PPq5pr9HwSeNrO/97pPg/oxQhTH9XiY/vGWqr1Juq9fo6DhYuAqRZ5raye8mJXha9dnVSK6G8jdDXysMstINp4/t+IckDxlnreCCzfZWsZX7DHJy2tiL42zIxlJ1+CzzTsYnOSyJ9d/mjnsAixvZkdIWgZ4m5ndkSF7LPA7M7uz2/2safdS4BCrSYQqaTxwmJltXl+yN4wU4/i8ZnZHjR900wqAXWRYGLhKMU/OOAGPJj8AD8T7uqRfNjEijqpZmnqejDxpktbBE9S9gBvIT8eXqkZJ2s3McuqWk+RuS30XfUh5McLJTSjYLX6N1+LZEDgCz9wwgYGl00Z8BPiCpCdwpVeJAev2bHVcrdLAG54oaVyX227KSFEcz6UcNQYgaTs8FUlPGMrA1av2O0Fyid0DeAf+IF7LzJ5Nhv4H8XrGQ3G5pCvwNCPg6R9yUn78Cs8LtCBwDbCJmd0mzwF2Fu4i3BQz+5E819AH8Rt/716PIEcqkrYCVgDuNbMr+tSNtc1sdUl3AZjZi5LmzJTdpIv9akSjXHaN0gD1hJGiOPbBq/itJOkfwGN4ltRe8ZOq98PGwFXIdsDPrSYxoZm9Kqmhd5qZfT1F71e8ak4wswsaySRmN7MrASQdYWa3pfM9pIxEbzW8ll6Gz/iCLiPp13jQ6C3AkZLWMrMj+9CV/8krUVYGjmPxGUhTzOyJlC7lg2lXT9KlAHdK+ryZnVi9U9KeDIPqhSPCxlEhGVlHNVuT72B7w9rAVYKkH5rZQc321Xw+1Pf/EPCPZt9fVQVoVFOMpna7yXkqXlUX4IorvKp6gKT7gNWqXNBv7JOheRd8lrs6vkS5HZ57rWmGZQ2kS6nEfG2ND3y6Gt+RvBAvwBN6VhTFeDzP2tb9duUfEYpD0kLAbnhG1emzLOtyBbrhbuAqod6DWk0SFbb7/SW9ycC68jwMzBQEzG1mcwwlW9tPwquq57Sj7LvQl5Vwb0YBV+d6BapP6VKq2v8IUHHiud/MrulFu80YKUtVlwG3AfeSOUXtEMPawJWDpC/io/V3pJuowgLAzfWlptPW9zez2Qq62gjhmXEr/I+B3ENB91ip6poRA9dQrwzM3rDbNx8zs2PltWE+JukZM/t3jjjwZtX2m/Tw2jGza4Frmx7YY0aK4pjbzBqWIe1Wuw0+67uBK5Pf44bsHwDV0aqvWKrh3IC+fn+lBI0M9qoCX24Ir6ru8+5+dyAxARiflk5/C1yCX9ebZsj2PV3KcGSkLFV9Bfg/4FIG56pq9uBrt92zgGuGMHB93Mx26Gb7nUDS6OSGW7d2RaPfsN/fv8ZGsiYDXlU3hFfVyKFyHUj6BvCamR0j6a5KTFeGfKRLqWGkKI59gO8B/2bALdbMbPkutzusDVw5SLrUzDaT9BgDRZEqNPwN+/39Sx4OwayLpNuBo4FD8fIKjzULAG5nwDQSGCmK4++4L/dzfWp/WBq4ekW/vr+kKcDPhvrczIb8LJh1kLQysDdu1D5L0nLADmZ2VAOZ2gHT9I/owaBzuDNSFMfFwI5m9v/bu9cYuas6jOPfhwqIViSARqPcgiIWTMFADW2jgGLEECMX5YVaRFGJiRcMIRpfgEYj4RYTvCAaS4KCiSgmKKKGalGgVMDewJgg1ijIpSBSkaCUxxfnLLs73d3OdHbmP5fnk2w6s7P/nTPp7p45t9+T/fsdqsP0Wdm+u19t6ZSkf1BqlM24mGm76RPNMcBqqZL9PIQ1tnptXDqO6ykHkX7N9DWOnm7HHQX1xPVsbPv4vjWmQ01u/4xJmjmX5V+U+OEvuc1ilfP0vM9rZ1eXGipyOOjGZVfVT5gsqR0dsH1c023oQrbcDoafU7axXlPvT4QoPQlcBfTqPNNJ8/A91kg6OpspphuLEQdArU1zKOUdyJ9s/7fhJg0dlTCkRUzP/R7YTBFJe4/7IuYg0AzRzROfk7TRA1ylWNK9wOuAzfS3yOFAG4sRh6R3At8C/kz5jz9I0sdst1NoLwBJ51MS/xZRDlSeSImQHdiOI53GwFgo6U227wCQtARYWB/reZVqlSrLl1POlewGLACesr1nG5c3VeRwoI1Fx0HZWXOc7fvg+ZOkP6O9Cq1RnAYsBv5g+8y61fY7DbcphsNZwHdrqRdRpqjOquU7vtKH5/8aZXrsh5Tt4CsoFXt3qBY5XA681vbKWiBx4Y6uG3Xj0nE8MtFpVPdT4kejfU+7ZH0/qxLo9Agw1lsSoz11feANKgFmain10Zd4Adv3SVpgexuwUtJt7VxXR9pHUaarVgK7At8Dls113agbl47jHkk3Un5IDbyHUrb4FADbP57r4gDgzlos8tuUw3z/piS6RcxJJXHxVGqRUdWS+La/2Kcm/Keuca6TdBEli+fFbV57MnAkcDeA7QclvaQ3zRweY7E4LmnlHA/b9px5EjFdLVC450wFDCNaSbqJsv32LqYUDLR9aZ+e/wDgYcr6xjmUYLBvtMxCzHbtWttLppQt6Wt13EE1Fh1HdK9maGzHLcFOEa12VN6jh8+7f7eH9ySdS4l5PoGyHvMh4Jpe53EMupHuOCSd5xIbejkzHATKAcD2qcTfTnghsAS4a5APAMZgkHQlcLntjX1+3qlFLn9k+9QOrt3d9jP19gnA2ykL+7+w/aueNHiIjPoax0RYy52NtmIEtIYuSdoPuKih5sRwWQ58sNZ9eob+nYWYegC0040ctwNvlHS17Q8AY99ZTDXSHYftG+q/yV6Yf39nsnBhxFyaOgvhWW63YzdJZwBLJzbRTPvGY76hZqQ7jgmSDgHOZfvo2EyztKllum8Xyk6T9c21KIZFPQuxmJKHAiV7vB8/O4slPUmNHq63YXLEM9cBwLOB9wF7sX1JFDOZQT6WRnqNY4Kk9cAVbL+r465ZL4ppaqbJRJTrY8Bm2zuKjo1A0qeAjzD5x/Zk4MphWGCW9GHbY5/412pcOo5UuNxJknYFLqactt1Mebf2cspi54WSjkwiWsyl5owfY/upen+otrRKWsr2sxUDW2qnH0Z6qmpKetcNkj5OSaPrW3TsiLgUeBFwgO2tUNLRgEskfRN4B3BQg+2LwSemjPTr7aGoXCzpauBgYB2Tr8EMcI22fhjpEccscacTxj7Fqx2S7qPU6XHL5xcAW4ATba9ppHExFCR9BjiD8sYN4N3AVba/2lyr2iPpj8Ci1p//cTfSIw7beSfcvedm+qWxvU3So+k0YkdsXybpN5RtuQLOHKLpzU3AKyhlSqIa6Y5D0tHA32w/VO+voNTM+StwQaaq2nKvpBWtc7qS3s/kOZmIGUnaBdhQT44PbMzwHPal/A6sZfo097uaa1LzRn2q6m7gbbYfryUzfgB8AjgCeL3t0xpt4BCQ9CrKbpinKbvSDBwN7AGcbPuBBpsXQ0DS94HPDWN2t6S3zPR526v73ZZBMuodx3rbi+vtrwOP2r6g3l9n+4gm2zdMJB1PyW0XcI/tmxtuUgwJSasobzbWUlL0gLxrH2YjPVUFLJD0AtvPAm8FPjrlsVF/7fPK9ipgVdPtiKH0haYb0ClJW5n5tHk7hwdH3qj/8bwWWC1pC2Wq5bcAkl5DKfMcET0i6dPArcCt9c3b0LA99pkbcxnpqSp4Pm/4lcAvpxxAOgRYaHsYF+sihoKkS4ClwKHABuA2SkdyezamDLeR7zgiolk1fe8oSidyTP14wvaiRhsWO23Up6oionl7AHtSkvdeCjwI9DWbI+ZXRhwR0RM1wOkwYCtwB7AGWGP7n402LLq2S9MNiIiRtT+wO/AQ8AAlw+WJRlsU8yIjjojoGUmijDqW1o/DgccpC+TnN9m22HnpOCKi5yS9GlhG6TxOAvaxvVezrYqdlY4jInpC0icpHcUy4H/Urbj13422n2uwedGF7KqKiF45ELgOOMd2qsuOkIw4IiKiI9lVFRERHUnHERERHUnHEdElSXvVTPudvf5GSdlhFEMjaxwRXZJ0IPDTmnIXMfIy4ojo3oXAwZLWSbq4fmyStFHS6QCSjpV0i6TrJd0r6Yoaq4qkzZL2rbdXSNogab2kqxt8TRGzynbciO59Fjjc9hGSTgXOBhZT8qp/L+mW+nVLgEWUzPubgFMo21UBkHQY8Hlgme0tkvbu42uIaFtGHBHzazlwre1tth8GVlNiUwHW2r7f9jZKyNjylmuPB66zvQUgmRUxqNJxRMwvzfFY64Ji633N8LmIgZOOI6J7W4GJqNFbgNMlLZD0MuDNwNr62BJJB9W1jdOB37V8n5uB90raByBTVTGo0nFEdMn2Y8CtkjZR0u02AOuBVcB5th+qX3o7ZSF9E/AX4PqW73MP8GVgtaT1wGX9eQURncl23Ig+kHQscK7tk5puS0S3MuKIiIiOZMQREREdyYgjIiI6ko4jIiI6ko4jIiI6ko4jIiI6ko4jIiI68n+ruqB4aKfY7wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Displaying the count of all 21 labels appearing in train_df\n",
    "sns.countplot(train_df['topic'])\n",
    "plt.xticks(rotation=90)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5959,)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now let us transform labels from text to integer labels\n",
    "le = LabelEncoder()\n",
    "tr_lbls = le.fit_transform(train_df['topic'])\n",
    "tr_lbls.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now since we have a Multilabel Classification problem to solve, but our dataset is provided in form of a Multiclass classification format, with some reviews occuring more than once with different labels, we should squeeze it to change its format to multilabel classification problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining function to check whether review at idx1 is same as that at idx2\n",
    "def isSameReview(df, idx1, idx2):\n",
    "        \n",
    "    if df.iloc[idx1, 1].strip() != df.iloc[idx2, 1].strip():\n",
    "        return False\n",
    "    else:\n",
    "        if df.iloc[idx1, 0].strip() != df.iloc[idx2, 0].strip():\n",
    "            return False\n",
    "        \n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checking in train_df which all are same reviews to reduce redundancy using above util function\n",
    "idx = 0\n",
    "tr_titles = []\n",
    "tr_reviews = []\n",
    "tr_labels = []\n",
    "tr_idx_list = [idx]\n",
    "\n",
    "while idx < len(train_df):\n",
    "    tr_titles.append(train_df.iloc[idx, 1])\n",
    "    tr_reviews.append(train_df.iloc[idx, 0])\n",
    "    labels = [tr_lbls[idx]]\n",
    "    \n",
    "    while idx<(len(train_df)-1) and isSameReview(idx, idx+1):\n",
    "        #print (idx)\n",
    "        labels.append(tr_lbls[idx+1])\n",
    "        idx += 1\n",
    "        \n",
    "    tr_labels.append(labels)\n",
    "    idx += 1\n",
    "    tr_idx_list.append(idx)\n",
    "    #print (idx)\n",
    "    #print ('------------------------------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checking in test_df which all are same reviews to reduce redundancy using above util function\n",
    "idx = 0\n",
    "te_titles = []\n",
    "te_reviews = []\n",
    "te_idx_list = [idx]\n",
    "\n",
    "while idx < len(test_df):\n",
    "    te_titles.append(test_df.iloc[idx, 1])\n",
    "    te_reviews.append(test_df.iloc[idx, 0])\n",
    "    \n",
    "    while idx<(len(test_df)-1) and isSameReview(idx, idx+1):\n",
    "        #print (idx)\n",
    "        idx += 1\n",
    "        \n",
    "    idx += 1\n",
    "    te_idx_list.append(idx)\n",
    "    #print (idx)\n",
    "    #print ('------------------------------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6000"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now we need to combine titles and reviews from train_df and test_df, so that we can create a tf-idf corpus on all data\n",
    "all_text = []\n",
    "\n",
    "for idx in range(len(tr_titles)):\n",
    "    all_text.append(tr_titles[idx] + ' ' + tr_reviews[idx])\n",
    "    \n",
    "for idx in range(len(te_titles)):\n",
    "    all_text.append(te_titles[idx] + ' ' + te_reviews[idx])\n",
    "\n",
    "len(all_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now we need to clean our data                                                                                                                                   1.  We need to remove encodings as <> which are present in many reviews                                                                             2. Next we need to remove URL addresses which are outside the <>                                                                            3. We need to remove Video IDs mentioned in some of the reviews                                                                                   4. We also need to remove quotation marks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Removing frequently present <br> tags from our corpus, this will also remove other useless HTML tags, if present\n",
    "clean = re.compile('<.*?>')\n",
    "for idx in range(len(all_text)):\n",
    "    all_text[idx] = re.sub(clean, '', all_text[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Removing quotation marks from data, otherwise words like don't get tokenized as [don, t], which makes no sense\n",
    "clean = re.compile(r\"'|\\\"|’\")\n",
    "for idx in range(len(all_text)):\n",
    "    all_text[idx] = re.sub(clean, '', all_text[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Removing Video ID links present in few reviews as these are just gibberish alphanumeric strings\n",
    "clean = re.compile(r\"\\S+VIDEOID\\S+\")\n",
    "for idx in range(len(all_text)):\n",
    "    all_text[idx] = re.sub(clean, '', all_text[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Removing web links present in few reviews. In some weblinks there is a space present in \"X BRAND\", so I have first\n",
    "# replaced \"X BRAND\" to \"XBRAND\" and then removed these weblinks\n",
    "clean = re.compile(r\"http.?://[^\\s]+[\\s]?\")\n",
    "for idx in range(len(all_text)):\n",
    "    all_text[idx] = re.sub('X BRAND', 'XBRAND', all_text[idx])\n",
    "\n",
    "for idx in range(len(all_text)):\n",
    "    all_text[idx] = re.sub(clean, '', all_text[idx])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now since we have finished our data cleaning, we can proceed to create a Tf-Idf Vectorizer using a tokenizer, stopwords and a stemmer. The ouput of this Vectorizer will be a 2D sparse array that we can feed to a classifier, coupled with the label ground truths. Entries in this 2D sparse matrix represent the tf-idf scores for each token in a review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We will be using a Regular-Expression Tokenizer, English stop-words list and Porter Stemmer from NLTK\n",
    "regexp_tok = RegexpTokenizer('[A-Za-z]+')\n",
    "stop_list = stopwords.words('english')[:100]\n",
    "stemmer = PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6000"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Creating a new corpus of stemmed words which we will provide to Tf-Idf Vectorizer\n",
    "stem_words = []\n",
    "for i, txt in enumerate(all_text):\n",
    "    tokens = regexp_tok.tokenize(txt)\n",
    "    new_txt = ' '.join(stemmer.stem(tok) for tok in tokens)\n",
    "    stem_words.append(new_txt)\n",
    "\n",
    "len(stem_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6000, 10072)"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Using Tf-Idf Vectorizer class from SkLearn which will transform our stemmed corpus to a 2D sparse matrix as explained above\n",
    "tfidf = TfidfVectorizer(tokenizer=regexp_tok.tokenize, sublinear_tf=True)\n",
    "all_arr = tfidf.fit_transform(all_text)\n",
    "all_arr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['useless', 'did', 'noth', 'for', 'me', 'didnt', 'help', 'lost', 'even', 'with', 'work', 'out', 'and', 'eat', 'healthi', 'curb', 'appetit', 'or', 'anyth', 'trash', 'do', 'not', 'buy', 'these', 'bag', 'it', 'a', 'waist', 'of', 'time', 'i', 'have', 'bought', 'immedi', 'open', 'one', 'put', 'in', 'some', 'the', 'wa', 'split', 'down', 'side', 'so', 'anoth', 'to', 'cover', 'also', 'x', 'dont', 'you', 'will', 'end', 'up', 'throw', 'your', 'box', 'away', 'recommend', 'gave', 'an', 'allerg', 'reaction', 'on', 'my', 'face', 'cant', 'tackl', 'big', 'mess', 'compar', 'name', 'brand', 'wipe', 'famili', 'littl', 'kid', 'we', 'use', 'lot', 'cut', 'through', 'sticki', 'theyr', 'small', 'veri', 'thin', 'might', 'be', 'ok', 'less', 'often', 'place', 'but', 'good', 'at', 'actual', 'care', 'lemon', 'scent', 'felt', 'too', 'strong', 'tast', 'horribl', 'no', 'choke', 'hazard', 'are', 'extrem', 'hard', 'swallow', 'pill', 'huge', 'sharp', 'serious', 'feel', 'like', 'can', 'them', 'know', 'about', 'their', 'effect', 'honestli', 'becaus', 'worth', 'pain', 'danger', 'bring', 'back', 'old', 'style', 'first', 'all', 'am', 'leav', 'review', 'is', 'solimo', 'incontin', 'maximum', 'absorb', 'long', 'length', 'individu', 'wrap', 'had', 'alway', 'still', 'search', 'someth', 'better', 'when', 'order', 'must', 'say', 'pleasantli', 'surpris', 'daytim', 'great', 'they', 'were', 'soft', 'wide', 'enough', 'those', 'gush', 'well', 'myself', 'realli', 'stuck', 'much', 'then', 'chang', 'outsid', 'wrapper', 'inwhich', 'would', 'take', 'off', 'peel', 'paper', 'strip', 'attach', 'now', 'what', 'done', 'outer', 'unwrap', 'that', 'stuff', 'come', 'same', 'bad', 'move', 'pictur', 'thi', 'go', 'into', 'bathroom', 'pad', 'new', 'part', 'as', 'somehow', 'hold', 'while', 'want', 'oop', 'fall', 'floor', 'everyth', 'abl', 'least', 'recycl', 'anymor', 'whi', 'just', 'thing', 'way', 'im', 'yet', 'product', 'crappi', 'happi', 'pleas', 'smell', 'ammonia', 'soon', 'bottl', 'hit', 'by', 'perm', 'solut', 'if', 'how', 'wrong', 'afraid', 'let', 'refund', 'purchas', 'wast', 'money', 'super', 'easi', 'definit', 'doesnt', 'give', 'energi', 'ill', 'stick', 'gnc', 'instead', 'sugar', 'mani', 'calori', 'multipl', 'dose', 'vitamin', 'light', 'disappoint', 'find', 'veggi', 'isnt', 'conveni', 'gummi', 'appeal', 'neither', 'dentist', 'nor', 'waistlin', 'consum', 'dosag', 'three', 'day', 'absolut', 'improv', 'health', 'ani', 'shape', 'form', 'expir', 'date', 'neck', 'surgeri', 'thought', 'stock', 'last', 'whe', 'n', 'second', 'pack', 'two', 'week', 'couldnt', 'drink', 'longer', 'shelf', 'life', 'obvious', 'get', 'rid', 'rate', 'high', 'differ', 'lavend', 'essenti', 'oil', 'from', 'qualiti', 'aura', 'cacia', 'cheaper', 'lesser', 'known', 'none', 'doe', 'stale', 'flower', 'after', 'pay', 'sad', 'return', 'sinc', 'alreadi', 'horrid', 'packag', 'rosehip', 'more', 'cook', 'fact', 'without', 'seal', 'protect', 'make', 'suspici', 'compani', 'ever', 'may', 'organ', 'approv', 'shadi', 'mix', 'other', 'hair', 'treatment', 'tonic', 'wish', 'could', 'zero', 'wors', 'poor', 'check', 'size', 'u', 'need', 'gallon', 'yucki', 'aw', 'piec', 'tri', 'kind', 'defeat', 'purpos', 'chewabl', 'love', 'unsur', 'diaper', 'ive', 'never', 'problem', 'notic', 'few', 'though', 'got', 'daughter', 'wear', 'which', 'weird', 'been', 'befor', 'apart', 'layer', 'idk', 'recent', 'mayb', 'realiz', 'weve', 'besid', 'leak', 'reliabl', 'finish', 'caus', 'dizzi', 'nausea', 'idea', 'such', 'liter', 'pass', 'possibl', 'els', 'onli', 'came', 'claim', 'item', 'rip', 'version', 'regular', 'raspberri', 'lemonad', 'frozen', 'bombsicl', 'price', 'increas', 'ha', 'gone', 'than', 'month', 'allergi', 'medic', 'inform', 'section', 'state', 'consid', 'tablet', 'each', 'costco', 'fals', 'advertis', 'expens', 'found', 'store', 'again', 'hate', 'enorm', 'hors', 'id', 'rather', 'smaller', 'arriv', 'ago', 'transit', 'option', 'receiv', 'abil', 'commun', 'protein', 'per', 'serv', 'meat', 'eater', 'half', 'intak', 'morn', 'shake', 'powder', 'base', 'food', 'ingredi', 'list', 'appreci', 'grass', 'fed', 'aspect', 'unflavor', 'add', 'textur', 'smoothi', 'guess', 'point', 'who', 'choos', 'person', 'vanilla', 'next', 'compliment', 'fruit', 'jug', 'handi', 'scoop', 'bottom', 'total', 'broken', 'complet', 'glass', 'spill', 'sure', 'made', 'shipment', 'healthli', 'nice', 'initi', 'dislik', 'pillow', 'nut', 'ate', 'okay', 'vegan', 'cheap', 'howev', 'sugari', 'whenev', 'job', 'onc', 'age', 'game', 'win', 'dri', 'plu', 'seem', 'concept', 'macro', 'oat', 'fine', 'ground', 'repeat', 'figur', 'whey', 'warm', 'hot', 'water', 'honest', 'blueberri', 'muffin', 'gulp', 'instantli', 'throat', 'cough', 'flavor', 'dryness', 'real', 'downfal', 'sit', 'think', 'reason', 'star', 'apolog', 'optimum', 'nutrit', 'fail', 'told', 'boyfriend', 'him', 'supplement', 'avoid', 'placebo', 'affect', 'he', 'mislead', 'mg', 'cranberri', 'concentr', 'expect', 'front', 'number', 'match', 'almost', 'sick', 'reclama', 'o', 'gain', 'weight', 'look', 'gawd', 'uncomfort', 'around', 'difficult', 'havent', 'slow', 'ship', 'where', 'somebodi', 'drop', 'ball', 'write', 'cost', 'benefit', 'weak', 'inner', 'lid', 'preworkout', 'yeah', 'energ', 'drank', 'vega', 'ga', 'took', 'terribl', 'bewar', 'provid', 'gardenoffail', 'edit', 'earn', 'gardenoffailgarden', 'direct', 'send', 'dm', 'twitter', 'respons', 'call', 'tweet', 'respond', 'block', 'garden', 'prefer', 'reward', 'activ', 'bait', 'switch', 'custom', 'reduc', 'measur', 'cup', 'content', 'within', 'ongo', 'decept', 'peopl', 'read', 'see', 'deal', 'integr', 'honor', 'respect', 'origin', 'across', 'folk', 'blend', 'tell', 'current', 'chocol', 'superb', 'smooth', 'fool', 'label', 'level', 'cc', 'includ', 'lie', 'teeth', 'despic', 'unfortun', 'pull', 'similar', 'stunt', 'perfect', 'green', 'theyv', 'snuck', 'replac', 'run', 'warn', 'whatsoev', 'trustworthi', 'keep', 'eye', 'enjoy', 'us', 'tea', 'tree', 'gotten', 'dropper', 'worst', 'abund', 'stench', 'smelt', 'should', 'listen', 'medicin', 'suppli', 'cannot', 'bite', 'cold', 'overnight', 'supper', 'bread', 'vitafus', 'softer', 'woohoonatur', 'forward', 'diy', 'trip', 'contain', 'itself', 'sleepi', 'asleep', 'bit', 'desir', 'oh', 'nightmar', 'fairli', 'hand', 'squar', 'imposs', 'grip', 'spent', 'hour', 'darn', 'final', 'access', 'pair', 'scissor', 'stab', 'pretti', 'sweati', 'singl', 'awak', 'annoy', 'ultim', 'wouldnt', 'hassl', 'seller', 'sent', 'promis', 'right', 'dha', 'miss', 'sweet', 'consist', 'coffe', 'drinker', 'unsweeten', 'almond', 'milk', 'thicker', 'gross', 'disgust', 'dear', 'gahhhhhhh', 'ib', 'symptom', 'awar', 'experi', 'case', 'happen', 'start', 'calcium', 'year', 'trader', 'joe', 'discontinu', 'taken', 'decad', 'togeth', 'chronic', 'diarrhea', 'issu', 'attribut', 'prescript', 'stress', 'develop', 'lactos', 'intoler', 'elimin', 'wheat', 'dairi', 'whole', 'elim', 'diet', 'celiac', 'diseas', 'test', 'tsh', 'doctor', 'ran', 'travel', 'disappear', 'focus', 'along', 'stop', 'daili', 'miseri', 'steer', 'clear', 'trust', 'impress', 'shea', 'butter', 'graini', 'appli', 'hope', 'continu', 'strongli', 'alcohol', 'lakanto', 'granul', 'monkfruit', 'sweeten', 'mostli', 'fake', 'overwhelm', 'nutri', 'dyn', 'dynam', 'mistak', 'odor', 'sour', 'breath', 'salt', 'caramel', 'dump', 'entir', 'anyon', 'insid', 'fallen', 'most', 'except', 'past', 'low', 'yuck', 'formula', 'until', 'legal', 'sell', 'pharmacist', 'close', 'ibuprofen', 'repackag', 'silica', 'drier', 'tape', 'cap', 'threw', 'garbag', 'four', 'gelcap', 'safeti', 'remov', 'polici', 'sold', 'contact', 'servic', 'offer', 'warehous', 'unattach', 'oct', 'fresh', 'window', 'temperatur', 'probabl', 'weaken', 'efficaci', 'mine', 'pit', 'amaz', 'messag', 'tummi', 'babi', 'gassi', 'fussi', 'her', 'our', 'shame', 'plastic', 'moisur', 'dog', 'sever', 'hive', 'broke', 'over', 'arm', 'lava', 'pour', 'bodi', 'upset', 'stomach', 'discolor', 'gum', 'pharmaci', 'thatll', 'teach', 'forgot', 'supposedli', 'boy', 'capsul', 'separ', 'troubl', 'melatonin', 'usual', 'everi', 'full', 'harder', 'chew', 'nasti', 'spend', 'buck', 'dec', 'pollen', 'count', 'here', 'ohio', 'coupl', 'zyrtec', 'cetirizin', 'hcl', 'shot', 'toward', 'march', 'sneez', 'head', 'blow', 'nose', 'rise', 'soo', 'twin', 'heap', 'teaspoon', 'tmg', 'guy', 'seriou', 'iphon', 'charg', 'cabl', 'usb', 'port', 'reshioo', 'intern', 'straight', 'scam', 'allow', 'joke', 'sucralos', 'understand', 'post', 'natur', 'bounti', 'ot', 'fir', 't', 'noti', 'wore', 'kept', 'tab', 'catch', 'talk', 'magnesium', 'ad', 'woke', 'nauseou', 'quit', 'sweat', 'laid', 'there', 'minut', 'nearli', 'lead', 'stretchi', 'clump', 'chunk', 'chemic', 'thrown', 'cancel', 'subscript', 'melt', 'goo', 'orang', 'red', 'pri', 'jar', 'rout', 'themselv', 'break', 'amino', 'metal', 'skip', 'far', 'biggest', 'pesticid', 'non', 'org', 'tend', 'opinion', 'rice', 'pea', 'corpor', 'save', 'overpow', 'aftertast', 'certain', 'extent', 'owe', 'seed', 'offend', 'winner', 'heard', 'wawaawaaa', 'prenat', 'strang', 'suppos', 'valu', 'urin', 'color', 'research', 'articl', 'averag', 'knowledg', 'went', 'throughout', 'bright', 'yellow', 'approxim', 'bed', 'typic', 'tp', 'email', 'ask', 'forgotten', 'lose', 'learn', 'lesson', 'bulk', 'leaf', 'stevia', 'pick', 'choic', 'anyway', 'liquid', 'unopen', 'credit', 'wonder', 'bloat', 'cramp', 'miser', 'log', 'free', 'pamper', 'sensit', 'skin', 'friendli', 'assur', 'larg', 'digest', 'bullet', 'left', 'basi', 'wasnt', 'result', 'boost', 'crash', 'caffein', 'system', 'tax', 'encount', 'formul', 'd', 'ingest', 'best', 'unseal', 'both', 'spray', 'hesit', 'refus', 'edibl', 'gym', 'decid', 'packet', 'condit', 'confus', 'white', 'brown', 'dark', 'werent', 'attempt', 'unabl', 'sorri', 'posit', 'toffe', 'twice', 'god', 'sore', 'workout', 'upon', 'curcuminoid', 'proprietari', 'curcumin', 'complex', 'compos', 'turmer', 'root', 'extract', 'standard', 'percentag', 'answer', 'therefor', 'surmis', 'fishi', 'random', 'fish', 'appar', 'rest', 'certainli', 'nd', 'keto', 'bust', 'probiot', 'today', 'thermogen', 'said', 'themupd', 'yard', 'jitter', 'lostsever', 'deliv', 'excit', 'bum', 'unscent', 'fragranc', 'although', 'moder', 'citrusi', 'headach', 'itchi', 'lemongrass', 'sourc', 'type', 'shampoo', 'rough', 'condition', 'crack', 'alot', 'fan', 'chewi', 'wont', 'either', 'favorit', 'damag', 'pound', 'wit', 'sickli', 'cloy', 'under', 'coconut', 'request', 'sodium', 'between', 'major', 'meal', 'exercis', 'woah', 'math', 'plan', 'clean', 'process', 'restaur', 'mind', 'heed', 'excess', 'blood', 'pressur', 'shred', 'friend', 'dietari', 'guidelin', 'addit', 'spice', 'easili', 'top', 'push', 'caveat', 'someon', 'electrolyt', 'intens', 'produc', 'replenish', 'beverag', 'balanc', 'potassium', 'address', 'splenda', 'appear', 'competitor', 'highest', 'given', 'amount', 'gram', 'crazi', 'potenti', 'artifici', 'aim', 'muscl', 'cleaner', 'easier', 'overal', 'ice', 'blender', 'bland', 'basic', 'ripe', 'banana', 'pop', 'absorpt', 'enzym', 'allud', 'vital', 'assist', 'proteas', 'bromelain', 'closer', 'tendenc', 'retain', 'fix', 'express', 'concern', 'surfac', 'previous', 'refil', 'hey', 'dollar', 'valuabl', 'invest', 'display', 'exact', 'lower', 'flush', 'drain', 'quantiti', 'st', 'track', 'show', 'detail', 'thick', 'tough', 'faulti', 'pump', 'word', 'matter', 'spun', 'unscrew', 'gel', 'clarison', 'brush', 'control', 'oilier', 'whom', 'inspir', 'confid', 'due', 'set', 'describ', 'unknown', 'directli', 'hopethi', 'unbias', 'thank', 'joint', 'constip', 'aid', 'fiber', 'fill', 'tear', 'buyit', 'terrifi', 'wake', 'tingli', 'brain', 'stroke', 'ruin', 'remain', 'sleep', 'lip', 'swollen', 'middl', 'mirror', 'saw', 'tongu', 'drowsi', 'benadryl', 'death', 'statin', 'coat', 'prime', 'trick', 'late', 'prone', 'gout', 'l', 'arginin', 'uric', 'acid', 'kidney', 'later', 'attack', 'knee', 'stone', 'cours', 'caution', 'wait', 'stopper', 'insert', 'incorrectli', 'fit', 'properli', 'ring', 'black', 'wintergreen', 'dissolv', 'allergen', 'mean', 'fructos', 'husk', 'psyllium', 'rash', 'burn', 'blotch', 'healthier', 'tern', 'dread', 'vomit', 'gentl', 'gener', 'cooki', 'creme', 'lb', 'tub', 'greatest', 'peanut', 'dens', 'sometim', 'vari', 'hershey', 'bar', 'anyhow', 'hang', 'seen', 'clari', 'sage', 'dumb', 'market', 'geniu', 'trio', 'somewher', 'china', 'robot', 'mass', 'world', 'inexpens', 'yiang', 'stimul', 'hell', 'thyme', 'oregano', 'pepper', 'jack', 'flat', 'perhap', 'common', 'substanc', 'oili', 'wet', 'chain', 'lastli', 'stand', 'sampl', 'welcom', 'materi', 'paperi', 'luv', 'print', 'cute', 'keg', 'babe', 'blowout', 'poop', 'chalki', 'moistur', 'holi', 'cow', 'giant', 'proper', 'requir', 'huggi', 'cheapest', 'dissapoint', 'reciev', 'scooper', 'inflat', 'downsid', 'doubl', 'psa', 'interest', 'suggest', 'flunctuat', 'shop', 'walgreen', 'c', 'flap', 'shrink', 'inch', 'loss', 'fat', 'reduct', 'drastic', 'slight', 'intermitt', 'fast', 'omad', 'cardio', 'strength', 'train', 'regimen', 'dure', 'suppress', 'believ', 'updat', 'hca', 'own', 'usa', 'pure', 'garcinia', 'cambogia', 'guest', 'delici', 'deliveri', 'truck', 'foul', 'mapl', 'syrup', 'reek', 'donut', 'favor', 'solid', 'butt', 'irrit', 'assum', 'hole', 'linger', 'mild', 'decent', 'struggl', 'five', 'transfer', 'pregnant', 'breastfeed', 'nourish', 'main', 'chose', 'iron', 'solv', 'futur', 'indigest', 'heartburn', 'mileag', 'complaint', 'bribe', 'hungrier', 'nauseat', 'mood', 'tire', 'htp', 'supplier', 'forth', 'walmart', 'itch', 'reactionwithin', 'load', 'g', 'reccomend', 'excel', 'lack', 'gallbladd', 'yogurt', 'dust', 'defect', 'import', 'intact', 'rememb', 'b', 'street', 'constant', 'ach', 'ceas', 'glutamin', 'stay', 'higher', 'med', 'everyon', 'forev', 'retail', 'earlier', 'niacinamid', 'usea', 'toxic', 'atp', 'asorb', 'crystal', 'mention', 'etc', 'overli', 'unimport', 'paperweight', 'unpleas', 'mouth', 'citru', 'gag', 'note', 'pregnanc', 'experienc', 'maltodextrin', 'carb', 'primari', 'carbohydr', 'glycem', 'index', 'tabl', 'bough', 'quickli', 'wreck', 'altogeth', 'industri', 'kinda', 'piss', 'normal', 'coff', 'partner', 'batch', 'chunki', 'clumpi', 'ye', 'tin', 'question', 'advic', 'shallow', 'chanc', 'sadli', 'pancak', 'submit', 'otherwis', 'savor', 'hype', 'sport', 'creatin', 'monohydr', 'juic', 'meh', 'queasi', 'heat', 'firm', 'upheld', 'despit', 'cool', 'bcaa', 'reorder', 'distributor', 'ineffect', 'tum', 'alongsid', 'paid', 'empti', 'space', 'composit', 'practic', 'storag', 'pantri', 'stung', 'par', 'glycerin', 'dilut', 'bitter', 'recip', 'avail', 'dessert', 'neighbor', 'diabet', 'exampl', 'abov', 'glycerit', 'wound', 'incred', 'tini', 'oz', 'tamper', 'depress', 'anxieti', 'mental', 'aom', 'book', 'calm', 'youtub', 'video', 'occor', 'vivid', 'explos', 'mail', 'lick', 'tarmac', 'dirt', 'lawn', 'freeway', 'slightli', 'nutrient', 'wholesom', 'tasti', 'truli', 'unsatisfi', 'abt', 'becuz', 'soooooo', 'shaker', 'upper', 'belli', 'region', 'vs', 'feb', 'nac', 'spot', 'unaccept', 'sort', 'rancid', 'limit', 'board', 'plump', 'catalog', 'glucon', 'roll', 'craftabl', 'perman', 'vinyl', 'asap', 'fuck', 'brew', 'unless', 'loser', 'carelessli', 'careless', 'manner', 'third', 'er', 'bled', 'everbadli', 'pee', 'prepar', 'th', 'perscript', 'she', 'nordic', 'elig', 'elsewher', 'vendor', 'method', 'ton', 'onlin', 'home', 'facil', 'restock', 'fee', 'yum', 'potenc', 'multivitamin', 'adhes', 'slide', 'toss', 'sulfur', 'gradual', 'onset', 'blown', 'yesterday', 'hous', 'unhappi', 'ick', 'icki', 'foami', 'gelatin', 'power', 'geez', 'convinc', 'confirm', 'scari', 'buyer', 'everyday', 'charmin', 'saus', 'revli', 'mcg', 'folat', 'folic', 'convert', 'unconvert', 'fa', 'circul', 'unmodifi', 'detriment', 'disrupt', 'metabol', 'evid', 'carcinogen', 'sulfat', 'combo', 'mint', 'photo', 'especi', 'support', 'cetaphil', 'cheek', 'chin', 'forehead', 'suddenli', 'acn', 'bump', 'teenag', 'regret', 's', 'ooz', 'angri', 'yellowish', 'makeup', 'poison', 'ivi', 'zinc', 'nail', 'ish', 'magic', 'minor', 'suck', 'discomfort', 'grab', 'randomli', 'drive', 'hurt', 'weirdli', 'line', 'infus', 'candi', 'olli', 'beauti', 'biotin', 'yummi', 'grapefruit', 'glucos', 'beet', 'citric', 'pectin', 'variou', 'veget', 'carnauba', 'wax', 'e', 'keratin', 'dv', 'establish', 'simpli', 'unlik', 'ps', 'chai', 'becam', 'imagin', 'deceiv', 'tripl', 'fathom', 'omiss', 'mom', 'anemia', 'short', 'similarli', 'oatmeal', 'structur', 'richer', 'live', 'loyal', 'member', 'worri', 'safe', 'foolishli', 'ahead', 'hadnt', 'cu', 'rude', 'treat', 'poorli', 'agent', 'inquir', 'manag', 'duct', 'mini', 'signific', 'afford', 'turn', 'heimlich', 'ridicul', 'adult', 'pinki', 'finger', 'wow', 'everywher', 'dandruff', 'wash', 'pre', 'loos', 'toothpast', 'creat', 'tradit', 'ident', 'frothi', 'af', 'independ', 'contractor', 'attitud', 'yell', 'phone', 'door', 'downstair', 'complexentr', 'complain', 'chat', 'hadprim', 'membership', 'blah', 'il', 'vhc', 'youll', 'subtl', 'tart', 'watermelon', 'achiev', 'sundown', 'herb', 'nutti', 'ashi', 'huh', 'combin', 'depreci', 'coincid', 'vision', 'touch', 'exchang', 'cristi', 'husband', 'room', 'vita', 'fusion', 'fault', 'pod', 'explod', 'round', 'term', 'yuk', 'burnt', 'candlewax', 'altern', 'palat', 'true', 'disguis', 'dang', 'raw', 'plain', 'awesom', 'afternoon', 'burst', 'toler', 'koolaid', 'grape', 'hardli', 'inconsist', 'freez', 'underwhelm', 'wild', 'rose', 'cleans', 'petrolatum', 'miner', 'feet', 'paraben', 'phthalat', 'compon', 'suffoc', 'moist', 'clog', 'pore', 'mother', 'dementia', 'shatter', 'bull', 'soak', 'greasi', 'wateri', 'stool', 'gritti', 'novemb', 'softgel', 'nope', 'schedul', 'zone', 'regul', 'substanti', 'flavour', 'palstic', 'niacin', 'fire', 'minim', 'crumbl', 'fist', 'household', 'girl', 'cycl', 'heck', 'asid', 'inconveni', 'categori', 'reformul', 'advis', 'extra', 'mp', 'assault', 'bang', 'newborn', 'hospit', 'kirkland', 'gentli', 'perfectli', 'babysitt', 'minu', 'bunch', 'purifi', 'wise', 'durabl', 'descent', 'deduct', 'born', 'rock', 'steam', 'refrigerator', 'eh', 'repurchas', 'perceiv', 'sneaki', 'dishonest', 'languag', 'talc', 'fave', 'grittier', 'blander', 'becom', 'particular', 'chalk', 'cardboard', 'link', 'report', 'neutral', 'gooey', 'correct', 'nastiest', 'obviou', 'dye', 'releas', 'theanin', 'nighttim', 'herbal', 'quick', 'faster', 'quiet', 'awok', 'night', 'blue', 'lightbulb', 'bedroom', 'com', 'gp', 'kzb', 'ref', 'ppx', 'yo', 'dt', 'asin', 'titl', 'ie', 'utf', 'psc', 'bedtim', 'bluebonnet', 'earthsweet', 'dp', 'ceeh', 'aa', 'sr', 'crid', 'v', 'cb', 'jsd', 'keyword', 'qid', 'gateway', 'sprefix', 'mel', 'descript', 'dription', 'instant', 'yike', 'stomachach', 'incompet', 'incorrect', 'reach', 'benefib', 'phenomen', 'hop', 'tbsp', 'substitut', 'omg', 'embarrass', 'ayurved', 'ear', 'sensat', 'facial', 'cream', 'knock', 'specif', 'firmer', 'hi', 'older', 'funki', 'palm', 'heart', 'catostroph', 'environ', 'dream', 'user', 'scientif', 'studi', 'accord', 'jelli', 'bean', 'flinston', 'uneat', 'fragment', 'crown', 'shell', 'beyond', 'belief', 'dental', 'reconsid', 'bucket', 'previou', 'k', 'bigger', 'viciou', 'sleev', 'act', 'larger', 'design', 'don', 'pleasant', 'rotten', 'gray', 'epiphani', 'panti', 'woman', 'tuna', 'handl', 'ignor', 'hood', 'aint', 'lumpi', 'grain', 'fenc', 'pb', 'strawberri', 'lunch', 'neg', 'wick', 'offic', 'honey', 'busi', 'june', 'till', 'tuesday', 'revis', 'inc', 'fashion', 'profession', 'envelop', 'dealer', 'drug', 'cultur', 'baggi', 'degre', 'resolv', 'fare', 'loud', 'fear', 'bacteri', 'contamin', 'forget', 'news', 'wat', 'asss', 'local', 'groceri', 'unus', 'mark', 'sam', 'club', 'polish', 'mthfr', 'inadvert', 'hoopla', 'row', 'sticker', 'mislabel', 'breaker', 'snack', 'arent', 'eventu', 'usp', 'hyperact', 'attent', 'disord', 'uni', 'bubbl', 'edg', 'pare', 'knife', 'compel', 'scroll', 'comment', 'parent', 'advisori', 'downright', 'tasteless', 'egg', 'theyd', 'youv', 'underwear', 'man', 'numb', 'troll', 'meet', 'appropri', 'translat', 'vitimin', 'towel', 'smear', 'counter', 'restor', 'build', 'adequ', 'critic', 'recoveri', 'enhanc', 'lean', 'drew', 'discov', 'depend', 'click', 'below', 'hide', 'dispens', 'diffus', 'manufactur', 'isol', 'incid', 'attemp', 'map', 'pw', 'avid', 'panel', 'complimentari', 'appl', 'step', 'goal', 'site', 'page', 'surprisingli', 'agre', 'tester', 'parti', 'puriti', 'decis', 'latest', 'fridg', 'chill', 'correctli', 'brownish', 'fri', 'toe', 'fungu', 'pricey', 'dip', 'paintbrush', 'sting', 'medium', 'lotion', 'goe', 'peppermint', 'menthol', 'heal', 'rosemari', 'camphor', 'antimicrobi', 'properti', 'regularli', 'applic', 'exclus', 'swell', 'uniqu', 'allegrad', 'asthma', 'stuf', 'soori', 'bear', 'children', 'tmi', 'alert', 'restroom', 'ounc', 'equal', 'junk', 'toilet', 'acesulfam', 'googl', 'sweeter', 'center', 'scienc', 'public', 'cspi', 'openli', 'queri', 'widespread', 'rais', 'cancer', 'earli', 'riski', 'whatev', 'wouldv', 'obnoxi', 'chip', 'growth', 'opposit', 'present', 'diminish', 'document', 'notabl', 'lighthead', 'preserv', 'bpa', 'estrogen', 'asthmat', 'scare', 'eaten', 'doubt', 'mailbox', 'brought', 'bother', 'secur', 'bm', 'reciv', 'demand', 'watch', 'intestin', 'distress', 'cutter', 'glorifi', 'breed', 'rare', 'ginger', 'ginseng', 'yerb', 'mate', 'strike', 'foam', 'repres', 'acia', 'berri', 'stawberri', 'um', 'remind', 'mainstream', 'whitewav', 'buzz', 'suitabl', 'memori', 'ehh', 'improperli', 'begin', 'summer', 'south', 'carolina', 'messi', 'cinnamon', 'flour', 'highli', 'testazon', 'skeptic', 'numer', 'voic', 'rep', 'herself', 'dead', 'odd', 'function', 'quest', 'fulli', 'compromis', 'casien', 'chug', 'forc', 'sens', 'glue', 'drywal', 'mud', 'pasti', 'spoon', 'spike', 'hollow', 'hunger', 'swear', 'area', 'vacat', 'overwhelmingli', 'cocoa', 'lockbox', 'mailroom', 'mailman', 'file', 'insur', 'mango', 'gmo', 'six', 'statu', 'met', 'stair', 'ly', 'carrier', 'destroy', 'recogn', 'eighti', 'french', 'press', 'bud', 'mixtur', 'sand', 'sickenli', 'popular', 'vast', 'undissolv', 'particl', 'thru', 'war', 'exactli', 'revie', 'alright', 'chap', 'calv', 'insomnia', 'gotta', 'satisfi', 'difficulti', 'm', 'picki', 'fight', 'portion', 'near', 'subscrib', 'rogu', 'driver', 'screen', 'porch', 'sun', 'rain', 'view', 'road', 'land', 'expos', 'visibl', 'failur', 'climat', 'lazi', 'prais', 'trod', 'onto', 'foot', 'plant', 'elev', 'palpit', 'froze', 'endocrin', 'edc', 'reluct', 'partak', 'sicker', 'fatter', 'poorer', 'mixabl', 'satiat', 'ol', 'mct', 'bpc', 'haul', 'easiest', 'yourself', 'ob', 'script', 'pep', 'q', 'tip', 'button', 'fulfil', 'crap', 'crush', 'agav', 'hairgrow', 'ammount', 'benifit', 'centrum', 'reevalu', 'sexual', 'ru', 'fishand', 'pinch', 'sideway', 'peac', 'air', 'styrofoam', 'w', 'overs', 'unnecessari', 'environment', 'unfriendli', 'auto', 'nearbi', 'discard', 'furthermor', 'indic', 'perish', 'shark', 'juul', 'mistook', 'feedback', 'smash', 'erythritol', 'rave', 'monk', 'react', 'remedi', 'matcha', 'savori', 'salti', 'cake', 'batter', 'emphas', 'websit', 'glob', 'float', 'stink', 'regrind', 'finer', 'confection', 'bake', 'crunchi', 'impart', 'unusu', 'kinder', 'gut', 'speak', 'aldi', 'spilt', 'rel', 'imo', 'cottag', 'chees', 'hungri', 'smh', 'prevent', 'freezer', 'superior', 'tom', 'dick', 'harri', 'driveway', 'smart', 'instruct', 'nestl', 'gobbl', 'shoulder', 'injuri', 'hammer', 'lucki', 'twist', 'listerin', 'onsal', 'windmil', 'atrium', 'brother', 'fizzl', 'chef', 'team', 'acquir', 'beach', 'stori', 'wife', 'knew', 'zing', 'halfway', 'semi', 'proof', 'tetra', 'pak', 'cherri', 'luden', 'habit', 'somewhat', 'productproduct', 'mushi', 'itd', 'stretch', 'regiment', 'gold', 'coast', 'speedi', 'depart', 'net', 'econom', 'oxid', 'crispi', 'lysin', 'unlucki', 'wrote', 'ty', 'exp', 'gift', 'foil', 'lift', 'shortli', 'scream', 'curiou', 'horrif', 'pic', 'cyanid', 'cyanocobalamin', 'cheapli', 'relax', 'aloud', 'torn', 'crave', 'cheeri', 'veryi', 'unanim', 'reflux', 'rolaid', 're', 'rich', 'optimim', 'frustrat', 'mold', 'cautiou', 'peculiar', 'comparison', 'hangov', 'trippi', 'btw', 'accustom', 'jibe', 'chemistri', 'massiv', 'comic', 'costum', 'youd', 'sip', 'heavi', 'slider', 'usabl', 'wheni', 'cracker', 'weigh', 'handheld', 'boat', 'motor', 'tablespoon', 'dutch', 'tremend', 'pan', 'season', 'omega', 'fron', 'lemoni', 'flaxse', 'soy', 'further', 'frequent', 'partial', 'residu', 'behind', 'manganes', 'clearcomplexion', 'rosacea', 'hasnt', 'challeng', 'men', 'women', 'plenti', 'burpless', 'canist', 'mojito', 'intend', 'bare', 'soap', 'deep', 'unexpect', 'woken', 'recollect', 'convers', 'moment', 'regardless', 'dufu', 'flimsi', 'unbroken', 'leaki', 'caught', 'luckili', 'gluten', 'intent', 'tinnitu', 'religi', 'error', 'luxuri', 'budget', 'target', 'compet', 'maca', 'hormon', 'earthi', 'patchouli', 'hint', 'dubai', 'uae', 'ignit', 'rank', 'yadgar', 'trade', 'potent', 'miracl', 'jello', 'dig', 'putti', 'harden', 'lard', 'granddaught', 'punctur', 'glad', 'refriger', 'jesu', 'colbert', 'factual', 'toddler', 'reject', 'anywher', 'headlin', 'twoand', 'apr', 'signatur', 'aller', 'tec', 'antihistamin', 'tabletsp', 'packsold', 'betterreturn', 'unimpress', 'tase', 'son', 'brocken', 'max', 'wherhous', 'wheelchair', 'fray', 'seventh', 'babygan', 'younger', 'pseudo', 'fruiti', 'brittl', 'breakag', 'postag', 'naturemad', 'fyi', 'pro', 'biotic', 'ined', 'april', 'dept', 'possess', 'stole', 'explain', 'accept', 'slice', 'yall', 'puke', 'dad', 'slim', 'grit', 'equat', 'share', 'thoght', 'trace', 'arsen', 'crevic', 'fli', 'usag', 'builder', 'shorter', 'lime', 'written', 'diaherra', 'fart', 'flip', 'bowel', 'movement', 'claritin', 'undrink', 'snake', 'synthet', 'cod', 'liver', 'sneakili', 'changenow', 'greedi', 'settl', 'min', 'revers', 'cloth', 'shiiddd', 'sacrif', 'faintli', 'play', 'doh', 'appet', 'kill', 'thigh', 'pant', 'senior', 'uncertain', 'held', 'began', 'winter', 'spring', 'task', 'motiv', 'restless', 'refer', 'ashwagandah', 'identifi', 'tract', 'deserv', 'summar', 'peach', 'against', 'baselin', 'volatil', 'migrat', 'sawdust', 'bitro', 'inprimetim', 'blame', 'grant', 'polic', 'soapi', 'oldest', 'youngest', 'poo', 'sag', 'insan', 'assassin', 'mask', 'ordinarili', 'particularli', 'nutrion', 'key', 'childbear', 'lanolin', 'hat', 'chapter', 'multi', 'violent', 'posion', 'ius', 'culprit', 'episod', 'falvor', 'eight', 'jogg', 'gainer', 'breakfast', 'glow', 'success', 'rcvd', 'co', 'nitrogen', 'europ', 'india', 'pitch', 'dawn', 'inspect', 'lover', 'shoe', 'leather', 'strongest', 'aggress', 'die', 'concord', 'outstand', 'versu', 'alon', 'liquefi', 'jolli', 'rancher', 'sound', 'darker', 'consciou', 'father', 'plasticki', 'prior', 'hyper', 'grown', 'immun', 'seeler', 'nutritionist', 'wallmart', 'badli', 'panic', 'gonna', 'freak', 'faint', 'sooooo', 'sluggish', 'macrobiot', 'counselor', 'guru', 'granola', 'termin', 'abysm', 'notif', 'notifi', 'sat', 'stump', 'hubbi', 'understat', 'mellow', 'ici', 'rocket', 'someplac', 'corn', 'subsid', 'flineston', 'deterg', 'wholefood', 'whim', 'jitteri', 'perform', 'wave', 'crippl', 'teen', 'vanila', 'hello', 'kick', 'comfort', 'postpartum', 'bleed', 'xl', 'wherea', 'birth', 'uteru', 'waistband', 'guard', 'coverag', 'wedgi', 'lay', 'discret', 'bulki', 'dent', 'undamag', 'inelig', 'rubber', 'lousi', 'pco', 'quarter', 'cruel', 'cedar', 'dirti', 'clearli', 'scatter', 'prick', 'swaddler', 'mama', 'nap', 'fun', 'sooooooooi', 'observ', 'deliber', 'shown', 'newer', 'krispi', 'kreme', 'locker', 'suffer', 'candida', 'killer', 'lymph', 'node', 'fever', 'sinu', 'infect', 'hidden', 'powderi', 'cloud', 'upto', 'dishwash', 'chunck', 'mosquito', 'absoooooolut', 'repel', 'burp', 'remaind', 'muscletech', 'shit', 'lol', 'rhodiola', 'reliev', 'adaptogen', 'expiri', 'golden', 'tumer', 'unbear', 'boron', 'arthriti', 'insult', 'remot', 'fluffi', 'airi', 'puck', 'assess', 'fortun', 'temporari', 'dissip', 'slowli', 'shock', 'hear', 'hundr', 'horrifi', 'throughli', 'tropic', 'blurri', 'reader', 'ink', 'font', 'led', 'flashlight', 'contract', 'verdict', 'whether', 'drawer', 'stronger', 'stolen', 'cliff', 'lamp', 'reminisc', 'lessen', 'inflamm', 'recov', 'decreas', 'weightlift', 'commiss', 'liqui', 'efffect', 'h', 'll', 'bent', 'flu', 'dr', 'tobia', 'strap', 'cash', 'letharg', 'distil', 'taco', 'bell', 'beef', 'mainli', 'filler', 'wood', 'pulp', 'feed', 'american', 'celucor', 'supp', 'pharm', 'jym', 'beta', 'alenin', 'meant', 'constantli', 'outing', 'child', 'jammi', 'underneath', 'droop', 'grow', 'bound', 'harm', 'toxin', 'exposur', 'consumpt', 'smoker', 'biolog', 'mushroom', 'chlorella', 'spirulina', 'hydroxocobalamin', 'methylcobalamin', 'aquacobalamin', 'info', 'ideal', 'varieti', 'readi', 'carri', 'tag', 'deform', 'ashwagandha', 'scott', 'commod', 'squarer', 'shall', 'plumber', 'speed', 'dial', 'backord', 'sheet', 'pipe', 'scalp', 'kit', 'phase', 'trigger', 'cure', 'mountain', 'pig', 'gen', 'everybodi', 'roger', 'saylor', 'fresher', 'vengeanc', 'steroid', 'necessarili', 'collagen', 'lake', 'darken', 'eyebrow', 'templ', 'patch', 'effort', 'intellig', 'sixth', 'tone', 'throghout', 'locat', 'consider', 'code', 'tan', 'jan', 'conceiv', 'carrageenan', 'coz', 'shaki', 'sooo', 'carageenan', 'shini', 'alga', 'corneal', 'ulcer', 'defens', 'agit', 'reintroduc', 'adam', 'grew', 'advers', 'cashier', 'station', 'promptli', 'fluke', 'needless', 'bellow', 'centigrad', 'frill', 'competit', 'vagu', 'fruitlik', 'meali', 'foremost', 'inde', 'hefti', 'presum', 'magnisium', 'rang', 'grace', 'sign', 'awe', 'aagain', 'unbeliev', 'nerd', 'teamtrib', 'solidifi', 'pralin', 'pecan', 'charm', 'sudden', 'gurgl', 'boil', 'cooler', 'account', 'bacteria', 'tobus', 'hydrat', 'boostbad', 'skyrocket', 'choc', 'ml', 'chamomil', 'genuin', 'volum', 'palmetto', 'capac', 'regain', 'ascorb', 'bug', 'happenst', 'cord', 'dupe', 'period', 'factor', 'molass', 'soooo', 'tutori', 'solar', 'keurig', 'adjust', 'seam', 'coq', 'grabbi', 'increment', 'shouldv', 'doze', 'couldv', 'sensori', 'depriv', 'tad', 'grumpi', 'chack', 'thursday', 'basement', 'dharma', 'bindu', 'ashoka', 'mahamadhya', 'bid', 'adieu', 'acharya', 'prep', 'rite', 'daastan', 'proud', 'hug', 'proudli', 'helena', 'mir', 'river', 'chosen', 'magadh', 'plot', 'gather', 'discipl', 'announc', 'sacrific', 'vain', 'enemi', 'ideolog', 'unit', 'keen', 'promot', 'drip', 'cat', 'counterfeit', 'basil', 'outcom', 'florastor', 'recomend', 'physician', 'seek', 'amp', 'largest', 'chop', 'bundl', 'vers', 'disaapoint', 'prioriti', 'dusti', 'rattl', 'unprotect', 'vertigo', 'courtesi', 'mingl', 'debri', 'karen', 'kish', 'hypoallergen', 'phenoxyethanol', 'ethanol', 'pocket', 'receivex', 'buckley', 'darkest', 'guarante', 'flax', 'reviv', 'hurri', 'chest', 'distinct', 'outdat', 'defici', 'sunflow', 'overcharg', 'gullet', 'unravel', 'ar', 'nitrotech', 'cholesterol', 'bothersom', 'deffici', 'exhaust', 'physic', 'partli', 'slimi', 'ng', 'receipt', 'lowest', 'deni', 'forthi', 'spasm', 'everytim', 'hr', 'cube', 'macadamia', 'misrepresent', 'premium', 'fair', 'appal', 'cbd', 'abus', 'itcoukd', 'sellerss', 'lug', 'dissatisfact', 'horrend', 'cleanser', 'mimic', 'advoc', 'rins', 'routin', 'toner', 'waterless', 'cleanli', 'among', 'debbi', 'lee', 'wesselmann', 'vile', 'pungent', 'cement', 'glop', 'scrape', 'gruel', 'punish', 'aback', 'kale', 'oatmilk', 'refrain', 'skittl', 'pinata', 'determind', 'disbelief', 'damn', 'ef', 'completli', 'luck', 'compens', 'frantic', 'nevertheless', 'liner', 'barrier', 'delic', 'undi', 'drippi', 'chafe', 'therapeut', 'coal', 'tar', 'pronounc', 'fade', 'lather', 'sud', 'band', 'spoil', 'funni', 'cheez', 'hmmm', 'echinacea', 'battl', 'ultra', 'lo', 'behold', 'shade', 'background', 'upgrad', 'viscou', 'evil', 'doorstep', 'splotch', 'harsh', 'smoke', 'smokey', 'sea', 'tastier', 'zzzquil', 'clay', 'birthday', 'mustv', 'extend', 'bummer', 'flexibl', 'chocolatey', 'offens', 'tylenol', 'pm', 'advil', 'relief', 'mover', 'sale', 'undesir', 'prove', 'apathet', 'unmotiv', 'tired', 'apathi', 'antidepress', 'action', 'ssri', 'om', 'human', 'craziest', 'understood', 'terrif', 'wanna', 'humong', 'rap', 'paw', 'frenzi', 'whew', 'virginia', 'cedarwood', 'texa', 'flea', 'risk', 'obeis', 'kg', 'height', 'skinner', 'shed', 'misrepres', 'cent', 'stinki', 'amd', 'east', 'havasu', 'worthless', 'aloecur', 'nexium', 'opramazol', 'stupid', 'dushav', 'un', 'unfair', 'pathet', 'decay', 'burlt', 'gasx', 'doenst', 'helo', 'transpar', 'blech', 'snap', 'tight', 'dextros', 'grade', 'steevia', 'kal', 'margarita', 'prompt', 'sciatica', 'shakeolog', 'teetg', 'doabl', 'boulardii', 'bunk', 'gnarli', 'drawn', 'optim', 'unsaf', 'fraud', 'illeg', 'swamp', 'decemb', 'f', 'fourth', 'repli', 'extremli', 'fantast', 'hay', 'coke', 'avg', 'english', 'bc', 'dot', 'polyethylen', 'interior', 'overpr', 'violat', 'clue', 'diann', 'r', 'ungermin', 'utterli', 'mediocr', 'aftercoat', 'reship', 'self', 'car', 'protocol', 'sucros', 'childlif', 'tour', 'livid', 'agreement', 'ugh', 'quo', 'fluoresc', 'anyclos', 'toobusi', 'arctic', 'snow', 'cone', 'flavorless', 'apprehens', 'punch', 'screw', 'vannila', 'moca', 'inaccur', 'recur', 'menopaus', 'carbon', 'drinkabl', 'blank', 'gimmick', 'thirsti', 'anxiou', 'soup', 'rapidli', 'otc', 'breach', 'oversight', 'discrep', 'smelli', 'inea', 'mag', 'crusti', 'sole', 'adaqu', 'reseal', 'zip', 'mil', 'select', 'badg', 'limeaid', 'wastedmi', 'diffrent', 'sink', 'narrow', 'sandi', 'pinpoint', 'vacuum', 'ten', 'group', 'treatabl', 'ordinari', 'roast', 'nowher', 'dumpster', 'draw', 'string', 'loo', 'fot', 'hugh', 'race', 'ferment', 'intuit', 'encourag', 'sadden', 'grime', 'film', 'thinner', 'leg', 'laundri', 'stain', 'unscrupul', 'uselesspil', 'deadlin', 'enrol', 'percent', 'impenetr', 'boulder', 'stanki', 'unsettl', 'shut', 'styrofoamish', 'breakabl', 'dispos', 'conduct', 'project', 'neglig', 'harvard', 'univers', 'atleast', 'hall', 'beg', 'tall', 'kitchen', 'overr', 'ninja', 'mixer', 'taffi', 'landfil', 'breakout', 'pwo', 'def', 'reimburs', 'jasmin', 'yea', 'hastl', 'inconvenientalso', 'weirdest', 'iodin', 'academi', 'pediatr', 'mildli', 'mykind', 'sept', 'productwhich', 'overbear', 'molten', 'veggicap', 'julep', 'mouthwash', 'squish', 'tho', 'follow', 'fertil', 'healthiest', 'turbo', 'multivitamn', 'northern', 'mid', 'southern', 'producto', 'incorrecto', 'alpha', 'refin', 'manafactur', 'ban', 'moldi', 'mildew', 'intedn', 'scan', 'foreign', 'object', 'squeez', 'investig', 'hdhdhd', 'tracker', 'ut', 'fluorid', 'qualit', 'genet', 'modifi', 'compound', 'jump', 'bridg', 'januari', 'abdomin', 'tissu', 'criticalof', 'grinder', 'grind', 'special', 'companion', 'school', 'oliv', 'tech', 'preform', 'pickup', 'puberti', 'monster', 'resort', 'satisfact', 'dnt', 'sulphur', 'ky', 'simultan', 'legitim', 'author', 'resel', 'whomev', 'nix', 'prebiot', 'worsen', 'penetr', 'cv', 'walgeen', 'knockoff', 'chunkc', 'zesti', 'grossest', 'disclaim', 'tray', 'lax', 'soften', 'glycol', 'miralax', 'osmot', 'polydextros', 'eas', 'defec', 'ratio', 'gi', 'propylen', 'glycolsom', 'creamsicl', 'nosi', 'naw', 'aukey', 'lockabl', 'lock', 'pysch', 'ct', 'recd', 'cheat', 'cholestoff', 'lil', 'impuls', 'ride', 'nuun', 'sunday', 'eve', 'ur', 'thicken', 'serum', 'alo', 'porcin', 'specifi', 'epic', 'coupon', 'messageyour', 'deliveredunfortun', 'damagedw', 'deriv', 'yeast', 'autoimmun', 'bill', 'fatigu', 'oppos', 'gatorad', 'y', 'cabinet', 'ptsd', 'bipolar', 'gripe', 'ketogen', 'advantag', 'fad', 'strictli', 'keton', 'tangi', 'ewweww', 'howeverthi', 'entertain', 'skunk', 'deaf', 'whopper', 'inositol', 'tonight', 'dismay', 'tact', 'perfor', 'minimum', 'crumb', 'nightshad', 'ashwaghanda', 'tomato', 'eggplant', 'potato', 'amber', 'granular', 'instanc', 'disturb', 'wad', 'disappont', 'whit', 'ritz', 'drag', 'nutribullet', 'swirl', 'purpl', 'yield', 'florida', 'ex', 'card', 'undeni', 'wafer', 'underton', 'incorpor', 'concret', 'porcelain', 'nightli', 'appoint', 'shakey', 'anti', 'town', 'tangerin', 'nonexist', 'swanson', 'soooooooo', 'delight', 'senna', 'eucalyptu', 'benefici', 'rubberi', 'pencil', 'eras', 'format', 'stun', 'complexion', 'dinner', 'wtf', 'carmel', 'ruffi', 'occas', 'strict', 'stall', 'corner', 'taint', 'megados', 'rda', 'congest', 'bodybuild', 'offici', 'bet', 'friday', 'cayenn', 'indest', 'spit', 'pointless', 'soybean', 'barf', 'shellfish', 'wost', 'mio', 'afterward', 'perfum', 'depth', 'qont', 'oyster', 'quicker', 'flew', 'countri', 'enclos', 'strangest', 'tightli', 'filthi', 'vitmin', 'octob', 'iskol', 'frustat', 'printer', 'blackberri', 'headl', 'tomorrow', 'whitley', 'azur', 'repuls', 'picolin', 'fell', 'gap', 'nerv', 'distribut', 'elast', 'soil', 'peed', 'outfit', 'execut', 'golif', 'dozen', 'clot', 'wha', 'temp', 'wqnt', 'nobodi', 'launch', 'subject', 'sandwich', 'spread', 'outga', 'wham', 'smack', 'diamet', 'lecithin', 'equival', 'sunk', 'gastro', 'sub', 'saltin', 'turtl', 'shipper', 'woohoo', 'pond', 'palett', 'naturalsfish', 'exit', 'alergi', 'coumarin', 'cinnamomum', 'burmannii', 'spoli', 'alter', 'diesel', 'fuel', 'element', 'trimest', 'realiti', 'kindli', 'piti', 'worm', 'clove', 'antacid', 'shaken', 'emptor', 'cheapo', 'rd', 'haphazard', 'christma', 'tornado', 'freshen', 'regim', 'rejuven', 'mechan', 'slime', 'whip', 'minti', 'condens', 'placement', 'dhea', 'bs', 'hiccup', 'disabl', 'incent', 'slightest', 'honesti', 'penni', 'pearl', 'epa', 'bloodwork', 'andv', 'ld', 'lieu', 'strain', 'thousand', 'comput', 'weekend', 'accumul', 'countless', 'exam', 'smoki', 'accur', 'tbh', 'introduc', 'program', 'pink', 'elderberri', 'seltzer', 'sambucu', 'telltal', 'slowest', 'evalu', 'oder', 'frankincens', 'methyl', 'undertast', 'sublingu', 'ath', 'supplemelt', 'thr', 'duti', 'pimpl', 'recib', 'nunca', 'el', 'pedido', 'solimno', 'pin', 'wall', 'maker', 'rush', 'shift', 'nurs', 'occasion', 'belch', 'bu', 'favorir', 'accid', 'yr', 'factori', 'comprehens', 'young', 'liar', 'pa', 'stumbl', 'familiar', 'rethink', 'mailer', 'lacklust', 'judg', 'male', 'pattern', 'bald', 'mile', 'tube', 'sunlight', 'fedex', 'rear', 'bumper', 'vehicl', 'walk', 'moral', 'resin', 'collect', 'regard', 'mar', 'glucosamin', 'exager', 'genit', 'inflam', 'contraind', 'associ', 'thiamin', 'riboflavin', 'chromium', 'pantothen', 'opt', 'expert', 'sprout', 'botan', 'assumpt', 'impact', 'balm', 'uterin', 'hydrocortison', 'owyn', 'displeas', 'bizzarr', 'useabl', 'inferior', 'correl', 'smartyp', 'util', 'undergo', 'dihydrofol', 'dhf', 'tetrahydrofol', 'thf', 'methylfol', 'untrustworthi', 'tangibl', 'capabl', 'expand', 'gol', 'underhand', 'tactic', 'misinform', 'scale', 'saturday', 'longest', 'dare', 'bacon', 'whiff', 'togetheraw', 'unpalat', 'weather', 'seven', 'influenc', 'soymilk', 'affili', 'recours', 'mega', 'nibbl', 'cri', 'mission', 'weightloss', 'gal', 'sweetner', 'aka', 'problemat', 'floss', 'nine', 'variabl', 'gasolin', 'wreak', 'approach', 'satur', 'squishi', 'turpentin', 'justifi', 'boom', 'tim', 'ei', 'natal', 'explan', 'mor', 'mysteri', 'blob', 'nitic', 'qc', 'bueno', 'phoni', 'knowingli', 'holiday', 'micron', 'awhil', 'groggi', 'refresh', 'pineappl', 'rx', 'shouldnt', 'ox', 'metamucil', 'bliss', 'unawar', 'recc', 'performix', 'scratch', 'arguabl', 'privat', 'alanin', 'ab', 'hardest', 'tew', 'anim', 'vit', 'emuls', 'insuffici', 'vegetarian', 'limb', 'specul', 'roman', 'legionnair', 'ration', 'backyard', 'slid', 'scali', 'abdomen', 'insulin', 'exfoli', 'detox', 'celeri', 'sparingli', 'trait', 'overlay', 'reflect', 'greater', 'beat', 'valid', 'trillion', 'transmitt', 'record', 'cellucor', 'freshest', 'indict', 'wareh', 'flow', 'nyc', 'shirt', 'therer', 'bf', 'welt', 'earl', 'labor', 'shard', 'accident', 'doc', 'consult', 'fibromyalgia', 'zipper', 'parad', 'beem', 'wheatgrass', 'spinach', 'vice', 'ocean', 'noooooo', 'postal', 'stingi', 'iserv', 'ki', 'bloodstream', 'starch', 'sift', 'unwel', 'mocha', 'cross', 'chickpea', 'enrout', 'estimai', 'brane', 'scre', 'bye', 'eeeeee', 'incomplet', 'netrush', 'sulfit', 'ourselv', 'misl', 'imho', 'summertim', 'palltabl', 'stomp', 'bonu', 'cell', 'furiou', 'idiot', 'wtfff', 'lasership', 'imag', 'briefli', 'tote', 'mv', 'spoke', 'expedit', 'safer', 'testosteron', 'reput', 'msg', 'via', 'chlorin', 'smth', 'alterard', 'tooth', 'chia', 'lifesav', 'unmix', 'subsequ', 'withdraw', 'bumblebe', 'torso', 'blog', 'pfizer', 'procedur', 'ordor', 'mect', 'rebiew', 'insul', 'entri', 'suspicion', 'swedish', 'oral', 'miggt', 'specteum', 'sonetim', 'prtein', 'statement', 'manor', 'retriev', 'dud', 'detect', 'tic', 'tac', 'repair', 'decubitu', 'patient', 'pouch', 'soupi', 'mucinex', 'nyquil', 'deturg', 'amunsur', 'eleg', 'flag', 'terror', 'bcca', 'evlut', 'toothpick', 'mde', 'fuse', 'junki', 'closest', 'septemb', 'bioavail', 'deem', 'lab', 'hemoglobin', 'dl', 'bisglycin', 'reportedli', 'pend', 'enlighten', 'rapid', 'acetaminophen', 'suit', 'licoric', 'frother', 'legit', 'dawg', 'rat', 'sing', 'daenyr', 'drogon', 'torch', 'westero', 'bath', 'cider', 'vinegar', 'graphic', 'radic', 'scar', 'houston', 'latt', 'fragil', 'brake', 'sauna', 'recal', 'flame', 'therapi', 'renew', 'slip', 'pulver', 'mash', 'thorough', 'coarser', 'overcom', 'nation', 'brag', 'thrill', 'tsa', 'bomb', 'detector', 'indianapoli', 'airport', 'greatli', 'devic', 'checkpoint', 'frisk', 'wand', 'flight', 'ziploc', 'unsurprisingli', 'cs', 'februari', 'pelvic', 'endometriosi', 'hiit', 'squirt', 'ew', 'sa', 'yourselv', 'whelm', 'teal', 'mike', 'obtain', 'roommat', 'focu', 'pang', 'ripoff', 'de', 'wildest', 'evenli', 'peak', 'creativ', 'sterat', 'stearatei', 'stearateit', 'knowi', 'youineseffect', 'stearateimag', 'healthyfocu', 'orgmagnesium', 'stearat', 'lubric', 'overdos', 'resembl', 'bat', 'pinkish', 'cuz', 'valerian', 'noteworthi', 'stubborn', 'plagu', 'dat', 'sidewalk', 'limeston', 'secondari', 'policiy', 'theyll', 'measli', 'discount', 'heartbroken', 'heaviest', 'stood', 'faith', 'knowest', 'comb', 'necessari', 'bold', 'dramat', 'scope', 'chuck', 'camomil', 'icepack', 'fifth', 'slept', 'magesium', 'nah', 'closur', 'tocopherol', 'acet', 'sucsin', 'determin', 'unnatur', 'occur', 'exist', 'tocotrienol', 'benedryl', 'swindl', 'authent', 'geranium', 'woodi', 'eo', 'reat', 'counteract', 'storefront', 'gess', 'pufa', 'pitti', 'sorta', 'cleanaherb', 'laxaherb', 'dissatisfi', 'buti', 'oxi', 'seafood', 'insist', 'event', 'delay', 'nest', 'naturalssup', 'conspiraci', 'theori', 'nozzl', 'heavili', 'stream', 'soda', 'chart', 'imbalanc', 'role', 'absent', 'pyridoxin', 'hydrochlorid', 'prescrib', 'bowl', 'runni', 'bike', 'pavement', 'weekli', 'kikland', 'logo', 'incredi', 'bargain', 'scrub', 'uncoat', 'cashew', 'goodby', 'bioperin', 'smarti', 'floati', 'lump', 'rotat', 'readili', 'pen', 'slipperi', 'elm', 'chocola', 'friendsi', 'cla', 'eatin', 'thoughtthat', 'visit', 'probe', 'camera', 'upload', 'necklac', 'bracelet', 'undilut', 'doterra', 'fiashi', 'marcona', 'distanc', 'virgin', 'sprig', 'folicacid', 'kool', 'mush', 'score', 'ugg', 'fo', 'motic', 'curv', 'bogu', 'footbal', 'trainer', 'basketbal', 'leagu', 'mma', 'stack', 'con', 'bcass', 'threeatin', 'juli', 'rst', 'anybodi', 'wallet', 'wider', 'undergar', 'outdoor', 'adher', 'fold', 'wing', 'whichev', 'brick', 'mortar', 'dull', 'letter', 'unread', 'maintain', 'meanwhil', 'bulkier', 'cappuccino', 'rougher', 'hed', 'receptor', 'mispul', 'rectifi', 'situat', 'bush', 'disgrac', 'john', 'carrigan', 'itemno', 'aroma', 'irresist', 'citizen', 'petit', 'jetlag', 'chicori', 'sketchi', 'effici', 'copi', 'advanc', 'mt', 'inabl', 'ensur', 'snug', 'longev', 'firmli', 'affix', 'glu', 'pastur', 'passion', 'disgustingli', 'unhealthi', 'enrich', 'undepend', 'anem', 'domin', 'disagre', 'urg', 'justic', 'jet', 'lag', 'rocki', 'retch', 'disintegr', 'lifestyl', 'compress', 'elbow', 'closeri', 'progress', 'cystic', 'behav', 'suspect', 'chelat', 'alkalin', 'stage', 'zen', 'electr', 'machin', 'ph', 'behavior', 'tate', 'separar', 'reli', 'fullfil', 'tingl', 'disjoint', 'fatti', 'glanc', 'breakdown', 'elud', 'blatantli', 'leakag', 'cowork', 'unprofession', 'unsanitari', 'md', 'significantli', 'bovin', 'silicon', 'dioxid', 'middlemen', 'inaccuraci', 'donat', 'plasma', 'steak', 'broccoli', 'flash', 'monday', 'overh', 'bone', 'spell', 'uggh', 'bht', 'butan', 'fd', 'earth', 'itll', 'caplet', 'gc', 'technic', 'milki', 'muski', 'gravi', 'anticip', 'wari', 'relat', 'cleanlabelproject', 'sunkist', 'pretzel', 'whoever', 'merchant', 'tore', 'drawstr', 'merchandis', 'wayyyi', 'unapp', 'visual', 'ra', 'anchovi', 'robberi', 'undercut', 'shave', 'protien', 'insect', 'argh', 'mete', 'puddl', 'amth', 'waaaay', 'jergen', 'lighter', 'syrupi', 'fisherman', 'jockstrap', 'abomin', 'flawless', 'boycott', 'vote', 'az', 'lengthwis', 'mckesson', 'coul', 'thatso', 'requirementswhich', 'takesi', 'vitaninsalso', 'pottasium', 'needl', 'gerd', 'reglan', 'fennel', 'oreganol', 'rxbar', 'thinkthin', 'disast', 'haphazardli', 'supermarket', 'noon', 'pooper', 'storm', 'probiotoc', 'inconten', 'halt', 'text', 'upstair', 'shower', 'unsalt', 'crisi', 'pallet', 'gastrointestin', 'manli', 'dfferenc', 'fpo', 'apo', 'untwist', 'mfd', 'casein', 'odorless', 'cloyingli', 'profil', 'moister', 'fewer', 'lengthi', 'ewg', 'benzoat', 'highchair', 'toy', 'discern', 'absenc', 'iffi', 'optimist', 'uc', 'ii', 'thu', 'sister', 'peer', 'snif', 'migrain', 'damp', 'gunshot', 'grassi', 'thee', 'declin', 'imit', 'former', 'latter', 'grabbingli', 'ador', 'whine', 'shockingli', 'ecstat', 'sue', 'tu', 'cosco', 'resid', 'hoov', 'proven', 'canari', 'simpl', 'overboard', 'fabul', 'smith', 'tighten', 'journal', 'immediatli', 'bp', 'kerosen', 'judgment', 'aspirin', 'daycar', 'sprinkl', 'patchi', 'gooood', 'deploy', 'kuwait', 'unsustain', 'diagnos', 'barcod', 'inoffens', 'memor', 'tang', 'casual', 'sirbitol', 'sorbitol', 'abandon', 'thunder', 'montg', 'neglect', 'adderal', 'pepto', 'agoni', 'baglid', 'bustedin', 'calcul', 'roughli', 'bot', 'yup', 'field', 'mmmgood', 'jojoba', 'furnitur', 'desk', 'wind', 'holist', 'irrespons', 'path', 'emergenzzzz', 'neuro', 'optinum', 'maggot', 'swerv', 'permeat', 'marin', 'whack', 'twitch', 'irrat', 'seeth', 'noropothi', 'fior', 'render', 'fog', 'preship', 'epson', 'slit', 'arizona', 'sake', 'takin', 'citrat', 'liabl', 'lilcritt', 'minion', 'zarbe', 'fend', 'unchart', 'territori', 'underlin', 'italic', 'centimet', 'plume', 'shoot', 'presto', 'ferrou', 'bi', 'glycin', 'samez', 'meter', 'flutter', 'drunk', 'tryna', 'epipen', 'expel', 'wheez', 'leucin', 'korintj', 'boswellia', 'flare', 'rebal', 'advocar', 'spark', 'yarf', 'bannana', 'pancrea', 'theater', 'nutriti', 'alzheim', 'denni', 'salad', 'blot', 'onion', 'greenleaf', 'lightli', 'assort', 'anesthet', 'gape', 'gigant', 'halv', 'offput', 'doorbel', 'rung', 'suppliment', 'neurotransmitt', 'phd', 'neuropharmacologist', 'precursor', 'descriptionkirkland', 'enter', 'spous', 'druggist', 'femal', 'omfg', 'garlic', 'absurdli', 'liabil', 'govern', 'buttol', 'lara', 'accompani', 'revisit', 'fraudul', 'turnoff', 'shortchang', 'disput', 'usless', 'vitmain', 'ewww', 'juven', 'rehab', 'scx', 'technolog', 'moron', 'vitex', 'menstrual', 'irregular', 'uneasi', 'og', 'aspart', 'swap', 'homebound', 'garabg', 'vera', 'dgl', 'monthli', 'plead', 'jillian', 'michael', 'drill', 'prohydrolas', 'tm', 'aceslfam', 'satisfactori', 'steal', 'suprem', 'distend', 'tread', 'trial', 'incas', 'unclear', 'deceit', 'ladi', 'surviv', 'revolt', 'grief', 'rachel', 'shepherd', 'pie', 'trifl', 'embodi', 'plug', 'tortur', 'preffer', 'nside', 'cnd', 'vinylux', 'topcoat', 'blast', 'internet', 'butterscotch', 'hazelnut', 'namesak', 'sunscreen', 'doorway', 'gala', 'saver', 'buffer', 'chicken', 'slimfast', 'inulin', 'consensu', 'krill', 'fishiest', 'cereal', 'kroger', 'meijer', 'stove', 'shew', 'benefiti', 'drugstor', 'lutein', 'endur', 'remark', 'electron', 'lt', 'zeaxanthin', 'isom', 'beeswax', 'gaia', 'chlorophyl', 'cellulos', 'mealsquantum', 'digit', 'bee', 'carob', 'allright', 'reishi', 'educ', 'essenc', 'dime', 'thereabout', 'po', 'plum', 'nornal', 'stabl', 'itthi', 'stamina', 'lookout', 'hunt', 'mannitol', 'sobitol', 'anhydrid', 'titanium', 'central', 'secret', 'gambl', 'outweigh', 'sock', 'mo', 'ile', 'emphasi', 'pudgi', 'bact', 'inf', 'nearolog', 'hallucin', 'wierd', 'nectar', 'ju', 'promin', 'callednatur', 'admit', 'border', 'larceni', 'pud', 'akin', 'iu', 'retinyl', 'palmit', 'caroten', 'ding', 'sigh', 'lenni', 'emerg', 'mad', 'manuka', 'peroxid', 'grandson', 'lame', 'gate', 'climb', 'barbwir', 'hypertens', 'sooth', 'braun', 'fraction', 'creamer', 'sank', 'marjoram', 'spici', 'blenderbottl', 'metric', 'secondli', 'dish', 'luke', 'salesmanship', 'booooooooo', 'thread', 'extens', 'profit', 'uber', 'congeal', 'overlook', 'cane', 'sight', 'allegedli', 'bow', 'ble', 'der', 'peolpl', 'ruptur', 'alaskan', 'salmon', 'stabil', 'bead', 'ny', 'drawback', 'labdoor', 'peachi', 'splurg', 'oarmeal', 'outbreak', 'nylon', 'metter', 'dodgi', 'mistaken', 'becu', 'frankli', 'disapoint', 'bricki', 'chief', 'hd', 'calor', 'strenuou', 'excercis', 'restrict', 'bleh', 'crunch', 'disolv', 'metamucel', 'stir', 'neutrogena', 'hydro', 'delivereda', 'zit', 'nickel', 'prohibit', 'smelliest', 'stunk', 'vanish', 'sh', 'whop', 'oxymoron', 'canola', 'sturdi', 'disclos', 'disclosur', 'sooner', 'rub', 'california', 'prop', 'traet', 'chaf', 'compliant', 'tempera', 'rashi', 'jb', 'predomin', 'virtual', 'putrid', 'op', 'debitt', 'beer', 'lactat', 'xray', 'cocktail', 'borag', 'saute', 'minc', 'toast', 'recogniz', 'asian', 'scinc', 'verifi', 'sho', 'nuff', 'dude', 'resend', 'iherb', 'prouduct', 'silver', 'everyong', 'opportun', 'purest', 'kcal', 'competitorfat', 'bothcarb', 'gprotein', 'gsodium', 'competitoringredi', 'competitior', 'built', 'conclus', 'prod', 'forklift', 'van', 'spin', 'consecut', 'armpit', 'phoney', 'ancient', 'contend', 'tuck', 'buyingth', 'hirribl', 'fitmiss', 'airplan', 'incom', 'plastici', 'glutathion', 'unpad', 'famous', 'pharma', 'wrinkli', 'ziplock', 'nervou', 'notrop', 'hs', 'student', 'switchet', 'unbeat', 'nootrop', 'boggl', 'lysol', 'hypothet', 'primarili', 'yuckier', 'aspartam', 'doodoo', 'flake', 'camt', 'farther', 'excruci', 'connect', 'rheumatoid', 'bergamot', 'sicken', 'infin', 'model', 'snail', 'farm', 'naiv', 'scenario', 'hastili', 'weav', 'iin', 'tasteexpensiveway', 'grame', 'laughabl', 'viva', 'suffici', 'mercuri', 'mistakenli', 'contin', 'vastlli', 'candyd', 'ceral', 'vastli', 'praulin', 'flaor', 'cowboy', 'redemt', 'vitamix', 'pois', 'pricq', 'greentea', 'bootl', 'cellophan', 'collar', 'bearabl', 'viabl', 'booti', 'borderlin', 'batteri', 'leisur', 'notch', 'despis', 'savag', 'basin', 'ceram', 'humidifi', 'sickingli', 'gumdrop', 'primros', 'greenish', 'sanitari', 'servingswhey', 'puf', 'parchment', 'attend', 'wed', 'url', 'lavandula', 'angustifolia', 'evapor', 'chines', 'la', 'mandalay', 'bay', 'hotel', 'annual', 'flavo', 'acryl', 'kevin', 'hutton', 'darci', 'forest', 'drivesilv', 'maryland', 'khutton', 'aol', 'sickeningli', 'soylent', 'chalkier', 'judgement', 'hush', 'puppi', 'poke', 'absorbt', 'magensium', 'fairlif', 'humid', 'chondroitin', 'msm', 'sorter', 'compart', 'gear', 'goof', 'butterbur', 'wth', 'lawsuit', 'rewrap', 'tastebud', 'lube', 'flang', 'breastpump', 'seri', 'instal', 'reinstal', 'irwin', 'honeztli', 'ersatz', 'unecessari', 'waaaaay', 'comm', 'ci', 'regrew', 'preciou', 'regrowth', 'cigarett', 'ashtray', 'inevit', 'fond', 'invent', 'wean', 'forti', 'astronom', 'wager', 'popcorn', 'brief', 'employe', 'vagina', 'blunt', 'nanni', 'ass', 'subsidiari', 'packer', 'supervis', 'cushion', 'featur', 'owner', 'dizz', 'spite', 'lodg', 'gwen', 'meriden', 'alexa', 'pat', 'christolini', 'pkchcc', 'cox', 'cuppa', 'danc', 'treadmil', 'intensifi', 'debt', 'dilig', 'crummi', 'retreiv', 'digreas', 'capcasin', 'diasappoint', 'publish', 'hip', 'anaconda', 'goat', 'dine', 'gastriti', 'carpet', 'mop', 'rot', 'rug', 'hardwood', 'rent', 'baffl', 'proceed', 'frame', 'gasket', 'tank', 'tinker', 'ooh', 'glide', 'smoothli', 'prank', 'proport', 'cling', 'dermat', 'hairlin', 'aggrav', 'disinfect', 'gastroparesi', 'greas', 'recommenc', 'nuktivitamin', 'similiar', 'branch', 'unidenti', 'naturewis', 'encapsul', 'browni', 'umm', 'shrug', 'kosher', 'trim', 'frond', 'core', 'pkg', 'nvr', 'wasdeliv', 'confer', 'amzn', 'merg', 'hung', 'lozeng', 'usal', 'sucker', 'breast', 'headachc', 'al', 'puffi', 'mehh', 'wrist', 'girlfriend', 'bedridden', 'pray', 'slushi', 'creami', 'vitamelt', 'suclar', 'shi', 'milkshak', 'billion', 'pronto', 'pharmci', 'utter', 'pwn', 'logist', 'baker', 'sludg', 'rope', 'attorney', 'recvd', 'grower', 'squat', 'ldl', 'abhorr', 'allegra', 'costli', 'ppl', 'gastrtiti', 'zma', 'allerclear', 'bounc', 'excus', 'truth', 'ididnt', 'unparallel', 'unregul', 'whose', 'mobil', 'app', 'changer', 'tenfold', 'alongwith', 'fetu', 'gyn', 'henc', 'prenatalsfin', 'thoughtsi', 'upfront', 'globe', 'fakespot', 'involv', 'orangey', 'emergenc', 'divid', 'realis', 'nonstop', 'esophag', 'interv', 'undoubtedli', 'germin', 'unsprout', 'unpleasur', 'america', 'ehhh', 'reflex', 'filter', 'senn', 'ley', 'horriabl', 'allevi', 'aliv', 'condom', 'barley', 'energet', 'usuallt', 'waxi', 'purport', 'iam', 'unknowli', 'garag', 'tougher', 'facemask', 'importantli', 'mist', 'charact', 'scummi', 'heavier', 'king', 'tryptophan', 'cleanest', 'rbgh', 'rbst', 'justm', 'anytim', 'idead', 'shorten', 'moth', 'hmm', 'addsom', 'financ', 'til', 'cellular', 'tjmaxx', 'marshal', 'epsom', 'musti', 'peppercorn', 'tellicherri', 'thew', 'milder', 'fuss', 'variant', 'xplode', 'sweeet', 'wherev', 'softest', 'sate', 'mejorar', 'dejarlo', 'en', 'casa', 'repartidor', 'tocaron', 'timbr', 'nosotro', 'estuvimo', 'todo', 'tiempo', 'ah', 'bolsa', 'estaba', 'abierta', 'aunqu', 'si', 'se', 'encontr', 'perinat', 'desicc', 'nil', 'steadi', 'goer', 'sjogren', 'syndrom', 'host', 'ojibwa', 'infant', 'butyon', 'vinilla', 'eww', 'inconclus', 'juri', 'sleeper', 'logic', 'awaken', 'sleepless', 'primal', 'wednesday', 'defrost', 'await', 'nonsolu', 'batman', 'dinosaur', 'preach', 'muddi', 'hadit', 'gaurente', 'unapologet', 'leas', 'flaw', 'bloodi', 'wholli', 'midwif', 'scentless', 'septic', 'flushabl', 'botll', 'evenvelop', 'dripper', 'pretend', 'esophagu', 'scammer', 'imvho', 'origina', 'fluctuat', 'hypersomnia', 'eagl', 'loooooovvv', 'jag', 'utensil', 'mucu', 'similac', 'enfamil', 'bottlei', 'firstli', 'tong', 'slot', 'frugal', 'hooray', 'yoo', 'ey', 'spore', 'snickerdoodl', 'sweent', 'sholder', 'kotek', 'uigi', 'vitacrav', 'bayer', 'cousin', 'attest', 'unchang', 'outrag', 'leaflet', 'rl', 'carton', 'sanit', 'fragrant', 'detach', 'shiva', 'shrieval', 'dedic', 'persist', 'kneecap', 'collarbon', 'anaphylaxi', 'threshold', 'trend', 'cluster', 'beforehand', 'horni', 'weed', 'nether', 'horsetail', 'punta', 'cana', 'evoo', 'insomniac', 'yawn', 'chi', 'repeatedli', 'amazo', 'submiss', 'guidelinesnew', 'wholemega', 'astaxanthin', 'sustain', 'david', 'bottlearriv', 'insofar', 'paquet', 'gentlemen', 'apt', 'cutmi', 'dietician', 'typo', 'ergocalciferol', 'ne', 'dfe', 'iodid', 'carrot', 'cucumb', 'overdr', 'simplifi', 'gyno', 'panoxyl', 'floral', 'unholi', 'trivial', 'roadkil', 'thereaft', 'multivit', 'quot', 'compris', 'brightli', 'complement', 'unremark', 'bout', 'orient', 'clip', 'arnica', 'blotchi', 'vegasport', 'overload', 'resist', 'west', 'crud', 'fl', 'cater', 'speck', 'orgain', 'cfu', 'vitaminshopp', 'inept', 'criteria', 'decidedli', 'desert', 'artic', 'swansson', 'combat', 'lethargi', 'troat', 'dislodg', 'remodel', 'sugarbearhair', 'neccesari', 'icey', 'crimin', 'cotton', 'accidentali', 'taiwan', 'booster', 'libido', 'jusst', 'chile', 'adress', 'malaga', 'condessantiagochileatt', 'gonzalo', 'cavada', 'orim', 'delet', 'crumpl', 'bankrupt', 'principl', 'scum', 'vanillaim', 'laugh', 'thoughi', 'shortcom', 'carbscon', 'involuntari', 'tortuou', 'debilit', 'dice', 'horror', 'goop', 'trashcan', 'tie', 'tempetur', 'ragwe', 'cannist', 'qa', 'interlock', 'chisel', 'lightn', 'curl', 'belong', 'swim', 'rebuild', 'ridg', 'takenot', 'tastecon', 'jarrow', 'acceler', 'timelin', 'whyyyy', 'loosen', 'impur', 'purif', 'massag', 'liber', 'allov', 'growingth', 'straw', 'camel', 'minoxidil', 'drown', 'antibiot', 'defraud', 'peptid', 'alkali', 'xanthan', 'enzyplex', 'papain', 'amylas', 'noooop', 'robust', 'tyrosin', 'panacea', 'histori', 'platinum', 'gaba', 'elit', 'conclud', 'desper', 'airway', 'gladli', 'jigsaw', 'magsrt', 'diagon', 'rib', 'supplimnet', 'oatmega', 'buff', 'diebet', 'massachusett', 'prud', 'musclemilk', 'projym', 'unison', 'sliver', 'pk', 'lust', 'beginn', 'cooti', 'hitch', 'movi', 'swig', 'bec', 'fda', 'wine', 'jewelleri', 'secondarili', 'zeal', 'carnitin', 'inquiri', 'sincer', 'enzymat', 'mannos', 'burb', 'durat', 'viru', 'twelv', 'forewarn', 'countin', 'lactic', 'matt', 'colour', 'labl', 'tapioca', 'premier', 'mi', 'slick', 'exterior', 'citranat', 'thatwont', 'cerav', 'cetyl', 'lauryl', 'stearyl', 'hydroxyacetophenon', 'hydroxid', 'withstand', 'warmer', 'motto', 'ahould', 'boroglucon', 'hypocalcemia', 'livestock', 'kicker', 'aspar', 'pediatrician', 'masti', 'dint', 'husbrand', 'kmow', 'purolog', 'costo', 'bullshit', 'harsher', 'superglu', 'cetapihl', 'unstick', 'protector', 'rant', 'pose', 'planet', 'carefre', 'pantilin', 'heavey', 'magnet', 'neodymium', 'sweep', 'phood', 'fianc', 'reced', 'beard', 'fuller', 'pace', 'incess', 'bumpi', 'allergist', 'incident', 'nonetheless', 'qith', 'grr', 'dock', 'drove', 'corridor', 'beaten', 'stare', 'peski', 'slower', 'irrelev', 'discuss', 'clock', 'gimmicki', 'capsaicin', 'devot', 'nutrab', 'kudo', 'jam', 'courag', 'degrad', 'keilman', 'salmonella', 'augment', 'buhh', 'anway', 'bin', 'superpump', 'dismiss', 'debat', 'eczema', 'noseble', 'harmless', 'traumat', 'hyster', 'endless', 'nitric', 'yassss', 'esspresso', 'cafien', 'midst', 'cacao', 'mug', 'workplac', 'bicycl', 'mastic', 'goug', 'wayyy', 'swoll', 'alll', 'cleansaherb', 'cloudi', 'journey', 'clen', 'graden', 'splint', 'withstood', 'stringi', 'printgood', 'slimmer', 'hypoallergenicaffordablecon', 'tatter', 'raggedi', 'immobil', 'strenghen', 'z', 'spout', 'starv', 'coment', 'ironish', 'fierc', 'ughhh', 'p', 'respiratori', 'comprehend', 'photograph', 'argu', 'coco', 'stank', 'cottenel', 'linti', 'oralbiot', 'attract', 'ensepar', 'strand', 'expans', 'madei', 'upcom', 'psyillium', 'delier', 'haent', 'mumer', 'teas', 'bulli', 'yellowbin', 'soggi', 'cheerio', 'milligram', 'payment', 'squeamish', 'snappi', 'j', 'ello', 'farmtot', 'lmao', 'doo', 'creepi', 'ghost', 'haunt', 'spirit', 'invis', 'projectil', 'ricochet', 'tamp', 'reallllli', 'paranorm', 'veng', 'dammit', 'ribey', 'tn', 'decor', 'ect', 'lung', 'mason', 'addict', 'moreov', 'broom', 'mow', 'surround', 'quantit', 'discourag', 'misalign', 'vcap', 'ankl', 'paint', 'adulter', 'tap', 'molar', 'stickier', 'moonshin', 'smoother', 'creamier', 'pork', 'erron', 'tel', 'overspend', 'shook', 'ala', 'greyish', 'superfood', 'cheesecak', 'tastr', 'wonki', 'acai', 'flabergast', 'overpack', 'overfil', 'nitro', 'comprabl', 'engeri', 'dehydr', 'begun', 'menstur', 'abras', 'lotteri', 'bo', 'disspiat', 'deo', 'lavelin', 'cholesteroff', 'uneth', 'homemad', 'aromatherapi', 'softbal', 'bsn', 'unchil', 'giver', 'shabbi', 'haha', 'quercetin', 'tendon', 'spec', 'gender', 'stiffer', 'densiti', 'pastel', 'nit', 'woulf', 'hve', 'changet', 'lye', 'et', 'sucrolos', 'stapl', 'cruis', 'seasick', 'sway', 'rental', 'speeddislik', 'coldpack', 'articifi', 'dough', 'applaud', 'commit', 'unheard', 'inadequ', 'resourc', 'nearest', 'citi', 'bleach', 'aminogen', 'chocolati', 'legendari', 'deeper', 'fortifi', 'tieraona', 'cvitamin', 'chlorid', 'infrequ', 'postmenopaus', 'synergi', 'ethic', 'glove', 'neurochemistri', 'neuropharmacolog', 'meaning', 'ugli', 'rework', 'tastei', 'againtast', 'goodi', 'ester', 'gaggi', 'downhil', 'crappin', 'deadlift', 'prostat', 'ubersweet', 'sender', 'inflammatori', 'coliti', 'foggi', 'overus', 'timefram', 'glowon', 'multivitaminsthes', 'sculpt', 'conjug', 'linol', 'modest', 'hustl', 'gm', 'peep', 'shove', 'sneak', 'indulg', 'chiari', 'seizur', 'happili', 'menstruat', 'prune', 'noisi', 'cutesofturin', 'stripetru', 'sizenot', 'huggiescon', 'nois', 'treasur', 'plantar', 'cheapcon', 'customercal', 'vaselin', 'hemorrhoid', 'revamp', 'yucko', 'salvag', 'trail', 'raisin', 'postnat', 'panick', 'verg', 'butteri', 'data', 'aug', 'quaker', 'quakergreat', 'endocrinologist', 'wayyyyy', 'crawl', 'escal', 'pixi', 'illustr', 'infinit', 'wisdom', 'straighten', 'fiasco', 'succeed', 'slipshod', 'effin', 'jip', 'puls', 'rage', 'nautio', 'class', 'feld', 'linen', 'graciou', 'san', 'rawfus', 'es', 'lymphat', 'drainag', 'dangerousthey', 'onlinethey', 'dateor', 'spare', 'headachetak', 'skimpi', 'unfavor', 'ochi', 'ouch', 'pood', 'walli', 'wrestl', 'drift', 'tooooo', 'producti', 'awfulso', 'clash', 'jun', 'sep', 'contribut', 'thaw', 'warp', 'rainbow', 'unwant', 'voicemail', 'noup', 'falsifi', 'sugarfre', 'liek', 'yeck', 'selfish', 'antioxid', 'closet', 'lint', 'rule', 'sayin', 'dove', 'adag', 'motel', 'sitck', 'difer', 'slammedit', 'sap', 'lok', 'barg', 'alaska', 'estim', 'hotdog', 'august', 'nn', 'whin', 'dealt', 'norm', 'diphenhydramin', 'circumst', 'unlock', 'gerald', 'chasin', 'chasingerald', 'gmail', 'easthampton', 'ewest', 'kratom', 'interrog', 'suffic', 'probelm', 'optimen', 'gluey', 'trepid', 'natru', 'hottest', 'nc', 'cupboard', 'sniff', 'german', 'nupro', 'kibbl', 'sabotag', 'scorch', 'atkin', 'boo', 'wagon', 'burger', 'cring', 'bond', 'ecxit', 'menatonin', 'gorgeou', 'ploy', 'standbi', 'consol', 'breez', 'rebuy', 'bruis', 'uffff', 'island', 'breastf', 'mamabear', 'contour', 've', 'toughen', 'prolong', 'gage', 'overkil', 'sheep', 'wool', 'shear', 'exclud', 'asham', 'lichen', 'disappointng', 'mill', 'tricki', 'nsaid', 'cholin', 'dealbreak', 'thyroid', 'mayber', 'sloppi', 'reg', 'rollersand', 'hypothyroid', 'mondi', 'brisk', 'lipton', 'adapt', 'exagger', 'grandmoth', 'germtrol', 'chili', 'liken', 'biscuit', 'zyme', 'esp', 'theee', 'enthusiast', 'worthi', 'paraplegia', 'consequenti', 'wk', 'tension', 'migran', 'pine', 'jitti', 'flask', 'phosphadidyl', 'bottim', 'vsg', 'redund', 'colon', 'upward', 'sinus', 'selenium', 'mesh', 'arthrit', 'cbo', 'unplan', 'worser', 'amazingli', 'runner', 'unsuccess', 'scratchi', 'costun', 'buildup', 'limp', 'worker', 'swole', 'progesteron', 'ben', 'gay', 'tacki', 'monstros', 'prodcut', 'disregard', 'discreet', 'blister', 'vibrant', 'froma', 'dermatologist', 'pimpli', 'comedogen', 'freewhen', 'opaqu', 'washcloth', 'amen', 'wen', 'ecleans', 'tome', 'tumric', 'albeit', 'pj', 'aveeno', 'sauv', 'clorox', 'laptop', 'mous', 'keyboard', 'countertop', 'roof', 'uh', 'joker', 'bbq', 'idealshak', 'session', 'chase', 'tug', 'mascara', 'endedup', 'mgo', 'tasteon', 'japan', 'radiat', 'nuke', 'meltdown', 'raid', 'noxema', 'sunburn', 'crisp', 'sarna', 'liquidi', 'spider', 'buggi', 'kin', 'mixerstay', 'gosh', 'receptionist', 'isagenix', 'evacu', 'kickstart', 'carniv', 'laboratori', 'behest', 'cumbersom', 'saccharin', 'momma', 'reccommend', 'eek', 'objection', 'dopamin', 'reserv', 'crucial', 'neurolog', 'neural', 'spina', 'bifida', 'anencephali', 'vet', 'anal', 'gland', 'tahini', 'salsa', 'caster', 'backpack', 'boot', 'camp', 'isopur', 'colder', 'freakin', 'implant', 'automat', 'permiss', 'client', 'metimucil', 'thc', 'cantaloup', 'neatli', 'proaduct', 'bing', 'cumul', 'mediciney', 'worn', 'niec', 'gall', 'bladder', 'tunnel', 'applianc', 'vodka', 'swallor', 'playtex', 'tampon', 'optimimum', 'inject', 'chaulki', 'johnson', 'clarif', 'inhabit', 'chemtrail', 'aluminum', 'barium', 'strontium', 'steril', 'foe', 'unmark', 'cast', 'uneligbl', 'cornstarch', 'seemingli', 'superflu', 'soundli', 'hydrolyz', 'steamer', 'chemotherapi', 'applesauc', 'depot', 'crepe', 'abid', 'valley', 'neighborhood', 'ovul', 'cohash', 'leach', 'ness', 'spf', 'commut', 'osteoporosi', 'gun', 'tick', 'ramon', 'jeez', 'commonli', 'mari', 'kansa', 'resent', 'capsulesbi', 'solgar', 'advert', 'deterr', 'wal', 'mart', 'colostrum', 'gmp', 'diagnosi', 'trashx', 'myrrh', 'jul', 'omgwhich', 'avocado', 'dishearten', 'conseal', 'startl', 'tender', 'ai', 'aweful', 'coach', 'oncologist', 'shopp', 'habpv', 'flavorbi', 'commend', 'divers', 'agh', 'planter', 'entranc', 'loop', 'wrought', 'poarch', 'conceal', 'reprimand', 'jcgroff', 'acv', 'interact', 'mute', 'bitch', 'physiolog', 'sweetnessconsid', 'quickeri', 'premad', 'stave', 'urinari', 'detract', 'nasal', 'transact', 'gyro', 'conterfeit', 'fest', 'flaovor', 'tampax', 'sharpish', 'mgomega', 'ketosi', 'howeven', 'toenail', 'heartbeat', 'imbal', 'acerola', 'gro', 'robotusin', 'knuckl', 'flavora', 'cohosh', 'pale', 'laneig', 'bank', 'stomacheach', 'drizzl', 'dy', 'impostor', 'instor', 'wretch', 'herbi', 'thumb', 'litter', 'fume', 'petrochem', 'gr', 'xn', 'gaug', 'lutemax', 'marigold', 'pinto', 'coars', 'aidleyco', 'unknow', 'afterimag', 'oj', 'fairoveral', 'immunecon', 'toeat', 'elmer', 'fudd', 'castor', 'ultrar', 'illegitim', 'ant', 'tx', 'sack', 'hourli', 'wage', 'pesto', 'bismol', 'uphappi', 'fav', 'mehhhh', 'taper', 'pricier', 'microbiom', 'damang', 'tinfoil', 'passabl', 'uneaten', 'administ', 'glowstick', 'discoveri', 'dryer', 'stat', 'artif', 'commom', 'knot', 'ymmv', 'fenugreek', 'mushroomi', 'recwiv', 'replic', 'deficit', 'apocalyps', 'linse', 'xtend', 'supercharg', 'phenoxythanol', 'wooden', 'span', 'pr', 'earwax', 'woodgrain', 'overrun', 'larri', 'sin', 'dillut', 'feasibl', 'adel', 'davi', 'awkward', 'cypress', 'cautious', 'jmaz', 'childhood', 'intellectu', 'solomino', 'spendi', 'merit', 'threaten', 'healthcar', 'foundat', 'overdo', 'mane', 'shudder', 'xssential', 'xssentail', 'speckl', 'writer', 'shizzz', 'parcel', 'cafe', 'campu', 'disovl', 'cassia', 'ceylon', 'cheesi', 'cheddar', 'optionsthey', 'institut', 'highlight', 'manufacturerwat', 'dimethicon', 'distearyldimonium', 'hydroxyethyl', 'urea', 'barbadensi', 'cyclopentasiloxan', 'cetearyl', 'behentrimonium', 'methosulf', 'glyceryl', 'gluceth', 'avena', 'sativa', 'kernel', 'chamomilla', 'recutita', 'matricaria', 'bisabolol', 'zingib', 'officinal', 'tocopheryl', 'crosspolym', 'ascorbyl', 'phosphat', 'polysorb', 'stearamidopropyl', 'pg', 'dimonium', 'butyrospermum', 'parkii', 'steareth', 'panthenol', 'butylen', 'disodium', 'edta', 'benzyl', 'serrata', 'dipropylen', 'triethyl', 'octinox', 'pollut', 'oxybenzon', 'dress', 'blueraspberri', 'laureth', 'supertast', 'cest', 'vie', 'butterfli', 'reveal', 'ca', 'fingerprint', 'unexpir', 'microbead', 'wildlif', 'oddli', 'salicyl', 'pubesc', 'schooler', 'absolutli', 'silomo', 'align', 'futil', 'harshli', 'grandma', 'sandpap', 'methylchorloisthiazolin', 'methylisohiazolinon', 'mice', 'err', 'kiss', 'bittersweet', 'micellular', 'suttl', 'anabol', 'overtim', 'bpi', 'shitti', 'interestingli', 'tween', 'proce', 'specialist', 'nutriplustea', 'nutiplustea', 'wire', 'zap', 'consequ', 'shiver', 'ceil', 'voluntarili', 'necess', 'iodiz', 'godhead', 'career', 'amt', 'razz', 'razzberri', 'swing', 'ewwww', 'thou', 'velvet', 'overlap', 'saint', 'wort', 'huel', 'churn', 'swore', 'cheapen', 'quietli', 'joy', 'deffrent', 'pagag', 'fasten', 'ini', 'oenni', 'researh', 'velcro', 'unseemli', 'predict', 'ooof', 'flintston', 'govt', 'avenu', 'abtain', 'increa', 'finest', 'bulb', 'tinctur', 'volumin', 'inconvenienc', 'gauz', 'creep', 'coolaid', 'caddi', 'snot', 'frito', 'mk', 'smother', 'miiiiight', 'media', 'nutiva', 'ubiquinol', 'imper', 'weil', 'introduct', 'soapmak', 'innov', 'waffl', 'flakey', 'dtore', 'thoroughli', 'seep', 'noooooooooo', 'runnfast', 'lifelong', 'safest', 'bizarr', 'hunk', 'shopper', 'spruce', 'conflict', 'stalebut', 'thet', 'psycholog', 'manual', 'bv', 'ti', 'fishoil', 'hoop', 'hardback', 'hose', 'tblsp', 'tsp', 'bypass', 'chaser', 'coward', 'wack', 'stray', 'brrrr', 'evenbeauti', 'maov', 'cotsco', 'transport', 'odour', 'bla', 'charcoal', 'malodextrin', 'flovor', 'againeven', 'oversea', 'insignific', 'oblig', 'unsuspect', 'nih', 'oleic', 'polyunsatur', 'monounsatur', 'larh', 'concur', 'maneuv', 'regrett', 'flaki', 'repack', 'coin', 'colloid', 'agreeabl', 'circu', 'slam', 'uhhh', 'southwest', 'puerto', 'rico', 'slap', 'discredit', 'pc', 'bitti', 'hypercrit', 'zyflam', 'proctor', 'founder', 'pursuit', 'delud', 'multin', 'crime', 'oper', 'million', 'takesever', 'specialti', 'bioavil', 'longvida', 'theracumin', 'tenth', 'vouch', 'tb', 'ointment', 'oak', 'blisteri', 'devolv', 'veto', 'atroci', 'grosssssss', 'fbshehsidjfnf', 'winc', 'dryest', 'witch', 'manos', 'paranoid', 'cmon', 'twinlab', 'scrambl', 'simmer', 'deciev', 'deck', 'ranch', 'waaaaaay', 'unexplain', 'microscop', 'lymphocyt', 'endoscopi', 'rebut', 'anticanc', 'cautionari', 'induc', 'senesc', 'vasculatur', 'dna', 'atm', 'reactiv', 'oxygen', 'speci', 'ro', 'postul', 'induct', 'majorli', 'monitor', 'rogain', 'appt', 'unmeasur', 'sibl', 'sterol', 'antifung', 'truffl', 'quart', 'chitlin', 'waaaaayyyi', 'tooo', 'greek', 'senc', 'tbd', 'gtf', 'etern', 'engin', 'hydrol', 'guar', 'iitem', 'flavorr', 'grandchildren', 'tweak', 'ginkgo', 'biloba', 'supervisor', 'tout', 'sup', 'margin', 'revolv', 'copper', 'meantim', 'wellcon', 'sediment', 'gunk', 'mucou', 'membran', 'decaf', 'ahem', 'invad', 'dangeri', 'grimac', 'ez', 'sweetz', 'surgar', 'freight', 'puritan', 'pride', 'egregi', 'melti', 'supplementth', 'tool', 'compat', 'len', 'lens', 'assail', 'toopen', 'alloc', 'cacium', 'yech', 'nada', 'precis', 'uhhhhggg', 'swallowi', 'gunki', 'outright', 'andd', 'scurri', 'laser', 'smallest', 'experiment', 'longtim', 'cognit', 'oab', 'caregiv', 'comyou', 'bb', 'goos', 'debra', 'cordenizdebrat', 'overhaul', 'guilti', 'gah', 'alarm', 'quasi', 'fetal', 'rubbish', 'nodul'])"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf.vocabulary_.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4224, 10072) (1776, 10072)\n"
     ]
    }
   ],
   "source": [
    "#Separating train and test arrays\n",
    "X_train = all_arr[:len(tr_titles), :]\n",
    "X_test = all_arr[len(tr_titles):, :]\n",
    "print (X_train.shape, X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4224, 21)"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For Maultilabel classification, we need to prepare out train ground truth labels as a 2D array of ones and zeros, which\n",
    "# is why we use MultiLabelBinarizer to convert 1D array (4224,) to 2D array (4224, 21)\n",
    "multi_label = MultiLabelBinarizer()\n",
    "y_train = multi_label.fit_transform(tr_labels)\n",
    "y_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now since we have prepared our training data and training labels, we can proceed to Model Training. I have used and experimented on four models - Gradient Boosted trees using XgBoost, Random Forest Classifiers using XgBoost, NB-SVM (Naive-Bayes Support Vector Machine) and Bidirectional LSTMs.                                    \n",
    "### For first 2 classifiers, I have used SkLearn's OneVsRest Classifier to convert them to MultiLabel Classifiers, however, NB-SVM and LSTMs can be used as they are.\n",
    "### Also for first 2 classifiers, I have fixed there parameters after running a RandomSearchCV over a large variety of parameters. For NB-SVM and LSTM, I manually experimented with different hyper-parameter values and have fixed the currently used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# These is the parameter space used for both Gradient Boosted Tree and Random Forest for RandomSearchCV api\n",
    "params = {\n",
    "        'estimator__max_depth' : [3, 4, 5],                           # learning parameter\n",
    "        'estimator__learning_rate': [0.3, 0.1, 0.05],                 # learning parameter\n",
    "        'estimator__n_estimators': [100, 200, 300],                   # learning parameter\n",
    "        'estimator__colsample_bytree': [0.6, 0.8, 1.0],               # regulazation parameter\n",
    "        'estimator__reg_alpha': [1, 2, 3]                             # l1 regularization parameter\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.3 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed: 14.7min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.3, score=0.818174133558749, total=14.6min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.3 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed: 28.0min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.3, score=0.8010726026110642, total=13.2min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.3 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.3, score=0.8124786324786325, total=13.0min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.3 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.3, score=0.8252521837137222, total=13.1min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.3 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.3, score=0.8246689987211313, total=13.5min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.01 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.01, score=0.70236071620687, total=21.6min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.01 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.01, score=0.7046623462008077, total=21.4min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.01 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.01, score=0.7129322813938197, total=21.7min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.01 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.01, score=0.6955574340189725, total=21.5min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.01 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.01, score=0.7005228315654854, total=21.6min\n",
      "[CV] estimator__n_estimators=100, estimator__learning_rate=0.05 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=100, estimator__learning_rate=0.05, score=0.7762637362637363, total= 7.3min\n",
      "[CV] estimator__n_estimators=100, estimator__learning_rate=0.05 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=100, estimator__learning_rate=0.05, score=0.7647299708838171, total= 7.5min\n",
      "[CV] estimator__n_estimators=100, estimator__learning_rate=0.05 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=100, estimator__learning_rate=0.05, score=0.7747187001033156, total= 7.4min\n",
      "[CV] estimator__n_estimators=100, estimator__learning_rate=0.05 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=100, estimator__learning_rate=0.05, score=0.7713816098431484, total= 7.6min\n",
      "[CV] estimator__n_estimators=100, estimator__learning_rate=0.05 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=100, estimator__learning_rate=0.05, score=0.7791976980365606, total= 7.4min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.3 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.3, score=0.8126120033812342, total=18.8min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.3 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.3, score=0.7967727998497229, total=19.4min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.3 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.3, score=0.8092213769136847, total=19.2min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.3 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.3, score=0.8157058326289095, total=18.9min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.3 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.3, score=0.8187448281050177, total=20.0min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.05 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.05, score=0.8127636466098004, total=13.9min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.05 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.05, score=0.8007570207570208, total=13.6min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.05 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.05, score=0.805231520616136, total=13.7min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.05 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.05, score=0.8089771766694843, total=13.9min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.05 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.05, score=0.812034529451591, total=14.3min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.1 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.1, score=0.8204052357898511, total=13.5min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.1 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.1, score=0.807711092326477, total=13.2min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.1 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.1, score=0.8168291537522306, total=13.1min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.1 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.1, score=0.8235841081994928, total=13.7min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.1 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.1, score=0.8251466937485894, total=13.7min\n",
      "[CV] estimator__n_estimators=100, estimator__learning_rate=0.3 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=100, estimator__learning_rate=0.3, score=0.818658264812111, total= 7.1min\n",
      "[CV] estimator__n_estimators=100, estimator__learning_rate=0.3 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=100, estimator__learning_rate=0.3, score=0.8043796374565605, total= 6.9min\n",
      "[CV] estimator__n_estimators=100, estimator__learning_rate=0.3 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=100, estimator__learning_rate=0.3, score=0.8105456936226166, total= 6.9min\n",
      "[CV] estimator__n_estimators=100, estimator__learning_rate=0.3 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=100, estimator__learning_rate=0.3, score=0.8198872921949845, total= 6.9min\n",
      "[CV] estimator__n_estimators=100, estimator__learning_rate=0.3 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=100, estimator__learning_rate=0.3, score=0.8211163770405475, total= 6.9min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.1 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.1, score=0.8214308768154922, total=20.1min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.1 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.1, score=0.812642058795905, total=19.6min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.1 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.1, score=0.8160120221658683, total=19.5min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.1 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.1, score=0.8251620174697099, total=28.2min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.1 .......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.1, score=0.8309260513051981, total=30.6min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.01 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.01, score=0.65141730064807, total=24.5min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.01 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.01, score=0.652140509063586, total=21.5min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.01 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.01, score=0.6630327791866254, total=21.5min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.01 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.01, score=0.6347228839536531, total=17.6min\n",
      "[CV] estimator__n_estimators=200, estimator__learning_rate=0.01 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=200, estimator__learning_rate=0.01, score=0.6496934476792298, total=13.9min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.05 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.05, score=0.815603883296191, total=26.9min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.05 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.05, score=0.8103033718418333, total=20.0min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.05 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.05, score=0.817144735606274, total=17.7min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.05 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.05, score=0.8168779938010707, total=26.6min\n",
      "[CV] estimator__n_estimators=300, estimator__learning_rate=0.05 ......\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed: 819.0min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__n_estimators=300, estimator__learning_rate=0.05, score=0.8196569623109907, total=23.0min\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=<generator object _BaseKFold.split at 0x000001911E376E08>,\n",
       "          error_score='raise',\n",
       "          estimator=OneVsRestClassifier(estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bynode=1, colsample_bytree=0.8, gamma=0,\n",
       "       learning_rate=0.1, max_delta_step=0, max_depth=5,\n",
       "       min_child_weight=1, missing=None, n_estimators=100, n_jobs=1,\n",
       "       ...1, scale_pos_weight=1,\n",
       "       seed=None, silent=None, subsample=1, verbosity=2),\n",
       "          n_jobs=4),\n",
       "          fit_params=None, iid=True, n_iter=10, n_jobs=1,\n",
       "          param_distributions={'estimator__learning_rate': [0.3, 0.1, 0.05, 0.01], 'estimator__n_estimators': [100, 200, 300]},\n",
       "          pre_dispatch='2*n_jobs', random_state=1001, refit=True,\n",
       "          return_train_score='warn', scoring='f1_samples', verbose=3)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Training Gradient Boosted Tree using XgBoost and applying Random Search over above mentioned parameter space for 10\n",
    "# iterations with 5 folds of CV. I ran above parameter space, in two folds, first checking only for the growth/learning\n",
    "# parameters and second time checking for parameters which prevent Over-Sampling\n",
    "n_folds = 5\n",
    "kf = KFold(n_splits=n_folds, shuffle = True, random_state = 1001)\n",
    "\n",
    "xgb_clf = XGBClassifier(learning_rate=0.1, n_estimators=300, max_depth=5, verbosity=2, objective='multi:softprob', num_class=21)\n",
    "onevsrest_clf = OneVsRestClassifier(xgb_clf, n_jobs=4)\n",
    "\n",
    "random_search = RandomizedSearchCV(onevsrest_clf, param_distributions=params, scoring='f1_samples', cv=kf.split(X_train, y_train), verbose=3, random_state=1001)\n",
    "random_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OneVsRestClassifier(estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bynode=1, colsample_bytree=1, gamma=0, learning_rate=0.1,\n",
       "       max_delta_step=0, max_depth=5, min_child_weight=1, missing=None,\n",
       "       n_estimators=300, n_jobs=1, nthread=None, num_class=21,\n",
       "       objective='multi:softprob', random_state=0, reg_alpha=0,\n",
       "       reg_lambda=1, scale_pos_weight=1, seed=None, silent=None,\n",
       "       subsample=1, verbosity=2),\n",
       "          n_jobs=4)"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Training Gradient Boosted Tree classifier with best parameters found using above Random search\n",
    "xgb_clf = XGBClassifier(learning_rate=0.1, n_estimators=300, max_depth=5, verbosity=2, objective='multi:softprob', num_class=21)\n",
    "onevsrest_clf = OneVsRestClassifier(xgb_clf, n_jobs=4)\n",
    "onevsrest_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predicting over the Test Dataset using trained XgBoost model\n",
    "preds_xgb = onevsrest_clf.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OneVsRestClassifier(estimator=XGBRFClassifier(base_score=0.5, colsample_bylevel=1, colsample_bynode=0.8,\n",
       "        colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "        max_depth=5, min_child_weight=1, missing=None, n_estimators=300,\n",
       "        n_jobs=1, nthread=None, num_class=21, objective='multi:softprob',\n",
       "        random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
       "        seed=None, silent=None, subsample=0.8, verbosity=2),\n",
       "          n_jobs=4)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Training Random Forest classifier with best parameters found using above Random search\n",
    "xgb_rf_clf = XGBRFClassifier(learning_rate=0.1, n_estimators=300, max_depth=5, verbosity=2, objective='multi:softprob', num_class=21)\n",
    "onevsrest_rf_clf = OneVsRestClassifier(xgb_rf_clf, n_jobs=4)\n",
    "onevsrest_rf_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predicting over the Test Dataset using trained Random Forest model\n",
    "preds_rf = onevsrest_rf_clf.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Below I have tried Naive Bayes - Support Vector Machines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is the Naive-Bayes basic equation\n",
    "def pr(y_i, y):\n",
    "    p = X_train[y==y_i].sum(0)\n",
    "    return (p+1) / ((y==y_i).sum()+1)\n",
    "\n",
    "def get_mdl(y):\n",
    "    r = np.log(pr(1,y) / pr(0,y))\n",
    "    m = LogisticRegression(C=4, dual=True)\n",
    "    x_nb = X_train.multiply(r)\n",
    "    return m.fit(x_nb, y), r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit 0\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'numpy.ndarray' object has no attribute 'multiply'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-131-dc29b0d888df>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m21\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0mprint\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m'fit '\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m     \u001b[0mm\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_mdl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m     \u001b[0mnbsvm_preds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmultiply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-130-1703693e2334>\u001b[0m in \u001b[0;36mget_mdl\u001b[1;34m(y)\u001b[0m\n\u001b[0;32m      7\u001b[0m     \u001b[0mr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m/\u001b[0m \u001b[0mpr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m     \u001b[0mm\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mLogisticRegression\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mC\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdual\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m     \u001b[0mx_nb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX_train\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmultiply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx_nb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mr\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'numpy.ndarray' object has no attribute 'multiply'"
     ]
    }
   ],
   "source": [
    "# We fit each label one at a time\n",
    "nbsvm_preds = np.zeros((1776, 21))\n",
    "\n",
    "for i in range(21):\n",
    "    print ('fit '+str(i))\n",
    "    m, r = get_mdl(y_train[:, i])\n",
    "    nbsvm_preds[:, i] = m.predict_proba(X_test.multiply(r))[:, 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Below I have prepared a Keras Bidirectional LSTM model. All the hyper-parameters, embedding length of tokens, input size for each sentence, layer size, different types of layers like 1D Convolution with/without LSTM were tweaked manually. But I still found a normal Bidirectional LSTM model working best for this problem. I did not use pre-trained embeddings because generally they don't work well in case of small datasets, pertaining to the fact that the common tokens turn out to be very less, which reduces the performance further."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4224, 200) (1776, 200)\n"
     ]
    }
   ],
   "source": [
    "keras_tok = text.Tokenizer()\n",
    "keras_tok.fit_on_texts(all_text)\n",
    "new_all_texts = keras_tok.texts_to_sequences(all_text)\n",
    "X = sequence.pad_sequences(new_all_texts, 200)\n",
    "X_train = X[:len(tr_titles), :]\n",
    "X_test = X[len(tr_titles):, :]\n",
    "print (X_train.shape, X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10722"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_features = len(keras_tok.word_counts)\n",
    "embed_size = 100\n",
    "max_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3801, 200) (3801, 21) (423, 200) (423, 21)\n"
     ]
    }
   ],
   "source": [
    "train_idx = random.sample([i for i in range(4224)], int(0.9*4224))\n",
    "valid_idx = list(set([i for i in range(4224)]) - set(train_idx))\n",
    "\n",
    "X_tr = X_train[train_idx, :]\n",
    "y_tr = y_train[train_idx, :]\n",
    "X_val = X_train[valid_idx, :]\n",
    "y_val = y_train[valid_idx, :]\n",
    "\n",
    "print (X_tr.shape, y_tr.shape, X_val.shape, y_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0811 23:40:32.845762 12000 deprecation_wrapper.py:119] From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "W0811 23:40:32.872495 12000 deprecation_wrapper.py:119] From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W0811 23:40:32.880112 12000 deprecation_wrapper.py:119] From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "W0811 23:40:33.135438 12000 deprecation_wrapper.py:119] From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:133: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
      "\n",
      "W0811 23:40:33.149546 12000 deprecation.py:506] From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 200)               0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 200, 100)          1072300   \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 200, 128)          84480     \n",
      "_________________________________________________________________\n",
      "global_average_pooling1d_1 ( (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 21)                1365      \n",
      "=================================================================\n",
      "Total params: 1,166,401\n",
      "Trainable params: 1,166,401\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "inp = Input((200,))\n",
    "embed = Embedding(max_features+1, embed_size, input_length=200)(inp)\n",
    "lstm = Bidirectional(LSTM(64, return_sequences=True, dropout=0.2, recurrent_dropout=0.2))(embed)\n",
    "avg_pool = GlobalAveragePooling1D()(lstm)\n",
    "dense = Dense(64, activation='relu')(avg_pool)\n",
    "drop = Dropout(0.1)(dense)\n",
    "out = Dense(21, activation='sigmoid')(dense)\n",
    "\n",
    "lstm_model = Model(inp, out)\n",
    "lstm_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top5(y_true, y_pred):\n",
    "    return top_k_categorical_accuracy(y_true, y_pred, k=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0811 23:40:56.492053 12000 deprecation_wrapper.py:119] From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "W0811 23:40:56.521557 12000 deprecation_wrapper.py:119] From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3376: The name tf.log is deprecated. Please use tf.math.log instead.\n",
      "\n",
      "W0811 23:40:56.536788 12000 deprecation.py:323] From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\nn_impl.py:180: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    }
   ],
   "source": [
    "lstm_model.compile(optimizer=Adam(lr=0.001), loss='binary_crossentropy', metrics=[top5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 3801 samples, validate on 423 samples\n",
      "Epoch 1/50\n",
      "3801/3801 [==============================] - 81s 21ms/step - loss: 0.3625 - top5: 0.3773 - val_loss: 0.2134 - val_top5: 0.6596\n",
      "Epoch 2/50\n",
      "3801/3801 [==============================] - 75s 20ms/step - loss: 0.2134 - top5: 0.6514 - val_loss: 0.2129 - val_top5: 0.6147\n",
      "Epoch 3/50\n",
      "3801/3801 [==============================] - 70s 18ms/step - loss: 0.2132 - top5: 0.6354 - val_loss: 0.2130 - val_top5: 0.6619\n",
      "Epoch 4/50\n",
      "3801/3801 [==============================] - 80s 21ms/step - loss: 0.2127 - top5: 0.6464 - val_loss: 0.2121 - val_top5: 0.6147\n",
      "Epoch 5/50\n",
      "3801/3801 [==============================] - 80s 21ms/step - loss: 0.2120 - top5: 0.6456 - val_loss: 0.2106 - val_top5: 0.6619\n",
      "Epoch 6/50\n",
      "3801/3801 [==============================] - 78s 21ms/step - loss: 0.2107 - top5: 0.6627 - val_loss: 0.2112 - val_top5: 0.6336\n",
      "Epoch 7/50\n",
      "3801/3801 [==============================] - 76s 20ms/step - loss: 0.2082 - top5: 0.6625 - val_loss: 0.2086 - val_top5: 0.6619\n",
      "Epoch 8/50\n",
      "3801/3801 [==============================] - 78s 21ms/step - loss: 0.2047 - top5: 0.6922 - val_loss: 0.2075 - val_top5: 0.6359\n",
      "Epoch 9/50\n",
      "3801/3801 [==============================] - 75s 20ms/step - loss: 0.1988 - top5: 0.7153 - val_loss: 0.2070 - val_top5: 0.6619\n",
      "Epoch 10/50\n",
      "3801/3801 [==============================] - 73s 19ms/step - loss: 0.1998 - top5: 0.7117 - val_loss: 0.2062 - val_top5: 0.6690\n",
      "Epoch 11/50\n",
      "3801/3801 [==============================] - 79s 21ms/step - loss: 0.1973 - top5: 0.7159 - val_loss: 0.2077 - val_top5: 0.6336\n",
      "Epoch 12/50\n",
      "3801/3801 [==============================] - 79s 21ms/step - loss: 0.1951 - top5: 0.7261 - val_loss: 0.2051 - val_top5: 0.6643\n",
      "Epoch 13/50\n",
      "3801/3801 [==============================] - 80s 21ms/step - loss: 0.1940 - top5: 0.7377 - val_loss: 0.2054 - val_top5: 0.6879\n",
      "Epoch 14/50\n",
      "3801/3801 [==============================] - 87s 23ms/step - loss: 0.1918 - top5: 0.7432 - val_loss: 0.2004 - val_top5: 0.6832\n",
      "Epoch 15/50\n",
      "3801/3801 [==============================] - 89s 23ms/step - loss: 0.1889 - top5: 0.7556 - val_loss: 0.2034 - val_top5: 0.6879\n",
      "Epoch 16/50\n",
      "3801/3801 [==============================] - 84s 22ms/step - loss: 0.1888 - top5: 0.7545 - val_loss: 0.1968 - val_top5: 0.7163\n",
      "Epoch 17/50\n",
      "3801/3801 [==============================] - 82s 22ms/step - loss: 0.1849 - top5: 0.7616 - val_loss: 0.1931 - val_top5: 0.6974\n",
      "Epoch 18/50\n",
      "3801/3801 [==============================] - 82s 22ms/step - loss: 0.1827 - top5: 0.7701 - val_loss: 0.1930 - val_top5: 0.7139\n",
      "Epoch 19/50\n",
      "3801/3801 [==============================] - 77s 20ms/step - loss: 0.1805 - top5: 0.7806 - val_loss: 0.1925 - val_top5: 0.7069\n",
      "Epoch 20/50\n",
      "3801/3801 [==============================] - 81s 21ms/step - loss: 0.1790 - top5: 0.7798 - val_loss: 0.1898 - val_top5: 0.7305\n",
      "Epoch 21/50\n",
      "3801/3801 [==============================] - 87s 23ms/step - loss: 0.1791 - top5: 0.7795 - val_loss: 0.1958 - val_top5: 0.6785\n",
      "Epoch 22/50\n",
      "3801/3801 [==============================] - 85s 22ms/step - loss: 0.1763 - top5: 0.7916 - val_loss: 0.1963 - val_top5: 0.7045\n",
      "Epoch 23/50\n",
      "3801/3801 [==============================] - 84s 22ms/step - loss: 0.1770 - top5: 0.7924 - val_loss: 0.1884 - val_top5: 0.7565\n",
      "Epoch 24/50\n",
      "3801/3801 [==============================] - 88s 23ms/step - loss: 0.1730 - top5: 0.7998 - val_loss: 0.1859 - val_top5: 0.7281\n",
      "Epoch 25/50\n",
      "3801/3801 [==============================] - 82s 21ms/step - loss: 0.1708 - top5: 0.8122 - val_loss: 0.1829 - val_top5: 0.7470\n",
      "Epoch 26/50\n",
      "3801/3801 [==============================] - 84s 22ms/step - loss: 0.1674 - top5: 0.8172 - val_loss: 0.1843 - val_top5: 0.7329\n",
      "Epoch 27/50\n",
      "3801/3801 [==============================] - 86s 23ms/step - loss: 0.1652 - top5: 0.8248 - val_loss: 0.1846 - val_top5: 0.7400\n",
      "Epoch 28/50\n",
      "3801/3801 [==============================] - 81s 21ms/step - loss: 0.1635 - top5: 0.8300 - val_loss: 0.1823 - val_top5: 0.7352\n",
      "Epoch 29/50\n",
      "3801/3801 [==============================] - 84s 22ms/step - loss: 0.1611 - top5: 0.8421 - val_loss: 0.1802 - val_top5: 0.7636\n",
      "Epoch 30/50\n",
      " 576/3801 [===>..........................] - ETA: 1:17 - loss: 0.1592 - top5: 0.8472"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-127-15afae7b2a2b>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mhist\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlstm_model\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_tr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_tr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m64\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m50\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_val\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_val\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mReduceLROnPlateau\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfactor\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpatience\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmin_lr\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.0000001\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mEarlyStopping\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpatience\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrestore_best_weights\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[0;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1039\u001b[1;33m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[0;32m   1040\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\engine\\training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[1;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[0;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    198\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 199\u001b[1;33m                 \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    201\u001b[0m                 \u001b[1;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mo\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2713\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2714\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2715\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2716\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2717\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2674\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2675\u001b[1;33m             \u001b[0mfetched\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2676\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2677\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1456\u001b[0m         ret = tf_session.TF_SessionRunCallable(self._session._session,\n\u001b[0;32m   1457\u001b[0m                                                \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1458\u001b[1;33m                                                run_metadata_ptr)\n\u001b[0m\u001b[0;32m   1459\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1460\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "hist = lstm_model.fit(X_tr, y_tr, batch_size=64, epochs=50, verbose=1, validation_data=(X_val, y_val), callbacks=[ReduceLROnPlateau(factor=0.1, patience=3, verbose=1, min_lr=0.0000001), EarlyStopping(patience=10, verbose=1, restore_best_weights=True)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predictions for LSTM model\n",
    "lstm_preds = lstm_model.predict(X_test, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Average predictions for all 4 models\n",
    "preds_avg = (preds + preds_xgb + preds_rf + lstm_preds) / 4.0\n",
    "\n",
    "#preds_geomean = (preds * preds_xgb * preds_rf * lstm_preds) ** 4.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Below I prepared a helper function which prepares a submission dataframe and saves it with provided name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "def subm_df(preds, subm_file):\n",
    "    \n",
    "    new_preds = []\n",
    "    for i in range(len(te_idx_list)-1):\n",
    "        start_idx = te_idx_list[i]\n",
    "        end_idx = te_idx_list[i+1]\n",
    "        diff = end_idx - start_idx\n",
    "        pred_idx = preds[i].argsort()[-diff:][::-1]\n",
    "        for pred in pred_idx: new_preds.append(pred)\n",
    "        \n",
    "    test_preds = le.inverse_transform(new_preds)\n",
    "    subm_df = pd.read_csv('test.csv')\n",
    "    subm_df['topic'] = test_preds\n",
    "    subm_df.to_csv(subm_file, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "subm_df(preds_geomean, 'preds_avg.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.0259676 , 0.43909261, 0.03770643, ..., 0.02002454, 0.02000235,\n",
       "        0.01255358],\n",
       "       [0.04284434, 0.02483947, 0.01258211, ..., 0.01230513, 0.01710293,\n",
       "        0.01687884],\n",
       "       [0.04895935, 0.06284668, 0.02404781, ..., 0.03273235, 0.02678353,\n",
       "        0.01814154],\n",
       "       ...,\n",
       "       [0.5602819 , 0.60670522, 0.02127513, ..., 0.01562155, 0.06931867,\n",
       "        0.0138615 ],\n",
       "       [0.08962463, 0.03897221, 0.01520684, ..., 0.01236469, 0.02083567,\n",
       "        0.01408974],\n",
       "       [0.01930879, 0.20055215, 0.03684748, ..., 0.02019732, 0.01813376,\n",
       "        0.01376362]])"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}