dataDaysChallenge2-Github.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Programming by Mojtaba Valipour @ SUTest-V1.0.0, vpcom.ir\n",
    "# Copyright 2019\n",
    "# Title: Deep Hierarchical Persian Text Classification based on hdlTex\n",
    "\n",
    "# Information about the environments\n",
    "# Environment: hdlTex, vpcomDesk -> hdlTex.yml\n",
    "# Anaconda\n",
    "# Python:3.5.6\n",
    "# Tensorflow: 1.10.0\n",
    "# Keras: 2.2.2\n",
    "# Pandas: 0.23.4\n",
    "# nltk: 3.3.0\n",
    "# numpy: 1.15.2\n",
    "# Cuda:9.0\n",
    "\n",
    "# github.com/mvpcom/ddCh2"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# RESOURCES:\n",
    "'''\n",
    "1- https://github.com/kk7nc/HDLTex\n",
    "2- https://nlp.stanford.edu/projects/glove/\n",
    "3- https://research.cafebazaar.ir/visage/divar_datasets/\n",
    "4- HDLTex: Hierarchical Deep Learning for Text Classification\n",
    "5- https://fasttext.cc/docs/en/crawl-vectors.html\n",
    "6- https://github.com/hadifar/PNLP\n",
    "7- https://datadays.ir\n",
    "8- https://github.com/philipperemy/keras-attention-mechanism\n",
    "'''"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "''' Challenge2_DivarDataset_DataDays, Sharif University\n",
    "بخش اول\n",
    "عنوان: پیش بینی دسته بندی\n",
    "امتیاز: ۳۰۰۰ امتیاز\n",
    "توانایی: یادگیری ماشین و تحلیل متن\n",
    "مسئله: پیشبینی دسته بندی آگهی از روی سایر ویژگی های آن\n",
    "\n",
    "توصیف: در این بخش شما یک دیتاست شامل ۲۰۰ هزار سطر دانلود میکنید که هر سطر حاوی اطلاعات مربوط به یک آگهی است. شما باید دسته بندی سلسله مراتبی هر آگهی را به دست آورید و در قالب یک فایل csv که شامل ۲۰۰ هزار سطر و سه ستون cat1, cat2, cat3 است آپلود کنید.\n",
    "ملاحظه مهم: ساختار پاسخ باید دقیقا به شکل اشاره شده باشد. ضمنا تمام دسته ها باید به همان شکلی که در دیتاست Train قرار دارد باشد. یک نمونه از پاسخ مطلوب در این فایل فایل پیوست شده است. \n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(947635, 17)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://research.cafebazaar.ir/visage/divar_datasets/\n",
    "dataset = pd.read_csv(\"./data/divar_posts_dataset.csv\")\n",
    "dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataset[dataset['cat3'].str.contains(\"art\")==True] # search query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat1</th>\n",
       "      <th>cat2</th>\n",
       "      <th>cat3</th>\n",
       "      <th>city</th>\n",
       "      <th>desc</th>\n",
       "      <th>price</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>furniture-and-home-decore</td>\n",
       "      <td>antiques-and-art</td>\n",
       "      <td>Mashhad</td>\n",
       "      <td>سلام,یک عدد گلدون نخل سه طبقه ی سالم دارم با پ...</td>\n",
       "      <td>30000</td>\n",
       "      <td>گلدون مصنوعی نخل</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>heavy</td>\n",
       "      <td>Mashhad</td>\n",
       "      <td>سریال 43j$NUMبدون شکستگی سه حلقه لاستیک نو یک ...</td>\n",
       "      <td>-1</td>\n",
       "      <td>لودر کاتر پیلار 950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>furniture-and-home-decore</td>\n",
       "      <td>sofa-armchair</td>\n",
       "      <td>Tehran</td>\n",
       "      <td>مبل راحتی هفت نفره شامل سه نفره یک عدد دونفره ...</td>\n",
       "      <td>600000</td>\n",
       "      <td>مبل راحتی هفت نفره بامیز جلو مبلی</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>personal</td>\n",
       "      <td>baby-and-toys</td>\n",
       "      <td>personal-toys</td>\n",
       "      <td>Karaj</td>\n",
       "      <td>شارژی کنترلی سویچ حمل تا 35 کیلو صندلی برای دو...</td>\n",
       "      <td>450000</td>\n",
       "      <td>ماشین شارژی</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           cat1                       cat2              cat3     city  \\\n",
       "1  for-the-home  furniture-and-home-decore  antiques-and-art  Mashhad   \n",
       "2      vehicles                       cars             heavy  Mashhad   \n",
       "3  for-the-home  furniture-and-home-decore     sofa-armchair   Tehran   \n",
       "4      personal              baby-and-toys     personal-toys    Karaj   \n",
       "\n",
       "                                                desc   price  \\\n",
       "1  سلام,یک عدد گلدون نخل سه طبقه ی سالم دارم با پ...   30000   \n",
       "2  سریال 43j$NUMبدون شکستگی سه حلقه لاستیک نو یک ...      -1   \n",
       "3  مبل راحتی هفت نفره شامل سه نفره یک عدد دونفره ...  600000   \n",
       "4  شارژی کنترلی سویچ حمل تا 35 کیلو صندلی برای دو...  450000   \n",
       "\n",
       "                               title  \n",
       "1                   گلدون مصنوعی نخل  \n",
       "2                لودر کاتر پیلار 950  \n",
       "3  مبل راحتی هفت نفره بامیز جلو مبلی  \n",
       "4                        ماشین شارژی  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.iloc[1:5,[3,4,5,6,8,13,14]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['for-the-home' 'vehicles' 'personal' 'electronic-devices' 'businesses'\n",
      " 'leisure-hobbies'] 6\n",
      "['furniture-and-home-decore' 'cars' 'baby-and-toys' 'parts-accessories'\n",
      " 'utensils-and-appliances' 'clothing-and-shoes' 'mobile-tablet'\n",
      " 'childrens-clothing-and-shoe' 'game-consoles-and-video-games'\n",
      " 'audio-video' 'building-and-garden' 'jewelry-and-watches'\n",
      " 'equipments-and-machinery' 'bicycle' 'animals' nan 'batch'\n",
      " 'musical-instruments' 'health-beauty' 'motorcycles' 'computers'\n",
      " 'sport-leisure' 'book-student-literature' 'utility' 'travel-packages'\n",
      " 'hobby-collectibles' 'leisure-hobbies-toys' 'phone'] 28\n",
      "['sofa-armchair' 'antiques-and-art' 'heavy' 'personal-toys' nan\n",
      " 'cookware-tableware' 'light' 'clothing' 'mobile-phones' 'tv-projector'\n",
      " 'garden-and-patio' 'watches' 'offices' 'kitchen' 'farm-animals' 'cat'\n",
      " 'video-dvdplayer' 'shoes-belt-bag' 'industrial' 'tv-and-stereo-furniture'\n",
      " 'birds' 'guitar-bass-amplifier' 'beds-bedroom' 'carpets'\n",
      " 'mobile-tablet-accessories' 'fridge-and-freezer' 'lighting'\n",
      " 'tables-and-chairs' 'strollers-and-accessories'\n",
      " 'modem-and-network-equipment' 'jewelry' 'stereo-surround'\n",
      " 'camera-camcoders' 'training' 'storage' 'stove-and-heating'\n",
      " 'barbershop-and-beautysalon' 'dishwasher' 'parts-and-accessories'\n",
      " 'cafe-and-restaurant' 'microwave-stove' 'washer-dryer' 'educational'\n",
      " 'childrens-furniture' 'piano-keyboard' 'desktops' 'shop-and-cash'\n",
      " 'laptops' 'rhinestones' 'bathroom-wc-sauna' 'mp3-player'\n",
      " 'textile-ornaments' 'tablet' 'historical-objects'\n",
      " 'instrument-cleaning-tailoring' 'fish' 'accessories' 'camping-outdoor'\n",
      " 'traditional' 'child-car-seat' 'printer-scaner-copier' 'rodents-rabbits'\n",
      " 'coin-stamp' 'ball-sports' 'repair-tool' 'winter-sports'\n",
      " 'drums-percussion'] 67\n"
     ]
    }
   ],
   "source": [
    "print(dataset.cat1.unique(), len(dataset.cat1.unique()))\n",
    "print(dataset.cat2.unique(), len(dataset.cat2.unique()))\n",
    "print(dataset.cat3.unique(), len(dataset.cat3.unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "for-the-home          290313\n",
      "vehicles              206260\n",
      "electronic-devices    166507\n",
      "personal              139164\n",
      "leisure-hobbies        83513\n",
      "businesses             61878\n",
      "Name: cat1, dtype: int64\n",
      "Num of classes, Cat1:  6\n"
     ]
    }
   ],
   "source": [
    "# cat1 informations\n",
    "cat1Classes = dataset.cat1.value_counts()\n",
    "print(cat1Classes)\n",
    "numClassesCat1 = cat1Classes.count()\n",
    "print('Num of classes, Cat1: ', numClassesCat1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['furniture-and-home-decore' 'cars' 'baby-and-toys' 'parts-accessories'\n",
      " 'utensils-and-appliances' 'clothing-and-shoes' 'mobile-tablet'\n",
      " 'childrens-clothing-and-shoe' 'game-consoles-and-video-games'\n",
      " 'audio-video' 'building-and-garden' 'jewelry-and-watches'\n",
      " 'equipments-and-machinery' 'bicycle' 'animals' nan 'batch'\n",
      " 'musical-instruments' 'health-beauty' 'motorcycles' 'computers'\n",
      " 'sport-leisure' 'book-student-literature' 'utility' 'travel-packages'\n",
      " 'hobby-collectibles' 'leisure-hobbies-toys' 'phone']\n",
      "furniture-and-home-decore        204445\n",
      "cars                             130443\n",
      "clothing-and-shoes                87096\n",
      "mobile-tablet                     76307\n",
      "utensils-and-appliances           58714\n",
      "parts-accessories                 55986\n",
      "animals                           50694\n",
      "equipments-and-machinery          50101\n",
      "game-consoles-and-video-games     31964\n",
      "audio-video                       29176\n",
      "computers                         26536\n",
      "baby-and-toys                     20502\n",
      "motorcycles                       19831\n",
      "building-and-garden               18915\n",
      "batch                             11777\n",
      "childrens-clothing-and-shoe       11433\n",
      "bicycle                           11283\n",
      "sport-leisure                     10374\n",
      "jewelry-and-watches               10316\n",
      "health-beauty                      9390\n",
      "utility                            8239\n",
      "musical-instruments                3854\n",
      "book-student-literature            3203\n",
      "hobby-collectibles                 2431\n",
      "phone                              1193\n",
      "travel-packages                    1071\n",
      "leisure-hobbies-toys                603\n",
      "Name: cat2, dtype: int64\n",
      "Num of classes, Cat2:  27\n"
     ]
    }
   ],
   "source": [
    "# cat1 informations\n",
    "print(dataset.cat2.unique())\n",
    "cat2Classes = dataset.cat2.value_counts()\n",
    "print(cat2Classes)\n",
    "numClassesCat2 = cat2Classes.count()\n",
    "print('Num of classes, Cat2: ', numClassesCat2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "light                            120451\n",
      "mobile-phones                     62176\n",
      "clothing                          53758\n",
      "sofa-armchair                     49363\n",
      "birds                             38511\n",
      "shoes-belt-bag                    33338\n",
      "tables-and-chairs                 30742\n",
      "antiques-and-art                  29057\n",
      "storage                           27702\n",
      "cookware-tableware                24433\n",
      "carpets                           22472\n",
      "industrial                        19985\n",
      "shop-and-cash                     16660\n",
      "beds-bedroom                      16650\n",
      "tv-projector                      16118\n",
      "tv-and-stereo-furniture           13688\n",
      "fridge-and-freezer                11152\n",
      "stove-and-heating                 11128\n",
      "childrens-furniture               10798\n",
      "parts-and-accessories             10179\n",
      "heavy                              9992\n",
      "mobile-tablet-accessories          9790\n",
      "training                           7720\n",
      "instrument-cleaning-tailoring      7387\n",
      "lighting                           7298\n",
      "textile-ornaments                  6499\n",
      "watches                            6285\n",
      "accessories                        5971\n",
      "microwave-stove                    5933\n",
      "stereo-surround                    5726\n",
      "                                  ...  \n",
      "modem-and-network-equipment        4406\n",
      "tablet                             4341\n",
      "kitchen                            4145\n",
      "farm-animals                       3981\n",
      "laptops                            3924\n",
      "strollers-and-accessories          3581\n",
      "printer-scaner-copier              3329\n",
      "educational                        3203\n",
      "barbershop-and-beautysalon         3117\n",
      "jewelry                            2586\n",
      "historical-objects                 1696\n",
      "guitar-bass-amplifier              1407\n",
      "garden-and-patio                   1240\n",
      "fish                               1221\n",
      "winter-sports                      1216\n",
      "dishwasher                         1143\n",
      "bathroom-wc-sauna                  1122\n",
      "child-car-seat                     1084\n",
      "piano-keyboard                     1059\n",
      "mp3-player                          908\n",
      "video-dvdplayer                     856\n",
      "ball-sports                         779\n",
      "rhinestones                         668\n",
      "camping-outdoor                     659\n",
      "rodents-rabbits                     538\n",
      "traditional                         472\n",
      "cat                                 472\n",
      "coin-stamp                          388\n",
      "drums-percussion                    354\n",
      "repair-tool                         346\n",
      "Name: cat3, Length: 66, dtype: int64\n",
      "Num of classes, Cat3:  66\n"
     ]
    }
   ],
   "source": [
    "# cat1 informations\n",
    "cat3Classes = dataset.cat3.value_counts()\n",
    "print(cat3Classes)\n",
    "numClassesCat3 = cat3Classes.count()\n",
    "print('Num of classes, Cat3: ', numClassesCat3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat1                cat2                         \n",
      "businesses          equipments-and-machinery          50101\n",
      "                    batch                             11777\n",
      "electronic-devices  mobile-tablet                     76307\n",
      "                    game-consoles-and-video-games     31964\n",
      "                    audio-video                       29176\n",
      "                    computers                         26536\n",
      "                    phone                              1193\n",
      "for-the-home        furniture-and-home-decore        204445\n",
      "                    utensils-and-appliances           58714\n",
      "                    building-and-garden               18915\n",
      "                    utility                            8239\n",
      "leisure-hobbies     animals                           50694\n",
      "                    bicycle                           11283\n",
      "                    sport-leisure                     10374\n",
      "                    musical-instruments                3854\n",
      "                    book-student-literature            3203\n",
      "                    hobby-collectibles                 2431\n",
      "                    travel-packages                    1071\n",
      "                    leisure-hobbies-toys                603\n",
      "personal            clothing-and-shoes                87096\n",
      "                    baby-and-toys                     20502\n",
      "                    childrens-clothing-and-shoe       11433\n",
      "                    jewelry-and-watches               10316\n",
      "                    health-beauty                      9390\n",
      "vehicles            cars                             130443\n",
      "                    parts-accessories                 55986\n",
      "                    motorcycles                       19831\n",
      "Name: cat2, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Hierarchical\n",
    "catClassesB = dataset.groupby(['cat1']).cat2.value_counts()\n",
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None):\n",
    "    print(catClassesB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat1                cat2                       cat3                         \n",
      "businesses          equipments-and-machinery   industrial                        19985\n",
      "                                               shop-and-cash                     16660\n",
      "                                               offices                            5148\n",
      "                                               cafe-and-restaurant                4670\n",
      "                                               barbershop-and-beautysalon         3117\n",
      "electronic-devices  audio-video                tv-projector                      16118\n",
      "                                               stereo-surround                    5726\n",
      "                                               camera-camcoders                   5147\n",
      "                                               mp3-player                          908\n",
      "                                               video-dvdplayer                     856\n",
      "                    computers                  parts-and-accessories             10179\n",
      "                                               desktops                           4698\n",
      "                                               modem-and-network-equipment        4406\n",
      "                                               laptops                            3924\n",
      "                                               printer-scaner-copier              3329\n",
      "                    mobile-tablet              mobile-phones                     62176\n",
      "                                               mobile-tablet-accessories          9790\n",
      "                                               tablet                             4341\n",
      "for-the-home        building-and-garden        stove-and-heating                 11128\n",
      "                                               kitchen                            4145\n",
      "                                               garden-and-patio                   1240\n",
      "                                               bathroom-wc-sauna                  1122\n",
      "                    furniture-and-home-decore  sofa-armchair                     49363\n",
      "                                               tables-and-chairs                 30742\n",
      "                                               antiques-and-art                  29057\n",
      "                                               storage                           27702\n",
      "                                               carpets                           22472\n",
      "                                               beds-bedroom                      16650\n",
      "                                               tv-and-stereo-furniture           13688\n",
      "                                               lighting                           7298\n",
      "                                               textile-ornaments                  6499\n",
      "                    utensils-and-appliances    cookware-tableware                24433\n",
      "                                               fridge-and-freezer                11152\n",
      "                                               microwave-stove                    5933\n",
      "                                               washer-dryer                       5396\n",
      "                                               dishwasher                         1143\n",
      "                    utility                    instrument-cleaning-tailoring      7387\n",
      "                                               repair-tool                         346\n",
      "leisure-hobbies     animals                    birds                             38511\n",
      "                                               accessories                        5971\n",
      "                                               farm-animals                       3981\n",
      "                                               fish                               1221\n",
      "                                               rodents-rabbits                     538\n",
      "                                               cat                                 472\n",
      "                    book-student-literature    educational                        3203\n",
      "                    hobby-collectibles         historical-objects                 1696\n",
      "                                               coin-stamp                          388\n",
      "                    musical-instruments        guitar-bass-amplifier              1407\n",
      "                                               piano-keyboard                     1059\n",
      "                                               traditional                         472\n",
      "                                               drums-percussion                    354\n",
      "                    sport-leisure              training                           7720\n",
      "                                               winter-sports                      1216\n",
      "                                               ball-sports                         779\n",
      "                                               camping-outdoor                     659\n",
      "personal            baby-and-toys              childrens-furniture               10798\n",
      "                                               personal-toys                      5039\n",
      "                                               strollers-and-accessories          3581\n",
      "                                               child-car-seat                     1084\n",
      "                    clothing-and-shoes         clothing                          53758\n",
      "                                               shoes-belt-bag                    33338\n",
      "                    jewelry-and-watches        watches                            6285\n",
      "                                               jewelry                            2586\n",
      "                                               rhinestones                         668\n",
      "vehicles            cars                       light                            120451\n",
      "                                               heavy                              9992\n",
      "Name: cat3, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Hierarchical\n",
    "catClasses = dataset.groupby(['cat1','cat2']).cat3.value_counts()\n",
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None):\n",
    "    print(catClasses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cat1Num \n",
    "# Hint: This will fix the NAN problem -> (NAN : -1)\n",
    "dataset['cat1'] = dataset['cat1'].astype('category')\n",
    "dataset['cat2'] = dataset['cat2'].astype('category')\n",
    "dataset['cat3'] = dataset['cat3'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat1</th>\n",
       "      <th>cat2</th>\n",
       "      <th>cat3</th>\n",
       "      <th>city</th>\n",
       "      <th>desc</th>\n",
       "      <th>price</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>furniture-and-home-decore</td>\n",
       "      <td>antiques-and-art</td>\n",
       "      <td>Mashhad</td>\n",
       "      <td>سلام,یک عدد گلدون نخل سه طبقه ی سالم دارم با پ...</td>\n",
       "      <td>30000</td>\n",
       "      <td>گلدون مصنوعی نخل</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>heavy</td>\n",
       "      <td>Mashhad</td>\n",
       "      <td>سریال 43j$NUMبدون شکستگی سه حلقه لاستیک نو یک ...</td>\n",
       "      <td>-1</td>\n",
       "      <td>لودر کاتر پیلار 950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>furniture-and-home-decore</td>\n",
       "      <td>sofa-armchair</td>\n",
       "      <td>Tehran</td>\n",
       "      <td>مبل راحتی هفت نفره شامل سه نفره یک عدد دونفره ...</td>\n",
       "      <td>600000</td>\n",
       "      <td>مبل راحتی هفت نفره بامیز جلو مبلی</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>personal</td>\n",
       "      <td>baby-and-toys</td>\n",
       "      <td>personal-toys</td>\n",
       "      <td>Karaj</td>\n",
       "      <td>شارژی کنترلی سویچ حمل تا 35 کیلو صندلی برای دو...</td>\n",
       "      <td>450000</td>\n",
       "      <td>ماشین شارژی</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           cat1                       cat2              cat3     city  \\\n",
       "1  for-the-home  furniture-and-home-decore  antiques-and-art  Mashhad   \n",
       "2      vehicles                       cars             heavy  Mashhad   \n",
       "3  for-the-home  furniture-and-home-decore     sofa-armchair   Tehran   \n",
       "4      personal              baby-and-toys     personal-toys    Karaj   \n",
       "\n",
       "                                                desc   price  \\\n",
       "1  سلام,یک عدد گلدون نخل سه طبقه ی سالم دارم با پ...   30000   \n",
       "2  سریال 43j$NUMبدون شکستگی سه حلقه لاستیک نو یک ...      -1   \n",
       "3  مبل راحتی هفت نفره شامل سه نفره یک عدد دونفره ...  600000   \n",
       "4  شارژی کنترلی سویچ حمل تا 35 کیلو صندلی برای دو...  450000   \n",
       "\n",
       "                               title  \n",
       "1                   گلدون مصنوعی نخل  \n",
       "2                لودر کاتر پیلار 950  \n",
       "3  مبل راحتی هفت نفره بامیز جلو مبلی  \n",
       "4                        ماشین شارژی  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.iloc[1:5,[3,4,5,6,8,13,14]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat1</th>\n",
       "      <th>cat2</th>\n",
       "      <th>cat3</th>\n",
       "      <th>city</th>\n",
       "      <th>desc</th>\n",
       "      <th>price</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>personal</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tehran</td>\n",
       "      <td>جنس چتر از ساتنه سایزش بزرگه\\nدسته چتر با بدنه...</td>\n",
       "      <td>15000</td>\n",
       "      <td>چتر ساتن بنفش</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>personal</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tehran</td>\n",
       "      <td>رنگ آبی اکبند دانشجویی</td>\n",
       "      <td>20000</td>\n",
       "      <td>خودکار یوروپن اصل</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>619</th>\n",
       "      <td>personal</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tehran</td>\n",
       "      <td>گوی اعصاب کاملا تمیز و نو خیلی کم استفاده شده</td>\n",
       "      <td>40000</td>\n",
       "      <td>گوی اعصاب</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>710</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tehran</td>\n",
       "      <td>ترموستات 900\\nدر جعبه نو نو</td>\n",
       "      <td>-1</td>\n",
       "      <td>ترموستات 900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1391</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tehran</td>\n",
       "      <td>کولر:موتورژن -3500</td>\n",
       "      <td>-1</td>\n",
       "      <td>دینام کولروماشین لباسشویی</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1432</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tehran</td>\n",
       "      <td>وارد کننده دوربینهای اوکای ویژن زیر قیمت واردات</td>\n",
       "      <td>60000</td>\n",
       "      <td>دوربین فروش کلی وجزعی</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2213</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tehran</td>\n",
       "      <td>لطفا قبل از تماس گرفتن اس ام اس بدهید</td>\n",
       "      <td>-1</td>\n",
       "      <td>یو پی اس سینوسی online</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3610</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mashhad</td>\n",
       "      <td>کیفیت 1mp\\nدیددرشب20m</td>\n",
       "      <td>49000</td>\n",
       "      <td>دوربین دید درشب AHD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3858</th>\n",
       "      <td>personal</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tehran</td>\n",
       "      <td>مارک فابل کستل \\nبا جعبه چوبی و فلزی \\nاصل المان</td>\n",
       "      <td>120000</td>\n",
       "      <td>مداد رنگی حرفه ای ٢٥ رنگ پلی کروم</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4557</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mashhad</td>\n",
       "      <td>دو عدد ای سی 100w واقعی و تا 150 هم میرسه به ش...</td>\n",
       "      <td>25900</td>\n",
       "      <td>ای سی 100w امپلی فایر</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5081</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mashhad</td>\n",
       "      <td>کاسیو ،نو میباشد</td>\n",
       "      <td>50000</td>\n",
       "      <td>ماشین حساب مهندسی،حرفه ای</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5152</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mashhad</td>\n",
       "      <td>بعد از خریدن پشیمون شدم و کاملا نویه برا همین ...</td>\n",
       "      <td>38000</td>\n",
       "      <td>ریموت کنترل چهار کانال نو</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5493</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tehran</td>\n",
       "      <td>باسلام این یوپی اس  550خودم خریدم اصلااستفاده ...</td>\n",
       "      <td>450000</td>\n",
       "      <td>یوپی اس</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6210</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tehran</td>\n",
       "      <td>DVRچهار کاناله \\nدو دوربین دید در شب یکی سقفی ...</td>\n",
       "      <td>600000</td>\n",
       "      <td>ست دوربین مدار بسته</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    cat1 cat2 cat3     city  \\\n",
       "99              personal  NaN  NaN   Tehran   \n",
       "164             personal  NaN  NaN   Tehran   \n",
       "619             personal  NaN  NaN   Tehran   \n",
       "710   electronic-devices  NaN  NaN   Tehran   \n",
       "1391  electronic-devices  NaN  NaN   Tehran   \n",
       "1432  electronic-devices  NaN  NaN   Tehran   \n",
       "2213  electronic-devices  NaN  NaN   Tehran   \n",
       "3610  electronic-devices  NaN  NaN  Mashhad   \n",
       "3858            personal  NaN  NaN   Tehran   \n",
       "4557  electronic-devices  NaN  NaN  Mashhad   \n",
       "5081  electronic-devices  NaN  NaN  Mashhad   \n",
       "5152  electronic-devices  NaN  NaN  Mashhad   \n",
       "5493  electronic-devices  NaN  NaN   Tehran   \n",
       "6210  electronic-devices  NaN  NaN   Tehran   \n",
       "\n",
       "                                                   desc   price  \\\n",
       "99    جنس چتر از ساتنه سایزش بزرگه\\nدسته چتر با بدنه...   15000   \n",
       "164                              رنگ آبی اکبند دانشجویی   20000   \n",
       "619       گوی اعصاب کاملا تمیز و نو خیلی کم استفاده شده   40000   \n",
       "710                         ترموستات 900\\nدر جعبه نو نو      -1   \n",
       "1391                                 کولر:موتورژن -3500      -1   \n",
       "1432    وارد کننده دوربینهای اوکای ویژن زیر قیمت واردات   60000   \n",
       "2213              لطفا قبل از تماس گرفتن اس ام اس بدهید      -1   \n",
       "3610                              کیفیت 1mp\\nدیددرشب20m   49000   \n",
       "3858   مارک فابل کستل \\nبا جعبه چوبی و فلزی \\nاصل المان  120000   \n",
       "4557  دو عدد ای سی 100w واقعی و تا 150 هم میرسه به ش...   25900   \n",
       "5081                                   کاسیو ،نو میباشد   50000   \n",
       "5152  بعد از خریدن پشیمون شدم و کاملا نویه برا همین ...   38000   \n",
       "5493  باسلام این یوپی اس  550خودم خریدم اصلااستفاده ...  450000   \n",
       "6210  DVRچهار کاناله \\nدو دوربین دید در شب یکی سقفی ...  600000   \n",
       "\n",
       "                                  title  \n",
       "99                        چتر ساتن بنفش  \n",
       "164                   خودکار یوروپن اصل  \n",
       "619                           گوی اعصاب  \n",
       "710                        ترموستات 900  \n",
       "1391          دینام کولروماشین لباسشویی  \n",
       "1432              دوربین فروش کلی وجزعی  \n",
       "2213             یو پی اس سینوسی online  \n",
       "3610                دوربین دید درشب AHD  \n",
       "3858  مداد رنگی حرفه ای ٢٥ رنگ پلی کروم  \n",
       "4557              ای سی 100w امپلی فایر  \n",
       "5081          ماشین حساب مهندسی،حرفه ای  \n",
       "5152          ریموت کنترل چهار کانال نو  \n",
       "5493                            یوپی اس  \n",
       "6210                ست دوربین مدار بسته  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[dataset.cat2.isnull()].iloc[1:15,[3,4,5,6,8,13,14]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat1Classes = dataset['cat1'].cat.categories\n",
    "cat2Classes = dataset['cat2'].cat.categories\n",
    "cat3Classes = dataset['cat3'].cat.categories\n",
    "dataset['Y1'] = dataset['cat1'].cat.codes\n",
    "dataset['Y2'] = dataset['cat2'].cat.codes\n",
    "dataset['Y3'] = dataset['cat3'].cat.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[50  1 26 42 -1 16 33 14 36 61 24 64 40 31 21 11 62 48 28 60  6 25  5 10\n",
      " 37 23 34 55 54 38 30 51  8 59 52 53  3 18 41  7 35 63 20 13 43 17 49 32\n",
      " 46  4 39 57 56 27 29 22  0  9 58 12 44 47 15  2 45 65 19]\n"
     ]
    }
   ],
   "source": [
    "print(dataset.Y3.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "newDataset = dataset.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save categories for later usage\n",
    "cat1Classes = newDataset['cat1'].cat.categories\n",
    "cat2Classes = newDataset['cat2'].cat.categories\n",
    "cat3Classes = newDataset['cat3'].cat.categories\n",
    "import json\n",
    "with open('categoriesDivar.json', 'w', encoding='utf8') as outfile:\n",
    "    json.dump({'cat1':cat1Classes.values.tolist(),'cat2':cat2Classes.values.tolist(),'cat3':cat3Classes.values.tolist()}, outfile, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "newDataset['Y1'] = newDataset['cat1'].cat.codes\n",
    "newDataset['Y2'] = newDataset['cat2'].cat.codes\n",
    "newDataset['Y3'] = newDataset['cat3'].cat.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat1</th>\n",
       "      <th>Y1</th>\n",
       "      <th>cat2</th>\n",
       "      <th>Y2</th>\n",
       "      <th>cat3</th>\n",
       "      <th>Y3</th>\n",
       "      <th>city</th>\n",
       "      <th>desc</th>\n",
       "      <th>price</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>2</td>\n",
       "      <td>furniture-and-home-decore</td>\n",
       "      <td>12</td>\n",
       "      <td>antiques-and-art</td>\n",
       "      <td>1</td>\n",
       "      <td>Mashhad</td>\n",
       "      <td>سلام,یک عدد گلدون نخل سه طبقه ی سالم دارم با پ...</td>\n",
       "      <td>30000</td>\n",
       "      <td>گلدون مصنوعی نخل</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>5</td>\n",
       "      <td>cars</td>\n",
       "      <td>7</td>\n",
       "      <td>heavy</td>\n",
       "      <td>26</td>\n",
       "      <td>Mashhad</td>\n",
       "      <td>سریال 43j$NUMبدون شکستگی سه حلقه لاستیک نو یک ...</td>\n",
       "      <td>-1</td>\n",
       "      <td>لودر کاتر پیلار 950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>2</td>\n",
       "      <td>furniture-and-home-decore</td>\n",
       "      <td>12</td>\n",
       "      <td>sofa-armchair</td>\n",
       "      <td>50</td>\n",
       "      <td>Tehran</td>\n",
       "      <td>مبل راحتی هفت نفره شامل سه نفره یک عدد دونفره ...</td>\n",
       "      <td>600000</td>\n",
       "      <td>مبل راحتی هفت نفره بامیز جلو مبلی</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>personal</td>\n",
       "      <td>4</td>\n",
       "      <td>baby-and-toys</td>\n",
       "      <td>2</td>\n",
       "      <td>personal-toys</td>\n",
       "      <td>42</td>\n",
       "      <td>Karaj</td>\n",
       "      <td>شارژی کنترلی سویچ حمل تا 35 کیلو صندلی برای دو...</td>\n",
       "      <td>450000</td>\n",
       "      <td>ماشین شارژی</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           cat1  Y1                       cat2  Y2              cat3  Y3  \\\n",
       "1  for-the-home   2  furniture-and-home-decore  12  antiques-and-art   1   \n",
       "2      vehicles   5                       cars   7             heavy  26   \n",
       "3  for-the-home   2  furniture-and-home-decore  12     sofa-armchair  50   \n",
       "4      personal   4              baby-and-toys   2     personal-toys  42   \n",
       "\n",
       "      city                                               desc   price  \\\n",
       "1  Mashhad  سلام,یک عدد گلدون نخل سه طبقه ی سالم دارم با پ...   30000   \n",
       "2  Mashhad  سریال 43j$NUMبدون شکستگی سه حلقه لاستیک نو یک ...      -1   \n",
       "3   Tehran  مبل راحتی هفت نفره شامل سه نفره یک عدد دونفره ...  600000   \n",
       "4    Karaj  شارژی کنترلی سویچ حمل تا 35 کیلو صندلی برای دو...  450000   \n",
       "\n",
       "                               title  \n",
       "1                   گلدون مصنوعی نخل  \n",
       "2                لودر کاتر پیلار 950  \n",
       "3  مبل راحتی هفت نفره بامیز جلو مبلی  \n",
       "4                        ماشین شارژی  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newDataset.iloc[1:5,[3,17,4,18,5,19,6,8,13,14]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TODO: max, avg and min sequence length (title and desc)\n",
    "#TODO: Build a dictionary, unique words\n",
    "newDataset['descLength'] = newDataset['desc'].apply(len)\n",
    "newDataset['titleLength'] = newDataset['title'].apply(len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Desc = Mean: 76.45529238578145 Min: 4 Max: 995\n",
      "Title = Mean: 17.69796071272167 Min: 3 Max: 50\n"
     ]
    }
   ],
   "source": [
    "print('Desc = Mean:',newDataset['descLength'].mean(), 'Min:',newDataset['descLength'].min(), 'Max:',newDataset['descLength'].max())\n",
    "print('Title = Mean:',newDataset['titleLength'].mean(), 'Min:',newDataset['titleLength'].min(), 'Max:',newDataset['titleLength'].max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#newDataset['context'] = newDataset.title + ' ' + newDataset.desc + ' ' + newDataset.desc + ' ' + newDataset.city + ' ' + newDataset.price.astype(str)\n",
    "#TODO: Check differenet combinations of hacks\n",
    "newDataset['context'] = newDataset.title + ' ' + newDataset.desc + ' ' + newDataset.city + ' ' + newDataset.price.astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "newDataset.context = newDataset.context.str.replace('\\n',' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper funcitons\n",
    "import re\n",
    "def clean_str(string):\n",
    "    \"\"\"\n",
    "    Tokenization/string cleaning for dataset\n",
    "    Every dataset is lower cased except\n",
    "    \"\"\"\n",
    "    string = re.sub(r\"\\\\\", \" \", string)\n",
    "    string = re.sub(r\"\\'\", \" \", string)\n",
    "    string = re.sub(r\"\\\"\", \" \", string)\n",
    "    string = re.sub(r\"/\", \" \", string)\n",
    "    string = re.sub(r\"//\", \" \", string)\n",
    "    string = re.sub(r\"$NUM\", \" \", string)\n",
    "    #string = re.sub(r'[^\\w\\s]', '', string, re.UNICODE)\n",
    "    string = re.sub(r'([a-z])\\1+', r'\\1', string, re.UNICODE)\n",
    "    return string.strip().lower()\n",
    "\n",
    "def text_cleaner(text):\n",
    "    text = text.replace(\":\", \" \")\n",
    "    text = text.replace(\";\", \" \")\n",
    "    text = text.replace(\".\", \" \")\n",
    "    text = text.replace(\"&\", \" \")\n",
    "    text = text.replace(\"%\", \" \")\n",
    "    text = text.replace(\"$\", \" \")\n",
    "    text = text.replace(\"#\", \" \")\n",
    "    text = text.replace(\"%\", \" \")\n",
    "    text = text.replace(\"@\", \" \")\n",
    "    text = text.replace(\"!\", \" \")\n",
    "    text = text.replace(\"+\", \" \")\n",
    "    text = text.replace(\"-\", \" \")\n",
    "    text = text.replace(\"_\", \" \")\n",
    "    text = text.replace(\"[\", \" \")\n",
    "    text = text.replace(\",\", \" \")\n",
    "    text = text.replace(\"،\", \" \")\n",
    "    text = text.replace(\"]\", \" \")\n",
    "    text = text.replace(\"(\", \" \")\n",
    "    text = text.replace(\")\", \" \")\n",
    "    text = text.replace(\"{\", \" \")\n",
    "    text = text.replace(\"}\", \" \")\n",
    "    text = text.replace(\"\\\"\", \"\")\n",
    "    text = text.replace(\"-\", \" \")\n",
    "    text = text.replace(\"=\", \" \")\n",
    "    text = text.replace(\"~\", \" \")\n",
    "    text = text.replace(\"<\", \" \")\n",
    "    text = text.replace(\">\", \" \")\n",
    "    text = text.replace(\"«\", \" \")\n",
    "    text = text.replace(\"*\", \" \")\n",
    "    text = text.replace(\"❌\", \" \")\n",
    "    text = text.replace(\"⚽\", \" \")\n",
    "    text = text.replace(\"✅\", \" \")\n",
    "    text = text.replace(\"⌛\", \" \")\n",
    "    text = text.replace(\"⑤\", \" \")\n",
    "    text = text.replace(\"•\", \" \")\n",
    "    text = text.replace(\"♧\", \" \")\n",
    "    text = text.replace(\"num\", \" \")\n",
    "    text = text.replace(u'\\u2013','')    \n",
    "    rules = [\n",
    "        {r'>\\s+': u'>'},  # remove spaces after a tag opens or closes\n",
    "        {r'\\s+': u' '},  # replace consecutive spaces\n",
    "        {r'\\s*<br\\s*/?>\\s*': u'\\n'},  # newline after a <br>\n",
    "        {r'</(div)\\s*>\\s*': u'\\n'},  # newline after </p> and </div> and <h1/>...\n",
    "        {r'</(p|h\\d)\\s*>\\s*': u'\\n\\n'},  # newline after </p> and </div> and <h1/>...\n",
    "        {r'<head>.*<\\s*(/head|body)[^>]*>': u''},  # remove <head> to </head>\n",
    "        {r'<a\\s+href=\"([^\"]+)\"[^>]*>.*</a>': r'\\1'},  # show links instead of texts\n",
    "        {r'[ \\t]*<[^<]*?/?>': u''},  # remove remaining tags\n",
    "        {r'^\\s+': u''}  # remove spaces at the beginning\n",
    "    ]\n",
    "    for rule in rules:\n",
    "        for (k, v) in rule.items():\n",
    "            regex = re.compile(k)\n",
    "            text = regex.sub(v, text)\n",
    "        text = text.rstrip()\n",
    "        text = text.strip()\n",
    "    text.lower().replace(\"num\", \" \")\n",
    "    text = re.sub(r'-?\\d+\\.?\\d*', ' ', text)\n",
    "    text = re.sub(u'\\u200c',' ', text)\n",
    "    text = re.sub(u'\\u200e',' ', text)\n",
    "    text = re.sub(u'\\xad',' ', text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['پراید',\n",
       " 'مدل',\n",
       " 'پراید',\n",
       " 'فنی',\n",
       " 'سالم',\n",
       " 'یکسال',\n",
       " 'بیمه',\n",
       " 'دوگانه',\n",
       " 'سوز',\n",
       " 'کارخانه',\n",
       " 'لاستیک',\n",
       " 'ماشین',\n",
       " 'فوق',\n",
       " 'به',\n",
       " 'نرخ',\n",
       " 'دور',\n",
       " 'رنگ',\n",
       " 'میباشد',\n",
       " 'شماره',\n",
       " 'تماس',\n",
       " 'kermanshah',\n",
       " '8900000']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample = newDataset.context[282]\n",
    "price = sample.split()[-1]\n",
    "sample = clean_str(sample)\n",
    "sample = text_cleaner(sample)\n",
    "sample = sample + price\n",
    "sample = re.split(r'([a-zA-Z]+)', sample)\n",
    "sample = \" \".join(str(item) for item in sample)\n",
    "sample.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build a dictionary\n",
    "wordDict = {}\n",
    "for idx,row in enumerate(newDataset.context):\n",
    "    price = row.split()[-1]\n",
    "    row = clean_str(row)\n",
    "    row = text_cleaner(row)\n",
    "    row = row + price\n",
    "    row = re.split(r'([a-zA-Z]+)', row)\n",
    "    row = \" \".join(str(item) for item in row)\n",
    "    words = row.split()\n",
    "    for wrd in words:\n",
    "        if wrd in wordDict:\n",
    "            wordDict[wrd] += 1\n",
    "        else:\n",
    "            wordDict[wrd] = 1\n",
    "    #if idx > 5000:\n",
    "    #    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "256492"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(wordDict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save dictionary to file\n",
    "import json\n",
    "with open('wordDict.json', 'w', encoding='utf8') as outfile:\n",
    "    json.dump(wordDict, outfile, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load dictionary\n",
    "with open(\"wordDict.json\", \"r\") as read_file:\n",
    "    wordDict = json.load(read_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sanitize(x):\n",
    "    price = x.split()[-1]\n",
    "    x = clean_str(x)\n",
    "    x = text_cleaner(x)\n",
    "    x = x + price\n",
    "    x = re.split(r'([a-zA-Z]+)', x)\n",
    "    x = \" \".join(str(item) for item in x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "newDataset['contextProcessed'] = newDataset.context.apply(lambda row: sanitize(row))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>contextProcessed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>نوکیا6303 سلام.یه گوشیه6303سالم که فقط دوتا خط...</td>\n",
       "      <td>نوکیا  سلام یه گوشیه سالم که فقط دوتا خط کوچیک...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>لباس های دخترانه 2تا9ساله لباس های دخترانه از3...</td>\n",
       "      <td>لباس های دخترانه  تا ساله لباس های دخترانه از ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>کمک فنر روغنی کمک فنر روغنی تعمیری Tehran 50000</td>\n",
       "      <td>کمک فنر روغنی کمک فنر روغنی تعمیری  tehran   5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>کلش اف کلنز لول ۳۳ تاون حال لول ۶دیوارها لول ۴...</td>\n",
       "      <td>کلش اف کلنز لول   تاون حال لول  دیوارها لول   ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              context  \\\n",
       "11  نوکیا6303 سلام.یه گوشیه6303سالم که فقط دوتا خط...   \n",
       "12  لباس های دخترانه 2تا9ساله لباس های دخترانه از3...   \n",
       "13    کمک فنر روغنی کمک فنر روغنی تعمیری Tehran 50000   \n",
       "14  کلش اف کلنز لول ۳۳ تاون حال لول ۶دیوارها لول ۴...   \n",
       "\n",
       "                                     contextProcessed  \n",
       "11  نوکیا  سلام یه گوشیه سالم که فقط دوتا خط کوچیک...  \n",
       "12  لباس های دخترانه  تا ساله لباس های دخترانه از ...  \n",
       "13  کمک فنر روغنی کمک فنر روغنی تعمیری  tehran   5...  \n",
       "14  کلش اف کلنز لول   تاون حال لول  دیوارها لول   ...  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newDataset.iloc[11:15,[22,23]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export dataset to text files\n",
    "import os\n",
    "outputPath = './dataset/'\n",
    "\n",
    "if not os.path.exists(outputPath):\n",
    "    os.makedirs(outputPath)\n",
    "    \n",
    "for c in newDataset.columns:\n",
    "    newDataset[c].to_csv(outputPath + c + '.txt', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "61"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# free memory\n",
    "import gc\n",
    "#del dataset, datasetNOW\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "pathDataset = './dataset'\n",
    "fname = os.path.join(pathDataset,\"contextProcessed.txt\")\n",
    "fnamek = os.path.join(pathDataset,\"Y1.txt\")\n",
    "fnameL2 = os.path.join(pathDataset,\"Y2.txt\")\n",
    "fnameL3 = os.path.join(pathDataset,\"Y3.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "content = pd.read_table(fname, header=None)\n",
    "content = content[0].apply(str.strip)\n",
    "\n",
    "contentk = pd.read_table(fnamek, header=None).values\n",
    "\n",
    "contentL2 = pd.read_table(fnameL2, header=None).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (X_train, y_train, X_test, y_test, content_L2_Train, L2_Train, content_L2_Test, L2_Test, number_of_classes_L2,word_index,embeddings_index,number_of_classes_L1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "Label = contentk\n",
    "Label_L2 = contentL2\n",
    "\n",
    "number_of_classes_L1 = np.max(Label)+1 #number of classes in Level 1\n",
    "number_of_classes_L2 = np.zeros(number_of_classes_L1,dtype=int)\n",
    "\n",
    "np.random.seed(7)\n",
    "\n",
    "Label = np.column_stack((Label, Label_L2))\n",
    "\n",
    " #number of classes in Level 2 that is 1D array with size of (number of classes in level one,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 256364 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "\n",
    "MAX_NB_WORDS = 55000\n",
    "tokenizer = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "tokenizer.fit_on_texts(content)\n",
    "sequences = tokenizer.texts_to_sequences(content)\n",
    "word_index = tokenizer.word_index\n",
    "\n",
    "print('Found %s unique tokens.' % len(word_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "MAX_SEQUENCE_LENGTH = 500\n",
    "content = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(947635, 500)\n"
     ]
    }
   ],
   "source": [
    "indices = np.arange(content.shape[0])\n",
    "np.random.shuffle(indices)\n",
    "content = content[indices]\n",
    "Label = Label[indices]\n",
    "print(content.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(content, Label, test_size=0.2, random_state=0)\n",
    "X_train, X_test, y_train, y_test  = train_test_split(content, Label, test_size=0.5,random_state= 0)\n",
    "X_val, X_test, y_val, y_test = train_test_split(X_test, y_test, test_size=0.5, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(473817, 500) (236909, 500) (236909, 500)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape, X_val.shape, X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(28,)\n"
     ]
    }
   ],
   "source": [
    "content = pd.read_table(fname, header=None)\n",
    "content = content[0].apply(str.strip)\n",
    "contentk = pd.read_table(fnamek, header=None).values\n",
    "contentL2 = pd.read_table(fnameL2, header=None).values\n",
    "contentL3 = pd.read_table(fnameL3, header=None).values\n",
    "\n",
    "Label = contentk\n",
    "Label_L2 = contentL2\n",
    "Label_L3 = contentL3\n",
    "\n",
    "np.random.seed(7)\n",
    "Label = np.column_stack((Label, Label_L2, Label_L3))\n",
    "LabelDF = pd.DataFrame(Label)    \n",
    "\n",
    "labelsL1 = LabelDF[0].unique()\n",
    "labelsL2 = LabelDF[1].unique()\n",
    "labelsL3 = LabelDF[2].unique()\n",
    "\n",
    "number_of_classes_L1 = len(labelsL1) #number of classes in Level 1\n",
    "number_of_classes_L2 = len(labelsL2)\n",
    "number_of_classes_L3 = len(labelsL3)\n",
    "\n",
    "classes_L1 = np.zeros((number_of_classes_L1,))\n",
    "classes_L2 = np.zeros((number_of_classes_L2,))\n",
    "classes_L3 = np.zeros((number_of_classes_L3,))\n",
    "\n",
    "print(classes_L2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15422"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp = pd.DataFrame(y_val)\n",
    "len(tmp[tmp[0]==0].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "number_of_classes_L2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LabelDF = pd.DataFrame(Label)\n",
    "labelsL2 = []\n",
    "labelsL2.append(LabelDF[LabelDF[0]==1][1].unique())\n",
    "len(labelsL2[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "labelsL1 = LabelDF[0].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 2\n",
      "1 5\n",
      "2 4\n",
      "3 1\n",
      "4 0\n",
      "5 3\n"
     ]
    }
   ],
   "source": [
    "labelsL2 = []\n",
    "for idx in range(number_of_classes_L1):\n",
    "    print(idx,labelsL1[idx])\n",
    "    LabelDF = pd.DataFrame(Label)    \n",
    "    labelsL2.append(LabelDF[LabelDF[0]==labelsL1[idx]][1].unique())\n",
    "    classes_L2[idx] = len(labelsL2[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([12, 25,  6, 26]),\n",
       " array([ 7, 21, 19]),\n",
       " array([ 2,  9,  8, 16, 14, -1]),\n",
       " array([18, 13,  1, -1, 10, 22]),\n",
       " array([11,  3]),\n",
       " array([ 4,  0, 20, 23,  5, 24, 15, 17])]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labelsL2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./fastText/cc.fa.300.vec\n",
      "Total 91286 word vectors.\n"
     ]
    }
   ],
   "source": [
    "embedder = 'fastTextEn'\n",
    "embeddings_index = {}\n",
    "'''\n",
    "For CNN and RNN, we used the text vector-space models using $100$ dimensions as described in Glove. A vector-space model is a mathematical mapping of the word space\n",
    "'''\n",
    "if embedder == 'glove':\n",
    "    Glove_path = os.path.join(GLOVE_DIR, 'glove.6B.300d.txt')\n",
    "    print(Glove_path)\n",
    "    f = open(Glove_path, encoding=\"utf8\")\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        try:\n",
    "            coefs = np.asarray(values[1:], dtype='float32')\n",
    "        except:\n",
    "            print(\"Warnning\"+str(values)+\" in\" + str(line))\n",
    "        embeddings_index[word] = coefs\n",
    "    f.close()\n",
    "    print('Total %s word vectors.' % len(embeddings_index))\n",
    "elif embedder == 'fastTextEn':\n",
    "    fastTextDir = './fastText/'\n",
    "    fastText_path = os.path.join(fastTextDir, 'cc.fa.300.vec')\n",
    "    print(fastText_path)\n",
    "    embeddings_index = {}\n",
    "    with open(fastText_path, encoding='utf8') as infile:\n",
    "        #for idx,line in enumerate(infile):\n",
    "        for line in infile:\n",
    "            #if idx > 1: # skip the first line\n",
    "            values = line.split()\n",
    "            word = values[0]\n",
    "            try:\n",
    "                coefs = np.asarray(values[1:], dtype='float32')\n",
    "            except:\n",
    "                print(\"Warnning\"+str(values)+\" in\" + str(line))\n",
    "            if word in wordDict: # need only embedding for words that are in corpus\n",
    "                embeddings_index[word] = coefs\n",
    "    #f = open(fastText_path, encoding=\"utf8\")\n",
    "    #fLines = f.readlines()\n",
    "    #firstLine = fLines.pop(0) # remove the first line\n",
    "    #f.close()\n",
    "    gc.collect()\n",
    "    print('Total %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert Keras model to layer based models\n",
    "EMBEDDING_DIM = 300 #embedding dimension you can change it to {25, 100, 150, and 300} but need to change glove version\n",
    "\n",
    "embedding_matrix = np.random.random((len(word_index) + 1, EMBEDDING_DIM))\n",
    "for word, i in word_index.items():\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Dense, Input, Flatten\n",
    "input1 = Input((MAX_SEQUENCE_LENGTH,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Conv1D, MaxPooling1D, Embedding, concatenate, Dropout, LSTM, GRU, Bidirectional,SimpleRNN\n",
    "layerM1Embedding = Embedding(len(word_index) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=True)(input1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "layerM1 = GRU(100,dropout=0.2, recurrent_dropout=0.2)(layerM1Embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'gru_1/TensorArrayReadV3:0' shape=(?, 100) dtype=float32>"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layerM1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "input2 = Input((1,)) # price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "layerM2 = Dense(100, activation='relu')(input2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'dense_1/Relu:0' shape=(?, 100) dtype=float32>"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layerM2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Add\n",
    "layer = Add()([layerM1,layerM2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'add_1/add:0' shape=(?, 100) dtype=float32>"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "nClasses = 3\n",
    "from keras.models import Model\n",
    "out = Dense(nClasses, activation='softmax')(layer)\n",
    "model = Model(inputs=[input1,input2], outputs=out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 500)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 500, 300)     76909500    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "gru_1 (GRU)                     (None, 100)          120300      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 100)          200         input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 100)          0           gru_1[0][0]                      \n",
      "                                                                 dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 3)            303         add_1[0][0]                      \n",
      "==================================================================================================\n",
      "Total params: 77,030,303\n",
      "Trainable params: 77,030,303\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='sparse_categorical_crossentropy',\n",
    "              optimizer='rmsprop',\n",
    "              metrics=['acc'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Persian Embedding (FastText)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "fastTextDir = './fastText/'\n",
    "fastText_path = os.path.join(fastTextDir, 'cc.fa.300.vec')\n",
    "print(fastText_path)\n",
    "f = open(fastText_path, encoding=\"utf8\")\n",
    "fLines = f.readlines()\n",
    "firstLine = fLines.pop(0) # remove the first line\n",
    "embeddings_index = {}\n",
    "print(firstLine)\n",
    "for line in fLines:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    try:\n",
    "        coefs = np.asarray(values[1:], dtype='float32')\n",
    "    except:\n",
    "        print(\"Warnning\"+str(values)+\" in\" + str(line))\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "print('Total %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "fastTextDir = './fastText/'\n",
    "fastText_path = os.path.join(fastTextDir, 'cc.en.300.vec')\n",
    "print(fastText_path)\n",
    "f = open(fastText_path, encoding=\"utf8\")\n",
    "fLines = f.readlines()\n",
    "firstLine = fLines.pop(0) # remove the first line\n",
    "embeddings_index = {}\n",
    "print(firstLine)\n",
    "for line in fLines:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    try:\n",
    "        coefs = np.asarray(values[1:], dtype='float32')\n",
    "    except:\n",
    "        print(\"Warnning\"+str(values)+\" in\" + str(line))\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "print('Total %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### You can ingore all the previous parts or use the main python code (dataDaysChallenge_BIGNet.py)\n",
    "#### This file is only for your reference. some parts are not compatible with the latest changes\n",
    "#### You need to prepare files like wordDict.json and all the preprocessed files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Start: Prepare Dataset for the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load dictionary\n",
    "import json\n",
    "with open(\"wordDict.json\", \"r\") as read_file:\n",
    "    wordDict = json.load(read_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ['KERAS_BACKEND'] = 'tensorflow'\n",
    "\n",
    "import re\n",
    "import numpy as np\n",
    "from keras.models import Sequential\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper funcitons\n",
    "\n",
    "def clean_str(string):\n",
    "    \"\"\"\n",
    "    Tokenization/string cleaning for dataset\n",
    "    Every dataset is lower cased except\n",
    "    \"\"\"\n",
    "    string = re.sub(r\"\\\\\", \" \", string)\n",
    "    string = re.sub(r\"\\'\", \" \", string)\n",
    "    string = re.sub(r\"\\\"\", \" \", string)\n",
    "    string = re.sub(r\"/\", \" \", string)\n",
    "    string = re.sub(r\"//\", \" \", string)\n",
    "    string = re.sub(r\"$NUM\", \" \", string)\n",
    "    #string = re.sub(r'[^\\w\\s]', '', string, re.UNICODE)\n",
    "    string = re.sub(r'([a-z])\\1+', r'\\1', string, re.UNICODE)\n",
    "    return string.strip().lower()\n",
    "\n",
    "def text_cleaner(text):\n",
    "    text = text.replace(\":\", \" \")\n",
    "    text = text.replace(\";\", \" \")\n",
    "    text = text.replace(\".\", \" \")\n",
    "    text = text.replace(\"&\", \" \")\n",
    "    text = text.replace(\"%\", \" \")\n",
    "    text = text.replace(\"$\", \" \")\n",
    "    text = text.replace(\"#\", \" \")\n",
    "    text = text.replace(\"%\", \" \")\n",
    "    text = text.replace(\"@\", \" \")\n",
    "    text = text.replace(\"!\", \" \")\n",
    "    text = text.replace(\"+\", \" \")\n",
    "    text = text.replace(\"-\", \" \")\n",
    "    text = text.replace(\"_\", \" \")\n",
    "    text = text.replace(\"[\", \" \")\n",
    "    text = text.replace(\",\", \" \")\n",
    "    text = text.replace(\"،\", \" \")\n",
    "    text = text.replace(\"]\", \" \")\n",
    "    text = text.replace(\"(\", \" \")\n",
    "    text = text.replace(\")\", \" \")\n",
    "    text = text.replace(\"{\", \" \")\n",
    "    text = text.replace(\"}\", \" \")\n",
    "    text = text.replace(\"\\\"\", \"\")\n",
    "    text = text.replace(\"-\", \" \")\n",
    "    text = text.replace(\"=\", \" \")\n",
    "    text = text.replace(\"~\", \" \")\n",
    "    text = text.replace(\"<\", \" \")\n",
    "    text = text.replace(\">\", \" \")\n",
    "    text = text.replace(\"«\", \" \")\n",
    "    text = text.replace(\"*\", \" \")\n",
    "    text = text.replace(\"❌\", \" \")\n",
    "    text = text.replace(\"⚽\", \" \")\n",
    "    text = text.replace(\"✅\", \" \")\n",
    "    text = text.replace(\"⌛\", \" \")\n",
    "    text = text.replace(\"⑤\", \" \")\n",
    "    text = text.replace(\"•\", \" \")\n",
    "    text = text.replace(\"♧\", \" \")\n",
    "    text = text.replace(\"num\", \" \")\n",
    "    text = text.replace(u'\\u2013','')    \n",
    "    rules = [\n",
    "        {r'>\\s+': u'>'},  # remove spaces after a tag opens or closes\n",
    "        {r'\\s+': u' '},  # replace consecutive spaces\n",
    "        {r'\\s*<br\\s*/?>\\s*': u'\\n'},  # newline after a <br>\n",
    "        {r'</(div)\\s*>\\s*': u'\\n'},  # newline after </p> and </div> and <h1/>...\n",
    "        {r'</(p|h\\d)\\s*>\\s*': u'\\n\\n'},  # newline after </p> and </div> and <h1/>...\n",
    "        {r'<head>.*<\\s*(/head|body)[^>]*>': u''},  # remove <head> to </head>\n",
    "        {r'<a\\s+href=\"([^\"]+)\"[^>]*>.*</a>': r'\\1'},  # show links instead of texts\n",
    "        {r'[ \\t]*<[^<]*?/?>': u''},  # remove remaining tags\n",
    "        {r'^\\s+': u''}  # remove spaces at the beginning\n",
    "    ]\n",
    "    for rule in rules:\n",
    "        for (k, v) in rule.items():\n",
    "            regex = re.compile(k)\n",
    "            text = regex.sub(v, text)\n",
    "        text = text.rstrip()\n",
    "        text = text.strip()\n",
    "    text.lower().replace(\"num\", \" \")\n",
    "    text = re.sub(r'-?\\d+\\.?\\d*', ' ', text)\n",
    "    text = re.sub(u'\\u200c',' ', text)\n",
    "    text = re.sub(u'\\u200e',' ', text)\n",
    "    text = re.sub(u'\\xad',' ', text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "def loadData_Tokenizer_Efficient(MAX_NB_WORDS,MAX_SEQUENCE_LENGTH, EMBEDDING_DIM = 100, embedder = 'fastTextEn'):\n",
    "\n",
    "    pathDataset = './dataset'\n",
    "    fname = os.path.join(pathDataset,\"contextProcessed.txt\")\n",
    "    fname2 = os.path.join(pathDataset,\"price.txt\")\n",
    "    fnamek = os.path.join(pathDataset,\"Y1.txt\")\n",
    "    fnameL2 = os.path.join(pathDataset,\"Y2.txt\")\n",
    "    fnameL3 = os.path.join(pathDataset,\"Y3.txt\")\n",
    "\n",
    "    content = pd.read_table(fname, header=None)\n",
    "    content = content[0].apply(str.strip)\n",
    "    content2 = pd.read_table(fname2, header=None, dtype='int64') # read price as integer\n",
    "\n",
    "    contentk = pd.read_table(fnamek, header=None).values\n",
    "    contentL2 = pd.read_table(fnameL2, header=None).values\n",
    "    contentL3 = pd.read_table(fnameL3, header=None).values\n",
    "\n",
    "    Label_L1 = contentk\n",
    "    Label_L2 = contentL2\n",
    "    Label_L3 = contentL3\n",
    "\n",
    "    np.random.seed(7)\n",
    "    Label = np.column_stack((Label_L1, Label_L2, Label_L3))\n",
    "    LabelDF = pd.DataFrame(Label)    \n",
    "\n",
    "    labelsL1 = LabelDF[0].unique()\n",
    "    labelsL2 = LabelDF[1].unique()\n",
    "    labelsL3 = LabelDF[2].unique()\n",
    "\n",
    "    number_of_classes_L1 = len(labelsL1) #number of classes in Level 1\n",
    "    number_of_classes_L2 = len(labelsL2)\n",
    "    number_of_classes_L3 = len(labelsL3)    \n",
    "    \n",
    "    tokenizer = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "    tokenizer.fit_on_texts(content)\n",
    "    sequences = tokenizer.texts_to_sequences(content)\n",
    "    word_index = tokenizer.word_index\n",
    "    print('Found %s unique tokens.' % len(word_index))\n",
    "\n",
    "    content = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "\n",
    "    indices = np.arange(content.shape[0])\n",
    "    np.random.shuffle(indices)\n",
    "    content = content[indices]\n",
    "    Label = Label[indices]\n",
    "    print(content.shape)\n",
    "\n",
    "    # join two inputs \n",
    "    content = np.concatenate((content,content2),axis=1)\n",
    "\n",
    "    #TODO: Balance dataset\n",
    "    X_train, X_test, y_train, y_test  = train_test_split(content, Label, test_size=0.3,random_state= 0, stratify=Label, shuffle=True)\n",
    "    X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=0, stratify=y_train, shuffle=True)\n",
    "    print(X_train.shape, X_val.shape, X_test.shape)\n",
    "\n",
    "    embeddings_index = {}\n",
    "    '''\n",
    "    For CNN and RNN, we used the text vector-space models using $100$ dimensions as described in Glove. A vector-space model is a mathematical mapping of the word space\n",
    "    '''\n",
    "    if embedder == 'glove':\n",
    "        Glove_path = os.path.join(GLOVE_DIR, 'glove.6B.300d.txt')\n",
    "        print(Glove_path)\n",
    "        f = open(Glove_path, encoding=\"utf8\")\n",
    "        for line in f:\n",
    "            values = line.split()\n",
    "            word = values[0]\n",
    "            try:\n",
    "                coefs = np.asarray(values[1:], dtype='float32')\n",
    "            except:\n",
    "                print(\"Warnning\"+str(values)+\" in\" + str(line))\n",
    "            embeddings_index[word] = coefs\n",
    "        f.close()\n",
    "        print('Total %s word vectors.' % len(embeddings_index))\n",
    "    elif embedder == 'fastTextEn':\n",
    "        fastTextDir = './fastText/'\n",
    "        embedderName = 'cc.fa.' + str(EMBEDDING_DIM) + '.vec'\n",
    "        fastText_path = os.path.join(fastTextDir, embedderName)\n",
    "        print(fastText_path)\n",
    "        embeddings_index = {}\n",
    "        with open(fastText_path, encoding='utf8') as infile:\n",
    "            #for idx,line in enumerate(infile):\n",
    "            for line in infile:\n",
    "                #if idx > 1: # skip the first line\n",
    "                values = line.split()\n",
    "                word = values[0]\n",
    "                try:\n",
    "                    coefs = np.asarray(values[1:], dtype='float32')\n",
    "                except:\n",
    "                    print(\"Warnning\"+str(values)+\" in\" + str(line))\n",
    "                if word in wordDict: # need only embedding for words that are in corpus\n",
    "                    embeddings_index[word] = coefs\n",
    "        gc.collect()\n",
    "        print('Total %s word vectors.' % len(embeddings_index))\n",
    "    return (tokenizer,LabelDF,X_train,y_train,X_val,y_val,X_test,y_test,labelsL1,labelsL2,labelsL3,number_of_classes_L1,number_of_classes_L2,number_of_classes_L3,word_index,embeddings_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.models import Model\n",
    "import numpy as np\n",
    "from keras.layers import Dense, Input, Flatten\n",
    "from keras.layers import Conv1D, MaxPooling1D, Embedding, concatenate, Dropout, LSTM, GRU, Bidirectional,SimpleRNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "buildModel_DNN(nFeatures, nClasses, nLayers=3,Numberof_NOde=100, dropout=0.5)\n",
    "Build Deep neural networks Model for text classification\n",
    "Shape is input feature space\n",
    "nClasses is number of classes\n",
    "nLayers is number of hidden Layer\n",
    "Number_Node is number of unit in each hidden layer\n",
    "dropout is dropout value for solving overfitting problem\n",
    "'''\n",
    "def buildModel_DNN(Shape, nClasses, nLayers=3,Number_Node=100, dropout=0.5):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(Number_Node, input_dim=Shape))\n",
    "    model.add(Dropout(dropout))\n",
    "    for i in range(0,nLayers):\n",
    "        model.add(Dense(Number_Node, activation='relu'))\n",
    "        model.add(Dropout(dropout))\n",
    "    model.add(Dense(nClasses, activation='softmax'))\n",
    "    model.compile(loss='sparse_categorical_crossentropy',\n",
    "                  optimizer='RMSprop',\n",
    "                  metrics=['accuracy'])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import multiply\n",
    "def attention_3d_block(inputs):\n",
    "    # inputs.shape = (batch_size, time_steps, input_dim)\n",
    "    input_dim = int(inputs.shape[2])\n",
    "    a = Permute((2, 1))(inputs)\n",
    "    a = Reshape((input_dim, TIME_STEPS))(a) # this line is not useful. It's just to know which dimension is what.\n",
    "    a = Dense(TIME_STEPS, activation='softmax')(a)\n",
    "    if SINGLE_ATTENTION_VECTOR:\n",
    "        a = Lambda(lambda x: K.mean(x, axis=1), name='dim_reduction')(a)\n",
    "        a = RepeatVector(input_dim)(a)\n",
    "    a_probs = Permute((2, 1), name='attention_vec')(a)\n",
    "    output_attention_mul = multiply([inputs, a_probs], name='attention_mul')\n",
    "    return output_attention_mul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "def buildModel_RNN(word_index, embeddings_index, nClasses, MAX_SEQUENCE_LENGTH, EMBEDDING_DIM):\n",
    "word_index in word index , \n",
    "embeddings_index is embeddings index, \n",
    "nClasses is number of classes, \n",
    "MAX_SEQUENCE_LENGTH is maximum lentgh of text sequences, \n",
    "EMBEDDING_DIM is an int value for dimention of word embedding \n",
    "'''\n",
    "import keras.backend as K\n",
    "from keras.layers import Concatenate\n",
    "from attention_utils import get_activations, get_data_recurrent\n",
    "def buildModel_RNN(word_index, embeddings_index, nClasses, MAX_SEQUENCE_LENGTH, EMBEDDING_DIM, type=0):\n",
    "    model = Sequential()\n",
    "    embedding_matrix = np.random.random((len(word_index) + 1, EMBEDDING_DIM))\n",
    "    for word, i in word_index.items():\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None:\n",
    "            # words not found in embedding index will be all-zeros.\n",
    "            embedding_matrix[i] = embedding_vector\n",
    "    '''\n",
    "    model.add(Embedding(len(word_index) + 1,\n",
    "                                EMBEDDING_DIM,\n",
    "                                weights=[embedding_matrix],\n",
    "                                input_length=MAX_SEQUENCE_LENGTH,\n",
    "                                trainable=True))\n",
    "    if type==0:\n",
    "        model.add(GRU(100,dropout=0.2, recurrent_dropout=0.2))\n",
    "    elif type==1:\n",
    "        model.add(Conv1D(filters=64, kernel_size=3, padding='same', activation='relu'))\n",
    "        model.add(MaxPooling1D(pool_size=2))\n",
    "        model.add(LSTM(200,dropout=0.2, recurrent_dropout=0.2))\n",
    "    model.add(Dense(nClasses, activation='softmax'))\n",
    "    '''\n",
    "    input1 = Input((MAX_SEQUENCE_LENGTH,),name='context')\n",
    "    layerM1Embedding = Embedding(len(word_index) + 1,\n",
    "                                EMBEDDING_DIM,\n",
    "                                weights=[embedding_matrix],\n",
    "                                input_length=MAX_SEQUENCE_LENGTH,\n",
    "                                trainable=True)(input1)\n",
    "    input2 = Input((1,),name='price') # price\n",
    "    if type==0:\n",
    "        layer = GRU(100,dropout=0.2, recurrent_dropout=0.2)(layerM1Embedding)\n",
    "    elif type==1:\n",
    "        layer = GRU(100,dropout=0.2, recurrent_dropout=0.2)(layerM1Embedding)\n",
    "        layer = Conv1D(filters=64, kernel_size=3, padding='same', activation='relu')(layer)\n",
    "        layer = MaxPooling1D(pool_size=2)(layer)\n",
    "        layer = LSTM(200,dropout=0.2, recurrent_dropout=0.2)(layer)\n",
    "    elif type==2:\n",
    "        layerM1 = GRU(100,dropout=0.2, recurrent_dropout=0.2)(layerM1Embedding)\n",
    "        layerM1 = Dense(nClasses, activation='softmax')(layerM1)\n",
    "        layerM2 = Dense(nClasses, activation='softmax')(input2)\n",
    "        layer = Concatenate()([layerM1,layerM2])     \n",
    "    elif type==3:\n",
    "        # attention\n",
    "        attentionMul = attention_3d_block(layerM1Embedding)\n",
    "        layer = GRU(100,dropout=0.2, recurrent_dropout=0.2)(attentionMul)\n",
    "    out = Dense(nClasses, activation='softmax')(layer)\n",
    "    model = Model(inputs=[input1,input2], outputs=out)\n",
    "    model.summary()\n",
    "    model.compile(loss='sparse_categorical_crossentropy',\n",
    "                  optimizer='rmsprop',\n",
    "                  metrics=['acc'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "def buildModel_CNN(word_index,embeddings_index,nClasses,MAX_SEQUENCE_LENGTH,EMBEDDING_DIM,Complexity=0):\n",
    "word_index in word index , \n",
    "embeddings_index is embeddings index,\n",
    "nClasses is number of classes, \n",
    "MAX_SEQUENCE_LENGTH is maximum length of text sequences, \n",
    "EMBEDDING_DIM is an int value for dimention of word embedding, \n",
    "Complexity we have two different CNN model as follows \n",
    "Complexity=0 is simple CNN with 3 hidden layer\n",
    "Complexity=2 is more complex model of CNN with filter_length of [3, 4, 5, 6, 7]\n",
    "'''\n",
    "def buildModel_CNN(word_index,embeddings_index,nClasses,MAX_SEQUENCE_LENGTH,EMBEDDING_DIM,Complexity=1):\n",
    "    if Complexity==0:\n",
    "        embedding_matrix = np.random.random((len(word_index) + 1, EMBEDDING_DIM))\n",
    "        for word, i in word_index.items():\n",
    "            embedding_vector = embeddings_index.get(word)\n",
    "            if embedding_vector is not None:\n",
    "                embedding_matrix[i] = embedding_vector\n",
    "        embedding_layer = Embedding(len(word_index) + 1,\n",
    "                                    EMBEDDING_DIM,\n",
    "                                    weights=[embedding_matrix],\n",
    "                                    input_length=MAX_SEQUENCE_LENGTH,\n",
    "                                    trainable=True)\n",
    "        sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,))\n",
    "        embedded_sequences = embedding_layer(sequence_input)\n",
    "\n",
    "        x = Conv1D(256, 5, activation='relu')(embedded_sequences)\n",
    "        x = MaxPooling1D(5)(x)\n",
    "        x = Conv1D(256, 5, activation='relu')(x)\n",
    "        x = MaxPooling1D(5)(x)\n",
    "        x = Conv1D(256, 5, activation='relu')(x)\n",
    "        x = MaxPooling1D(35)(x)  # global max pooling\n",
    "        x = Flatten()(x)\n",
    "        x = Dense(256, activation='relu')(x)\n",
    "        preds = Dense(nClasses, activation='softmax')(x)\n",
    "\n",
    "        model = Model(sequence_input, preds)\n",
    "        model.compile(loss='sparse_categorical_crossentropy',\n",
    "                      optimizer='rmsprop',\n",
    "                      metrics=['acc'])\n",
    "    else:\n",
    "        embedding_matrix = np.random.random((len(word_index) + 1, EMBEDDING_DIM))\n",
    "        for word, i in word_index.items():\n",
    "            embedding_vector = embeddings_index.get(word)\n",
    "            if embedding_vector is not None:\n",
    "                # words not found in embedding index will be all-zeros.\n",
    "                embedding_matrix[i] = embedding_vector\n",
    "\n",
    "        embedding_layer = Embedding(len(word_index) + 1,\n",
    "                                    EMBEDDING_DIM,\n",
    "                                    weights=[embedding_matrix],\n",
    "                                    input_length=MAX_SEQUENCE_LENGTH,\n",
    "                                    trainable=True)\n",
    "\n",
    "        convs = []\n",
    "        filter_sizes = [3, 4, 5, 6, 7]\n",
    "\n",
    "        sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "        embedded_sequences = embedding_layer(sequence_input)\n",
    "\n",
    "        for fsz in filter_sizes:\n",
    "            l_conv = Conv1D(128, filter_length=fsz, activation='relu')(embedded_sequences)\n",
    "            l_pool = MaxPooling1D(5)(l_conv)\n",
    "            convs.append(l_pool)\n",
    "\n",
    "        l_merge = concatenate(convs,axis=1) # Merge(mode='concat', concat_axis=1)(convs)\n",
    "        l_cov1 = Conv1D(128, 5, activation='relu')(l_merge)\n",
    "        l_pool1 = MaxPooling1D(5)(l_cov1)\n",
    "        l_cov2 = Conv1D(128, 5, activation='relu')(l_pool1)\n",
    "        l_pool2 = MaxPooling1D(30)(l_cov2)\n",
    "        l_flat = Flatten()(l_pool2)\n",
    "        l_dense = Dense(128, activation='relu')(l_flat)\n",
    "        preds = Dense(nClasses, activation='softmax')(l_dense)\n",
    "        model = Model(sequence_input, preds)\n",
    "        model.compile(loss='sparse_categorical_crossentropy',\n",
    "                      optimizer='rmsprop',\n",
    "                      metrics=['acc'])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TODO: balance dataset with weighted loss \n",
    "MEMORY_MB_MAX = 1200000 # maximum memory you can use\n",
    "MAX_SEQUENCE_LENGTH = 100 # Maximum sequance lentgh 500 words\n",
    "MAX_NB_WORDS = 55000 # Maximum number of unique words\n",
    "EMBEDDING_DIM = 300 #embedding dimension you can change it to {25, 100, 150, and 300} but need to change glove version\n",
    "batch_size_L1 = int(3048/2) # batch size in Level 1\n",
    "batch_size_L2 = int(3048/2) # batch size in Level 2\n",
    "batch_size_L3 = int(3048/2) # batch size in Level 3\n",
    "epochs = 10\n",
    "rnnType = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 256364 unique tokens.\n",
      "(947635, 100)\n",
      "(530675, 101) (132669, 101) (284291, 101)\n",
      "./fastText/cc.fa.300.vec\n",
      "Total 91286 word vectors.\n",
      "Loading Data is Done\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "#TODO: For now only RNN is working perfectly, need to change others later\n",
    "L1_model = 2 # 0 is DNN, 1 is CNN, and 2 is RNN for Level 1\n",
    "L2_model = 2 # 0 is DNN, 1 is CNN, and 2 is RNN for Level 2\n",
    "L3_model = 2 # 0 is DNN, 1 is CNN, and 2 is RNN for Level 2\n",
    "\n",
    "np.set_printoptions(threshold=np.inf)\n",
    "'''\n",
    "location of input data in two ways \n",
    "1: Tokenizer that is using GLOVE or FastText\n",
    "1: loadData that is using couting words or tf-idf\n",
    "'''\n",
    "\n",
    "#X_train, y_train, X_test, y_test, content_L2_Train, L2_Train, content_L2_Test, L2_Test, number_of_classes_L2,word_index, embeddings_index,number_of_classes_L1 =  \\\n",
    "#        loadData_Tokenizer(MAX_NB_WORDS,MAX_SEQUENCE_LENGTH)\n",
    "\n",
    "#X_train_DNN, y_train_DNN, X_test_DNN, y_test_DNN, content_L2_Train_DNN, L2_Train_DNN, content_L2_Test_DNN, L2_Test_DNN, number_of_classes_L2_DNN, number_of_classes_L1 = loadData()\n",
    "\n",
    "tokenizer,LabelDF,X_train,y_train,X_val,y_val,X_test,y_test,labelsL1,labelsL2,labelsL3,number_of_classes_L1,number_of_classes_L2,number_of_classes_L3,word_index,embeddings_index = loadData_Tokenizer_Efficient(MAX_NB_WORDS,MAX_SEQUENCE_LENGTH,EMBEDDING_DIM)\n",
    "\n",
    "print(\"Loading Data is Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2.55240198 0.94855199 0.54403431 1.8911209  1.13492491 0.76572501]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.utils import class_weight\n",
    "classWeights = class_weight.compute_class_weight('balanced',np.unique(y_train[:,0]),y_train[:,0])\n",
    "print(classWeights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Add"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Create model of RNN\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "context (InputLayer)            (None, 100)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 100, 300)     76909500    context[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "permute_1 (Permute)             (None, 300, 100)     0           embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "reshape_1 (Reshape)             (None, 300, 100)     0           permute_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 300, 100)     10100       reshape_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "attention_vec (Permute)         (None, 100, 300)     0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "attention_mul (Multiply)        (None, 100, 300)     0           embedding_1[0][0]                \n",
      "                                                                 attention_vec[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "gru_1 (GRU)                     (None, 100)          120300      attention_mul[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 6)            606         gru_1[0][0]                      \n",
      "==================================================================================================\n",
      "Total params: 77,040,506\n",
      "Trainable params: 77,040,506\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 530675 samples, validate on 132669 samples\n",
      "Epoch 1/10\n",
      "530675/530675 [==============================] - 112s 211us/step - loss: 0.9763 - acc: 0.6399 - val_loss: 0.4030 - val_acc: 0.8724\n",
      "Epoch 2/10\n",
      "530675/530675 [==============================] - 108s 203us/step - loss: 0.3215 - acc: 0.9005 - val_loss: 0.2393 - val_acc: 0.9283\n",
      "Epoch 3/10\n",
      "530675/530675 [==============================] - 108s 204us/step - loss: 0.2214 - acc: 0.9345 - val_loss: 0.1920 - val_acc: 0.9425\n",
      "Epoch 4/10\n",
      "530675/530675 [==============================] - 109s 206us/step - loss: 0.1852 - acc: 0.9449 - val_loss: 0.1742 - val_acc: 0.9466\n",
      "Epoch 5/10\n",
      "530675/530675 [==============================] - 110s 208us/step - loss: 0.1652 - acc: 0.9503 - val_loss: 0.1630 - val_acc: 0.9488\n",
      "Epoch 6/10\n",
      "530675/530675 [==============================] - 110s 207us/step - loss: 0.1518 - acc: 0.9539 - val_loss: 0.1549 - val_acc: 0.9518\n",
      "Epoch 7/10\n",
      "530675/530675 [==============================] - 110s 207us/step - loss: 0.1427 - acc: 0.9566 - val_loss: 0.1485 - val_acc: 0.9536\n",
      "Epoch 8/10\n",
      "530675/530675 [==============================] - 110s 208us/step - loss: 0.1340 - acc: 0.9586 - val_loss: 0.1483 - val_acc: 0.9534\n",
      "Epoch 9/10\n",
      "530675/530675 [==============================] - 110s 207us/step - loss: 0.1279 - acc: 0.9603 - val_loss: 0.1432 - val_acc: 0.9552\n",
      "Epoch 10/10\n",
      "530675/530675 [==============================] - 110s 208us/step - loss: 0.1222 - acc: 0.9618 - val_loss: 0.1401 - val_acc: 0.9558\n"
     ]
    }
   ],
   "source": [
    "#######################RNN Level 1########################\n",
    "from keras.layers.core import *\n",
    "from sklearn.utils import class_weight\n",
    "TIME_STEPS = MAX_SEQUENCE_LENGTH\n",
    "SINGLE_ATTENTION_VECTOR = False\n",
    "if L1_model == 2:\n",
    "    print('Create model of RNN')\n",
    "    model = buildModel_RNN(word_index, embeddings_index,number_of_classes_L1,MAX_SEQUENCE_LENGTH,EMBEDDING_DIM,rnnType)\n",
    "    classWeights = class_weight.compute_class_weight('balanced',np.unique(y_train[:,0]),y_train[:,0])\n",
    "    model.fit([X_train[:,:-1],X_train[:,-1]], y_train[:,0],\n",
    "              validation_data=([X_val[:,:-1],X_val[:,-1]], y_val[:,0]),\n",
    "              epochs=epochs,\n",
    "              batch_size=batch_size_L1,\n",
    "              class_weight=classWeights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "modelL1Filename = './models/modelL1.h5'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(modelL1Filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load model\n",
    "model = load_model(modelL1Filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0, 11, 28],\n",
       "       [ 0, 11,  7],\n",
       "       [ 0, 11, 49],\n",
       "       [ 0, 11, 40]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[y_train[:,0]==0,:][1:5,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 5, 4, 1, 0, 3])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labelsL1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8663, 100)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_val[y_val[:,0]==0,:].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8663, 100)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_val[y_val[:,0]==0,:].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Create Sub model of  0\n",
      "[11  3]  Number of samples:  61878\n",
      "Train on 34652 samples, validate on 8663 samples\n",
      "Epoch 1/10\n",
      "34652/34652 [==============================] - 6s 183us/step - loss: 0.4070 - acc: 0.7957 - val_loss: 0.2019 - val_acc: 0.9209\n",
      "Epoch 2/10\n",
      "34652/34652 [==============================] - 4s 114us/step - loss: 0.1614 - acc: 0.9429 - val_loss: 0.1218 - val_acc: 0.9620\n",
      "Epoch 3/10\n",
      "34652/34652 [==============================] - 4s 114us/step - loss: 0.1103 - acc: 0.9627 - val_loss: 0.1184 - val_acc: 0.9614\n",
      "Epoch 4/10\n",
      "34652/34652 [==============================] - 4s 113us/step - loss: 0.0885 - acc: 0.9694 - val_loss: 0.1192 - val_acc: 0.9606\n",
      "Epoch 5/10\n",
      "34652/34652 [==============================] - 4s 114us/step - loss: 0.0787 - acc: 0.9731 - val_loss: 0.1082 - val_acc: 0.9663\n",
      "Epoch 6/10\n",
      "34652/34652 [==============================] - 4s 114us/step - loss: 0.0659 - acc: 0.9785 - val_loss: 0.1047 - val_acc: 0.9664\n",
      "Epoch 7/10\n",
      "34652/34652 [==============================] - 4s 114us/step - loss: 0.0641 - acc: 0.9790 - val_loss: 0.1075 - val_acc: 0.9659\n",
      "Epoch 8/10\n",
      "34652/34652 [==============================] - 4s 115us/step - loss: 0.0562 - acc: 0.9815 - val_loss: 0.1135 - val_acc: 0.9656\n",
      "Epoch 9/10\n",
      "34652/34652 [==============================] - 4s 117us/step - loss: 0.0523 - acc: 0.9820 - val_loss: 0.1125 - val_acc: 0.9642\n",
      "Epoch 10/10\n",
      "34652/34652 [==============================] - 4s 115us/step - loss: 0.0453 - acc: 0.9849 - val_loss: 0.1181 - val_acc: 0.9639\n",
      "Create Sub model of  1\n",
      "[18 13  1 -1 10 22]  Number of samples:  166507\n",
      "Train on 93243 samples, validate on 23313 samples\n",
      "Epoch 1/10\n",
      "93243/93243 [==============================] - 13s 144us/step - loss: 0.8895 - acc: 0.6329 - val_loss: 0.3596 - val_acc: 0.8783\n",
      "Epoch 2/10\n",
      "93243/93243 [==============================] - 11s 115us/step - loss: 0.2362 - acc: 0.9337 - val_loss: 0.1623 - val_acc: 0.9569\n",
      "Epoch 3/10\n",
      "93243/93243 [==============================] - 11s 115us/step - loss: 0.1520 - acc: 0.9572 - val_loss: 0.1366 - val_acc: 0.9587\n",
      "Epoch 4/10\n",
      "93243/93243 [==============================] - 11s 115us/step - loss: 0.1208 - acc: 0.9638 - val_loss: 0.1156 - val_acc: 0.9634\n",
      "Epoch 5/10\n",
      "93243/93243 [==============================] - 11s 116us/step - loss: 0.0914 - acc: 0.9719 - val_loss: 0.1013 - val_acc: 0.9685\n",
      "Epoch 6/10\n",
      "93243/93243 [==============================] - 11s 116us/step - loss: 0.0734 - acc: 0.9780 - val_loss: 0.0867 - val_acc: 0.9730\n",
      "Epoch 7/10\n",
      "93243/93243 [==============================] - 11s 115us/step - loss: 0.0626 - acc: 0.9814 - val_loss: 0.0880 - val_acc: 0.9724\n",
      "Epoch 8/10\n",
      "93243/93243 [==============================] - 11s 115us/step - loss: 0.0543 - acc: 0.9839 - val_loss: 0.0810 - val_acc: 0.9753\n",
      "Epoch 9/10\n",
      "93243/93243 [==============================] - 11s 115us/step - loss: 0.0478 - acc: 0.9857 - val_loss: 0.0819 - val_acc: 0.9752\n",
      "Epoch 10/10\n",
      "93243/93243 [==============================] - 11s 115us/step - loss: 0.0431 - acc: 0.9872 - val_loss: 0.0820 - val_acc: 0.9753\n",
      "Create Sub model of  2\n",
      "[12 25  6 26]  Number of samples:  290313\n",
      "Train on 162574 samples, validate on 40643 samples\n",
      "Epoch 1/10\n",
      "162574/162574 [==============================] - 22s 133us/step - loss: 0.4733 - acc: 0.8304 - val_loss: 0.2244 - val_acc: 0.9277\n",
      "Epoch 2/10\n",
      "162574/162574 [==============================] - 19s 115us/step - loss: 0.1555 - acc: 0.9474 - val_loss: 0.1048 - val_acc: 0.9668\n",
      "Epoch 3/10\n",
      "162574/162574 [==============================] - 19s 115us/step - loss: 0.0966 - acc: 0.9689 - val_loss: 0.0884 - val_acc: 0.9709\n",
      "Epoch 4/10\n",
      "162574/162574 [==============================] - 19s 115us/step - loss: 0.0788 - acc: 0.9737 - val_loss: 0.0851 - val_acc: 0.9714\n",
      "Epoch 5/10\n",
      "162574/162574 [==============================] - 19s 115us/step - loss: 0.0693 - acc: 0.9764 - val_loss: 0.0792 - val_acc: 0.9728\n",
      "Epoch 6/10\n",
      "162574/162574 [==============================] - 19s 116us/step - loss: 0.0629 - acc: 0.9779 - val_loss: 0.0818 - val_acc: 0.9719\n",
      "Epoch 7/10\n",
      "162574/162574 [==============================] - 19s 116us/step - loss: 0.0577 - acc: 0.9796 - val_loss: 0.0769 - val_acc: 0.9731\n",
      "Epoch 8/10\n",
      "162574/162574 [==============================] - 19s 116us/step - loss: 0.0536 - acc: 0.9809 - val_loss: 0.0773 - val_acc: 0.9726\n",
      "Epoch 9/10\n",
      "162574/162574 [==============================] - 19s 116us/step - loss: 0.0512 - acc: 0.9818 - val_loss: 0.0769 - val_acc: 0.9727\n",
      "Epoch 10/10\n",
      "162574/162574 [==============================] - 19s 116us/step - loss: 0.0478 - acc: 0.9828 - val_loss: 0.0780 - val_acc: 0.9736\n",
      "Create Sub model of  3\n",
      "[ 4  0 20 23  5 24 15 17]  Number of samples:  83513\n",
      "Train on 46769 samples, validate on 11690 samples\n",
      "Epoch 1/10\n",
      "46769/46769 [==============================] - 8s 172us/step - loss: 0.9576 - acc: 0.7276 - val_loss: 0.5735 - val_acc: 0.7887\n",
      "Epoch 2/10\n",
      "46769/46769 [==============================] - 5s 116us/step - loss: 0.4753 - acc: 0.8424 - val_loss: 0.3860 - val_acc: 0.8660\n",
      "Epoch 3/10\n",
      "46769/46769 [==============================] - 5s 116us/step - loss: 0.3508 - acc: 0.8842 - val_loss: 0.3070 - val_acc: 0.9021\n",
      "Epoch 4/10\n",
      "46769/46769 [==============================] - 5s 116us/step - loss: 0.2506 - acc: 0.9165 - val_loss: 0.2143 - val_acc: 0.9292\n",
      "Epoch 5/10\n",
      "46769/46769 [==============================] - 5s 116us/step - loss: 0.1711 - acc: 0.9443 - val_loss: 0.1522 - val_acc: 0.9556\n",
      "Epoch 6/10\n",
      "46769/46769 [==============================] - 5s 115us/step - loss: 0.1248 - acc: 0.9639 - val_loss: 0.1194 - val_acc: 0.9674\n",
      "Epoch 7/10\n",
      "46769/46769 [==============================] - 5s 115us/step - loss: 0.0915 - acc: 0.9752 - val_loss: 0.1022 - val_acc: 0.9723\n",
      "Epoch 8/10\n",
      "46769/46769 [==============================] - 5s 116us/step - loss: 0.0673 - acc: 0.9813 - val_loss: 0.0823 - val_acc: 0.9758\n",
      "Epoch 9/10\n",
      "46769/46769 [==============================] - 5s 115us/step - loss: 0.0512 - acc: 0.9849 - val_loss: 0.0755 - val_acc: 0.9796\n",
      "Epoch 10/10\n",
      "46769/46769 [==============================] - 5s 116us/step - loss: 0.0404 - acc: 0.9891 - val_loss: 0.0695 - val_acc: 0.9814\n",
      "Create Sub model of  4\n",
      "[ 2  9  8 16 14 -1]  Number of samples:  139164\n",
      "Train on 77931 samples, validate on 19484 samples\n",
      "Epoch 1/10\n",
      "77931/77931 [==============================] - 12s 152us/step - loss: 0.9351 - acc: 0.6803 - val_loss: 0.6399 - val_acc: 0.7540\n",
      "Epoch 2/10\n",
      "77931/77931 [==============================] - 9s 118us/step - loss: 0.5283 - acc: 0.7911 - val_loss: 0.4132 - val_acc: 0.8340\n",
      "Epoch 3/10\n",
      "77931/77931 [==============================] - 9s 117us/step - loss: 0.3294 - acc: 0.8830 - val_loss: 0.2254 - val_acc: 0.9211\n",
      "Epoch 4/10\n",
      "77931/77931 [==============================] - 9s 117us/step - loss: 0.2023 - acc: 0.9370 - val_loss: 0.1926 - val_acc: 0.9403\n",
      "Epoch 5/10\n",
      "77931/77931 [==============================] - 9s 118us/step - loss: 0.1490 - acc: 0.9546 - val_loss: 0.1543 - val_acc: 0.9502\n",
      "Epoch 6/10\n",
      "77931/77931 [==============================] - 9s 118us/step - loss: 0.1263 - acc: 0.9613 - val_loss: 0.1420 - val_acc: 0.9541\n",
      "Epoch 7/10\n",
      "77931/77931 [==============================] - 9s 117us/step - loss: 0.1115 - acc: 0.9646 - val_loss: 0.1288 - val_acc: 0.9578\n",
      "Epoch 8/10\n",
      "77931/77931 [==============================] - 9s 117us/step - loss: 0.0998 - acc: 0.9682 - val_loss: 0.1352 - val_acc: 0.9547\n",
      "Epoch 9/10\n",
      "77931/77931 [==============================] - 9s 118us/step - loss: 0.0909 - acc: 0.9701 - val_loss: 0.1195 - val_acc: 0.9593\n",
      "Epoch 10/10\n",
      "77931/77931 [==============================] - 9s 118us/step - loss: 0.0843 - acc: 0.9720 - val_loss: 0.1213 - val_acc: 0.9614\n",
      "Create Sub model of  5\n",
      "[ 7 21 19]  Number of samples:  206260\n",
      "Train on 115506 samples, validate on 28876 samples\n",
      "Epoch 1/10\n",
      "115506/115506 [==============================] - 16s 139us/step - loss: 0.3792 - acc: 0.8551 - val_loss: 0.0933 - val_acc: 0.9702\n",
      "Epoch 2/10\n",
      "115506/115506 [==============================] - 13s 117us/step - loss: 0.0769 - acc: 0.9753 - val_loss: 0.0732 - val_acc: 0.9756\n",
      "Epoch 3/10\n",
      "115506/115506 [==============================] - 14s 118us/step - loss: 0.0492 - acc: 0.9843 - val_loss: 0.0476 - val_acc: 0.9849\n",
      "Epoch 4/10\n",
      "115506/115506 [==============================] - 14s 118us/step - loss: 0.0373 - acc: 0.9883 - val_loss: 0.0413 - val_acc: 0.9864\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5/10\n",
      "115506/115506 [==============================] - 13s 115us/step - loss: 0.0320 - acc: 0.9901 - val_loss: 0.0397 - val_acc: 0.9882\n",
      "Epoch 6/10\n",
      "115506/115506 [==============================] - 13s 117us/step - loss: 0.0274 - acc: 0.9915 - val_loss: 0.0391 - val_acc: 0.9877\n",
      "Epoch 7/10\n",
      "115506/115506 [==============================] - 13s 116us/step - loss: 0.0235 - acc: 0.9929 - val_loss: 0.0389 - val_acc: 0.9875\n",
      "Epoch 8/10\n",
      "115506/115506 [==============================] - 13s 116us/step - loss: 0.0205 - acc: 0.9938 - val_loss: 0.0400 - val_acc: 0.9881\n",
      "Epoch 9/10\n",
      "115506/115506 [==============================] - 13s 117us/step - loss: 0.0183 - acc: 0.9944 - val_loss: 0.0375 - val_acc: 0.9884\n",
      "Epoch 10/10\n",
      "115506/115506 [==============================] - 13s 116us/step - loss: 0.0167 - acc: 0.9948 - val_loss: 0.0393 - val_acc: 0.9890\n"
     ]
    }
   ],
   "source": [
    "import keras.backend as K\n",
    "from sklearn.utils import class_weight\n",
    "HDLTex = [] # Level 2 models is list of Deep Structure\n",
    "######################RNN Level 2################################\n",
    "if L2_model == 2:\n",
    "    for idx in range(0, number_of_classes_L1):\n",
    "        print('Create Sub model of ', idx)\n",
    "        classes = LabelDF[LabelDF[0]==idx][1].unique()\n",
    "        #classesL2.append(classes)\n",
    "        numberSamples = len(LabelDF[LabelDF[0]==idx])\n",
    "        print(classes, ' Number of samples: ', numberSamples)\n",
    "        #HDLTex.append(Sequential()) # memory limitation\n",
    "        model = Sequential()\n",
    "        #HDLTex[idx] = buildModel_RNN(word_index, embeddings_index,len(classes),MAX_SEQUENCE_LENGTH,EMBEDDING_DIM)\n",
    "        model = buildModel_RNN(word_index, embeddings_index,len(classes),MAX_SEQUENCE_LENGTH,EMBEDDING_DIM)\n",
    "        labelTrain = y_train[y_train[:,0]==idx,1]\n",
    "        for clsIdx,cls in enumerate(classes):\n",
    "            labelTrain[labelTrain==cls] = clsIdx\n",
    "        labelVal = y_val[y_val[:,0]==idx,1]\n",
    "        for clsIdx,cls in enumerate(classes):\n",
    "            labelVal[labelVal==cls] = clsIdx\n",
    "        #HDLTex[idx].fit(X_train[y_train[:,0]==idx,:], labelTrain,\n",
    "        classWeights = class_weight.compute_class_weight('balanced',np.unique(labelTrain),labelTrain)\n",
    "        model.fit(X_train[y_train[:,0]==idx,:], labelTrain,\n",
    "                      validation_data=(X_val[y_val[:,0]==idx,:], labelVal),\n",
    "                      epochs=epochs,\n",
    "                      batch_size=batch_size_L2,\n",
    "                      class_weight = classWeights)\n",
    "        # save model\n",
    "        modelL2Filename = './models/modelL2_'+ str(idx)+'.h5'\n",
    "        #HDLTex[idx].save(modelL2Filename)\n",
    "        model.save(modelL2Filename)\n",
    "        del model\n",
    "        K.clear_session()\n",
    "        gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[12, 4], [19, 14, 2, 0, 11, 23], [13, 26, 7, 27], [5, 1, 21, 24, 6, 25, 16, 18], [3, 10, 9, 17, 15, 0], [8, 22, 20]]\n"
     ]
    }
   ],
   "source": [
    "classesL2 = []\n",
    "for idx in range(0, number_of_classes_L1):\n",
    "        classes = LabelDF[LabelDF[0]==idx][1].unique()\n",
    "        classesL2.append(list(classes))\n",
    "print(classesL2)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "classesL2 = [[11,3],[18,13,1,-1,10,22],[12,25,6,26],[4,0,20,23,5,24,15,17],[2,9,8,16,14,-1],[7,21,19]],"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28\n",
      "28\n"
     ]
    }
   ],
   "source": [
    "classes = LabelDF[1].unique()\n",
    "print(len(classes))\n",
    "print(number_of_classes_L2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([21, 11,  6, 22,  0, 47])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LabelDF[LabelDF[1]==0][2].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "760830"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# free memory\n",
    "from keras import backend as K\n",
    "K.clear_session()\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[21, 11, 6, 22, 0, 47], [61, 62, 51, 8, 39, -1], [42, 54, 13, 12], [-1], [-1], [20], [24, 31, 53, 4, -1], [26, 33], [-1], [14, 48], [38, 41, 17, 32, 44], [40, 28, 3, 7, 49, -1], [50, 1, 60, 5, 10, 34, 55, 52, 57, -1], [-1], [-1], [27, 15, -1], [64, 30, 46, -1], [-1], [36, 37, 56], [-1], [25, 43, -1, 58, 19], [-1], [-1], [59, 9, 2, 65], [-1], [16, 23, -1, 18, 35, 63], [-1, 29, 45], []]\n"
     ]
    }
   ],
   "source": [
    "classesL3 = []\n",
    "for idx in range(0, number_of_classes_L2):\n",
    "        classes = LabelDF[LabelDF[1]==idx][2].unique()\n",
    "        classesL3.append(list(classes))\n",
    "print(classesL3)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# classesL3 = [[21, 11, 6, 22, 0, 47], [61, 62, 51, 8, 39, -1], [42, 54, 13, 12], [-1], [-1], [20], \n",
    "# [24, 31, 53, 4, -1], [26, 33], [-1], [14, 48], [38, 41, 17, 32, 44], [40, 28, 3, 7, 49, -1], \n",
    "# [50, 1, 60, 5, 10, 34, 55, 52, 57, -1], [-1], [-1], [27, 15, -1], [64, 30, 46, -1], [-1], [36, 37, 56],\n",
    "# [-1], [25, 43, -1, 58, 19], [-1], [-1], [59, 9, 2, 65], [-1], [16, 23, -1, 18, 35, 63], [-1, 29, 45], []]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Create Sub model of  16\n",
      "[64 30 46 -1]  Number of samples:  10316\n",
      "Train on 5776 samples, validate on 1445 samples\n",
      "Epoch 1/20\n",
      "5776/5776 [==============================] - 4s 628us/step - loss: 1.2634 - acc: 0.4697 - val_loss: 1.0605 - val_acc: 0.6166\n",
      "Epoch 2/20\n",
      "5776/5776 [==============================] - 1s 125us/step - loss: 1.0091 - acc: 0.6189 - val_loss: 0.9408 - val_acc: 0.6215\n",
      "Epoch 3/20\n",
      "5776/5776 [==============================] - 1s 140us/step - loss: 0.9175 - acc: 0.6390 - val_loss: 0.8613 - val_acc: 0.6913\n",
      "Epoch 4/20\n",
      "5776/5776 [==============================] - 1s 131us/step - loss: 0.8144 - acc: 0.6904 - val_loss: 0.7295 - val_acc: 0.7280\n",
      "Epoch 5/20\n",
      "5776/5776 [==============================] - 1s 128us/step - loss: 0.7328 - acc: 0.7254 - val_loss: 0.7009 - val_acc: 0.7751\n",
      "Epoch 6/20\n",
      "5776/5776 [==============================] - 1s 144us/step - loss: 0.5926 - acc: 0.8068 - val_loss: 0.5236 - val_acc: 0.8166\n",
      "Epoch 7/20\n",
      "5776/5776 [==============================] - 1s 140us/step - loss: 0.4472 - acc: 0.8423 - val_loss: 0.4158 - val_acc: 0.8464\n",
      "Epoch 8/20\n",
      "5776/5776 [==============================] - 1s 132us/step - loss: 0.3598 - acc: 0.8603 - val_loss: 0.3453 - val_acc: 0.8651\n",
      "Epoch 9/20\n",
      "5776/5776 [==============================] - 1s 127us/step - loss: 0.3043 - acc: 0.8779 - val_loss: 0.3025 - val_acc: 0.8747\n",
      "Epoch 10/20\n",
      "5776/5776 [==============================] - 1s 125us/step - loss: 0.3052 - acc: 0.8809 - val_loss: 0.3212 - val_acc: 0.8886\n",
      "Epoch 11/20\n",
      "5776/5776 [==============================] - 1s 135us/step - loss: 0.2439 - acc: 0.9077 - val_loss: 0.2565 - val_acc: 0.9010\n",
      "Epoch 12/20\n",
      "5776/5776 [==============================] - 1s 131us/step - loss: 0.2147 - acc: 0.9186 - val_loss: 0.2380 - val_acc: 0.9163\n",
      "Epoch 13/20\n",
      "5776/5776 [==============================] - 1s 125us/step - loss: 0.1954 - acc: 0.9299 - val_loss: 0.2241 - val_acc: 0.9253\n",
      "Epoch 14/20\n",
      "5776/5776 [==============================] - 1s 138us/step - loss: 0.1818 - acc: 0.9410 - val_loss: 0.2023 - val_acc: 0.9266\n",
      "Epoch 15/20\n",
      "5776/5776 [==============================] - 1s 126us/step - loss: 0.1599 - acc: 0.9501 - val_loss: 0.2098 - val_acc: 0.9370\n",
      "Epoch 16/20\n",
      "5776/5776 [==============================] - 1s 140us/step - loss: 0.1514 - acc: 0.9560 - val_loss: 0.1748 - val_acc: 0.9412\n",
      "Epoch 17/20\n",
      "5776/5776 [==============================] - 1s 129us/step - loss: 0.1297 - acc: 0.9655 - val_loss: 0.1890 - val_acc: 0.9377\n",
      "Epoch 18/20\n",
      "5776/5776 [==============================] - 1s 133us/step - loss: 0.1272 - acc: 0.9626 - val_loss: 0.1531 - val_acc: 0.9529\n",
      "Epoch 19/20\n",
      "5776/5776 [==============================] - 1s 135us/step - loss: 0.1027 - acc: 0.9725 - val_loss: 0.1486 - val_acc: 0.9550\n",
      "Epoch 20/20\n",
      "5776/5776 [==============================] - 1s 129us/step - loss: 0.0910 - acc: 0.9761 - val_loss: 0.1461 - val_acc: 0.9543\n",
      "Create Sub model of  17\n",
      "Create Sub model of  18\n",
      "[36 37 56]  Number of samples:  76307\n",
      "Train on 42731 samples, validate on 10684 samples\n",
      "Epoch 1/20\n",
      "42731/42731 [==============================] - 8s 190us/step - loss: 0.5792 - acc: 0.7917 - val_loss: 0.4160 - val_acc: 0.8425\n",
      "Epoch 2/20\n",
      "42731/42731 [==============================] - 5s 125us/step - loss: 0.3569 - acc: 0.8762 - val_loss: 0.2271 - val_acc: 0.9131\n",
      "Epoch 3/20\n",
      "42731/42731 [==============================] - 5s 121us/step - loss: 0.2080 - acc: 0.9290 - val_loss: 0.1304 - val_acc: 0.9569\n",
      "Epoch 4/20\n",
      "42731/42731 [==============================] - 5s 115us/step - loss: 0.1306 - acc: 0.9572 - val_loss: 0.1479 - val_acc: 0.9549\n",
      "Epoch 5/20\n",
      "42731/42731 [==============================] - 5s 115us/step - loss: 0.1013 - acc: 0.9678 - val_loss: 0.0854 - val_acc: 0.9737\n",
      "Epoch 6/20\n",
      "42731/42731 [==============================] - 5s 114us/step - loss: 0.0760 - acc: 0.9770 - val_loss: 0.0828 - val_acc: 0.9772\n",
      "Epoch 7/20\n",
      "42731/42731 [==============================] - 5s 115us/step - loss: 0.0640 - acc: 0.9820 - val_loss: 0.0672 - val_acc: 0.9812\n",
      "Epoch 8/20\n",
      "42731/42731 [==============================] - 5s 115us/step - loss: 0.0497 - acc: 0.9859 - val_loss: 0.0625 - val_acc: 0.9813\n",
      "Epoch 9/20\n",
      "42731/42731 [==============================] - 5s 115us/step - loss: 0.0450 - acc: 0.9878 - val_loss: 0.0620 - val_acc: 0.9834\n",
      "Epoch 10/20\n",
      "42731/42731 [==============================] - 5s 114us/step - loss: 0.0398 - acc: 0.9895 - val_loss: 0.0590 - val_acc: 0.9830\n",
      "Epoch 11/20\n",
      "42731/42731 [==============================] - 5s 115us/step - loss: 0.0348 - acc: 0.9910 - val_loss: 0.0594 - val_acc: 0.9830\n",
      "Epoch 12/20\n",
      "42731/42731 [==============================] - 5s 115us/step - loss: 0.0308 - acc: 0.9917 - val_loss: 0.0607 - val_acc: 0.9827\n",
      "Epoch 13/20\n",
      "42731/42731 [==============================] - 5s 115us/step - loss: 0.0275 - acc: 0.9927 - val_loss: 0.0613 - val_acc: 0.9840\n",
      "Epoch 14/20\n",
      "42731/42731 [==============================] - 5s 115us/step - loss: 0.0263 - acc: 0.9931 - val_loss: 0.0688 - val_acc: 0.9823\n",
      "Epoch 15/20\n",
      "42731/42731 [==============================] - 5s 115us/step - loss: 0.0223 - acc: 0.9939 - val_loss: 0.0582 - val_acc: 0.9836\n",
      "Epoch 16/20\n",
      "42731/42731 [==============================] - 5s 115us/step - loss: 0.0241 - acc: 0.9938 - val_loss: 0.0581 - val_acc: 0.9838\n",
      "Epoch 17/20\n",
      "42731/42731 [==============================] - 5s 115us/step - loss: 0.0202 - acc: 0.9945 - val_loss: 0.0571 - val_acc: 0.9842\n",
      "Epoch 18/20\n",
      "42731/42731 [==============================] - 5s 115us/step - loss: 0.0185 - acc: 0.9951 - val_loss: 0.0718 - val_acc: 0.9821\n",
      "Epoch 19/20\n",
      "42731/42731 [==============================] - 5s 117us/step - loss: 0.0184 - acc: 0.9953 - val_loss: 0.0623 - val_acc: 0.9836\n",
      "Epoch 20/20\n",
      "42731/42731 [==============================] - 5s 115us/step - loss: 0.0165 - acc: 0.9957 - val_loss: 0.0614 - val_acc: 0.9832\n",
      "Create Sub model of  19\n",
      "Create Sub model of  20\n",
      "[25 43 -1 58 19]  Number of samples:  3854\n",
      "Train on 2157 samples, validate on 540 samples\n",
      "Epoch 1/20\n",
      "2157/2157 [==============================] - 3s 1ms/step - loss: 1.6023 - acc: 0.1841 - val_loss: 1.4844 - val_acc: 0.3630\n",
      "Epoch 2/20\n",
      "2157/2157 [==============================] - 0s 211us/step - loss: 1.4768 - acc: 0.3681 - val_loss: 1.4611 - val_acc: 0.3667\n",
      "Epoch 3/20\n",
      "2157/2157 [==============================] - 0s 210us/step - loss: 1.4406 - acc: 0.3709 - val_loss: 1.4413 - val_acc: 0.4000\n",
      "Epoch 4/20\n",
      "2157/2157 [==============================] - 0s 204us/step - loss: 1.4151 - acc: 0.4186 - val_loss: 1.4220 - val_acc: 0.3981\n",
      "Epoch 5/20\n",
      "2157/2157 [==============================] - 0s 210us/step - loss: 1.3804 - acc: 0.4182 - val_loss: 1.4149 - val_acc: 0.4426\n",
      "Epoch 6/20\n",
      "2157/2157 [==============================] - 0s 212us/step - loss: 1.3578 - acc: 0.5044 - val_loss: 1.3978 - val_acc: 0.4000\n",
      "Epoch 7/20\n",
      "2157/2157 [==============================] - 0s 213us/step - loss: 1.3280 - acc: 0.4219 - val_loss: 1.3348 - val_acc: 0.4648\n",
      "Epoch 8/20\n",
      "2157/2157 [==============================] - 0s 211us/step - loss: 1.2494 - acc: 0.5141 - val_loss: 1.3428 - val_acc: 0.4241\n",
      "Epoch 9/20\n",
      "2157/2157 [==============================] - 0s 210us/step - loss: 1.2487 - acc: 0.4738 - val_loss: 1.2590 - val_acc: 0.5537\n",
      "Epoch 10/20\n",
      "2157/2157 [==============================] - 0s 209us/step - loss: 1.1256 - acc: 0.6124 - val_loss: 1.1322 - val_acc: 0.5741\n",
      "Epoch 11/20\n",
      "2157/2157 [==============================] - 0s 208us/step - loss: 0.9883 - acc: 0.6152 - val_loss: 1.2501 - val_acc: 0.4685\n",
      "Epoch 12/20\n",
      "2157/2157 [==============================] - 0s 207us/step - loss: 1.0794 - acc: 0.5526 - val_loss: 0.9831 - val_acc: 0.6296\n",
      "Epoch 13/20\n",
      "2157/2157 [==============================] - 0s 209us/step - loss: 0.8528 - acc: 0.6616 - val_loss: 0.8988 - val_acc: 0.6593\n",
      "Epoch 14/20\n",
      "2157/2157 [==============================] - 0s 211us/step - loss: 0.7820 - acc: 0.7140 - val_loss: 0.8763 - val_acc: 0.6426\n",
      "Epoch 15/20\n",
      "2157/2157 [==============================] - 0s 206us/step - loss: 0.7396 - acc: 0.7056 - val_loss: 0.9130 - val_acc: 0.6722\n",
      "Epoch 16/20\n",
      "2157/2157 [==============================] - 0s 213us/step - loss: 0.7488 - acc: 0.7487 - val_loss: 0.8435 - val_acc: 0.7185\n",
      "Epoch 17/20\n",
      "2157/2157 [==============================] - 0s 208us/step - loss: 0.6727 - acc: 0.7946 - val_loss: 0.8413 - val_acc: 0.7241\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18/20\n",
      "2157/2157 [==============================] - 0s 209us/step - loss: 0.6550 - acc: 0.8206 - val_loss: 0.7098 - val_acc: 0.7481\n",
      "Epoch 19/20\n",
      "2157/2157 [==============================] - 0s 210us/step - loss: 0.5564 - acc: 0.8183 - val_loss: 0.6980 - val_acc: 0.7241\n",
      "Epoch 20/20\n",
      "2157/2157 [==============================] - 0s 210us/step - loss: 0.5715 - acc: 0.7923 - val_loss: 0.6997 - val_acc: 0.7370\n",
      "Create Sub model of  21\n",
      "Create Sub model of  22\n",
      "Create Sub model of  23\n",
      "[59  9  2 65]  Number of samples:  10374\n",
      "Train on 5809 samples, validate on 1452 samples\n",
      "Epoch 1/20\n",
      "5809/5809 [==============================] - 3s 576us/step - loss: 1.2238 - acc: 0.5147 - val_loss: 0.9090 - val_acc: 0.7445\n",
      "Epoch 2/20\n",
      "5809/5809 [==============================] - 1s 122us/step - loss: 0.8589 - acc: 0.7442 - val_loss: 0.7886 - val_acc: 0.7445\n",
      "Epoch 3/20\n",
      "5809/5809 [==============================] - 1s 121us/step - loss: 0.7582 - acc: 0.7442 - val_loss: 0.7479 - val_acc: 0.7445\n",
      "Epoch 4/20\n",
      "5809/5809 [==============================] - 1s 121us/step - loss: 0.7282 - acc: 0.7514 - val_loss: 0.6572 - val_acc: 0.7514\n",
      "Epoch 5/20\n",
      "5809/5809 [==============================] - 1s 119us/step - loss: 0.6072 - acc: 0.7616 - val_loss: 0.5544 - val_acc: 0.7803\n",
      "Epoch 6/20\n",
      "5809/5809 [==============================] - 1s 120us/step - loss: 0.5220 - acc: 0.8050 - val_loss: 0.5583 - val_acc: 0.7837\n",
      "Epoch 7/20\n",
      "5809/5809 [==============================] - 1s 122us/step - loss: 0.4966 - acc: 0.8201 - val_loss: 0.4313 - val_acc: 0.8285\n",
      "Epoch 8/20\n",
      "5809/5809 [==============================] - 1s 121us/step - loss: 0.3846 - acc: 0.8404 - val_loss: 0.3804 - val_acc: 0.8430\n",
      "Epoch 9/20\n",
      "5809/5809 [==============================] - 1s 120us/step - loss: 0.3441 - acc: 0.8487 - val_loss: 0.3721 - val_acc: 0.8457\n",
      "Epoch 10/20\n",
      "5809/5809 [==============================] - 1s 122us/step - loss: 0.3243 - acc: 0.8495 - val_loss: 0.3421 - val_acc: 0.8450\n",
      "Epoch 11/20\n",
      "5809/5809 [==============================] - 1s 119us/step - loss: 0.3046 - acc: 0.8559 - val_loss: 0.3417 - val_acc: 0.8512\n",
      "Epoch 12/20\n",
      "5809/5809 [==============================] - 1s 117us/step - loss: 0.2882 - acc: 0.8606 - val_loss: 0.3259 - val_acc: 0.8526\n",
      "Epoch 13/20\n",
      "5809/5809 [==============================] - 1s 119us/step - loss: 0.2837 - acc: 0.8662 - val_loss: 0.3500 - val_acc: 0.8623\n",
      "Epoch 14/20\n",
      "5809/5809 [==============================] - 1s 122us/step - loss: 0.2859 - acc: 0.8740 - val_loss: 0.3114 - val_acc: 0.8567\n",
      "Epoch 15/20\n",
      "5809/5809 [==============================] - 1s 120us/step - loss: 0.2635 - acc: 0.8862 - val_loss: 0.3134 - val_acc: 0.8643\n",
      "Epoch 16/20\n",
      "5809/5809 [==============================] - 1s 119us/step - loss: 0.2481 - acc: 0.8929 - val_loss: 0.3069 - val_acc: 0.8712\n",
      "Epoch 17/20\n",
      "5809/5809 [==============================] - 1s 120us/step - loss: 0.2299 - acc: 0.9113 - val_loss: 0.2849 - val_acc: 0.8781\n",
      "Epoch 18/20\n",
      "5809/5809 [==============================] - 1s 121us/step - loss: 0.2161 - acc: 0.9203 - val_loss: 0.2671 - val_acc: 0.8829\n",
      "Epoch 19/20\n",
      "5809/5809 [==============================] - 1s 119us/step - loss: 0.2343 - acc: 0.9110 - val_loss: 0.3016 - val_acc: 0.8809\n",
      "Epoch 20/20\n",
      "5809/5809 [==============================] - 1s 123us/step - loss: 0.1954 - acc: 0.9315 - val_loss: 0.2521 - val_acc: 0.8946\n",
      "Create Sub model of  24\n",
      "Create Sub model of  25\n",
      "[16 23 -1 18 35 63]  Number of samples:  58714\n",
      "Train on 32879 samples, validate on 8220 samples\n",
      "Epoch 1/20\n",
      "32879/32879 [==============================] - 6s 197us/step - loss: 1.4082 - acc: 0.4733 - val_loss: 1.0910 - val_acc: 0.5835\n",
      "Epoch 2/20\n",
      "32879/32879 [==============================] - 4s 120us/step - loss: 0.9639 - acc: 0.5957 - val_loss: 0.8058 - val_acc: 0.6235\n",
      "Epoch 3/20\n",
      "32879/32879 [==============================] - 4s 119us/step - loss: 0.7473 - acc: 0.6801 - val_loss: 0.6238 - val_acc: 0.7349\n",
      "Epoch 4/20\n",
      "32879/32879 [==============================] - 4s 120us/step - loss: 0.5807 - acc: 0.7545 - val_loss: 0.5208 - val_acc: 0.7830\n",
      "Epoch 5/20\n",
      "32879/32879 [==============================] - 4s 123us/step - loss: 0.4928 - acc: 0.7874 - val_loss: 0.4329 - val_acc: 0.8162\n",
      "Epoch 6/20\n",
      "32879/32879 [==============================] - 4s 124us/step - loss: 0.4179 - acc: 0.8177 - val_loss: 0.4394 - val_acc: 0.8047\n",
      "Epoch 7/20\n",
      "32879/32879 [==============================] - 4s 120us/step - loss: 0.3846 - acc: 0.8363 - val_loss: 0.4329 - val_acc: 0.8091\n",
      "Epoch 8/20\n",
      "32879/32879 [==============================] - 4s 122us/step - loss: 0.3681 - acc: 0.8409 - val_loss: 0.3926 - val_acc: 0.8212\n",
      "Epoch 9/20\n",
      "32879/32879 [==============================] - 4s 122us/step - loss: 0.3437 - acc: 0.8530 - val_loss: 0.3667 - val_acc: 0.8344\n",
      "Epoch 10/20\n",
      "32879/32879 [==============================] - 4s 123us/step - loss: 0.3376 - acc: 0.8534 - val_loss: 0.3990 - val_acc: 0.8260\n",
      "Epoch 11/20\n",
      "32879/32879 [==============================] - 4s 122us/step - loss: 0.3261 - acc: 0.8594 - val_loss: 0.3733 - val_acc: 0.8337\n",
      "Epoch 12/20\n",
      "32879/32879 [==============================] - 4s 122us/step - loss: 0.3139 - acc: 0.8642 - val_loss: 0.4008 - val_acc: 0.8282\n",
      "Epoch 13/20\n",
      "32879/32879 [==============================] - 4s 122us/step - loss: 0.3052 - acc: 0.8688 - val_loss: 0.3654 - val_acc: 0.8343\n",
      "Epoch 14/20\n",
      "32879/32879 [==============================] - 4s 123us/step - loss: 0.2982 - acc: 0.8688 - val_loss: 0.3611 - val_acc: 0.8376\n",
      "Epoch 15/20\n",
      "32879/32879 [==============================] - 4s 121us/step - loss: 0.2901 - acc: 0.8749 - val_loss: 0.3635 - val_acc: 0.8344\n",
      "Epoch 16/20\n",
      "32879/32879 [==============================] - 4s 121us/step - loss: 0.2840 - acc: 0.8777 - val_loss: 0.3624 - val_acc: 0.8378\n",
      "Epoch 17/20\n",
      "32879/32879 [==============================] - 4s 121us/step - loss: 0.2778 - acc: 0.8785 - val_loss: 0.3750 - val_acc: 0.8365\n",
      "Epoch 18/20\n",
      "32879/32879 [==============================] - 4s 121us/step - loss: 0.2686 - acc: 0.8845 - val_loss: 0.3759 - val_acc: 0.8341\n",
      "Epoch 19/20\n",
      "32879/32879 [==============================] - 4s 121us/step - loss: 0.2620 - acc: 0.8872 - val_loss: 0.3782 - val_acc: 0.8347\n",
      "Epoch 20/20\n",
      "32879/32879 [==============================] - 4s 120us/step - loss: 0.2531 - acc: 0.8929 - val_loss: 0.4123 - val_acc: 0.8241\n",
      "Create Sub model of  26\n",
      "[-1 29 45]  Number of samples:  8239\n",
      "Train on 4614 samples, validate on 1153 samples\n",
      "Epoch 1/20\n",
      "4614/4614 [==============================] - 3s 723us/step - loss: 0.9307 - acc: 0.6580 - val_loss: 0.5389 - val_acc: 0.8968\n",
      "Epoch 2/20\n",
      "4614/4614 [==============================] - 1s 150us/step - loss: 0.4826 - acc: 0.8966 - val_loss: 0.4042 - val_acc: 0.8968\n",
      "Epoch 3/20\n",
      "4614/4614 [==============================] - 1s 147us/step - loss: 0.3895 - acc: 0.8966 - val_loss: 0.3701 - val_acc: 0.8968\n",
      "Epoch 4/20\n",
      "4614/4614 [==============================] - 1s 149us/step - loss: 0.3541 - acc: 0.8966 - val_loss: 0.3521 - val_acc: 0.8968\n",
      "Epoch 5/20\n",
      "4614/4614 [==============================] - 1s 151us/step - loss: 0.3432 - acc: 0.8966 - val_loss: 0.3178 - val_acc: 0.8968\n",
      "Epoch 6/20\n",
      "4614/4614 [==============================] - 1s 149us/step - loss: 0.2880 - acc: 0.8971 - val_loss: 0.2804 - val_acc: 0.8968\n",
      "Epoch 7/20\n",
      "4614/4614 [==============================] - 1s 150us/step - loss: 0.2491 - acc: 0.9016 - val_loss: 0.2830 - val_acc: 0.9046\n",
      "Epoch 8/20\n",
      "4614/4614 [==============================] - 1s 148us/step - loss: 0.2254 - acc: 0.9161 - val_loss: 0.2094 - val_acc: 0.9107\n",
      "Epoch 9/20\n",
      "4614/4614 [==============================] - 1s 148us/step - loss: 0.1601 - acc: 0.9298 - val_loss: 0.1608 - val_acc: 0.9228\n",
      "Epoch 10/20\n",
      "4614/4614 [==============================] - 1s 150us/step - loss: 0.1255 - acc: 0.9499 - val_loss: 0.1506 - val_acc: 0.9350\n",
      "Epoch 11/20\n",
      "4614/4614 [==============================] - 1s 149us/step - loss: 0.1733 - acc: 0.9467 - val_loss: 0.1521 - val_acc: 0.9315\n",
      "Epoch 12/20\n",
      "4614/4614 [==============================] - 1s 149us/step - loss: 0.1100 - acc: 0.9556 - val_loss: 0.1390 - val_acc: 0.9350\n",
      "Epoch 13/20\n",
      "4614/4614 [==============================] - 1s 150us/step - loss: 0.1009 - acc: 0.9567 - val_loss: 0.1368 - val_acc: 0.9410\n",
      "Epoch 14/20\n",
      "4614/4614 [==============================] - 1s 148us/step - loss: 0.0889 - acc: 0.9627 - val_loss: 0.1254 - val_acc: 0.9462\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 15/20\n",
      "4614/4614 [==============================] - 1s 148us/step - loss: 0.0861 - acc: 0.9634 - val_loss: 0.1390 - val_acc: 0.9428\n",
      "Epoch 16/20\n",
      "4614/4614 [==============================] - 1s 145us/step - loss: 0.0812 - acc: 0.9690 - val_loss: 0.1503 - val_acc: 0.9402\n",
      "Epoch 17/20\n",
      "4614/4614 [==============================] - 1s 150us/step - loss: 0.0793 - acc: 0.9699 - val_loss: 0.1547 - val_acc: 0.9454\n",
      "Epoch 18/20\n",
      "4614/4614 [==============================] - 1s 148us/step - loss: 0.0743 - acc: 0.9738 - val_loss: 0.1677 - val_acc: 0.9445\n",
      "Epoch 19/20\n",
      "4614/4614 [==============================] - 1s 148us/step - loss: 0.0774 - acc: 0.9716 - val_loss: 0.1301 - val_acc: 0.9532\n",
      "Epoch 20/20\n",
      "4614/4614 [==============================] - 1s 147us/step - loss: 0.0688 - acc: 0.9751 - val_loss: 0.1541 - val_acc: 0.9514\n",
      "Create Sub model of  27\n"
     ]
    }
   ],
   "source": [
    "epochs = 20\n",
    "import keras.backend as K\n",
    "from sklearn.utils import class_weight\n",
    "HDLTexL3 = [] # Level 3 models is list of Deep Structure\n",
    "######################RNN Level 3################################\n",
    "L3_model = 2\n",
    "if L3_model == 2:\n",
    "    for idx in range(0, number_of_classes_L2):\n",
    "        print('Create Sub model of ', idx)\n",
    "        classes = LabelDF[LabelDF[1]==idx][2].unique()\n",
    "        if len(classes) < 2:\n",
    "            continue\n",
    "        #classesL3.append(list(classes))\n",
    "        numberSamples = len(LabelDF[LabelDF[1]==idx])\n",
    "        print(classes, ' Number of samples: ', numberSamples)\n",
    "        #HDLTexL3.append(Sequential())\n",
    "        model = Sequential()\n",
    "        #HDLTexL3[idx] = buildModel_RNN(word_index, embeddings_index,len(classes),MAX_SEQUENCE_LENGTH,EMBEDDING_DIM)\n",
    "        model = buildModel_RNN(word_index, embeddings_index,len(classes),MAX_SEQUENCE_LENGTH,EMBEDDING_DIM)\n",
    "        labelTrain = y_train[y_train[:,1]==idx,2]\n",
    "        for clsIdx,cls in enumerate(classes):\n",
    "            labelTrain[labelTrain==cls] = clsIdx\n",
    "        labelVal = y_val[y_val[:,1]==idx,2]\n",
    "        for clsIdx,cls in enumerate(classes):\n",
    "            labelVal[labelVal==cls] = clsIdx\n",
    "        #HDLTexL3[idx].fit(X_train[y_train[:,1]==idx,:], labelTrain,\n",
    "        classWeights = class_weight.compute_class_weight('balanced',np.unique(labelTrain),labelTrain)\n",
    "        model.fit(X_train[y_train[:,1]==idx,:], labelTrain,\n",
    "                      validation_data=(X_val[y_val[:,1]==idx,:], labelVal),\n",
    "                      epochs=epochs,\n",
    "                      batch_size=batch_size_L3,\n",
    "                      class_weight = classWeights)\n",
    "        # save model\n",
    "        modelL3Filename = './models/modelL3_'+ str(idx)+'.h5'\n",
    "        #HDLTexL3[idx].save(modelL3Filename)\n",
    "        model.save(modelL3Filename)\n",
    "        del model\n",
    "        K.clear_session()\n",
    "        gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Time to Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5, 100) (5, 3)\n"
     ]
    }
   ],
   "source": [
    "from keras.models import load_model\n",
    "index = 120\n",
    "numberSamples = 5\n",
    "xSample = X_test[index:index+numberSamples]\n",
    "ySample = y_test[index:index+numberSamples]\n",
    "print(xSample.shape, ySample.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-1 -1 -1]\n",
      " [-1 -1 -1]\n",
      " [-1 -1 -1]\n",
      " [-1 -1 -1]\n",
      " [-1 -1 -1]]\n"
     ]
    }
   ],
   "source": [
    "results = -np.ones_like(ySample)\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 2 -1 -1]\n",
      " [ 2 -1 -1]\n",
      " [ 2 -1 -1]\n",
      " [ 4 -1 -1]\n",
      " [ 0 -1 -1]]\n",
      "[[ 2 12 10]\n",
      " [ 2 25 63]\n",
      " [ 2 12 60]\n",
      " [ 4  9 14]\n",
      " [ 0 11 40]]\n"
     ]
    }
   ],
   "source": [
    "# Level 1 Test\n",
    "# load model\n",
    "modelFilename = 'modelL1'+'.h5'\n",
    "model = load_model(modelFilename)\n",
    "# test model\n",
    "yPred = model.predict(xSample)\n",
    "predL1Class = yPred.argmax(axis=1)\n",
    "results[:,0] = predL1Class\n",
    "print(results)\n",
    "print(ySample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 12:2 25:2 12:4 9:0 11:"
     ]
    }
   ],
   "source": [
    "# Level 2 Test\n",
    "#print(classesL2)\n",
    "#predL2Class = []\n",
    "predL1Class = results[:,0]\n",
    "for idx, smp in enumerate(predL1Class):\n",
    "    #print(smp)\n",
    "    if len(classesL2[smp]) < 2:\n",
    "        # return -1 or the class number\n",
    "        if classesL2[smp][0]==-1 or classesL2[smp][0] is None:\n",
    "            #predL2Class.append(-1)\n",
    "            results[idx,1] = -1\n",
    "        else:\n",
    "            #predL2Class.append(classesL2[smp][0])\n",
    "            results[idx,1] = classesL2[smp][0]\n",
    "        continue\n",
    "    # load related model \n",
    "    modelFilename = 'modelL2_'+ str(smp)+ '.h5'\n",
    "    model = load_model(modelFilename)\n",
    "    yPred = model.predict(xSample[idx:idx+1,:])\n",
    "    predClass = yPred.argmax()\n",
    "    #predL2Class.append(classesL2[smp][predClass])\n",
    "    results[idx,1] = classesL2[smp][predClass]\n",
    "    print(smp, results[idx,1], end=':', flush=True)\n",
    "    del model\n",
    "    K.clear_session()\n",
    "    gc.collect()\n",
    "#results[idx,1] = predL2Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 2 12 -1]\n",
      " [ 2 25 -1]\n",
      " [ 2 12 -1]\n",
      " [ 4  9 -1]\n",
      " [ 0 11 -1]]\n",
      "[[ 2 12 10]\n",
      " [ 2 25 63]\n",
      " [ 2 12 60]\n",
      " [ 4  9 14]\n",
      " [ 0 11 40]]\n"
     ]
    }
   ],
   "source": [
    "print(results)\n",
    "print(ySample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12 10:25 63:12 60:9 14:11 40:"
     ]
    }
   ],
   "source": [
    "# Level 3 Test\n",
    "#print(classesL3)\n",
    "#predL3Class = []\n",
    "predL2Class = results[:,1]\n",
    "for idx, smp in enumerate(predL2Class):\n",
    "    #print(smp)\n",
    "    if len(classesL3[smp]) < 2:\n",
    "        # return -1 or the class number\n",
    "        if classesL3[smp][0]==-1 or classesL3[smp][0] is None:\n",
    "            #predL3Class.append(-1)\n",
    "            results[idx,2] = -1\n",
    "        else:\n",
    "            #predL3Class.append(classesL3[smp][0])\n",
    "            results[idx,2] = classesL3[smp][0]\n",
    "        continue\n",
    "    # load related model \n",
    "    modelFilename = './models/modelL3_'+ str(smp)+ '.h5'\n",
    "    model = load_model(modelFilename)\n",
    "    yPred = model.predict(xSample[idx:idx+1,:])\n",
    "    predClass = yPred.argmax()\n",
    "    #predL3Class.append(classesL3[smp][predClass])\n",
    "    results[idx,2] = classesL3[smp][predClass]\n",
    "    print(smp, results[idx,2], end=':', flush=True)\n",
    "    del model\n",
    "    K.clear_session()\n",
    "    gc.collect()\n",
    "#results[:,2] = predL2Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 2 12 10]\n",
      " [ 2 25 63]\n",
      " [ 2 12 60]\n",
      " [ 4  9 14]\n",
      " [ 0 11 40]]\n",
      "[[ 2 12 10]\n",
      " [ 2 25 63]\n",
      " [ 2 12 60]\n",
      " [ 4  9 14]\n",
      " [ 0 11 40]]\n"
     ]
    }
   ],
   "source": [
    "print(results)\n",
    "print(ySample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1928 / 236909\n"
     ]
    }
   ],
   "source": [
    "print(idx,'/',len(predL1Class))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['./models/modelL3_20.h5', './models/modelL3_7.h5', './models/modelL3_16.h5', './models/modelL3_2.h5', './models/modelL3_6.h5', './models/modelL2_4.h5', './models/modelL3_9.h5', './models/modelL3_26.h5', './models/modelL3_18.h5', './models/modelL3_25.h5', './models/modelL3_23.h5', './models/modelL3_10.h5', './models/modelL2_5.h5', './models/modelL3_15.h5', './models/modelL1.h5', './models/modelL3_12.h5', './models/modelL2_2.h5', './models/modelL3_1.h5', './models/modelL2_3.h5', './models/modelL3_0.h5', './models/modelL2_1.h5', './models/modelL3_11.h5', './models/modelL2_0.h5']\n"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "modelExists = glob.glob('./models/*.h5') \n",
    "print(modelExists)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(284291, 100) (284291, 3)\n",
      "284291/284291 [==============================] - 43s 152us/step\n",
      "Level 1 is done\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm_notebook as tqdm\n",
    "# all test data\n",
    "#X_test, y_test\n",
    "x = X_test\n",
    "y = y_test\n",
    "print(x.shape,y.shape)\n",
    "results = -np.ones_like(y)\n",
    "\n",
    "# Level 1 Test\n",
    "# load model\n",
    "modelFilename = './models/modelL1'+'.h5'\n",
    "model = load_model(modelFilename)\n",
    "# test model\n",
    "yPred = model.predict(x, verbose=1, batch_size=256)\n",
    "predL1Class = yPred.argmax(axis=1)\n",
    "results[:,0] = predL1Class\n",
    "print('Level 1 is done')\n",
    "\n",
    "# TODO: Parallel Computing, batchSize\n",
    "# Level 2 Test\n",
    "predL1Class = results[:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 5,  7, 33],\n",
       "       [ 4,  9, 48],\n",
       "       [ 5,  7, 33],\n",
       "       [ 1, 13, -1],\n",
       "       [ 4, 16, 64],\n",
       "       [ 2, 12, 60],\n",
       "       [ 2, 12, 50],\n",
       "       [ 1, 18, 36],\n",
       "       [ 5,  7, 33],\n",
       "       [ 5,  7, 33]])"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[40:50,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 5,  7, 33],\n",
       "       [ 4,  9, 48],\n",
       "       [ 5,  7, 33],\n",
       "       [ 1,  1, -1],\n",
       "       [ 4, 16, 64],\n",
       "       [ 2, 12, 60],\n",
       "       [ 2, 12, 50],\n",
       "       [ 1, 18, 36],\n",
       "       [ 5,  7, 33],\n",
       "       [ 5,  7, 33]])"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[40:50,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(284291, 100) (284291, 3)\n",
      "284291/284291 [==============================] - 9s 30us/step\n",
      "Level 1 is done\n",
      "Selected Indices for class  0 :  16350 / 284291\n",
      "16350/16350 [==============================] - 1s 44us/step\n",
      "Selected Indices for class  1 :  49927 / 284291\n",
      "49927/49927 [==============================] - 2s 34us/step\n",
      "Selected Indices for class  2 :  88507 / 284291\n",
      "88507/88507 [==============================] - 3s 32us/step\n",
      "Selected Indices for class  3 :  24650 / 284291\n",
      "24650/24650 [==============================] - 1s 39us/step\n",
      "Selected Indices for class  4 :  42747 / 284291\n",
      "42747/42747 [==============================] - 1s 34us/step\n",
      "Selected Indices for class  5 :  62110 / 284291\n",
      "62110/62110 [==============================] - 2s 32us/step\n",
      "Level 2 is done\n",
      "Selected Indices for class  -1 :  387 / 284291\n",
      "Selected Indices for class  0 :  18570 / 284291\n",
      "18570/18570 [==============================] - 1s 41us/step\n",
      "Selected Indices for class  1 :  18448 / 284291\n",
      "18448/18448 [==============================] - 1s 40us/step\n",
      "Selected Indices for class  3 :  2499 / 284291\n",
      "Selected Indices for class  6 :  5747 / 284291\n",
      "5747/5747 [==============================] - 0s 63us/step\n",
      "Selected Indices for class  7 :  39104 / 284291\n",
      "39104/39104 [==============================] - 1s 35us/step\n",
      "Selected Indices for class  8 :  9830 / 284291\n",
      "Selected Indices for class  9 :  26578 / 284291\n",
      "26578/26578 [==============================] - 1s 36us/step\n",
      "Selected Indices for class  10 :  7814 / 284291\n",
      "7814/7814 [==============================] - 0s 51us/step\n",
      "Selected Indices for class  11 :  13851 / 284291\n",
      "13851/13851 [==============================] - 1s 45us/step\n",
      "Selected Indices for class  12 :  62063 / 284291\n",
      "62063/62063 [==============================] - 2s 32us/step\n",
      "Selected Indices for class  14 :  3000 / 284291\n",
      "Selected Indices for class  15 :  613 / 284291\n",
      "613/613 [==============================] - 0s 325us/step\n",
      "Selected Indices for class  16 :  3302 / 284291\n",
      "3302/3302 [==============================] - 0s 86us/step\n",
      "Selected Indices for class  17 :  80 / 284291\n",
      "Selected Indices for class  18 :  22943 / 284291\n",
      "22943/22943 [==============================] - 1s 38us/step\n",
      "Selected Indices for class  19 :  5922 / 284291\n",
      "Selected Indices for class  20 :  1107 / 284291\n",
      "1107/1107 [==============================] - 0s 179us/step\n",
      "Selected Indices for class  21 :  17084 / 284291\n",
      "Selected Indices for class  22 :  372 / 284291\n",
      "Selected Indices for class  23 :  3006 / 284291\n",
      "3006/3006 [==============================] - 0s 91us/step\n",
      "Selected Indices for class  24 :  1274 / 284291\n",
      "Selected Indices for class  25 :  17944 / 284291\n",
      "17944/17944 [==============================] - 1s 40us/step\n",
      "Selected Indices for class  26 :  2753 / 284291\n",
      "2753/2753 [==============================] - 0s 89us/step\n",
      "Level 3 is done\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm_notebook as tqdm\n",
    "# all test data\n",
    "#X_test, y_test\n",
    "x = X_test\n",
    "y = y_test\n",
    "print(x.shape,y.shape)\n",
    "results = -np.ones_like(y)\n",
    "\n",
    "# Level 1 Test\n",
    "# load model\n",
    "modelFilename = './models/modelL1'+'.h5'\n",
    "model = load_model(modelFilename)\n",
    "# test model\n",
    "yPred = model.predict(x, verbose=1, batch_size=2048)\n",
    "predL1Class = yPred.argmax(axis=1)\n",
    "results[:,0] = predL1Class\n",
    "print('Level 1 is done')\n",
    "\n",
    "# Level 2 Test\n",
    "predL1Class = results[:,0]\n",
    "for cls in np.unique(predL1Class):\n",
    "    indexes = predL1Class==cls\n",
    "    print('Selected Indices for class ',cls,': ',len(indexes[indexes]),'/',len(indexes))\n",
    "    # load related model \n",
    "    modelFilename = './models/modelL2_'+ str(cls)+ '.h5'\n",
    "    if modelFilename in modelExists:\n",
    "        model = load_model(modelFilename)  \n",
    "        yPred = model.predict(x[indexes,:], verbose=1, batch_size=2048)\n",
    "        predClasses = yPred.argmax(axis=1)\n",
    "        for idx, value in enumerate(classesL2[cls]):\n",
    "            predClasses[predClasses==idx] = value\n",
    "        results[indexes,1] = predClasses\n",
    "        del model\n",
    "        K.clear_session()\n",
    "        gc.collect()\n",
    "    else:\n",
    "        if len(classesL2[cls]) < 2:\n",
    "            if classesL2[cls][0]==-1 or classesL2[cls][0] is None:\n",
    "                results[indexes,1] = -1\n",
    "            else:\n",
    "                results[indexes,1] = classesL2[cls][0]\n",
    "print('Level 2 is done')  \n",
    "\n",
    "# Level 3 Test\n",
    "predL2Class = results[:,1]\n",
    "for cls in np.unique(predL2Class):\n",
    "    indexes = predL2Class==cls\n",
    "    print('Selected Indices for class ',cls,': ',len(indexes[indexes]),'/',len(indexes))\n",
    "    # load related model \n",
    "    modelFilename = './models/modelL3_'+ str(cls)+ '.h5'\n",
    "    if modelFilename in modelExists:\n",
    "        model = load_model(modelFilename)  \n",
    "        yPred = model.predict(x[indexes,:], verbose=1, batch_size=2048)\n",
    "        predClasses = yPred.argmax(axis=1)\n",
    "        for idx, value in enumerate(classesL3[cls]):\n",
    "            predClasses[predClasses==idx] = value\n",
    "        results[indexes,2] = predClasses\n",
    "        del model\n",
    "        K.clear_session()\n",
    "        gc.collect()\n",
    "    else:\n",
    "        if cls == -1:\n",
    "            results[indexes,2] = -1\n",
    "            continue\n",
    "        if len(classesL3[cls]) < 2:\n",
    "            if classesL3[cls][0]==-1 or classesL3[cls][0] is None:\n",
    "                results[indexes,2] = -1\n",
    "            else:\n",
    "                results[indexes,2] = classesL3[cls][0]\n",
    "print('Level 3 is done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8200224417937958 0.9599846636017321 0.8726445789701398 0.824774614743344\n"
     ]
    }
   ],
   "source": [
    "# calculate accuracy\n",
    "eq = y == results\n",
    "\n",
    "cat1Acc = len(eq[eq[:,0]==True])/len(eq[:,0])\n",
    "cat2Acc = len(eq[eq[:,1]==True])/len(eq[:,1])\n",
    "cat3Acc = len(eq[eq[:,2]==True])/len(eq[:,2])\n",
    "\n",
    "eqS = eq[eq[:,0]==True,:]\n",
    "eqS = eqS[eqS[:,1]==True,:]\n",
    "eqS = eqS[eqS[:,2]==True,:]\n",
    "totallAcc =  len(eqS)/len(eq)\n",
    "\n",
    "print(totallAcc,cat1Acc,cat2Acc,cat3Acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prevRes = [0.8216361556547028 0.9590982191474363 0.8725037883744391 0.831395177051104]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Challenge Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200000, 10)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load Test data\n",
    "dataset = pd.read_csv(\"../data/phase_2_dataset.csv\")\n",
    "dataset.shape "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>desc</th>\n",
       "      <th>price</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tehran</td>\n",
       "      <td>تبلت GALAXY TAB A\\r\\r\\r\\n١٠ اینچ ١٦ گیگ فول اچ...</td>\n",
       "      <td>800000</td>\n",
       "      <td>تبلت سامسونگ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Karaj</td>\n",
       "      <td>بدلیل جابجایی فروش فوری</td>\n",
       "      <td>200000</td>\n",
       "      <td>میز تلویزیون و  نمای دکوری</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Mashhad</td>\n",
       "      <td>رو دسته های قیچی کارشوده و خیلی خوش دست و تیز ...</td>\n",
       "      <td>90000</td>\n",
       "      <td>قیچی جگوار  نو و سالم</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Tehran</td>\n",
       "      <td>111سالم بی رنگ فنی سالم تخفیف بیمه کامل بیمه ت...</td>\n",
       "      <td>16500000</td>\n",
       "      <td>111سالم نقد و اقساط</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      city                                               desc     price  \\\n",
       "1   Tehran  تبلت GALAXY TAB A\\r\\r\\r\\n١٠ اینچ ١٦ گیگ فول اچ...    800000   \n",
       "2    Karaj                            بدلیل جابجایی فروش فوری    200000   \n",
       "3  Mashhad  رو دسته های قیچی کارشوده و خیلی خوش دست و تیز ...     90000   \n",
       "4   Tehran  111سالم بی رنگ فنی سالم تخفیف بیمه کامل بیمه ت...  16500000   \n",
       "\n",
       "                        title  \n",
       "1                تبلت سامسونگ  \n",
       "2  میز تلویزیون و  نمای دکوری  \n",
       "3       قیچی جگوار  نو و سالم  \n",
       "4         111سالم نقد و اقساط  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.iloc[1:5,[2,4,8,9]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "newDataset = dataset.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "newDataset['context'] = newDataset.title + ' ' + newDataset.desc + ' ' + newDataset.city + ' ' + newDataset.price.astype(str)\n",
    "#newDataset['context'] = newDataset.title + ' ' + newDataset.desc + ' ' + newDataset.desc + ' ' + newDataset.city + ' ' + newDataset.price.astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    تبلت سامسونگ تبلت GALAXY TAB A\\r\\r\\r\\n١٠ اینچ ...\n",
       "2    میز تلویزیون و  نمای دکوری بدلیل جابجایی فروش ...\n",
       "3    قیچی جگوار  نو و سالم رو دسته های قیچی کارشوده...\n",
       "4    111سالم نقد و اقساط 111سالم بی رنگ فنی سالم تخ...\n",
       "Name: context, dtype: object"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newDataset.iloc[1:5,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sanitize(x):\n",
    "    price = x.split()[-1]\n",
    "    x = clean_str(x)\n",
    "    x = text_cleaner(x)\n",
    "    x = x + price\n",
    "    x = re.split(r'([a-zA-Z]+)', x)\n",
    "    x = \" \".join(str(item) for item in x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "newDataset['contextProcessed'] = newDataset.context.apply(lambda row: sanitize(str(row)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    تبلت سامسونگ تبلت  galaxy   tab   a    اینچ   ...\n",
       "2    میز تلویزیون و نمای دکوری بدلیل جابجایی فروش ف...\n",
       "3    قیچی جگوار نو و سالم رو دسته های قیچی کارشوده ...\n",
       "4     سالم نقد و اقساط  سالم بی رنگ فنی سالم تخفیف ...\n",
       "Name: contextProcessed, dtype: object"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newDataset.iloc[1:5,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export dataset to text files\n",
    "outputPath = './dataChallenge/'\n",
    "\n",
    "if not os.path.exists(outputPath):\n",
    "    os.makedirs(outputPath)\n",
    "\n",
    "for c in newDataset.columns:\n",
    "    newDataset[c].to_csv(outputPath + c + '.txt', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Utilized 256364 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "fname = os.path.join(outputPath,\"contextProcessed.txt\")\n",
    "content = pd.read_table(fname, header=None)\n",
    "content = content[0].apply(str.strip)\n",
    "word_index = tokenizer.word_index\n",
    "print('Utilized %s unique tokens.' % len(word_index))\n",
    "sequences = tokenizer.texts_to_sequences(content)\n",
    "content = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200000, 100)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "content.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[11, 3],\n",
       " [18, 13, 1, -1, 10, 22],\n",
       " [12, 25, 6, 26],\n",
       " [4, 0, 20, 23, 5, 24, 15, 17],\n",
       " [2, 9, 8, 16, 14, -1],\n",
       " [7, 21, 19]]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classesL2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(200000, 100) (200000, 3)\n",
      "200000/200000 [==============================] - 31s 154us/step\n",
      "Level 1 is done\n",
      "Selected Indices for class  0 :  9257 / 200000\n",
      "9257/9257 [==============================] - 2s 178us/step\n",
      "Selected Indices for class  1 :  31433 / 200000\n",
      "31433/31433 [==============================] - 5s 158us/step\n",
      "Selected Indices for class  2 :  49766 / 200000\n",
      "49766/49766 [==============================] - 8s 156us/step\n",
      "Selected Indices for class  3 :  17061 / 200000\n",
      "17061/17061 [==============================] - 3s 162us/step\n",
      "Selected Indices for class  4 :  26370 / 200000\n",
      "26370/26370 [==============================] - 4s 159us/step\n",
      "Selected Indices for class  5 :  66113 / 200000\n",
      "66113/66113 [==============================] - 10s 156us/step\n",
      "Level 2 is done\n",
      "Selected Indices for class  -1 :  259 / 200000\n",
      "Selected Indices for class  0 :  13168 / 200000\n",
      "13168/13168 [==============================] - 2s 161us/step\n",
      "Selected Indices for class  1 :  9953 / 200000\n",
      "9953/9953 [==============================] - 2s 167us/step\n",
      "Selected Indices for class  3 :  1448 / 200000\n",
      "Selected Indices for class  6 :  3859 / 200000\n",
      "3859/3859 [==============================] - 1s 196us/step\n",
      "Selected Indices for class  7 :  50614 / 200000\n",
      "50614/50614 [==============================] - 8s 155us/step\n",
      "Selected Indices for class  8 :  6086 / 200000\n",
      "Selected Indices for class  9 :  15712 / 200000\n",
      "15712/15712 [==============================] - 3s 163us/step\n",
      "Selected Indices for class  10 :  5263 / 200000\n",
      "5263/5263 [==============================] - 1s 183us/step\n",
      "Selected Indices for class  11 :  7809 / 200000\n",
      "7809/7809 [==============================] - 1s 172us/step\n",
      "Selected Indices for class  12 :  33959 / 200000\n",
      "33959/33959 [==============================] - 5s 156us/step\n",
      "Selected Indices for class  14 :  1756 / 200000\n",
      "Selected Indices for class  15 :  434 / 200000\n",
      "434/434 [==============================] - 0s 529us/step\n",
      "Selected Indices for class  16 :  2795 / 200000\n",
      "2795/2795 [==============================] - 1s 199us/step\n",
      "Selected Indices for class  17 :  39 / 200000\n",
      "Selected Indices for class  18 :  15807 / 200000\n",
      "15807/15807 [==============================] - 3s 164us/step\n",
      "Selected Indices for class  19 :  5481 / 200000\n",
      "Selected Indices for class  20 :  722 / 200000\n",
      "722/722 [==============================] - 0s 415us/step\n",
      "Selected Indices for class  21 :  10018 / 200000\n",
      "Selected Indices for class  22 :  172 / 200000\n",
      "Selected Indices for class  23 :  1538 / 200000\n",
      "1538/1538 [==============================] - 0s 266us/step\n",
      "Selected Indices for class  24 :  1160 / 200000\n",
      "Selected Indices for class  25 :  10528 / 200000\n",
      "10528/10528 [==============================] - 2s 175us/step\n",
      "Selected Indices for class  26 :  1420 / 200000\n",
      "1420/1420 [==============================] - 0s 268us/step\n",
      "Level 3 is done\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm_notebook as tqdm\n",
    "# all test data\n",
    "#X_test, y_test\n",
    "x = content\n",
    "y = np.zeros((x.shape[0],3), dtype=int)\n",
    "print(x.shape,y.shape)\n",
    "results = -np.ones_like(y)\n",
    "\n",
    "# Level 1 Test\n",
    "# load model\n",
    "modelFilename = './models/modelL1'+'.h5'\n",
    "model = load_model(modelFilename)\n",
    "# test model\n",
    "yPred = model.predict(x, verbose=1, batch_size=256)\n",
    "predL1Class = yPred.argmax(axis=1)\n",
    "results[:,0] = predL1Class\n",
    "print('Level 1 is done')\n",
    "\n",
    "# Level 2 Test\n",
    "predL1Class = results[:,0]\n",
    "for cls in np.unique(predL1Class):\n",
    "    indexes = predL1Class==cls\n",
    "    print('Selected Indices for class ',cls,': ',len(indexes[indexes]),'/',len(indexes))\n",
    "    # load related model \n",
    "    modelFilename = './models/modelL2_'+ str(cls)+ '.h5'\n",
    "    if modelFilename in modelExists:\n",
    "        model = load_model(modelFilename)  \n",
    "        yPred = model.predict(x[indexes,:], verbose=1, batch_size=256)\n",
    "        predClasses = yPred.argmax(axis=1)\n",
    "        for idx, value in enumerate(classesL2[cls]):\n",
    "            predClasses[predClasses==idx] = value\n",
    "        results[indexes,1] = predClasses\n",
    "        del model\n",
    "        K.clear_session()\n",
    "        gc.collect()\n",
    "    else:\n",
    "        if len(classesL2[cls]) < 2:\n",
    "            if classesL2[cls][0]==-1 or classesL2[cls][0] is None:\n",
    "                results[indexes,1] = -1\n",
    "            else:\n",
    "                results[indexes,1] = classesL2[cls][0]\n",
    "print('Level 2 is done')  \n",
    "\n",
    "# Level 3 Test\n",
    "predL2Class = results[:,1]\n",
    "for cls in np.unique(predL2Class):\n",
    "    indexes = predL2Class==cls\n",
    "    print('Selected Indices for class ',cls,': ',len(indexes[indexes]),'/',len(indexes))\n",
    "    # load related model \n",
    "    modelFilename = './models/modelL3_'+ str(cls)+ '.h5'\n",
    "    if modelFilename in modelExists:\n",
    "        model = load_model(modelFilename)  \n",
    "        yPred = model.predict(x[indexes,:], verbose=1, batch_size=256)\n",
    "        predClasses = yPred.argmax(axis=1)\n",
    "        for idx, value in enumerate(classesL3[cls]):\n",
    "            predClasses[predClasses==idx] = value\n",
    "        results[indexes,2] = predClasses\n",
    "        del model\n",
    "        K.clear_session()\n",
    "        gc.collect()\n",
    "    else:\n",
    "        if cls == -1:\n",
    "            results[indexes,2] = -1\n",
    "            continue\n",
    "        if len(classesL3[cls]) < 2:\n",
    "            if classesL3[cls][0]==-1 or classesL3[cls][0] is None:\n",
    "                results[indexes,2] = -1\n",
    "            else:\n",
    "                results[indexes,2] = classesL3[cls][0]\n",
    "print('Level 3 is done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1, 18, 56],\n",
       "       [ 2, 12, 60],\n",
       "       [ 2, 26, 29],\n",
       "       [ 5,  7, 33],\n",
       "       [ 3,  0,  6],\n",
       "       [ 4, 14, -1],\n",
       "       [ 1,  1, 61],\n",
       "       [ 2, 12,  1],\n",
       "       [ 5,  7, 33]])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[1:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load categories\n",
    "with open(\"categoriesDivar.json\", \"r\") as read_file:\n",
    "    catDict = json.load(read_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['cat1', 'cat2', 'cat3'])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "catDict.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   0   1   2\n",
      "1  1  18  56\n",
      "2  2  12  60\n",
      "3  2  26  29\n",
      "4  5   7  33\n"
     ]
    }
   ],
   "source": [
    "# Export to the file\n",
    "resultsDF = pd.DataFrame(results)\n",
    "print(resultsDF.iloc[1:5,:])\n",
    "resultsDF['cat1'] = resultsDF[0] \n",
    "resultsDF['cat2'] = resultsDF[1] \n",
    "resultsDF['cat3'] = resultsDF[2] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>cat1</th>\n",
       "      <th>cat2</th>\n",
       "      <th>cat3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>60</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>26</td>\n",
       "      <td>29</td>\n",
       "      <td>2</td>\n",
       "      <td>26</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>33</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0   1   2  cat1  cat2  cat3\n",
       "1  1  18  56     1    18    56\n",
       "2  2  12  60     2    12    60\n",
       "3  2  26  29     2    26    29\n",
       "4  5   7  33     5     7    33"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resultsDF.iloc[1:5,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx,value in enumerate(catDict['cat1']):\n",
    "    resultsDF.loc[resultsDF[0]==idx, 'cat1'] = catDict['cat1'][idx]\n",
    "for idx,value in enumerate(catDict['cat2']):\n",
    "    resultsDF.loc[resultsDF[1]==idx, 'cat2'] = catDict['cat2'][idx]\n",
    "for idx,value in enumerate(catDict['cat3']):\n",
    "    resultsDF.loc[resultsDF[2]==idx, 'cat3'] = catDict['cat3'][idx]\n",
    "resultsDF.loc[resultsDF.cat1==-1,'cat1'] = ''\n",
    "resultsDF.loc[resultsDF.cat2==-1,'cat2'] = ''\n",
    "resultsDF.loc[resultsDF.cat3==-1,'cat3'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>cat1</th>\n",
       "      <th>cat2</th>\n",
       "      <th>cat3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>56</td>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>mobile-tablet</td>\n",
       "      <td>tablet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>60</td>\n",
       "      <td>for-the-home</td>\n",
       "      <td>furniture-and-home-decore</td>\n",
       "      <td>tv-and-stereo-furniture</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>26</td>\n",
       "      <td>29</td>\n",
       "      <td>for-the-home</td>\n",
       "      <td>utility</td>\n",
       "      <td>instrument-cleaning-tailoring</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>33</td>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>leisure-hobbies</td>\n",
       "      <td>animals</td>\n",
       "      <td>birds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>-1</td>\n",
       "      <td>personal</td>\n",
       "      <td>health-beauty</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>61</td>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>audio-video</td>\n",
       "      <td>tv-projector</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>for-the-home</td>\n",
       "      <td>furniture-and-home-decore</td>\n",
       "      <td>antiques-and-art</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>33</td>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0   1   2                cat1                       cat2  \\\n",
       "1  1  18  56  electronic-devices              mobile-tablet   \n",
       "2  2  12  60        for-the-home  furniture-and-home-decore   \n",
       "3  2  26  29        for-the-home                    utility   \n",
       "4  5   7  33            vehicles                       cars   \n",
       "5  3   0   6     leisure-hobbies                    animals   \n",
       "6  4  14  -1            personal              health-beauty   \n",
       "7  1   1  61  electronic-devices                audio-video   \n",
       "8  2  12   1        for-the-home  furniture-and-home-decore   \n",
       "9  5   7  33            vehicles                       cars   \n",
       "\n",
       "                            cat3  \n",
       "1                         tablet  \n",
       "2        tv-and-stereo-furniture  \n",
       "3  instrument-cleaning-tailoring  \n",
       "4                          light  \n",
       "5                          birds  \n",
       "6                                 \n",
       "7                   tv-projector  \n",
       "8               antiques-and-art  \n",
       "9                          light  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resultsDF.iloc[1:10,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "resultsDF['title'] = dataset['title']\n",
    "resultsDF['desc'] = dataset['desc']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat1</th>\n",
       "      <th>cat2</th>\n",
       "      <th>cat3</th>\n",
       "      <th>title</th>\n",
       "      <th>desc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>motorcycles</td>\n",
       "      <td></td>\n",
       "      <td>ویو مدل 94 انژکتور</td>\n",
       "      <td>هندا ویو انژکتور\\r\\r\\r\\nتمیزومرتب سفارشی\\r\\r\\r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>سمندef7دوگانه مدل90در نی ریز</td>\n",
       "      <td>***این خودرو در نی ریز هست***\\r\\r\\r\\nخوش قیمت\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>parts-accessories</td>\n",
       "      <td></td>\n",
       "      <td>لوازم یدکى پیکان</td>\n",
       "      <td>تعدادى لوازم یدکى پیکان \\r\\r\\r\\r\\nساخت انگلیس ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>پرایددوگانه</td>\n",
       "      <td>پرایدسفید مدل 84 بیمه تا اخرسال دوگانه دستی ول...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>204</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>تیپ 2 - سفارشی کارخانه</td>\n",
       "      <td>تیپ 2 - داخل چرم و سینه قهوه ای - بدون رنگ - خ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>پراید81 مدادی دوگانه</td>\n",
       "      <td>رنگ مدادی بدونه ضربه . بیمه 18 همین برج تمام ش...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>mobile-tablet</td>\n",
       "      <td>mobile-phones</td>\n",
       "      <td>اپل ۶. گری. ۶۴ گیگ</td>\n",
       "      <td>گوشی در حد آک هست.گوشی سالم به شرط میدم که هرج...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>furniture-and-home-decore</td>\n",
       "      <td>tv-and-stereo-furniture</td>\n",
       "      <td>میزال سی دی</td>\n",
       "      <td>تولیدوپخش وپذیرش نمایندگی</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>building-and-garden</td>\n",
       "      <td>stove-and-heating</td>\n",
       "      <td>بخاری گازی</td>\n",
       "      <td>هخامنشی بزرگ بسیار سالم</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>audio-video</td>\n",
       "      <td>camera-camcoders</td>\n",
       "      <td>Ps3اسلیم 250 گیگ</td>\n",
       "      <td>سلام ps3اسلیم 250 گیگ ابدیت 4/80,با حدود 20 با...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210</th>\n",
       "      <td>leisure-hobbies</td>\n",
       "      <td>animals</td>\n",
       "      <td>fish</td>\n",
       "      <td>ماهی فلاور</td>\n",
       "      <td>تخفیف عالی به دلیل جمع کردن اکواریوم دو جفت نر...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>فروش پراید</td>\n",
       "      <td>بعضی از قسمتهای پایین اتومبیل رنگ شده ولی جلو ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212</th>\n",
       "      <td>leisure-hobbies</td>\n",
       "      <td>sport-leisure</td>\n",
       "      <td>ball-sports</td>\n",
       "      <td>چادر مسافرتی</td>\n",
       "      <td>در حد نو عالی \\r\\r\\r\\nرنگ قهوای و آبی</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>building-and-garden</td>\n",
       "      <td></td>\n",
       "      <td>درب پارکینگ</td>\n",
       "      <td>درب پارکینگ همراه با موتور و ریل شانه ای ...بس...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>audio-video</td>\n",
       "      <td>camera-camcoders</td>\n",
       "      <td>فروش هلی شات phantom 3 satandard</td>\n",
       "      <td>فانتوم 3 استاندارد \\r\\r\\r\\r\\nبه همراه تعدادی م...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>leisure-hobbies</td>\n",
       "      <td>animals</td>\n",
       "      <td>birds</td>\n",
       "      <td>سر دم سبز</td>\n",
       "      <td>سلام،کفتر صیده غروبه. به شرط پرش خیلی پرشی لاز...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>پراید صندوق دار مدل ۹۳</td>\n",
       "      <td>سلام پراید سفید صندوق دار ۱۳۱ se فول دو ایربک ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217</th>\n",
       "      <td>personal</td>\n",
       "      <td>clothing-and-shoes</td>\n",
       "      <td>clothing</td>\n",
       "      <td>لباس مجلسی سایز بزرگ</td>\n",
       "      <td>لباس مجلسی سایز بزرگ 2xlولی بزرگ تر هم میشه چو...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>furniture-and-home-decore</td>\n",
       "      <td>antiques-and-art</td>\n",
       "      <td>سماور (قلم زنی...)</td>\n",
       "      <td>سماور قلم زنی و زیبا\\r\\r\\r\\nگازسوز \\r\\r\\r\\nکام...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>computers</td>\n",
       "      <td>desktops</td>\n",
       "      <td>کامپیوتربا میز</td>\n",
       "      <td>سلام.من چیزی از کامپیتور سر در نمیار فقط در حد...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>leisure-hobbies</td>\n",
       "      <td>animals</td>\n",
       "      <td>birds</td>\n",
       "      <td>2تابلبل خرما</td>\n",
       "      <td>2تاباهم 30هزارتومن-دستی نیستن-میخونن-بدون قفس</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>اردی دو گانه</td>\n",
       "      <td>دوگانه عقب جلو سالم بدونه ایراد تمام تخفیف برج...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>furniture-and-home-decore</td>\n",
       "      <td>textile-ornaments</td>\n",
       "      <td>اویز پرده</td>\n",
       "      <td>اویز پرده قیمت مقطوع اس ام اس و تلگرام پاسخگو ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>computers</td>\n",
       "      <td>printer-scaner-copier</td>\n",
       "      <td>پرینتر لیزری چهار کاره1214hp</td>\n",
       "      <td>پرینتر لیزری چهار کاره اچ پی همراه با گوشی \\r\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>224</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>utensils-and-appliances</td>\n",
       "      <td>cookware-tableware</td>\n",
       "      <td>سرویس قابلمه ارکید</td>\n",
       "      <td>۳ قابلمه ویک تابه ‌.کف سرامیک.جنس عالی .امتحان...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>سمند Lxبسیار تمیز مدل۸۴</td>\n",
       "      <td>فنی و بدنه درحد۹۰ بدون تصادف و رنگ شدگی بیمه و...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>mobile-tablet</td>\n",
       "      <td>mobile-phones</td>\n",
       "      <td>آیفون سیکس</td>\n",
       "      <td>٩ ماه کار کرده بدون تعویض قطعه  با حافظه ی ١٦ گیگ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>227</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>پژو پارس سفید مدل ۹۰</td>\n",
       "      <td>فابریک \\r\\r\\r\\nبیمه یک سال\\r\\r\\r\\nروکش +کف پوش...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>228</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>audio-video</td>\n",
       "      <td>camera-camcoders</td>\n",
       "      <td>Call Of Duty Black Ops 3, PS4</td>\n",
       "      <td>Call Of Duty Black Ops 3\\r\\r\\r\\nبسیار تمیز و ب...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>personal</td>\n",
       "      <td>childrens-clothing-and-shoe</td>\n",
       "      <td></td>\n",
       "      <td>کمد بچه درحد نو</td>\n",
       "      <td>با سلام و تشکر از کارمندان دیوار \\r\\r\\r\\nیک عد...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270</th>\n",
       "      <td>businesses</td>\n",
       "      <td>equipments-and-machinery</td>\n",
       "      <td>shop-and-cash</td>\n",
       "      <td>دکوراسیون کامل مغازه استثنایی</td>\n",
       "      <td>دکوراسیون کامل مغازه با کاربرد انواع کار با قی...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>271</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>خودرو پراید</td>\n",
       "      <td>از نظر شاسی و موتور کاملا سالم،تخفیف بیمه چهار...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>furniture-and-home-decore</td>\n",
       "      <td>lighting</td>\n",
       "      <td>سرویس کامل خواب</td>\n",
       "      <td>این سرویس قهویه تیر با میز ارایش وکمدوکشو بسیا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>273</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>پژو ٨٥ بژ دوگانه</td>\n",
       "      <td>ماشین بسیار خوش رخ فنی سالم ١ سال بیمه کامل تخ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>ریو سفید درجه یک</td>\n",
       "      <td>ریو مدل 86  .بیمه یک سال .5سال تخفیف بیمه  .لا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>275</th>\n",
       "      <td>personal</td>\n",
       "      <td>jewelry-and-watches</td>\n",
       "      <td>watches</td>\n",
       "      <td>FORTISضد خط وخش</td>\n",
       "      <td>دور شیشه+خود شیشه ضد خط وخش،\\r\\r\\r\\nمابقی قاب ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>276</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>mobile-tablet</td>\n",
       "      <td>mobile-phones</td>\n",
       "      <td>گوشی htc دیزایر 530 دو سیم در حد آک</td>\n",
       "      <td>با سلام\\r\\r\\r\\r\\nگوشی htc Desire 530  سفید خاک...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>277</th>\n",
       "      <td>businesses</td>\n",
       "      <td>equipments-and-machinery</td>\n",
       "      <td>offices</td>\n",
       "      <td>___فایل ۳کشو ___ژاریک</td>\n",
       "      <td>»»»تماس$NUM«««\\r\\r\\r\\n(مبلمان اداری و دفتری ژا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278</th>\n",
       "      <td>personal</td>\n",
       "      <td>clothing-and-shoes</td>\n",
       "      <td>shoes-belt-bag</td>\n",
       "      <td>دامنهاى زنانه</td>\n",
       "      <td>دامنهاى زنانه نو</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>خودرو هاچ بک 83</td>\n",
       "      <td>باسلام پراید مدل 83 انژوکتور فنی درحد موتوری ه...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>furniture-and-home-decore</td>\n",
       "      <td>lighting</td>\n",
       "      <td>1 عدد تخت فرفورژه فقط...</td>\n",
       "      <td>باسلام 1 عدد تخت فرفورژه بدون تشک... فقط تماسس...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>سمند ال ایکس مدل ۸۸ فابریک</td>\n",
       "      <td>سمند ال ایکس سفید مدل ۸۸\\r\\r\\r\\nتمام فابریک بد...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>پراید 131 مدل 93 فول تک سوز</td>\n",
       "      <td>سفید، فول کامل\\r\\r\\r\\n2ایربگ،هیدرولیک و ABS \\r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>motorcycles</td>\n",
       "      <td></td>\n",
       "      <td>کمک فنر</td>\n",
       "      <td>کمک فنر عقب هندا 125 در حد نو</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>284</th>\n",
       "      <td>leisure-hobbies</td>\n",
       "      <td>musical-instruments</td>\n",
       "      <td>traditional</td>\n",
       "      <td>ساز سنتی - تار</td>\n",
       "      <td>ساز سنتی - تار- دارای جعبه و کاور دستی فوری فروشی</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>285</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>پژو۴۰۵</td>\n",
       "      <td>سلام ماشین تحویلی برج یازده بیمه یکسال کامل چه...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>286</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>utensils-and-appliances</td>\n",
       "      <td>dishwasher</td>\n",
       "      <td>Magic dishwasher_KOR-2185</td>\n",
       "      <td>ظرف شویی 8 نفره_اصل کره_سالم ,در حدنو</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>287</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>parts-accessories</td>\n",
       "      <td></td>\n",
       "      <td>ضبط فابریکی٢٠٦ فلش خور</td>\n",
       "      <td>در حد نو\\r\\r\\r\\nفلش خور سی دی خور خروجی ای یو ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>پارس تندر93-فروش استثنائی</td>\n",
       "      <td>پارس تندر93-بیرنگ-دوربین دنده عقب -فول امکانات...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>پژو 405 مدل 84</td>\n",
       "      <td>فنی بسیار سالم.دو گلگیر عقب رنگ مابقی بدون رنگ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>پراید92دوگانه نقدواقساط</td>\n",
       "      <td>دوگانه فابریک کارخانه جلوعقب پلمپ بدون خط وخش ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>291</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>furniture-and-home-decore</td>\n",
       "      <td>carpets</td>\n",
       "      <td>سه تخته فرش زرشکی</td>\n",
       "      <td>باسلام\\r\\r\\r\\nسه تخته فرش دارم.که یکیش سالم سا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>292</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>utensils-and-appliances</td>\n",
       "      <td>dishwasher</td>\n",
       "      <td>فروش عمده ظرفشویی ال جی</td>\n",
       "      <td>فروش عمده ظرفشویی ال جی\\r\\r\\r\\n14 نفره ، ساخت ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>furniture-and-home-decore</td>\n",
       "      <td>storage</td>\n",
       "      <td>میز آرایش چهار کشو</td>\n",
       "      <td>باکس آینه ریلی و چهار کشو عمیق جادار ریلی.سفار...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>mobile-tablet</td>\n",
       "      <td>mobile-phones</td>\n",
       "      <td>گوشی اپل 5 ۳۲ گیگ</td>\n",
       "      <td>یک عدد گوشی اپل ۵ ۳۲ گیگ سالم و تمیز فقط خط و ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>آردی 84سالم وتمیییییز</td>\n",
       "      <td>7سال تخفیف بیمه تابرج11بیمه داره،لاستیک90درصد،...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>296</th>\n",
       "      <td>electronic-devices</td>\n",
       "      <td>computers</td>\n",
       "      <td>modem-and-network-equipment</td>\n",
       "      <td>مودم همراه در حد نو ، سیم کارت خور</td>\n",
       "      <td>مودم در حد نو می باشد</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>297</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>cars</td>\n",
       "      <td>light</td>\n",
       "      <td>خودرو پژو ۴۰۵ متالیک</td>\n",
       "      <td>کم کار تمیز بسیار عالی رنگ نقری</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>for-the-home</td>\n",
       "      <td>utensils-and-appliances</td>\n",
       "      <td></td>\n",
       "      <td>آبمیوه گیری استیل CN TRONIC</td>\n",
       "      <td>آبمیوه گیری برقی سی ان ترونیک استیل با کار کرد...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>vehicles</td>\n",
       "      <td>motorcycles</td>\n",
       "      <td></td>\n",
       "      <td>موتور 125 رهرو</td>\n",
       "      <td>موتور شامل 10 ماه بیمه سالم بسیار تمیز ب رنگ آ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   cat1                         cat2  \\\n",
       "200            vehicles                  motorcycles   \n",
       "201            vehicles                         cars   \n",
       "202            vehicles            parts-accessories   \n",
       "203            vehicles                         cars   \n",
       "204            vehicles                         cars   \n",
       "205            vehicles                         cars   \n",
       "206  electronic-devices                mobile-tablet   \n",
       "207        for-the-home    furniture-and-home-decore   \n",
       "208        for-the-home          building-and-garden   \n",
       "209  electronic-devices                  audio-video   \n",
       "210     leisure-hobbies                      animals   \n",
       "211            vehicles                         cars   \n",
       "212     leisure-hobbies                sport-leisure   \n",
       "213        for-the-home          building-and-garden   \n",
       "214  electronic-devices                  audio-video   \n",
       "215     leisure-hobbies                      animals   \n",
       "216            vehicles                         cars   \n",
       "217            personal           clothing-and-shoes   \n",
       "218        for-the-home    furniture-and-home-decore   \n",
       "219  electronic-devices                    computers   \n",
       "220     leisure-hobbies                      animals   \n",
       "221            vehicles                         cars   \n",
       "222        for-the-home    furniture-and-home-decore   \n",
       "223  electronic-devices                    computers   \n",
       "224        for-the-home      utensils-and-appliances   \n",
       "225            vehicles                         cars   \n",
       "226  electronic-devices                mobile-tablet   \n",
       "227            vehicles                         cars   \n",
       "228  electronic-devices                  audio-video   \n",
       "229            personal  childrens-clothing-and-shoe   \n",
       "..                  ...                          ...   \n",
       "270          businesses     equipments-and-machinery   \n",
       "271            vehicles                         cars   \n",
       "272        for-the-home    furniture-and-home-decore   \n",
       "273            vehicles                         cars   \n",
       "274            vehicles                         cars   \n",
       "275            personal          jewelry-and-watches   \n",
       "276  electronic-devices                mobile-tablet   \n",
       "277          businesses     equipments-and-machinery   \n",
       "278            personal           clothing-and-shoes   \n",
       "279            vehicles                         cars   \n",
       "280        for-the-home    furniture-and-home-decore   \n",
       "281            vehicles                         cars   \n",
       "282            vehicles                         cars   \n",
       "283            vehicles                  motorcycles   \n",
       "284     leisure-hobbies          musical-instruments   \n",
       "285            vehicles                         cars   \n",
       "286        for-the-home      utensils-and-appliances   \n",
       "287            vehicles            parts-accessories   \n",
       "288            vehicles                         cars   \n",
       "289            vehicles                         cars   \n",
       "290            vehicles                         cars   \n",
       "291        for-the-home    furniture-and-home-decore   \n",
       "292        for-the-home      utensils-and-appliances   \n",
       "293        for-the-home    furniture-and-home-decore   \n",
       "294  electronic-devices                mobile-tablet   \n",
       "295            vehicles                         cars   \n",
       "296  electronic-devices                    computers   \n",
       "297            vehicles                         cars   \n",
       "298        for-the-home      utensils-and-appliances   \n",
       "299            vehicles                  motorcycles   \n",
       "\n",
       "                            cat3                                title  \\\n",
       "200                                                ویو مدل 94 انژکتور   \n",
       "201                        light         سمندef7دوگانه مدل90در نی ریز   \n",
       "202                                                  لوازم یدکى پیکان   \n",
       "203                        light                          پرایددوگانه   \n",
       "204                        light               تیپ 2 - سفارشی کارخانه   \n",
       "205                        light                 پراید81 مدادی دوگانه   \n",
       "206                mobile-phones                   اپل ۶. گری. ۶۴ گیگ   \n",
       "207      tv-and-stereo-furniture                          میزال سی دی   \n",
       "208            stove-and-heating                           بخاری گازی   \n",
       "209             camera-camcoders                     Ps3اسلیم 250 گیگ   \n",
       "210                         fish                           ماهی فلاور   \n",
       "211                        light                           فروش پراید   \n",
       "212                  ball-sports                         چادر مسافرتی   \n",
       "213                                                       درب پارکینگ   \n",
       "214             camera-camcoders     فروش هلی شات phantom 3 satandard   \n",
       "215                        birds                            سر دم سبز   \n",
       "216                        light               پراید صندوق دار مدل ۹۳   \n",
       "217                     clothing                 لباس مجلسی سایز بزرگ   \n",
       "218             antiques-and-art                   سماور (قلم زنی...)   \n",
       "219                     desktops                       کامپیوتربا میز   \n",
       "220                        birds                         2تابلبل خرما   \n",
       "221                        light                         اردی دو گانه   \n",
       "222            textile-ornaments                            اویز پرده   \n",
       "223        printer-scaner-copier         پرینتر لیزری چهار کاره1214hp   \n",
       "224           cookware-tableware                   سرویس قابلمه ارکید   \n",
       "225                        light              سمند Lxبسیار تمیز مدل۸۴   \n",
       "226                mobile-phones                           آیفون سیکس   \n",
       "227                        light                 پژو پارس سفید مدل ۹۰   \n",
       "228             camera-camcoders        Call Of Duty Black Ops 3, PS4   \n",
       "229                                                   کمد بچه درحد نو   \n",
       "..                           ...                                  ...   \n",
       "270                shop-and-cash        دکوراسیون کامل مغازه استثنایی   \n",
       "271                        light                          خودرو پراید   \n",
       "272                     lighting                      سرویس کامل خواب   \n",
       "273                        light                     پژو ٨٥ بژ دوگانه   \n",
       "274                        light                     ریو سفید درجه یک   \n",
       "275                      watches                      FORTISضد خط وخش   \n",
       "276                mobile-phones  گوشی htc دیزایر 530 دو سیم در حد آک   \n",
       "277                      offices                ___فایل ۳کشو ___ژاریک   \n",
       "278               shoes-belt-bag                        دامنهاى زنانه   \n",
       "279                        light                      خودرو هاچ بک 83   \n",
       "280                     lighting             1 عدد تخت فرفورژه فقط...   \n",
       "281                        light           سمند ال ایکس مدل ۸۸ فابریک   \n",
       "282                        light          پراید 131 مدل 93 فول تک سوز   \n",
       "283                                                           کمک فنر   \n",
       "284                  traditional                       ساز سنتی - تار   \n",
       "285                        light                               پژو۴۰۵   \n",
       "286                   dishwasher            Magic dishwasher_KOR-2185   \n",
       "287                                            ضبط فابریکی٢٠٦ فلش خور   \n",
       "288                        light            پارس تندر93-فروش استثنائی   \n",
       "289                        light                       پژو 405 مدل 84   \n",
       "290                        light              پراید92دوگانه نقدواقساط   \n",
       "291                      carpets                    سه تخته فرش زرشکی   \n",
       "292                   dishwasher              فروش عمده ظرفشویی ال جی   \n",
       "293                      storage                   میز آرایش چهار کشو   \n",
       "294                mobile-phones                    گوشی اپل 5 ۳۲ گیگ   \n",
       "295                        light                آردی 84سالم وتمیییییز   \n",
       "296  modem-and-network-equipment   مودم همراه در حد نو ، سیم کارت خور   \n",
       "297                        light                 خودرو پژو ۴۰۵ متالیک   \n",
       "298                                       آبمیوه گیری استیل CN TRONIC   \n",
       "299                                                    موتور 125 رهرو   \n",
       "\n",
       "                                                  desc  \n",
       "200  هندا ویو انژکتور\\r\\r\\r\\nتمیزومرتب سفارشی\\r\\r\\r...  \n",
       "201  ***این خودرو در نی ریز هست***\\r\\r\\r\\nخوش قیمت\\...  \n",
       "202  تعدادى لوازم یدکى پیکان \\r\\r\\r\\r\\nساخت انگلیس ...  \n",
       "203  پرایدسفید مدل 84 بیمه تا اخرسال دوگانه دستی ول...  \n",
       "204  تیپ 2 - داخل چرم و سینه قهوه ای - بدون رنگ - خ...  \n",
       "205  رنگ مدادی بدونه ضربه . بیمه 18 همین برج تمام ش...  \n",
       "206  گوشی در حد آک هست.گوشی سالم به شرط میدم که هرج...  \n",
       "207                          تولیدوپخش وپذیرش نمایندگی  \n",
       "208                            هخامنشی بزرگ بسیار سالم  \n",
       "209  سلام ps3اسلیم 250 گیگ ابدیت 4/80,با حدود 20 با...  \n",
       "210  تخفیف عالی به دلیل جمع کردن اکواریوم دو جفت نر...  \n",
       "211  بعضی از قسمتهای پایین اتومبیل رنگ شده ولی جلو ...  \n",
       "212              در حد نو عالی \\r\\r\\r\\nرنگ قهوای و آبی  \n",
       "213  درب پارکینگ همراه با موتور و ریل شانه ای ...بس...  \n",
       "214  فانتوم 3 استاندارد \\r\\r\\r\\r\\nبه همراه تعدادی م...  \n",
       "215  سلام،کفتر صیده غروبه. به شرط پرش خیلی پرشی لاز...  \n",
       "216  سلام پراید سفید صندوق دار ۱۳۱ se فول دو ایربک ...  \n",
       "217  لباس مجلسی سایز بزرگ 2xlولی بزرگ تر هم میشه چو...  \n",
       "218  سماور قلم زنی و زیبا\\r\\r\\r\\nگازسوز \\r\\r\\r\\nکام...  \n",
       "219  سلام.من چیزی از کامپیتور سر در نمیار فقط در حد...  \n",
       "220      2تاباهم 30هزارتومن-دستی نیستن-میخونن-بدون قفس  \n",
       "221  دوگانه عقب جلو سالم بدونه ایراد تمام تخفیف برج...  \n",
       "222  اویز پرده قیمت مقطوع اس ام اس و تلگرام پاسخگو ...  \n",
       "223  پرینتر لیزری چهار کاره اچ پی همراه با گوشی \\r\\...  \n",
       "224  ۳ قابلمه ویک تابه ‌.کف سرامیک.جنس عالی .امتحان...  \n",
       "225  فنی و بدنه درحد۹۰ بدون تصادف و رنگ شدگی بیمه و...  \n",
       "226  ٩ ماه کار کرده بدون تعویض قطعه  با حافظه ی ١٦ گیگ  \n",
       "227  فابریک \\r\\r\\r\\nبیمه یک سال\\r\\r\\r\\nروکش +کف پوش...  \n",
       "228  Call Of Duty Black Ops 3\\r\\r\\r\\nبسیار تمیز و ب...  \n",
       "229  با سلام و تشکر از کارمندان دیوار \\r\\r\\r\\nیک عد...  \n",
       "..                                                 ...  \n",
       "270  دکوراسیون کامل مغازه با کاربرد انواع کار با قی...  \n",
       "271  از نظر شاسی و موتور کاملا سالم،تخفیف بیمه چهار...  \n",
       "272  این سرویس قهویه تیر با میز ارایش وکمدوکشو بسیا...  \n",
       "273  ماشین بسیار خوش رخ فنی سالم ١ سال بیمه کامل تخ...  \n",
       "274  ریو مدل 86  .بیمه یک سال .5سال تخفیف بیمه  .لا...  \n",
       "275  دور شیشه+خود شیشه ضد خط وخش،\\r\\r\\r\\nمابقی قاب ...  \n",
       "276  با سلام\\r\\r\\r\\r\\nگوشی htc Desire 530  سفید خاک...  \n",
       "277  »»»تماس$NUM«««\\r\\r\\r\\n(مبلمان اداری و دفتری ژا...  \n",
       "278                                   دامنهاى زنانه نو  \n",
       "279  باسلام پراید مدل 83 انژوکتور فنی درحد موتوری ه...  \n",
       "280  باسلام 1 عدد تخت فرفورژه بدون تشک... فقط تماسس...  \n",
       "281  سمند ال ایکس سفید مدل ۸۸\\r\\r\\r\\nتمام فابریک بد...  \n",
       "282  سفید، فول کامل\\r\\r\\r\\n2ایربگ،هیدرولیک و ABS \\r...  \n",
       "283                      کمک فنر عقب هندا 125 در حد نو  \n",
       "284  ساز سنتی - تار- دارای جعبه و کاور دستی فوری فروشی  \n",
       "285  سلام ماشین تحویلی برج یازده بیمه یکسال کامل چه...  \n",
       "286              ظرف شویی 8 نفره_اصل کره_سالم ,در حدنو  \n",
       "287  در حد نو\\r\\r\\r\\nفلش خور سی دی خور خروجی ای یو ...  \n",
       "288  پارس تندر93-بیرنگ-دوربین دنده عقب -فول امکانات...  \n",
       "289  فنی بسیار سالم.دو گلگیر عقب رنگ مابقی بدون رنگ...  \n",
       "290  دوگانه فابریک کارخانه جلوعقب پلمپ بدون خط وخش ...  \n",
       "291  باسلام\\r\\r\\r\\nسه تخته فرش دارم.که یکیش سالم سا...  \n",
       "292  فروش عمده ظرفشویی ال جی\\r\\r\\r\\n14 نفره ، ساخت ...  \n",
       "293  باکس آینه ریلی و چهار کشو عمیق جادار ریلی.سفار...  \n",
       "294  یک عدد گوشی اپل ۵ ۳۲ گیگ سالم و تمیز فقط خط و ...  \n",
       "295  7سال تخفیف بیمه تابرج11بیمه داره،لاستیک90درصد،...  \n",
       "296                              مودم در حد نو می باشد  \n",
       "297                    کم کار تمیز بسیار عالی رنگ نقری  \n",
       "298  آبمیوه گیری برقی سی ان ترونیک استیل با کار کرد...  \n",
       "299  موتور شامل 10 ماه بیمه سالم بسیار تمیز ب رنگ آ...  \n",
       "\n",
       "[100 rows x 5 columns]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resultsDF.iloc[200:300,[3,4,5,6,7]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export to file\n",
    "resultsDF.to_csv('./resultsChallenge2.csv',columns=['cat1','cat2','cat3'])"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}