Quora Question Pairs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict if pairs of questions from Quora contain two questions with the same semantics. \n",
    "Dataset: https://www.kaggle.com/c/quora-question-pairs/data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import h5py\n",
    "import pickle\n",
    "from sklearn.model_selection import train_test_split\n",
    "from numpy.random import seed\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import optimizers\n",
    "from keras import activations\n",
    "from keras import losses\n",
    "from keras import metrics\n",
    "from keras.utils import to_categorical\n",
    "from keras.models import load_model\n",
    "from keras import layers\n",
    "from keras import activations\n",
    "from keras import losses\n",
    "from keras import optimizers\n",
    "from keras import metrics\n",
    "from keras import Input\n",
    "from keras.models import Model\n",
    "import keras.backend as K\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfOriginal = pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>What is the story of Kohinoor (Koh-i-Noor) Dia...</td>\n",
       "      <td>What would happen if the Indian government sto...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>How can I increase the speed of my internet co...</td>\n",
       "      <td>How can Internet speed be increased by hacking...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>Why am I mentally very lonely? How can I solve...</td>\n",
       "      <td>Find the remainder when [math]23^{24}[/math] i...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>Which one dissolve in water quikly sugar, salt...</td>\n",
       "      <td>Which fish would survive in salt water?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>Astrology: I am a Capricorn Sun Cap moon and c...</td>\n",
       "      <td>I'm a triple Capricorn (Sun, Moon and ascendan...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>Should I buy tiago?</td>\n",
       "      <td>What keeps childern active and far from phone ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>How can I be a good geologist?</td>\n",
       "      <td>What should I do to be a great geologist?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>When do you use シ instead of し?</td>\n",
       "      <td>When do you use \"&amp;\" instead of \"and\"?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "      <td>Motorola (company): Can I hack my Charter Moto...</td>\n",
       "      <td>How do I hack Motorola DCX3400 for free internet?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  qid1  qid2                                          question1  \\\n",
       "0   0     1     2  What is the step by step guide to invest in sh...   \n",
       "1   1     3     4  What is the story of Kohinoor (Koh-i-Noor) Dia...   \n",
       "2   2     5     6  How can I increase the speed of my internet co...   \n",
       "3   3     7     8  Why am I mentally very lonely? How can I solve...   \n",
       "4   4     9    10  Which one dissolve in water quikly sugar, salt...   \n",
       "5   5    11    12  Astrology: I am a Capricorn Sun Cap moon and c...   \n",
       "6   6    13    14                                Should I buy tiago?   \n",
       "7   7    15    16                     How can I be a good geologist?   \n",
       "8   8    17    18                    When do you use シ instead of し?   \n",
       "9   9    19    20  Motorola (company): Can I hack my Charter Moto...   \n",
       "\n",
       "                                           question2  is_duplicate  \n",
       "0  What is the step by step guide to invest in sh...             0  \n",
       "1  What would happen if the Indian government sto...             0  \n",
       "2  How can Internet speed be increased by hacking...             0  \n",
       "3  Find the remainder when [math]23^{24}[/math] i...             0  \n",
       "4            Which fish would survive in salt water?             0  \n",
       "5  I'm a triple Capricorn (Sun, Moon and ascendan...             1  \n",
       "6  What keeps childern active and far from phone ...             0  \n",
       "7          What should I do to be a great geologist?             1  \n",
       "8              When do you use \"&\" instead of \"and\"?             0  \n",
       "9  How do I hack Motorola DCX3400 for free internet?             0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfOriginal.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "404290"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dfOriginal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id              0\n",
       "qid1            0\n",
       "qid2            0\n",
       "question1       1\n",
       "question2       2\n",
       "is_duplicate    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfOriginal.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfOriginal.dropna(axis=0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "404287"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dfOriginal)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example of two questions which share the same semantics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('How can I be a good geologist?', 'What should I do to be a great geologist?')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfOriginal['question1'][7], dfOriginal['question2'][7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    255024\n",
       "1    149263\n",
       "Name: is_duplicate, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfOriginal['is_duplicate'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = dfOriginal.sample(n=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    6263\n",
       "1    3737\n",
       "Name: is_duplicate, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['is_duplicate'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vectorize the dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's consider only the top 1000 words appearing in the questions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxWords = 1000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need to decide what's the maximum length of the questions that we shall consider.\n",
    "\n",
    "Let's get the sizes of each question."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "sizeQuestionOne = df['question1'].str.len().tolist()\n",
    "sizeQuestionTwo = df['question2'].str.len().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEKCAYAAAAcgp5RAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAFbBJREFUeJzt3X+QXeV93/H3p/z0ryB+LFSVRAS1xrHraTDeYrlu3QQ5iREdi86gDJlOUBh11GlJGtftBDmeqZOZZgZ3UhOTcciohlikrg0hZtAkJDEjoJn+Abaw+WmMWWOMNlKRHIMcl9gJ9rd/3GfNzXJXe1e6u3f36P2auXPPec5z7373SPdzzz733OekqpAkddffG3cBkqTFZdBLUscZ9JLUcQa9JHWcQS9JHWfQS1LHGfSS1HEGvSR1nEEvSR138rgLADjnnHNq/fr14y5DklaUhx566JtVNTFfv2UR9OvXr2ffvn3jLkOSVpQk3ximn0M3ktRxBr0kdZxBL0kdZ9BLUscZ9JLUcQa9JHWcQS9JHWfQS1LHGfSS1HHL4puxK9X6nX88sP3Z6y9f4kokaW4e0UtSxxn0ktRxBr0kddxQQZ/kPyZ5IsnjST6d5PQkFyR5MMnTSW5Lcmrre1pbn2rb1y/mLyBJOrp5gz7JGuA/AJNV9VbgJOAq4CPADVW1AXgB2N4esh14oareCNzQ+kmSxmTYoZuTgdckORl4LXAQuBS4o23fDVzRlre0ddr2TUkymnIlSQs1b9BX1V8Avwk8Ry/gjwAPAS9W1cut2zSwpi2vAfa3x77c+p89+3mT7EiyL8m+w4cPH+/vIUmawzBDN2fSO0q/APgHwOuAywZ0rZmHHGXbKw1Vu6pqsqomJybmvRKWJOkYDTN08x7g61V1uKr+Fvgs8E+BVW0oB2AtcKAtTwPrANr2M4BvjbRqSdLQhgn654CNSV7bxto3AV8G7gOubH22AXe15T1tnbb93qp61RG9JGlpDDNG/yC9D1W/CDzWHrMLuA74QJIpemPwN7eH3Ayc3do/AOxchLolSUMaaq6bqvow8OFZzc8Alwzo+11g6/GXJkkaBb8ZK0kdZ9BLUscZ9JLUcQa9JHWcQS9JHWfQS1LHGfSS1HEGvSR1nEEvSR1n0EtSxxn0ktRxQ811o2O3fucf/3D52esvH2Mlkk5UHtFLUscZ9JLUcQa9JHXcMNeMfVOSh/tu307y/iRnJbknydPt/szWP0luTDKV5NEkFy/+ryFJmsswV5h6qqouqqqLgLcDLwF30rty1N6q2gDs5ZUrSV0GbGi3HcBNi1G4JGk4Cx262QR8raq+AWwBdrf23cAVbXkLcGv1PEDvIuKrR1KtJGnBFhr0VwGfbsvnVdVBgHZ/bmtfA+zve8x0a5MkjcHQQZ/kVOB9wB/M13VAWw14vh1J9iXZd/jw4WHLkCQt0EKO6C8DvlhVz7f152eGZNr9odY+Dazre9xa4MDsJ6uqXVU1WVWTExMTC69ckjSUhQT9z/HKsA3AHmBbW94G3NXXfnU7+2YjcGRmiEeStPSGmgIhyWuBnwL+bV/z9cDtSbYDzwFbW/vdwGZgit4ZOteMrFpJ0oINFfRV9RJw9qy2v6R3Fs7svgVcO5LqJEnHzW/GSlLHOXvlEnImS0nj4BG9JHWcQS9JHWfQS1LHGfSS1HEGvSR1nEEvSR1n0EtSxxn0ktRxBr0kdZxBL0kdZ9BLUscZ9JLUcQa9JHWcQS9JHTdU0CdZleSOJF9J8mSSdyY5K8k9SZ5u92e2vklyY5KpJI8muXhxfwVJ0tEMOx/9x4A/raork5wKvBb4VWBvVV2fZCewE7iO3kXEN7TbO4Cb2v2KtdB55Pv7S9K4zXtEn+RHgHcDNwNU1d9U1YvAFmB367YbuKItbwFurZ4HgFVJVo+8cknSUIYZurkQOAz8XpIvJflEktcB51XVQYB2f27rvwbY3/f46dYmSRqDYYL+ZOBi4Kaqehvw/+gN08wlA9rqVZ2SHUn2Jdl3+PDhoYqVJC3cMEE/DUxX1YNt/Q56wf/8zJBMuz/U139d3+PXAgdmP2lV7aqqyaqanJiYONb6JUnzmDfoq+r/AvuTvKk1bQK+DOwBtrW2bcBdbXkPcHU7+2YjcGRmiEeStPSGPevml4BPtTNungGuofcmcXuS7cBzwNbW925gMzAFvNT6SpLGZKigr6qHgckBmzYN6FvAtcdZlyRpRPxmrCR1nEEvSR1n0EtSxxn0ktRxw551o8Z5bCStNB7RS1LHGfSS1HEGvSR1nEEvSR1n0EtSxxn0ktRxBr0kdZxBL0kd5xemxmShFxyXpGPlEb0kdZxBL0kdN1TQJ3k2yWNJHk6yr7WdleSeJE+3+zNbe5LcmGQqyaNJLl7MX0CSdHQLOaL/yaq6qKpmrjS1E9hbVRuAvW0d4DJgQ7vtAG4aVbGSpIU7nqGbLcDutrwbuKKv/dbqeQBYlWT1cfwcSdJxGDboC/hckoeS7Ght51XVQYB2f25rXwPs73vsdGuTJI3BsKdXvquqDiQ5F7gnyVeO0jcD2upVnXpvGDsAzj///CHLkCQt1FBH9FV1oN0fAu4ELgGenxmSafeHWvdpYF3fw9cCBwY8566qmqyqyYmJiWP/DSRJRzVv0Cd5XZI3zCwDPw08DuwBtrVu24C72vIe4Op29s1G4MjMEI8kaekNM3RzHnBnkpn+/6uq/jTJF4Dbk2wHngO2tv53A5uBKeAl4JqRVy1JGtq8QV9VzwA/PqD9L4FNA9oLuHYk1UmSjpvfjJWkjjPoJanjDHpJ6jiDXpI6zqCXpI4z6CWp4wx6Seo4LyU4h/5L/UnSSuYRvSR1nEEvSR1n0EtSxxn0ktRxBr0kdZxBL0kdZ9BLUscZ9JLUcUMHfZKTknwpyR+19QuSPJjk6SS3JTm1tZ/W1qfa9vWLU7okaRgLOaL/ZeDJvvWPADdU1QbgBWB7a98OvFBVbwRuaP0kSWMyVNAnWQtcDnyirQe4FLijddkNXNGWt7R12vZNrb8kaQyGPaL/LeBXgB+09bOBF6vq5bY+Daxpy2uA/QBt+5HWX5I0BvMGfZJ/CRyqqof6mwd0rSG29T/vjiT7kuw7fPjwUMVKkhZumNkr3wW8L8lm4HTgR+gd4a9KcnI7al8LHGj9p4F1wHSSk4EzgG/NftKq2gXsApicnHzVG8GJqn/WzGevv3yMlUjqinmP6Kvqg1W1tqrWA1cB91bVvwbuA65s3bYBd7XlPW2dtv3eqjLIJWlMjuc8+uuADySZojcGf3Nrvxk4u7V/ANh5fCVKko7Hgi48UlX3A/e35WeASwb0+S6wdQS1nfAcxpE0Cn4zVpI6zqCXpI4z6CWp4wx6Seo4g16SOm5BZ91ocfSfXSNJo+YRvSR1nEEvSR1n0EtSxxn0ktRxBr0kdZxBL0kdZ9BLUscZ9JLUcQa9JHWcQS9JHTfMxcFPT/L5JI8keSLJr7f2C5I8mOTpJLclObW1n9bWp9r29Yv7K0iSjmaYI/rvAZdW1Y8DFwHvTbIR+AhwQ1VtAF4Atrf+24EXquqNwA2tnyRpTIa5OHhV1Xfa6intVsClwB2tfTdwRVve0tZp2zclycgqliQtyFBj9ElOSvIwcAi4B/ga8GJVvdy6TANr2vIaYD9A236E3sXDJUljMFTQV9X3q+oiYC29C4K/eVC3dj/o6L1mNyTZkWRfkn2HDx8etl5J0gIt6KybqnoRuB/YCKxKMjOf/VrgQFueBtYBtO1nAN8a8Fy7qmqyqiYnJiaOrXpJ0ryGOetmIsmqtvwa4D3Ak8B9wJWt2zbgrra8p63Ttt9bVa86opckLY1hrjC1Gtid5CR6bwy3V9UfJfky8Jkk/xX4EnBz638z8PtJpugdyV+1CHVLkoY0b9BX1aPA2wa0P0NvvH52+3eBrSOpTgP1X3rw2esvH2MlklYCvxkrSR1n0EtSxxn0ktRxBr0kdZxBL0kdZ9BLUscZ9JLUccN8YUrLmOfUS5qPR/SS1HEGvSR1nEEvSR1n0EtSx/lh7ArR/6GrJC2ER/SS1HEGvSR1nEEvSR037xh9knXArcDfB34A7KqqjyU5C7gNWA88C/xsVb2QJMDHgM3AS8AvVNUXF6f80XIcXFIXDXNE/zLwn6rqzfQuCn5tkrcAO4G9VbUB2NvWAS4DNrTbDuCmkVctSRravEFfVQdnjsir6q/oXRh8DbAF2N267QauaMtbgFur5wFgVZLVI69ckjSUBZ1emWQ9vevHPgicV1UHofdmkOTc1m0NsL/vYdOt7eDxFqvhOQeOpBlDB32S1wN/CLy/qr7dG4of3HVAWw14vh30hnY4//zzhy1j5ByXl9R1Q511k+QUeiH/qar6bGt+fmZIpt0fau3TwLq+h68FDsx+zqraVVWTVTU5MTFxrPVLkuYxb9C3s2huBp6sqo/2bdoDbGvL24C7+tqvTs9G4MjMEI8kaekNM3TzLuDngceSPNzafhW4Hrg9yXbgOWBr23Y3vVMrp+idXnnNSCuWJC3IvEFfVf+HwePuAJsG9C/g2uOsSyPkB7PSic1JzTrED5YlDeIUCJLUcQa9JHWcQS9JHWfQS1LHGfSS1HEGvSR1nEEvSR1n0EtSxxn0ktRxBr0kdZxBL0kdZ9BLUscZ9JLUcc5eKeDVM186nbHUHQb9Cca56aUTzzCXErwlyaEkj/e1nZXkniRPt/szW3uS3JhkKsmjSS5ezOIlSfMbZoz+k8B7Z7XtBPZW1QZgb1sHuAzY0G47gJtGU6Yk6VgNcynBP0+yflbzFuAn2vJu4H7gutZ+a7uc4ANJViVZ7cXBlyevSCWdGI71rJvzZsK73Z/b2tcA+/v6Tbc2SdKYjPr0ykEXEa+BHZMdSfYl2Xf48OERlyFJmnGsQf98ktUA7f5Qa58G1vX1WwscGPQEVbWrqiaranJiYuIYy5AkzedYg34PsK0tbwPu6mu/up19sxE44vi8JI3XvB/GJvk0vQ9ez0kyDXwYuB64Pcl24Dlga+t+N7AZmAJeAq5ZhJolSQswzFk3PzfHpk0D+hZw7fEWpeXLL1xJK88J981YTykcjoEudccJF/QanbneDHyTkJYXZ6+UpI4z6CWp4xy60bz8XENa2Tyil6SO84heIzHXUb8fzErjZ9BrRfANQzp2Br3GYq6/AOY6TVPSsXOMXpI6ziN6LSsexUuj5xG9JHWcR/RaMh6tS+PhEb0kdZxBL0kdd0IM3ThkIOlEtihH9Enem+SpJFNJdi7Gz5AkDWfkR/RJTgI+DvwUvYuFfyHJnqr68qh/lk5MfktWWpjFGLq5BJiqqmcAknwG2AIsadA7XHPi8Q1AGmwxgn4NsL9vfRp4xyL8HGmoydQWarGmYRjmzed43qyWyxvdcqlDr0jvet4jfMJkK/AzVfVv2vrPA5dU1S/N6rcD2NFW3wQ8dQw/7hzgm8dR7lJZCXVa42ishBphZdRpjfP70aqamK/TYhzRTwPr+tbXAgdmd6qqXcCu4/lBSfZV1eTxPMdSWAl1WuNorIQaYWXUaY2jsxhn3XwB2JDkgiSnAlcBexbh50iShjDyI/qqejnJLwJ/BpwE3FJVT4z650iShrMoX5iqqruBuxfjuWc5rqGfJbQS6rTG0VgJNcLKqNMaR2TkH8ZKkpYX57qRpI5bsUG/XKdZSPJskseSPJxkX2s7K8k9SZ5u92eOoa5bkhxK8nhf28C60nNj27ePJrl4jDX+WpK/aPvz4SSb+7Z9sNX4VJKfWaIa1yW5L8mTSZ5I8sutfdnsy6PUuGz2ZZLTk3w+ySOtxl9v7RckebDtx9vaCR0kOa2tT7Xt68dY4yeTfL1vP17U2sfyuhlKVa24G70Peb8GXAicCjwCvGXcdbXangXOmdX234CdbXkn8JEx1PVu4GLg8fnqAjYDfwIE2Ag8OMYafw34zwP6vqX9u58GXND+P5y0BDWuBi5uy28AvtpqWTb78ig1Lpt92fbH69vyKcCDbf/cDlzV2n8X+Hdt+d8Dv9uWrwJuW4L9OFeNnwSuHNB/LK+bYW4r9Yj+h9MsVNXfADPTLCxXW4DdbXk3cMVSF1BVfw58a1bzXHVtAW6tngeAVUlWj6nGuWwBPlNV36uqrwNT9P5fLKqqOlhVX2zLfwU8Se/b4MtmXx6lxrks+b5s++M7bfWUdivgUuCO1j57P87s3zuATUkyphrnMpbXzTBWatAPmmbhaP+Rl1IBn0vyUPv2L8B5VXUQei9C4NyxVfd3zVXXctu/v9j+FL6lb9hr7DW24YO30TvSW5b7claNsIz2ZZKTkjwMHALuofeXxItV9fKAOn5YY9t+BDh7qWusqpn9+BttP96Q5LTZNQ6of6xWatAPeidfLqcPvauqLgYuA65N8u5xF3QMltP+vQn4h8BFwEHgv7f2sdaY5PXAHwLvr6pvH63rgLYlqXNAjctqX1bV96vqInrfnr8EePNR6lgWNSZ5K/BB4MeAfwKcBVw3zhqHsVKDfqhpFsahqg60+0PAnfT+Az8/8ydcuz80vgr/jrnqWjb7t6qeby+2HwD/g1eGFMZWY5JT6AXop6rqs615We3LQTUux33Z6noRuJ/euPaqJDPf7+mv44c1tu1nMPww3yhrfG8bGquq+h7weyyT/Xg0KzXol+U0C0lel+QNM8vATwOP06ttW+u2DbhrPBW+ylx17QGubmcRbASOzAxLLLVZY5z/it7+hF6NV7WzMS4ANgCfX4J6AtwMPFlVH+3btGz25Vw1Lqd9mWQiyaq2/BrgPfQ+S7gPuLJ1m70fZ/bvlcC91T4BXeIav9L3hh56nyH078dl8bp5lXF/GnysN3qfcH+V3rjeh8ZdT6vpQnpnLzwCPDFTF72xxL3A0+3+rDHU9ml6f67/Lb0jj+1z1UXvT9CPt337GDA5xhp/v9XwKL0X0uq+/h9qNT4FXLZENf4zen+OPwo83G6bl9O+PEqNy2ZfAv8Y+FKr5XHgv7T2C+m9yUwBfwCc1tpPb+tTbfuFY6zx3rYfHwf+J6+cmTOW180wN78ZK0kdt1KHbiRJQzLoJanjDHpJ6jiDXpI6zqCXpI4z6HVCS/KhNjPho20mwnck+USSt4y7NmlUPL1SJ6wk7wQ+CvxEVX0vyTnAqdW+3Sx1hUf0OpGtBr5Zva+yU1XfrKoDSe5PMpnkfX1zjj+V5OsASd6e5H+3iev+bLnMUCjNxaDXiexzwLokX03yO0n+Rf/GqtpTVRdVb1KrR4DfbHPI/Da9+cjfDtwC/MaSVy4twKJcHFxaCarqO0neDvxz4CeB2zLgamVJfgX466r6eJu98K3APW069JPoTdsgLVsGvU5oVfV9erMS3p/kMV6ZOAuAJJuArfSufgW9+UyeqKp3LmWd0vFw6EYnrCRvSrKhr+ki4Bt9238U+B3gZ6vqr1vzU8BE+yCXJKck+UdLVbN0LDyi14ns9cBvt6loX6Y3M+IOXrmU3S/Qm5XyzjZMc6CqNie5ErgxyRn0XkO/RW+2UmlZ8vRKSeo4h24kqeMMeknqOINekjrOoJekjjPoJanjDHpJ6jiDXpI6zqCXpI77/xVKYlYq2ZrhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1cd01465ef0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(sizeQuestionOne,bins=100)\n",
    "plt.xlabel('Size');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEKCAYAAAD5MJl4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAESRJREFUeJzt3X3M3WV9x/H3Z0VwUydFCukALSyNGZoMseFh7gHHVqAY0QQWiBkdY+myYaLbEi3jDzaNCW7OGYyibHbigiBTGQ10w6aTmf0hUjbGg4jcAkIto2U4nMMY0e/+ONcNh153ez/2vs8p71dycn6/7+8651xXfzfnw+/xpKqQJGnYTy11ByRJo8dwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUuegpe7Avhx++OG1atWqpe6GJI2VO++888mqWjGf9xjpcFi1ahXbt29f6m5I0lhJ8u35voe7lSRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJnZG+QnoxrNp4y3PTj1xx9hL2RJJGh1sOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqTOtOGQ5JgkX05yf5L7kryr1Q9LsjXJg+15easnyZVJJpLcneTEofda39o/mGT9/huWJGk+ZrLl8CzwJ1X1C8ApwCVJjgc2AtuqajWwrc0DnAWsbo8NwFUwCBPgcuBk4CTg8slAkSSNlmnDoaoer6p/b9P/C9wPHAWcA1zTml0DvK1NnwN8pga+ChyaZCVwBrC1qp6qqu8CW4EzF3Q0kqQFMatjDklWAW8AbgeOrKrHYRAgwBGt2VHAY0Mv29Fqe6tLkkbMjMMhycuBLwDvrqrv7avpFLXaR33Pz9mQZHuS7bt3755p9yRJC2hG4ZDkJQyC4dqq+mIrP9F2F9Ged7X6DuCYoZcfDezcR/0FqurqqlpTVWtWrFgxm7FIkhbITM5WCvAp4P6q+vDQos3A5BlH64GbhuoXtrOWTgGebrudbgXWJlneDkSvbTVJ0og5aAZt3gT8NnBPkrta7U+BK4AbklwMPAqc15ZtAdYBE8AzwEUAVfVUkvcDd7R276uqpxZkFJKkBTVtOFTVvzH18QKA06doX8Ale3mvTcCm2XRQkrT4vEJaktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJnZlcIX3AWbXxlqXugiSNNLccJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1HlRnsq6N8OnuD5yxdlL2BNJWlpuOUiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOtOGQ5JNSXYluXeo9mdJvpPkrvZYN7Ts0iQTSR5IcsZQ/cxWm0iyceGHIklaKDPZcvg0cOYU9b+uqhPaYwtAkuOB84HXtdd8PMmyJMuAjwFnAccDF7S2kqQRdNB0DarqK0lWzfD9zgGur6ofAg8nmQBOassmquohgCTXt7Zfn3WPJUn73XyOObwzyd1tt9PyVjsKeGyozY5W21tdkjSC5hoOVwE/D5wAPA78Vatnira1j3onyYYk25Ns37179xy7J0majzmFQ1U9UVU/rqqfAH/D87uOdgDHDDU9Gti5j/pU7311Va2pqjUrVqyYS/ckSfM0p3BIsnJo9u3A5JlMm4HzkxyS5FhgNfA14A5gdZJjkxzM4KD15rl3W5K0P017QDrJdcBpwOFJdgCXA6clOYHBrqFHgN8HqKr7ktzA4EDzs8AlVfXj9j7vBG4FlgGbquq+BR+NJGlBzORspQumKH9qH+0/AHxgivoWYMuseidJWhJeIS1J6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6kwbDkk2JdmV5N6h2mFJtiZ5sD0vb/UkuTLJRJK7k5w49Jr1rf2DSdbvn+EsnFUbb3nuIUkvNjPZcvg0cOYetY3AtqpaDWxr8wBnAavbYwNwFQzCBLgcOBk4Cbh8MlAkSaNn2nCoqq8AT+1RPge4pk1fA7xtqP6ZGvgqcGiSlcAZwNaqeqqqvgtspQ8cSdKImOsxhyOr6nGA9nxEqx8FPDbUbker7a3eSbIhyfYk23fv3j3H7kmS5mOhD0hnilrto94Xq66uqjVVtWbFihUL2jlJ0szMNRyeaLuLaM+7Wn0HcMxQu6OBnfuoS5JG0FzDYTMwecbReuCmofqF7aylU4Cn226nW4G1SZa3A9FrW02SNIIOmq5BkuuA04DDk+xgcNbRFcANSS4GHgXOa823AOuACeAZ4CKAqnoqyfuBO1q791XVnge5JUkjYtpwqKoL9rLo9CnaFnDJXt5nE7BpVr2TJC0Jr5CWJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHWm/SU4waqNtzw3/cgVZy9hTyRpcbjlIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpM68wiHJI0nuSXJXku2tdliSrUkebM/LWz1JrkwykeTuJCcuxAAkSQtvIbYc3lxVJ1TVmja/EdhWVauBbW0e4CxgdXtsAK5agM+WJO0H++NnQs8BTmvT1wC3Ae9t9c9UVQFfTXJokpVV9fh+6ENn+Kc+JUn7Nt8thwK+lOTOJBta7cjJL/z2fESrHwU8NvTaHa0mSRox891yeFNV7UxyBLA1yTf20TZT1KprNAiZDQCvfvWr59k9SdJczGvLoap2tuddwI3AScATSVYCtOddrfkO4Jihlx8N7JziPa+uqjVVtWbFihXz6Z4kaY7mHA5JXpbkFZPTwFrgXmAzsL41Ww/c1KY3Axe2s5ZOAZ5erOMNkqTZmc9upSOBG5NMvs9nq+qfk9wB3JDkYuBR4LzWfguwDpgAngEumsdnS5L2ozmHQ1U9BPziFPX/Bk6fol7AJXP9PEnS4vEKaUlSZ39c53BAG75e4pErzl7CnkjS/uOWgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySp43UO8+A1D5IOVG45SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqeNFcAvEC+IkHUjccpAkdQwHSVLH3Ur7gbuYJI07txwkSR3DQZLUcbfSInJ3k6Rx4ZaDJKnjlsN+Nry1IEnjwi0HSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdRY9HJKcmeSBJBNJNi7250uSpreo4ZBkGfAx4CzgeOCCJMcvZh8kSdNb7NtnnARMVNVDAEmuB84Bvr4/PmyUb10x2755oz5Ji2mxw+Eo4LGh+R3AyYvch7E00zAZDpGZvGZv7Q0j6cVtscMhU9TqBQ2SDcCGNvv9JA/M8bMOB56c42tH1bRjygdn94Z7az/b95mHF+V6GkOOaTxMjuk1832jxQ6HHcAxQ/NHAzuHG1TV1cDV8/2gJNuras1832eUOKbx4JjGg2Pat8U+W+kOYHWSY5McDJwPbF7kPkiSprGoWw5V9WySdwK3AsuATVV132L2QZI0vUX/sZ+q2gJsWYSPmveuqRHkmMaDYxoPjmkfUlXTt5Ikvah4+wxJUueAC4dxvT1HkmOSfDnJ/UnuS/KuVj8sydYkD7bn5a2eJFe2cd6d5MSlHcHeJVmW5D+S3Nzmj01yexvT59rJCSQ5pM1PtOWrlrLfe5Pk0CSfT/KNtr5OHff1lOSP2t/dvUmuS/LScVxPSTYl2ZXk3qHarNdNkvWt/YNJ1i/FWIb6MtWY/rL9/d2d5MYkhw4tu7SN6YEkZwzVZ/fdWFUHzIPBQe5vAccBBwP/CRy/1P2aYd9XAie26VcA32Rwi5G/ADa2+kbgg216HfBPDK4dOQW4fanHsI+x/THwWeDmNn8DcH6b/gTwB236D4FPtOnzgc8tdd/3Mp5rgN9r0wcDh47zemJwcerDwE8PrZ/fGcf1BPwqcCJw71BtVusGOAx4qD0vb9PLR2xMa4GD2vQHh8Z0fPveOwQ4tn0fLpvLd+OSr8wF/kc8Fbh1aP5S4NKl7tccx3IT8JvAA8DKVlsJPNCmPwlcMNT+uXaj9GBwLcs24NeBm9t/iE8O/WE/t84YnMV2aps+qLXLUo9hj/H8bPsizR71sV1PPH/ngsPav/vNwBnjup6AVXt8kc5q3QAXAJ8cqr+g3SiMaY9lbweubdMv+M6bXFdz+W480HYrTXV7jqOWqC9z1jbT3wDcDhxZVY8DtOcjWrNxGetHgPcAP2nzrwL+p6qebfPD/X5uTG350639KDkO2A38XdtV9rdJXsYYr6eq+g7wIeBR4HEG/+53Mt7radhs183Ir7M9/C6DLSBYwDEdaOEw7e05Rl2SlwNfAN5dVd/bV9MpaiM11iRvAXZV1Z3D5Sma1gyWjYqDGGziX1VVbwD+j8Guir0Z+TG1ffDnMNgN8XPAyxjcOXlP47SeZmJv4xib8SW5DHgWuHayNEWzOY3pQAuHaW/PMcqSvIRBMFxbVV9s5SeSrGzLVwK7Wn0cxvom4K1JHgGuZ7Br6SPAoUkmr7EZ7vdzY2rLXwk8tZgdnoEdwI6qur3Nf55BWIzzevoN4OGq2l1VPwK+CPwS472ehs123YzDOqMdKH8L8I5q+4pYwDEdaOEwtrfnSBLgU8D9VfXhoUWbgcmzJdYzOBYxWb+wnXFxCvD05KbzqKiqS6vq6KpaxWBd/EtVvQP4MnBua7bnmCbHem5rP1L/x1ZV/wU8luS1rXQ6g1vOj+16YrA76ZQkP9P+DifHNLbraQ+zXTe3AmuTLG9bVWtbbWQkORN4L/DWqnpmaNFm4Px2RtmxwGrga8zlu3GpDx7thwM36xic6fMt4LKl7s8s+v3LDDbz7gbuao91DPblbgMebM+HtfZh8MNJ3wLuAdYs9RimGd9pPH+20nHtD3YC+AfgkFZ/aZufaMuPW+p+72UsJwDb27r6RwZntIz1egL+HPgGcC/w9wzOdhm79QRcx+C4yY8Y/N/yxXNZNwz240+0x0UjOKYJBscQJr8rPjHU/rI2pgeAs4bqs/pu9AppSVLnQNutJElaAIaDJKljOEiSOoaDJKljOEiSOoaDNIUkl7W7lN6d5K4kJ7dbZRy/1H2TFoOnskp7SHIq8GHgtKr6YZLDgYOrauSukpX2F7ccpN5K4Mmq+iFAVT1ZVTuT3JZkTZK3tq2Ju9r98R8GSPLGJP+a5M4kt07eskEaR4aD1PsScEySbyb5eJJfG15YVZur6oSqOoHBffE/1O6L9VHg3Kp6I7AJ+MCi91xaIAdN30R6camq7yd5I/ArwJuBz031y1lJ3gP8oKo+luT1wOuBrYPbE7GMwS0PpLFkOEhTqKofA7cBtyW5h+dv3AZAktOB8xj8ShcM7tNzX1Wdupj9lPYXdytJe0jy2iSrh0onAN8eWv4a4OPAb1XVD1r5AWBFO5hNkpcked1i9VlaaG45SL2XAx9tP9r+LIM7YG5g8NsNMPh95VcBN7ZdSDural2Sc4Erk7ySwX9bHwHuW+S+SwvCU1klSR13K0mSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKnz/wp2In623nUhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1cd017cfa58>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(sizeQuestionTwo,bins=100)\n",
    "plt.xlabel('Size');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 4, 370, 1151)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min(sizeQuestionOne), min(sizeQuestionTwo), max(sizeQuestionOne), max(sizeQuestionTwo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "questionOne = np.asarray(df['question1'].astype(str).tolist(), dtype='object')\n",
    "questionTwo = np.asarray(df['question2'].astype(str).tolist(), dtype='object')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Process the labels**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = np.array(df['is_duplicate'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "indices = np.arange(len(df))\n",
    "np.random.shuffle(indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "questionOne = questionOne[indices]\n",
    "questionTwo = questionTwo[indices]\n",
    "labels = labels[indices]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Tokenize the questions**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words=maxWords)\n",
    "tokenizer.fit_on_texts(questionOne + questionTwo)\n",
    "sequenceOne = tokenizer.texts_to_sequences(questionOne)\n",
    "sequenceTwo = tokenizer.texts_to_sequences(questionTwo)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Pad the questions**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxQuestionSize = 150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataOne = pad_sequences(sequenceOne, maxlen=maxQuestionSize)\n",
    "dataTwo = pad_sequences(sequenceTwo, maxlen=maxQuestionSize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((10000, 150), (10000, 150))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataOne.shape, dataTwo.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000,)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split into training, validation and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainingSetSize = 8000\n",
    "validationSetSize = 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainingSetOne = dataOne[:trainingSetSize]\n",
    "trainingSetTwo = dataTwo[:trainingSetSize]\n",
    "trainingLabels = labels[:trainingSetSize]\n",
    "\n",
    "validationSetOne = dataOne[trainingSetSize: trainingSetSize + validationSetSize]\n",
    "validationSetTwo = dataTwo[trainingSetSize: trainingSetSize + validationSetSize]\n",
    "validationLabels = labels[trainingSetSize: trainingSetSize + validationSetSize]\n",
    "\n",
    "testSetOne = dataOne[trainingSetSize + validationSetSize:]\n",
    "testSetTwo = dataTwo[trainingSetSize + validationSetSize:]\n",
    "testLabels = labels[trainingSetSize + validationSetSize:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build the model\n",
    "We will use a Siamese network in order to compute the similarity between the two questions.\n",
    "\n",
    "A Siamese network consists of two identical neural networks or layers, each taking one of the two input questions. The last layers of the two networks are then fed to a contrastive loss function , which calculates the similarity between the two questions."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Manhattan Distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exponent_neg_manhattan_distance(left, right):\n",
    "    return K.exp(-K.sum(K.abs(left-right), axis=1, keepdims=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create the embedding layer**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddingDimensions = 150\n",
    "embeddingLayer = layers.Embedding(embeddingDimensions, maxQuestionSize)\n",
    "\n",
    "leftInput = Input(shape=(maxQuestionSize,), dtype='int32', name=\"questionOne\")\n",
    "rightInput = Input(shape=(maxQuestionSize,), dtype='int32', name=\"questionTwo\")\n",
    "\n",
    "embeddedQuestionOne = embeddingLayer(leftInput)\n",
    "embeddedQuestionTwo = embeddingLayer(rightInput)\n",
    "\n",
    "lstm = layers.Bidirectional(layers.LSTM(64))\n",
    "\n",
    "leftOutput = lstm(embeddedQuestionOne)\n",
    "rightOutput = lstm(embeddedQuestionTwo)\n",
    "\n",
    "# Calculates the distance as defined by the MaLSTM model\n",
    "distance = layers.Merge(mode=lambda x: exponent_neg_manhattan_distance(x[0], x[1]), \n",
    "                        output_shape=lambda x: (x[0][0], 1))([leftOutput, rightOutput])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Model([leftInput, rightInput], distance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optimizers.Adam()\n",
    "model.compile(optimizer=optimizer,\n",
    "             loss=losses.MSE,\n",
    "             metrics=[metrics.binary_accuracy])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "questionOne (InputLayer)        (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "questionTwo (InputLayer)        (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_11 (Embedding)        (None, 150, 150)     22500       questionOne[0][0]                \n",
      "                                                                 questionTwo[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 128)          110080      embedding_11[0][0]               \n",
      "                                                                 embedding_11[1][0]               \n",
      "__________________________________________________________________________________________________\n",
      "merge_11 (Merge)                (None, 1)            0           bidirectional_1[0][0]            \n",
      "                                                                 bidirectional_1[1][0]            \n",
      "==================================================================================================\n",
      "Total params: 132,580\n",
      "Trainable params: 132,580\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8000 samples, validate on 1000 samples\n",
      "Epoch 1/50\n",
      "8000/8000 [==============================] - 100s 12ms/step - loss: 0.1756 - binary_accuracy: 0.7728 - val_loss: 0.2203 - val_binary_accuracy: 0.6850\n",
      "Epoch 2/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1750 - binary_accuracy: 0.7719 - val_loss: 0.2186 - val_binary_accuracy: 0.6940\n",
      "Epoch 3/50\n",
      "8000/8000 [==============================] - 99s 12ms/step - loss: 0.1745 - binary_accuracy: 0.7756 - val_loss: 0.2197 - val_binary_accuracy: 0.6850\n",
      "Epoch 4/50\n",
      "8000/8000 [==============================] - 100s 12ms/step - loss: 0.1735 - binary_accuracy: 0.7720 - val_loss: 0.2189 - val_binary_accuracy: 0.6840\n",
      "Epoch 5/50\n",
      "8000/8000 [==============================] - 100s 12ms/step - loss: 0.1734 - binary_accuracy: 0.7766 - val_loss: 0.2192 - val_binary_accuracy: 0.6800\n",
      "Epoch 6/50\n",
      "8000/8000 [==============================] - 99s 12ms/step - loss: 0.1723 - binary_accuracy: 0.7781 - val_loss: 0.2195 - val_binary_accuracy: 0.6810\n",
      "Epoch 7/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1718 - binary_accuracy: 0.7799 - val_loss: 0.2192 - val_binary_accuracy: 0.6840\n",
      "Epoch 8/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1710 - binary_accuracy: 0.7782 - val_loss: 0.2183 - val_binary_accuracy: 0.6830\n",
      "Epoch 9/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1716 - binary_accuracy: 0.7788 - val_loss: 0.2178 - val_binary_accuracy: 0.6870\n",
      "Epoch 10/50\n",
      "8000/8000 [==============================] - 97s 12ms/step - loss: 0.1704 - binary_accuracy: 0.7827 - val_loss: 0.2179 - val_binary_accuracy: 0.6830\n",
      "Epoch 11/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1696 - binary_accuracy: 0.7855 - val_loss: 0.2192 - val_binary_accuracy: 0.6810\n",
      "Epoch 12/50\n",
      "8000/8000 [==============================] - 97s 12ms/step - loss: 0.1694 - binary_accuracy: 0.7853 - val_loss: 0.2188 - val_binary_accuracy: 0.6800\n",
      "Epoch 13/50\n",
      "8000/8000 [==============================] - 97s 12ms/step - loss: 0.1686 - binary_accuracy: 0.7876 - val_loss: 0.2183 - val_binary_accuracy: 0.6820\n",
      "Epoch 14/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1680 - binary_accuracy: 0.7896 - val_loss: 0.2193 - val_binary_accuracy: 0.6840\n",
      "Epoch 15/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1676 - binary_accuracy: 0.7871 - val_loss: 0.2178 - val_binary_accuracy: 0.6870\n",
      "Epoch 16/50\n",
      "8000/8000 [==============================] - 97s 12ms/step - loss: 0.1671 - binary_accuracy: 0.7900 - val_loss: 0.2189 - val_binary_accuracy: 0.6750\n",
      "Epoch 17/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1664 - binary_accuracy: 0.7933 - val_loss: 0.2191 - val_binary_accuracy: 0.6820\n",
      "Epoch 18/50\n",
      "8000/8000 [==============================] - 97s 12ms/step - loss: 0.1666 - binary_accuracy: 0.7894 - val_loss: 0.2178 - val_binary_accuracy: 0.6820\n",
      "Epoch 19/50\n",
      "8000/8000 [==============================] - 99s 12ms/step - loss: 0.1659 - binary_accuracy: 0.7921 - val_loss: 0.2177 - val_binary_accuracy: 0.6810\n",
      "Epoch 20/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1655 - binary_accuracy: 0.7943 - val_loss: 0.2178 - val_binary_accuracy: 0.6800\n",
      "Epoch 21/50\n",
      "8000/8000 [==============================] - 97s 12ms/step - loss: 0.1650 - binary_accuracy: 0.7935 - val_loss: 0.2192 - val_binary_accuracy: 0.6770\n",
      "Epoch 22/50\n",
      "8000/8000 [==============================] - 97s 12ms/step - loss: 0.1641 - binary_accuracy: 0.7983 - val_loss: 0.2187 - val_binary_accuracy: 0.6780\n",
      "Epoch 23/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1639 - binary_accuracy: 0.7996 - val_loss: 0.2180 - val_binary_accuracy: 0.6950\n",
      "Epoch 24/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1632 - binary_accuracy: 0.7986 - val_loss: 0.2180 - val_binary_accuracy: 0.6830\n",
      "Epoch 25/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1633 - binary_accuracy: 0.7956 - val_loss: 0.2183 - val_binary_accuracy: 0.6810\n",
      "Epoch 26/50\n",
      "8000/8000 [==============================] - 97s 12ms/step - loss: 0.1624 - binary_accuracy: 0.8000 - val_loss: 0.2204 - val_binary_accuracy: 0.6780\n",
      "Epoch 27/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1621 - binary_accuracy: 0.8005 - val_loss: 0.2183 - val_binary_accuracy: 0.6840\n",
      "Epoch 28/50\n",
      "8000/8000 [==============================] - 97s 12ms/step - loss: 0.1611 - binary_accuracy: 0.8014 - val_loss: 0.2207 - val_binary_accuracy: 0.6750\n",
      "Epoch 29/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1614 - binary_accuracy: 0.7999 - val_loss: 0.2198 - val_binary_accuracy: 0.6740\n",
      "Epoch 30/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1607 - binary_accuracy: 0.8049 - val_loss: 0.2174 - val_binary_accuracy: 0.6900\n",
      "Epoch 31/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1597 - binary_accuracy: 0.8026 - val_loss: 0.2192 - val_binary_accuracy: 0.6850\n",
      "Epoch 32/50\n",
      "8000/8000 [==============================] - 97s 12ms/step - loss: 0.1596 - binary_accuracy: 0.8077 - val_loss: 0.2201 - val_binary_accuracy: 0.6790\n",
      "Epoch 33/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1593 - binary_accuracy: 0.8073 - val_loss: 0.2203 - val_binary_accuracy: 0.6710\n",
      "Epoch 34/50\n",
      "8000/8000 [==============================] - 97s 12ms/step - loss: 0.1587 - binary_accuracy: 0.8054 - val_loss: 0.2212 - val_binary_accuracy: 0.6700\n",
      "Epoch 35/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1589 - binary_accuracy: 0.8076 - val_loss: 0.2206 - val_binary_accuracy: 0.6750\n",
      "Epoch 36/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1580 - binary_accuracy: 0.8089 - val_loss: 0.2205 - val_binary_accuracy: 0.6780\n",
      "Epoch 37/50\n",
      "8000/8000 [==============================] - 97s 12ms/step - loss: 0.1571 - binary_accuracy: 0.8134 - val_loss: 0.2202 - val_binary_accuracy: 0.6740\n",
      "Epoch 38/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1568 - binary_accuracy: 0.8144 - val_loss: 0.2210 - val_binary_accuracy: 0.6720\n",
      "Epoch 39/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1563 - binary_accuracy: 0.8150 - val_loss: 0.2212 - val_binary_accuracy: 0.6730\n",
      "Epoch 40/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1560 - binary_accuracy: 0.8126 - val_loss: 0.2224 - val_binary_accuracy: 0.6720\n",
      "Epoch 41/50\n",
      "8000/8000 [==============================] - 99s 12ms/step - loss: 0.1555 - binary_accuracy: 0.8124 - val_loss: 0.2213 - val_binary_accuracy: 0.6710\n",
      "Epoch 42/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1548 - binary_accuracy: 0.8157 - val_loss: 0.2207 - val_binary_accuracy: 0.6750\n",
      "Epoch 43/50\n",
      "8000/8000 [==============================] - 99s 12ms/step - loss: 0.1544 - binary_accuracy: 0.8170 - val_loss: 0.2203 - val_binary_accuracy: 0.6750\n",
      "Epoch 44/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1542 - binary_accuracy: 0.8154 - val_loss: 0.2219 - val_binary_accuracy: 0.6700\n",
      "Epoch 45/50\n",
      "8000/8000 [==============================] - 97s 12ms/step - loss: 0.1541 - binary_accuracy: 0.8165 - val_loss: 0.2216 - val_binary_accuracy: 0.6750\n",
      "Epoch 46/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1531 - binary_accuracy: 0.8207 - val_loss: 0.2213 - val_binary_accuracy: 0.6760\n",
      "Epoch 47/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1525 - binary_accuracy: 0.8214 - val_loss: 0.2214 - val_binary_accuracy: 0.6680\n",
      "Epoch 48/50\n",
      "8000/8000 [==============================] - 97s 12ms/step - loss: 0.1524 - binary_accuracy: 0.8229 - val_loss: 0.2210 - val_binary_accuracy: 0.6830\n",
      "Epoch 49/50\n",
      "8000/8000 [==============================] - 98s 12ms/step - loss: 0.1517 - binary_accuracy: 0.8220 - val_loss: 0.2222 - val_binary_accuracy: 0.6700\n",
      "Epoch 50/50\n",
      "8000/8000 [==============================] - 99s 12ms/step - loss: 0.1515 - binary_accuracy: 0.8225 - val_loss: 0.2213 - val_binary_accuracy: 0.6790\n"
     ]
    }
   ],
   "source": [
    "history = model.fit([trainingSetOne, trainingSetTwo], \n",
    "                    trainingLabels, \n",
    "                    epochs=50, \n",
    "                    batch_size=64,\n",
    "                    validation_data=([validationSetOne, validationSetTwo], validationLabels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3XmYFNW5x/HvyyariuCOrK7sjiMkEXdENHHfIJqoREk0GKMxxusSeYyKxjVuuaIm0UhEEhUxcd81asKggA5cEXHQEZeRRUFQGHjvH6dmaIaZ6YWp7p7u3+d5+pmuqlPVp7qh3jprmbsjIiLSmBa5zoCIiOQ/BQsREUlKwUJERJJSsBARkaQULEREJCkFCxERSUrBQlJmZi3NbIWZdW/KtLlkZjubWZP3Hzez4WZWkbD8rpntm0raDD7rbjO7ONP9RVLRKtcZkPiY2YqExfbAt8DaaPmn7j4pneO5+1qgY1OnLQbuvltTHMfMzgBOcfcDEo59RlMcW6QxChYFzN1rL9bRnesZ7v5sQ+nNrJW7V2cjbyLJ6N9jflE1VBEzsyvN7EEze8DMlgOnmNl3zewNM1tmZp+Y2S1m1jpK38rM3Mx6Rsv3R9ufMLPlZva6mfVKN220/TAzm2dmX5rZrWb2bzM7rYF8p5LHn5rZfDNbama3JOzb0sxuMrPFZvY+MLKR7+dSM5tcZ93tZnZj9P4MM5sbnc/70V1/Q8eqNLMDovftzeyvUd7Kgb3q+dwF0XHLzezIaP0A4DZg36iK74uE73Z8wv4/i859sZlNNbPtU/lu0vmea/JjZs+a2RIz+9TMLkz4nMui7+QrMyszsx3qq/Izs1drfufo+3w5+pwlwKVmtouZvRCdyxfR97ZFwv49onOsirb/wczaRnneIyHd9ma20sy6NHS+koS761UEL6ACGF5n3ZXAauAIwo1DO2BvYCih1NkbmAeMi9K3AhzoGS3fD3wBlAKtgQeB+zNIuw2wHDgq2nY+sAY4rYFzSSWPjwJbAD2BJTXnDowDyoFuQBfg5fDfoN7P6Q2sADokHPtzoDRaPiJKY8BBwCpgYLRtOFCRcKxK4IDo/fXAi0BnoAcwp07aE4Hto9/kh1Eeto22nQG8WCef9wPjo/cjojwOBtoCdwDPp/LdpPk9bwF8BpwLbAZsDgyJtv0PMAvYJTqHwcBWwM51v2vg1ZrfOTq3auAsoCXh3+OuwMFAm+jfyb+B6xPO553o++wQpd8n2jYRuCrhc34FPJLr/4fN+ZXzDOiVpR+64WDxfJL9LgD+Hr2vLwD8b0LaI4F3Mkg7BnglYZsBn9BAsEgxj99J2P4wcEH0/mVCdVzNtsPrXsDqHPsN4IfR+8OAeY2k/Sfw8+h9Y8Hiw8TfAjg7MW09x30H+H70PlmwuBe4OmHb5oR2qm7Jvps0v+cfAWUNpHu/Jr911qcSLBYkycPxwPTo/b7Ap0DLetLtA3wAWLQ8Ezi2qf9fFdNL1VDyUeKCme1uZv+KqhW+Aq4Aujay/6cJ71fSeKN2Q2l3SMyHh//dlQ0dJMU8pvRZwMJG8gvwN2B09P6HQG2nADP7gZn9J6qGWUa4q2/su6qxfWN5MLPTzGxWVJWyDNg9xeNCOL/a47n7V8BSYMeENCn9Zkm+552A+Q3kYSdCwMhE3X+P25nZFDP7OMrDX+rkocJDZ4oNuPu/CaWUYWbWH+gO/CvDPAlqs5Bwp5noTsKd7M7uvjnwW8Kdfpw+Idz5AmBmxoYXt7o2JY+fEC4yNZJ17X0QGG5m3QjVZH+L8tgO+AcwgVBFtCXwdIr5+LShPJhZb+CPhKqYLtFx/y/huMm6+S4iVG3VHK8Tobrr4xTyVVdj3/NHQJ8G9mto29dRntonrNuuTpq653ctoRffgCgPp9XJQw8za9lAPu4DTiGUgqa4+7cNpJMUKFhIXZ2AL4GvowbCn2bhM/8JlJjZEWbWilAPvnVMeZwC/NLMdowaO3/TWGJ3/4xQVfJn4F13fy/atBmhHr0KWGtmPyDUraeah4vNbEsL41DGJWzrSLhgVhHi5hmEkkWNz4BuiQ3NdTwA/MTMBprZZoRg9oq7N1hSa0Rj3/M0oLuZjTOzNma2uZkNibbdDVxpZn0sGGxmWxGC5KeEjhQtzWwsCYGtkTx8DXxpZjsRqsJqvA4sBq620GmgnZntk7D9r4Rqqx8SAodsAgULqetXwKmEBuc7CXfWsYouyCcBNxL+8/cB3iLcUTZ1Hv8IPAe8DUwnlA6S+RuhDeJvCXleBpwHPEJoJD6eEPRScTmhhFMBPEHChczdZwO3AP+N0uwO/Cdh32eA94DPzCyxOqlm/ycJ1UWPRPt3B05OMV91Nfg9u/uXwCHAcYQG9XnA/tHm64CphO/5K0Jjc9uoevFM4GJCZ4ed65xbfS4HhhCC1jTgoYQ8VAM/APYglDI+JPwONdsrCL/zand/Lc1zlzpqGn9E8kZUrbAION7dX8l1fqT5MrP7CI3m43Odl+ZOg/IkL5jZSEK1wjeErpfVhLtrkYxE7T9HAQNynZdCoGooyRfDgAWE6omRwNFqkJRMmdkEwliPq939w1znpxCoGkpERJJSyUJERJIqmDaLrl27es+ePXOdDRGRZmXGjBlfuHtjXdWBAgoWPXv2pKysLNfZEBFpVsws2SwGgKqhREQkBQoWIiKSlIKFiIgkVTBtFvVZs2YNlZWVfPPNN7nOijSibdu2dOvWjdatG5ruSERyraCDRWVlJZ06daJnz56EiUwl37g7ixcvprKykl69eiXfQURyoqCrob755hu6dOmiQJHHzIwuXbqo9CeSgUmToGdPaNEi/J00KdkemSvokgWgQNEM6DcSSd+kSTB2LKxcGZYXLgzLACdnOs9wIwq6ZCEiUqguuWR9oKixcmVYHwcFixgtXryYwYMHM3jwYLbbbjt23HHH2uXVq1endIzTTz+dd999t9E0t99+O5PiLH+KSN75sIHpERtav6kKvhoqHZMmhaj84YfQvTtcddWmFee6dOnCzJkzARg/fjwdO3bkggsu2CBN7cPQW9Qft//85z8n/Zyf//znmWdSRJql7t1D1VN96+OgkkWkpv5v4UJwX1//F8cN+/z58+nfvz8/+9nPKCkp4ZNPPmHs2LGUlpbSr18/rrjiitq0w4YNY+bMmVRXV7Plllty0UUXMWjQIL773e/y+eefA3DppZdy880316a/6KKLGDJkCLvtthuvvRYeEPb1119z3HHHMWjQIEaPHk1paWltIEt0+eWXs/fee9fmr2ZW4nnz5nHQQQcxaNAgSkpKqKioAODqq69mwIABDBo0iEviKv+KNEPpNj6nm/6qq6B9+w3XtW8f1sei5s62ub/22msvr2vOnDkbrWtIjx7uIUxs+OrRI+VDNOryyy/36667zt3d33vvPTcz/+9//1u7ffHixe7uvmbNGh82bJiXl5e7u/s+++zjb731lq9Zs8YBf/zxx93d/bzzzvMJEya4u/sll1ziN910U236Cy+80N3dH330UT/00EPd3X3ChAl+9tlnu7v7zJkzvUWLFv7WW29tlM+afKxbt85HjRpV+3klJSU+bdo0d3dftWqVf/311z5t2jQfNmyYr1y5coN9M5HObyWS7+6/3719+w2vJe3bh/VNkT5xvx493M3C32Tp6wOUeQrXWJUsItmu/+vTpw9777137fIDDzxASUkJJSUlzJ07lzlz5my0T7t27TjssMMA2GuvvWrv7us69thjN0rz6quvMmrUKAAGDRpEv3796t33ueeeY8iQIQwaNIiXXnqJ8vJyli5dyhdffMERRxwBhEF07du359lnn2XMmDG0a9cOgK222ir9L0KkAKXb+JxpY/XJJ0NFBaxbF/7G0QuqhoJFpKF6vrjq/zp06FD7/r333uMPf/gDzz//PLNnz2bkyJH1jjto06ZN7fuWLVtSXV1d77E322yzjdJ4Cg+5WrlyJePGjeORRx5h9uzZjBkzpjYf9XVvdXd1e5VmJ5OxCenuk+7NZ7ZvVjOhYBHJev1fgq+++opOnTqx+eab88knn/DUU081+WcMGzaMKVOmAPD222/XW3JZtWoVLVq0oGvXrixfvpyHHnoIgM6dO9O1a1cee+wxIAx2XLlyJSNGjOCee+5h1apVACxZsqTJ8y2STDoX8kzaJjPZJ92bz2zfrGZCwSJy8skwcSL06AFm4e/EifEW62qUlJTQt29f+vfvz5lnnsk+++zT5J9xzjnn8PHHHzNw4EBuuOEG+vfvzxZbbLFBmi5dunDqqafSv39/jjnmGIYOHVq7bdKkSdxwww0MHDiQYcOGUVVVxQ9+8ANGjhxJaWkpgwcP5qabbmryfIs0Jt0LeSbVPZnsk+7NZy5vVlOWSsNGc3htagN3oVuzZo2vWrXK3d3nzZvnPXv29DVr1uQ4V+vpt5JMpNsxxaz+9GYNf0Ym+7in3/jcFI3VmSDFBm6NsygSK1as4OCDD6a6uhp3584776RVK/380rylW9efydiETMcznHxyejUT6abPNlVDFYktt9ySGTNmMGvWLGbPns2IESNynSUpcNmY5C7duv5MqnuaRRVRFihYiEiTy9Yg13Qv5Jm0TeayPTOfmKfQpbI5KC0t9bKysg3WzZ07lz322CNHOZJ06LcqLD171l9106NHGA/QlJp6mp5iY2Yz3L00WTpVWotIk8vmuIF8r+svFLFWQ5nZSDN718zmm9lF9Wy/ycxmRq95ZrYsYVt3M3vazOaa2Rwz6xlnXkWk6WQ6biCbD/OR9MQWLMysJXA7cBjQFxhtZn0T07j7ee4+2N0HA7cCDydsvg+4zt33AIYAn8eV17gccMABGw2wu/nmmzn77LMb3a9jx44ALFq0iOOPP77BY9etdqvr5ptvZmVCB/HDDz+cZcuWNbKHSNPIpFE4m5N5SvriLFkMAea7+wJ3Xw1MBo5qJP1o4AGAKKi0cvdnANx9hbuvbGTfvDR69GgmT568wbrJkyczevTolPbfYYcd+Mc//pHx59cNFo8//jhbbrllxscTSVUmjcLZfpiPpCfOYLEj8FHCcmW0biNm1gPoBTwfrdoVWGZmD5vZW2Z2XVRSqbvfWDMrM7OyqqqqJs7+pjv++OP55z//ybfffgtARUUFixYtYtiwYbXjHkpKShgwYACPPvroRvtXVFTQv39/IEzFMWrUKAYOHMhJJ51UO8UGwFlnnVU7vfnll18OwC233MKiRYs48MADOfDAAwHo2bMnX3zxBQA33ngj/fv3p3///rXTm1dUVLDHHntw5pln0q9fP0aMGLHB59R47LHHGDp0KHvuuSfDhw/ns88+A8JYjtNPP50BAwYwcODA2ulCnnzySUpKShg0aBAHH3xwk3y3sl6+Vt2kO8ldc5gfqailMnIvkxdwAnB3wvKPgFsbSPubxG3A8cCXQG9CI/xDwE8a+7xkI7jPPdd9//2b9nXuuY0OjHR398MPP9ynTp3q7mGa8AsuuMDdw4jqL7/80t3dq6qqvE+fPr5u3Tp3d+/QoYO7u3/wwQfer18/d3e/4YYb/PTTT3d391mzZnnLli19+vTp7r5+avDq6mrff//9fdasWe7u3qNHD6+qqqrNS81yWVmZ9+/f31esWOHLly/3vn37+ptvvukffPCBt2zZsnbq8hNOOMH/+te/bnROS5Ysqc3rXXfd5eeff767u1944YV+bsKXsmTJEv/888+9W7duvmDBgg3yWlehjuCOe1RuplNb56O4HxMg9SMPpiivBHZKWO4GLGog7SiiKqiEfd/yUIVVDUwFSmLJZcwSq6ISq6DcnYsvvpiBAwcyfPhwPv7449o79Pq8/PLLnHLKKQAMHDiQgQMH1m6bMmUKJSUl7LnnnpSXl9c7SWCiV199lWOOOYYOHTrQsWNHjj32WF555RUAevXqxeDBg4GGp0GvrKzk0EMPZcCAAVx33XWUl5cD8Oyzz27w1L7OnTvzxhtvsN9++9GrVy+guKYxz0YdfKZVN/lYGtHgt/wWZ9fZ6cAuZtYL+JgQEH5YN5GZ7QZ0Bl6vs29nM9va3auAg4DGW3OTiGpasu7oo4/m/PPP580332TVqlWUlISYN2nSJKqqqpgxYwatW7emZ8+e9U5Lnqi+6cA/+OADrr/+eqZPn07nzp057bTTkh7HGxlbUzO9OYQpzuurhjrnnHM4//zzOfLII3nxxRcZP3587XHr5rG+dcWisQt5U3X1zKTqpiaI1eStJohB4/mKezxDzbE0ZiI/xVayiEoE44CngLnAFHcvN7MrzOzIhKSjgcmecAVz97XABcBzZvY2YMBdceU1Th07duSAAw5gzJgxGzRsf/nll2yzzTa0bt2aF154gYX1jWBKsN9++zEpuv175513mD17NhCmN+/QoQNbbLEFn332GU888UTtPp06dWL58uX1Hmvq1KmsXLmSr7/+mkceeYR999035XP68ssv2XHH0Px077331q4fMWIEt912W+3y0qVL+e53v8tLL73EBx98ABTXNObZqIPPpItqJqWRbPVUyubDfCQ9sY6zcPfH3X1Xd+/j7ldF637r7tMS0ox3943GYLj7M+4+0N0HuPtpHnpUNUujR49m1qxZtU+qAzj55JMpKyujtLSUSZMmsfvuuzd6jLPOOosVK1YwcOBAfv/73zNkyBAgPPVuzz33pF+/fowZM2aD6c3Hjh3LYYcdVtvAXaOkpITTTjuNIUOGMHToUM444wz23HPPlM9n/PjxnHDCCey777507dq1dv2ll17K0qVL6d+/P4MGDeKFF15g6623ZuLEiRx77LEMGjSIk046KeXPae6y8YyCTKpuMgli6qkkOZ9avKlemqK8eSvE3ypbjc/pNqJn0pCc6TTdkv/IgwZukaKWrQno0q26yaQ00hye5CbxUrAQiVEmdfBx91TKJIipp5IU/ESCXsS9cZoLb6R3VrHJtKdSujJ5MA+op1IxK+iSRdu2bVm8eLEuRnnM3Vm8eDFt27bNdVaSysbYhHxuSFZPpeJW0CWLbt26UVlZST5OBSLrtW3blm7duuU6G43K1h2/pryQfFXQDz+S5ikfH2aTrYf5ZPOhQSKQ+sOPCroaSpqffJ2mOlt3/GpIlnylYCF5JV/r7LPVdVTPe5Z8pWAheSVbd/DpNlZn845fDcmSjxQsJK9k43GcmVR16Y5fip0auCWv1O11BOEOvrELc7r7qBFZZD01cEuzlI3Hcap7qkj6CnqchTRP6Y4uTvfi3717/SULzXMk0jCVLKTZS7edQ91TRdKnYCGxysYUGele/NVYLZI+VUNJbLI5KR6kN+o73aoukWKn3lASG/U6Esl/6g0lOadeRyKFQ8FCYqOnq4kUDgULiY16HYkUDgULSUs6vZvU60ikcKg3lKQsk95N6nUkUhhUsihi6Y6ByNfpw0UkfipZFKlMSgnq3SRSvGItWZjZSDN718zmm9lF9Wy/ycxmRq95ZraszvbNzexjM7stznwWo0xKCerdJFK8YgsWZtYSuB04DOgLjDazvolp3P08dx/s7oOBW4GH6xzmd8BLceWxmGVSSlDvJpHiFWfJYggw390XuPtqYDJwVCPpRwMP1CyY2V7AtsDTMeaxaGVSSlDvJpHiFWew2BH4KGG5Mlq3ETPrAfQCno+WWwA3AL9u7APMbKyZlZlZWVVVVZNkOl/EPQFfpqUEPfJTpDjFGSysnnUNTUQ1CviHu6+Nls8GHnf3jxpIHw7mPtHdS929dOutt96ErOaXTB77mS6VEkQkHXEGi0pgp4TlbsCiBtKOIqEKCvguMM7MKoDrgR+b2TVxZDIfZdL4nElJRKUEEUlVnF1npwO7mFkv4GNCQPhh3URmthvQGXi9Zp27n5yw/TSg1N036k1VqNJtfM7WVOAiUrxiK1m4ezUwDngKmAtMcfdyM7vCzI5MSDoamOyFMld6E0i38VmD5UQkbrGOs3D3x919V3fv4+5XRet+6+7TEtKMb6zU4O5/cfdxceYzbulWEaXb+KzBciISN033EbNMGqvTbXzWYDkRiZuelBezbDwtrm6bBYSSiHo3iUgyelJenshGFZG6wYpI3DSRYMy6d6+/ZNHUVUSaClxE4qSSRcw0n5KIFIKiDxZxT6uhKiIRKQRFXQ2VrcFsqiISkeauqEsWGswmIpKaog4WGswmIpKaog4WGswmIpKaog4W6qkkIpKaog4W6qkkIpKaou4NBeqpJCKSiqIuWYiISGoULEREJCkFCxERSUrBIk1xTw8iIpKPir6BOx161rWIFCuVLNKg6UFEpFgpWKRB04OISLFSsEiDpgcRkWKlYJEGTQ8iIsUqabAws3Fm1jkbmcl3mh5ERIpVKr2htgOmm9mbwJ+Ap9zd481W/tL0ICJSjJKWLNz9UmAX4B7gNOA9M7vazPok29fMRprZu2Y238wuqmf7TWY2M3rNM7Nl0frBZva6mZWb2WwzOyntMxMRkSaT0jgLd3cz+xT4FKgGOgP/MLNn3P3C+vYxs5bA7cAhQCWhdDLN3eckHPe8hPTnAHtGiyuBH7v7e2a2AzDDzJ5y92Xpn6KIiGyqVNosfmFmM4DfA/8GBrj7WcBewHGN7DoEmO/uC9x9NTAZOKqR9KOBBwDcfZ67vxe9XwR8DmydwvmIiEgMUilZdAWOdfeFiSvdfZ2Z/aCR/XYEPkpYrgSG1pfQzHoAvYDn69k2BGgDvJ9CXkVEJAapdJ19HFhSs2BmncxsKIC7z21kP6tnXUMN46OAf7j72g0OYLY98FfgdHdft9EHmI01szIzK6uqqkpyGiIikqlUgsUfgRUJy19H65KpBHZKWO4GLGog7SiiKqgaZrY58C/gUnd/o76d3H2iu5e6e+nWW6uWSkQkLqkEC0vsKhvd4adSfTUd2MXMeplZG0JAmLbRwc12IzSYv56wrg3wCHCfu/89hc8SEZEYpRIsFkSN3K2j17nAgmQ7uXs1MA54CpgLTHH3cjO7wsyOTEg6GphcZ+zGicB+wGkJXWsHp3xWIiLSpCzZ+Doz2wa4BTiI0ObwHPBLd/88/uylrrS01MvKynKdDRGRZsXMZrh7abJ0SauToqAwqklyJSIizVLSYGFmbYGfAP2AtjXr3X1MjPkSEZE8kkqbxV8J80MdCrxE6NW0PM5MiYhIfkklWOzs7pcBX7v7vcD3gQHxZktERPJJKsFiTfR3mZn1B7YAesaWIxERyTupjJeYGD3P4lLCOImOwGWx5kpERPJKo8HCzFoAX7n7UuBloHdWciUiInml0WqoaLT2uCzlRURE8lQqbRbPmNkFZraTmW1V84o9ZyIikjdSabOoGU/x84R1jqqkRESKRiojuHtlIyMiIpK/UhnB/eP61rv7fU2fHRERyUepVEPtnfC+LXAw8CagYCEiUiRSqYY6J3HZzLYgTAEiIiJFIpXeUHWtBHZp6oyIiEj+SqXN4jHWPzu7BdAXmBJnpkREJL+k0mZxfcL7amChu1fGlB8REclDqQSLD4FP3P0bADNrZ2Y93b0i1pyJiEjeSKXN4u/AuoTltdE6EREpEqkEi1buvrpmIXrfJr4siYhIvkklWFSZ2ZE1C2Z2FPBFfFkSEZF8k0qbxc+ASWZ2W7RcCdQ7qltERApTKoPy3ge+Y2YdAXN3PX9bRKTIJK2GMrOrzWxLd1/h7svNrLOZXZmNzImISH5Ipc3iMHdfVrMQPTXv8PiyJCIi+SaVYNHSzDarWTCzdsBmjaSvZWYjzexdM5tvZhfVs/0mM5sZveaZ2bKEbaea2XvR69RUPk9EROKRSgP3/cBzZvbnaPl04N5kO5lZS+B24BBCo/h0M5vm7nNq0rj7eQnpzwH2jN5vBVwOlBKmGpkR7bs0pbMSEZEmlbRk4e6/B64E9iDMC/Uk0COFYw8B5rv7gmhsxmTgqEbSjwYeiN4fCjzj7kuiAPEMMDKFzxQRkRikOuvsp4RR3McRnmcxN4V9dgQ+SliujNZtxMx6AL2A59PZ18zGmlmZmZVVVVWlkCUREclEg9VQZrYrMIpwx78YeJDQdfbAFI9t9azzetYRfc4/3H1tOvu6+0RgIkBpaWlDxxYRkU3UWMni/wiliCPcfZi730qYFypVlcBOCcvdgEUNpB3F+iqodPcVEZGYNRYsjiNUP71gZneZ2cHUf8ffkOnALmbWy8zaEALCtLqJzGw3oDPwesLqp4AR0ZiOzsCIaJ2IiORAg8HC3R9x95OA3YEXgfOAbc3sj2Y2ItmB3b0aGEe4yM8Fprh7uZldkTjXFKGaa7K7e8K+S4DfEQLOdOCKaJ2IiOSAJVyjkycOXVpPAE5y94Niy1UGSktLvaysLNfZEBFpVsxshruXJkuX1jO4o66sd+ZboBARkXilFSxERKQ4KViIiEhSChYiIpKUgoWIiCSlYCEiIkkpWIiISFIKFiIikpSChYiIJKVgISIiSSlYiIhIUgoWIiKSlIKFiIgkpWAhIiJJKViIiEhSChYiIpKUgoWIiCSlYCEiEoOVK+Gii+CNN3Kdk6ahYCEi0sRWr4bjjoNrr4X99oO77sp1jjadgoWISBNauxZOOQWefBJuvBEOOgjGjoWf/SwEkeZKwUJEpIm4w09/Cn//O1x/PZx3HvzrX6E66s474cAD4ZNPcp3LzChYiIg0AXe44AK45x649FL41a/C+pYtYcIEmDIFZs6EvfaC11/PbV4zoWAhItIErroqVDuNGwdXXLHx9hNOCI3d7drB/vs3v3YMBQsRkU10661w2WXwox/BH/4AZvWnGzAApk8P1VHNrR1DwUJEZBPcdx/84hdw9NHwpz9BiyRX1a22gscfh9/8pnm1Y8QaLMxspJm9a2bzzeyiBtKcaGZzzKzczP6WsP730bq5ZnaLWUOxWkQkN6ZOhTFj4OCD4YEHoFWr1PZr2RKuuQYefHDT2jGqq6GiAsrL0983XSmeWvrMrCVwO3AIUAlMN7Np7j4nIc0uwP8A+7j7UjPbJlr/PWAfYGCU9FVgf+DFuPIrIpKO556Dk06C0tIQNNq2Tf8YJ54Iu+8OxxwT2jFuvx3OPHPDNMuWwYIF9b8WLgwBY+jQ+Af/xRYsgCHAfHdfAGBmk4GjgDkJac4EbndaqW1vAAAS1ElEQVT3pQDu/nm03oG2QBvAgNbAZzHmVUQkZW+8AUcdBbvtFqqUOnbM/FgDB4Z2jNGjQzvGE09A69YhGLz/PixdumH6rl2hd2/Ye+8QrHr3hj322LTzSUWcwWJH4KOE5UpgaJ00uwKY2b+BlsB4d3/S3V83sxeATwjB4jZ3n1v3A8xsLDAWoHv37k1/BiIidcyeDYcdBtttB08/HdogNlVNO8all8Jtt4Vj9+kTAkLv3hu+Nt980z8vE3EGi/raGLyez98FOADoBrxiZv2BrsAe0TqAZ8xsP3d/eYODuU8EJgKUlpbWPbaIFIEPPwwD4fbaC844A3r2jO+z5s+HESOgQwd49tlwUW8qNeMxJkxoumM2pTgbuCuBnRKWuwGL6knzqLuvcfcPgHcJweMY4A13X+HuK4AngO/EmFcRaYY+/xwOOQRefDFcZHv3Dnf9jzwCa9Y07WdVVsLw4WE6j2eeiTco5aM4g8V0YBcz62VmbYBRwLQ6aaYCBwKYWVdCtdQC4ENgfzNrZWatCY3bG1VDiUjxWrYMDj0UPvoo3OVXVMBvfwtvvw3HHgs9eoRqnYqKzI5fXQ3/+U8YbHfAASEQLV0KTz2VnTaCfBNbsHD3amAc8BThQj/F3cvN7AozOzJK9hSw2MzmAC8Av3b3xcA/gPeBt4FZwCx3fyyuvIpI8/L11/CDH4Quo488AvvsAzvtBOPHh+AwbVqolkqntOEO8+bBHXeE3kldu8J3vhMCzldfwS9/Ca+9BiUl2TrL/GLuhVHVX1pa6mVlZbnOhojE7Ntv4cgjQ2niwQfh+OMbTvvRR2Guprvvho8/hu23D+Miato2PvsMnn8+VCs9+2xID6FUcsghodrpoINg662zcmo5YWYz3L00aToFCxFpLqqrYdQoeOihEATGjEl9vyeegIkTQ68j99DbaP78sL1z5xAUhg8Prz59Gp6yo9CkGizi7A0lItJk1q0L4xAeeghuuin1QAFhZPURR4RXTWmjrCwcY/jwULXUsmV8eS8EChYikvfcw5Tff/4zXH55aD/IVE3bhqRHEwmKSN773e/g5pvDhH2XX57r3BQnBQsRyWu33BICxKmnhuqnYmlLyDcKFiKSt+69F849N3Rlvfvu5NN/S3z01YtIXnr44dAAfcgh6U3/LfHQ1y9SwKqrwzQV9U1v/cEHYdDZHXeERt988swzYRbWoUPDYLrNNst1jkTBQiTPuIeRwitWpLff8uUhANQEg/ffX/+8gxqtWoXBaL17Q9++8Pe/Q//+4dnRY8bkR3vAa6+Fp87tvjv8619h0j7JPQ3KE8kjK1aEi/bf/575Mbp02XBK6z591r/fcccNq3Pefx9+8hN46SUYOTIMWstVKaO6Gq6/PjRmd+8Or74K226bm7wUEw3Kk1gsXhwuKEceCf365To3heX998Md9Zw5YfK6Aw9Mb/927aBXL9hii9T36dMnTHdxxx3hmdC5KmXMmQOnnRYeAnT88SE/hTzFRrPk7gXx2muvvVzis3y5++9+57755u7gPmiQe3V1rnNVOJ54wn3LLd07d3Z/6qnc5GH+fPf99w+/78iR7h9+GP9nrlnjPmGCe5s27l27uk+ZEv9nyoaAMk/hGqveUNKo1avDc4F33hkuuyzMnzNhAsyaBX/6U65zl11r1sD554cZTp94IrQtbCp3uOYaOPzwUP1TVhYerpMLNaWMW2+Fl18OpYx77mma86zPnDnwve/B//xPKKmWl8MJJ8TzWdIEUokozeGlkkXTWrvWfdIk9969w53mfvu5v/Za2LZunfuwYe5bb+2+dGlu85ktn37qvu++4bvYdtvwd//93V9/PfNjLl/ufsIJ4Vgnnui+YkWTZXeTxVnKqFuaePDBpju2pI8USxY5v8g31UvBommsW+f++OOhmqmmuunxx8P6RDNmuJu5n39+bvKZTf/9r3u3bu7t2oUA+u237rffvj5oHH20e3l5esecP999wAD3Fi3cr7124+83H6xd637rre7t24fqx7vv3vR8lpe77713+N6OO879s8+aJq+SOQULSdtrr4USBIQSxaRJ4YLRkJ/8xL1VK/f/+7/s5THb/vxn9802c+/Rw/3NNzfcltiO06KF++mnuy9cmPyYTz4Z2iZy2T6Rjvnz1/+7GDHC/c473Z95xv39991Xr07tGCpN5C8FC0lZebn7UUeFfw3bbON+223h7jmZTz9179TJ/fvfjz+P2bZ6tfu4ceE7Oegg96qqhtNWVYUS1mabhdf557t/8cXG6datc7/mmhBYBgwIF9vmoqaU0alT+E5qXi1bhhuL4cPdx44N5zdlintZ2foqSpUm8puChSS1cGG4G27RIlwEfve7cLecjuuuC/+KnnginjzmQmL7xPnnh7viVCR+n5tvvuH3mc/tE+morg7n+cIL7vfc437JJe6jR7sPHRrasBIDCYTSk0oT+S3VYKFBeUXoiy/g6qtDLyeAn/8cLr44PHM4XatXh/EWrVrB7NnQunXT5jUdq1aFB9i0aZP5MaZPh2OPDeNJ7r4bfvjD9I8xZw5ccglMnRoGlV1wAdx3X+jtM2EC/PrX+TFSOg5ffbXhKPIFC8K/jUsugW22yXXupD6pDsrLeYmgqV4qWSSXaR17MtOmhbvIm2/e9GNlYu3aUHXWoUN4HX64+403ur/9dnoNso21T2Ti9dfX9yhqLu0TUnxQNVT+qK52f/pp93/9K/Uqjab07bfhYprYe+edd5ru+OvWhYbPLbdsvG4/DgsWuB9wQDivQw5xP/ts9113XV8Nst127ief7P6Xv7h/9FH9x0infSJd69a5v/RSdga4iWRCwSIPfPxxuJPv0WP9xWuHHdwvu8y9oiL+z29srERTKy8PjZ1nnRXP8etKLE106uR+110bliIWLgx16qNHb1iXvvvu7uec4/7oo+7LlmXePiFSKBQscqS6OoxLOProcPGE0FNkyhT3qVNDFYlZeB1+eFjX1BeoVMdKNLVzzgnVW7Nmxfs5Cxa4H3ig13blTFaVtnZtyNP114cBZu3be21Pni22WD9+QqQYKVhkWd1SxDbbuP/mN+7vvbdx2oqKULrYYQdv0tLGunUb1pP36pV8rERTWrzYfautwoU8jsC0dm0YDNdQaSJV33zj/uKLoSfPCSc0TfuESHOlYJEFjZUiUhmnsGZN+qWNb791nzcvDOy64w73X/3K/ZhjQumhpg/8NtuEPvGp5KGp3XZbyMPDDzftcdMtTYhIalINFrF2nTWzkcAfgJbA3e5+TT1pTgTGAw7McvcfRuu7A3cDO0XbDnf3ioY+K5tdZ1esgJtvDl0rFy4MXQJPPx3OOCNMuJeJhQvDpG333AOLFsEOO4Qpmzt0WP8gmwULwlPP1q1bv1/btmFa6prnFfTtC6ecAh07Nsmppq26GgYPDt1Yy8tD/jbFunXwv/8LF14Ynr98443h+QuF2vVUJNty3nWWECDeB3oDbYBZQN86aXYB3gI6R8vbJGx7ETgket8RaN/Y52WrZLFy5fqpD9IpRaSqbmmjpkfP977nfsop7r/9rfu997q/8kqo+spWFVM6nn025HvChE07jkoTIvEjxZJFnA8/GgLMd/cFAGY2GTgKmJOQ5kzgdndfGgWuz6O0fYFW7v5MtD7NB0zGY/Xq8GCWV14JD5AfNarpP6NVKzjqqPBasiQ8e7i5PVby4IND/q+8Ek49FbbfPr39Fy0KJaxrrw2libvuUmlCJNfifJ7FjsBHCcuV0bpEuwK7mtm/zeyNqNqqZv0yM3vYzN4ys+vMrGXdDzCzsWZWZmZlVVVVsZxEjbVr4cc/hscfD9UicQSKurbaqvkFihrXXx+e/3DxxamlX7s2PCPimGPCIzV/+9vw7Ix33gnVewoUIrkVZ7Co77933QaSVoSqqAOA0cDdZrZltH5f4AJgb0JV1mkbHcx9oruXunvp1jE+g9Edzj4bHnwQfv97GDs2to8qGDvvDOedB3/5S5hCoyGLFoUSSJ8+4QFA//43/OpX8N57MG1aCBwikntxBotKQuN0jW7AonrSPOrua9z9A+BdQvCoBN5y9wXuXg1MBUpizGuD3MOziSdODHfJv/51LnLRPF1ySZgb6dxzN3za2tq18OST60sRl10WgsuUKaEB/9prM+8oICLxiDNYTAd2MbNeZtYGGAVMq5NmKnAggJl1JVQ/LYj27WxmNcWFg9iwrSNrrrkGrrsulCyuvDIXOWi+OnUKE+e9/npo41m0CK66KpQiDjtsw1LEs8+GR2puyiSAIhKf2Bq43b3azMYBTxF6Rv3J3cvN7ApC6/u0aNsIM5sDrAV+7e6LAczsAuA5MzNgBnBXXHltyB13hNLEKaeE5xKr3jx9p54aZrf96U9Dd9q1a0MD+HXXhUZwBQeR5kFTlDfg/vvhRz+CI46Ahx7K7dTbzd306TBmTGiTOPNMVTGJ5JNUx1nE2XW22Zo2LQyIO/DAUI+uQLFp9t4b3n4717kQkU0RZ5tFs/T883DiiVBSAo8+uukjkEVECoGCRYL//AeOPDJUkzzxRGigFRERBYta77wTeuhsuy08/TR06ZLrHImI5A8FC8IkfYccAu3ahS6cO+yQ6xyJiOSXom/gXrQIhg8PU1O8/HKYwVVERDZU9CWLjh2hf/8worhv31znRkQkPxV9yWLzzeGxx3KdCxGR/Fb0JQsREUlOwUJERJJSsBARkaQULEREJCkFCxERSUrBQkREklKwEBGRpBQsREQkqYJ5+JGZVQELo8WuwBc5zE4uFfO5Q3GffzGfOxT3+W/Kufdw962TJSqYYJHIzMpSefJTISrmc4fiPv9iPnco7vPPxrmrGkpERJJSsBARkaQKNVhMzHUGcqiYzx2K+/yL+dyhuM8/9nMvyDYLERFpWoVashARkSakYCEiIkkVVLAws5Fm9q6ZzTezi3Kdn2wzswoze9vMZppZWa7zEzcz+5OZfW5m7ySs28rMnjGz96K/nXOZx7g0cO7jzezj6PefaWaH5zKPcTGznczsBTOba2blZnZutL7gf/tGzj32375g2izMrCUwDzgEqASmA6PdfU5OM5ZFZlYBlLp7UQxMMrP9gBXAfe7eP1r3e2CJu18T3TB0dvff5DKfcWjg3McDK9z9+lzmLW5mtj2wvbu/aWadgBnA0cBpFPhv38i5n0jMv30hlSyGAPPdfYG7rwYmA0flOE8SI3d/GVhSZ/VRwL3R+3sJ/5EKTgPnXhTc/RN3fzN6vxyYC+xIEfz2jZx77AopWOwIfJSwXEmWvsQ84sDTZjbDzMbmOjM5sq27fwLhPxawTY7zk23jzGx2VE1VcNUwdZlZT2BP4D8U2W9f59wh5t++kIKF1bOuMOrYUrePu5cAhwE/j6oqpHj8EegDDAY+AW7IbXbiZWYdgYeAX7r7V7nOTzbVc+6x//aFFCwqgZ0SlrsBi3KUl5xw90XR38+BRwhVc8Xms6het6Z+9/Mc5ydr3P0zd1/r7uuAuyjg39/MWhMulpPc/eFodVH89vWdezZ++0IKFtOBXcysl5m1AUYB03Kcp6wxsw5Rgxdm1gEYAbzT+F4FaRpwavT+VODRHOYlq2oulJFjKNDf38wMuAeY6+43Jmwq+N++oXPPxm9fML2hAKLuYjcDLYE/uftVOc5S1phZb0JpAqAV8LdCP38zewA4gDA982fA5cBUYArQHfgQOMHdC64huIFzP4BQDeFABfDTmjr8QmJmw4BXgLeBddHqiwl19wX92zdy7qOJ+bcvqGAhIiLxKKRqKBERiYmChYiIJKVgISIiSSlYiIhIUgoWIiKSlIKFSBJmtjZhNs+ZTTmjsZn1TJw5ViRftcp1BkSagVXuPjjXmRDJJZUsRDIUPT/kWjP7b/TaOVrfw8yeiyZ1e87MukfrtzWzR8xsVvT6XnSolmZ2V/R8gqfNrF2U/hdmNic6zuQcnaYIoGAhkop2daqhTkrY9pW7DwFuI8weQPT+PncfCEwCbonW3wK85O6DgBKgPFq/C3C7u/cDlgHHResvAvaMjvOzuE5OJBUawS2ShJmtcPeO9ayvAA5y9wXR5G6funsXM/uC8ICaNdH6T9y9q5lVAd3c/duEY/QEnnH3XaLl3wCt3f1KM3uS8ICjqcBUd18R86mKNEglC5FN4w28byhNfb5NeL+W9W2J3wduB/YCZpiZ2hglZxQsRDbNSQl/X4/ev0aY9RjgZODV6P1zwFkQHgNsZps3dFAzawHs5O4vABcCWwIblW5EskV3KiLJtTOzmQnLT7p7TffZzczsP4Qbr9HRul8AfzKzXwNVwOnR+nOBiWb2E0IJ4izCg2rq0xK438y2IDzY6yZ3X9ZkZySSJrVZiGQoarModfcvcp0XkbipGkpERJJSyUJERJJSyUJERJJSsBARkaQULEREJCkFCxERSUrBQkREkvp/sxDJd6dKYXQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1ce3118cc50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3XmYFOW59/HvzQ6CgEAEQTY1KijriBsqKiYQFdDgguCCeohb3BM5YqJy9HJDMSS+KiaQGDG4HbdExQQJ4ElEVkeBIIggIwgjCopsM3C/fzw10Iwz08tMT8/0/D7X1dd0Vz1VfVc31N3PUk+ZuyMiIlKWWpkOQEREqj4lCxERiUvJQkRE4lKyEBGRuJQsREQkLiULERGJS8lCKoWZ1TazLWbWviLLZpKZHWpmFT723Mz6m9mqmNfLzOykRMqm8F6/N7PbU92+jP3eY2Z/rOj9SubUyXQAUjWZ2ZaYl42AHcCu6PXP3H1KMvtz911A44ouWxO4++EVsR8zuxIY4e79YvZ9ZUXsW7KfkoWUyN33nKyjX65Xuvs/SitvZnXcvbAyYhORyqdmKElJ1MzwnJn9xcy+BUaY2fFm9p6ZbTKzdWY2wczqRuXrmJmbWcfo9TPR+jfN7Fsz+7eZdUq2bLR+oJl9bGabzey3ZvZ/ZnZZKXEnEuPPzGyFmX1tZhNitq1tZuPNbKOZfQIMKOPzucPMphZb9piZPRI9v9LMlkbH80n0q7+0feWZWb/oeSMz+3MU22KgdwnvuzLa72IzGxQtPxr4HXBS1MT3Zcxne1fM9ldFx77RzF4xszaJfDbxmNmQKJ5NZvaOmR0es+52M1trZt+Y2X9ijvU4M1sQLV9vZg8l+n6SBu6uhx5lPoBVQP9iy+4BdgJnE350NASOAY4l1Fg7Ax8D10Xl6wAOdIxePwN8CeQAdYHngGdSKPsD4FtgcLTuZqAAuKyUY0kkxleBpkBH4KuiYweuAxYD7YAWwKzwX6jE9+kMbAH2i9n3BiAnen12VMaA04BtQLdoXX9gVcy+8oB+0fNxwD+B5kAHYEmxsucDbaLv5KIohgOjdVcC/ywW5zPAXdHzH0Ux9gAaAP8PeCeRz6aE478H+GP0/MgojtOi7+j26HOvC3QFVgOto7KdgM7R87nAsOh5E+DYTP9fqMkP1SykPN5199fdfbe7b3P3ue4+x90L3X0lMBE4pYztX3T3ee5eAEwhnKSSLXsWsMjdX43WjScklhIlGON97r7Z3VcRTsxF73U+MN7d89x9I3B/Ge+zEviIkMQAzgA2ufu8aP3r7r7Sg3eA6UCJndjFnA/c4+5fu/tqQm0h9n2fd/d10XfyLCHR5ySwX4DhwO/dfZG7bwdGA6eYWbuYMqV9NmW5EHjN3d+JvqP7gf0JSbuQkJi6Rk2Zn0afHYSkf5iZtXD3b919ToLHIWmgZCHlsSb2hZkdYWZ/M7MvzOwbYCzQsoztv4h5vpWyO7VLK3tQbBzu7oRf4iVKMMaE3ovwi7gszwLDoucXEZJcURxnmdkcM/vKzDYRftWX9VkVaVNWDGZ2mZl9EDX3bAKOSHC/EI5vz/7c/Rvga6BtTJlkvrPS9rub8B21dfdlwC2E72FD1KzZOio6EugCLDOz983sJwkeh6SBkoWUR/Fho08Sfk0f6u77A78mNLOk0zpCsxAAZmbse3IrrjwxrgMOjnkdb2jvc0D/6Jf5YELywMwaAi8C9xGaiJoBbycYxxelxWBmnYHHgauBFtF+/xOz33jDfNcSmraK9teE0Nz1eQJxJbPfWoTv7HMAd3/G3U8kNEHVJnwuuPsyd7+Q0NT4MPCSmTUoZyySIiULqUhNgM3Ad2Z2JPCzSnjPvwK9zOxsM6sD3AC0SlOMzwM3mllbM2sB3FZWYXdfD7wLTAaWufvyaFV9oB6QD+wys7OA05OI4XYza2bhOpTrYtY1JiSEfELevJJQsyiyHmhX1KFfgr8AV5hZNzOrTzhpz3b3UmtqScQ8yMz6Re/9C0I/0xwzO9LMTo3eb1v02EU4gIvNrGVUE9kcHdvucsYiKVKykIp0C3Ap4UTwJOGXdVpFJ+QLgEeAjcAhwELCdSEVHePjhL6FDwmdry8msM2zhA7rZ2Ni3gTcBLxM6CQeSkh6ibiTUMNZBbwJPB2z31xgAvB+VOYIILad/+/AcmC9mcU2JxVt/xahOejlaPv2hH6McnH3xYTP/HFCIhsADIr6L+oDDxL6mb4g1GTuiDb9CbDUwmi7ccAF7r6zvPFIaiw08YpkBzOrTWj2GOruszMdj0i2UM1Cqj0zG2BmTaOmjF8RRti8n+GwRLKKkoVkg77ASkJTxgBgiLuX1gwlIilQM5SIiMSlmoWIiMSV1okEzWwA8BvC2Onfu/v9xdbfTJiCoJAwSuJyd19tZh2A/422qwv81t2fKOu9WrZs6R07dqz4gxARyWLz58//0t3LGm4OpLEZKhqV8jFhmoM89s7zsiSmzKnAHHffamZXE+a/ucDM6kWx7TCzxoSLqE5w97WlvV9OTo7PmzcvLcciIpKtzGy+u8edEiadzVB9gBXR/Dc7gansnScHAHef4e5bo5fvEV2J6+47Yzoo66c5ThERiSOdJ+G27DuHTR5lT8NwBeEiIwDM7GAzy4328UBJtQozG2Vm88xsXn5+fgWFLSIixaUzWZQ0z02JbV5mNoIwM+ae+erdfY27dwMOBS41swO/tzP3ie6e4+45rVrFbXITEZEUpTNZ5LHvhGftCFfW7sPM+gNjCJf/f29sfFSjWExi0zeLiEgapDNZzCXMRd8p6rC+EHgttoCZ9STMzzPI3TfELG8XzcyJmTUHTgSWpTFWEREpQ9qGzrp7oZldB0wjDIGd5O6LzWwsMM/dXyM0OzUGXggzS/OZuw8i3FnrYTNzQnPWOHf/MF2xiohI2bLmCm4NnRURSV6iQ2fTelFedeAOv/wlnHkmnHwy1Kqmg3S//RZmzoT588MxmYVjKetRVKZXLzjhhEwfgYhUZTU+WXz6KTz5JIwbB+3bw8UXh8fhh2c6srIVFMCcOfCPf4THnDlQWJjavszgf/4Hbr89PBcRKa6a/o6uOJ07wxdfwJQpcOSRcN99cMQRcNxx8Pjj8NVXmY4wcIcPP4Tx40MtqHlzOOmkcJLfuRN+8QuYPh22bYPdu2HXrpBQduwIy777LtQ+Nm+GTZvCcX35JaxdC8OGwR13wPnnw5YtmT5SEamK1GdRzNq18Oyz8Kc/wUcfQd26cPbZcMklMHAg1KtXAcEmYOdO+OwzmD071BymT4f168O6H/4Q+vcPj379QuIoD3d45JHQHNelC7zyChxySLkPQUSqgUT7LJQsSuEOH3wATz8dah0bNkCLFuFX+KWXQu/eqTfZFBbCunWwZs2+j7y8vc/Xrw8xABx4YEgMp58eHu3bV9hh7uPtt+HCC8Pz556DM85Iz/uky6pV0LIlNG6c6UhEqg8liwpUUBBOpE8/Da++Gpp2Dj4YmjQJHcS1a+/9G/u8+N9vvgkJYe3a0FQUq3HjsM/ijz59oGvXyutL+OQTGDIEliyBBx+Em2+u2v0Yu3fDG2+EmtGMGdCqVWiau+IKqFPje+RE4lOySJNNm+CFF+Cdd0ISKeofSORvo0YlJ4R27aBp06pzUt6yBS67DF56CYYPh6eegoYNU9/f8uXw/vthxFWnThUT49atIXmPHw8ffwxt28KoUaHJbvZsOOqokEAyXTv68kv4wx+gZ0/40Y8yG4tISRJNFrh7Vjx69+7tUnF273a/5x53M/devdxXr05u+2XLwvbdu7uHBrXw6NrVffRo9//7P/fCwuTjWrvWfcwY9wMOCPvr3dv92Wfdd+7cG/eLL7p37hzW/+Qn7kuWJP8+5bVypft117k3bLj32MeMSe2YRdKJcJF03HNsxk/yFfVQskiP1193339/91at3P/5z7LLlpQgjj/e/ZFH3N9/3338ePfTTnOvUyesa9nS/ZJL3F94wX3z5rL3vWiR+6WXutetGxLYkCHus2aF5FCS7dvdH3ooxF67tvu117rn56f0ESRlwQL3YcPCe9at6z5ypPv8+e5XXBGO+bTT3L/4Iv1xiCRKyUIqzH/+43744eEk/7vf7XuCLi1BjB/v/tlnJe/v66/dp051Hz7cvXnzsE3duu5nnOE+YYL7p5+Gcrt2uf/1r+EEC+6NGoVf68uXJx77hg3u11wTTt5Nm7qPGxcSSUXavdv9H/9w/9GPQpxNmrjfeqt7Xt6+5SZNcm/QwL1NG/eZMys2BpFUKVlIhdq0yf2ss8K/mJEjv58gTjih7ARRmoKCcOK89Vb3I47wfZqrDj88PG/b1v2BB9y/+ir1+Bcvdh84MOzvkEPcX3qp9FpJMrFPnRqa6cC9dWv3++8PybA0H3zgfthhIXk98ED5YxApLyULqXC7drn/6lf7JohHH3Vfs6bi3mP58tBsdeqp7iee6D5lyt7+iIrw1lvuXbqE+E8+2f2990Ii3L498RP3d9+5P/bY3n6RH/7Q/amn3LdtS2z7zZvdhw4N2w4aVL4kKFJeiSYLjYaSpC1ZAvvvH0ZxVUeFhfD738OvfhVGK8WqVw8aNID69cOj6Hnsso8+gvx8OPZYuO02GDw4+TnF3OG3v4Vbbgkj4l54IVy7I1LZNHRWJI7Nm8NJ+ptvYPv2cP3Mjh3xnx94INxwA/TtW/7hzv/+d5hmZcMGmDAhDP+tKkOopWbQrLMicTRtCldemdkYjj8eFi6EESPgqqvg3XfhiSdgv/0yG5dIcTV+IkGRTGvZMlyFPnZsmFqmTx9YujTTUYnsS8lCpAqoVSv0obz9dugPOeYY+PnPQ61DpCpQshCpQvr3Dwli0KAwzUqvXmGqkN/+FjZuzGxsu3aFCTB37MhsHJIZ6uAWqaK++gr+8heYPDncAbFevTDJ4+WXh6RSu3bFv6d7GCH28cfhsWzZ3ucrVoREYQZt2kCHDtCx4/f/tm8f5kGT6kGjoUSyyAcfhKTxzDOhhtGuXZjs8bLLkr/3yM6dYQr8devCnSKLkkHRY9OmvWXr1g37/+EPw6NDh/D+q1eHKeFXrw73XSl+l8Yf/CCU7dABuncPTWpNm5bzQ6gBli6Fl18OQ6rr16+c91SyEMlCO3bA66/DpEkwbVqYzbhfv1DbOOOMUBtZty7c/bHoEft63bqS7/7Yvv3ehBD76NAh/lTvRc1TRcmj+N/ly8O9YMaOhf/6L00dX5q33oILLghDuc85B55/vnI+KyULkSz3+edhmvZJk0ITUUkaNAhNRq1b7/0b++jYEQ49tHxT0MezYAHcdBPMmhXuzfLww/DjH6fv/aobd/jd7+DGG+Hoo0NT4913h1rjH/6Q/AWfydJ1FiJZrm1b+O//htGjwz08PvggNP8UJYI2bcINujJ9kV+vXvDPf4bb9f7iFzBgQHg8/HC4jW9NVlAQLvB8/PEwqGHKlHAjtFq14M47oVmzcF+WTH+HgOaGEpHKs317mPm3adMwmeI114SZgZO1e3e4T8nDD7v37x+mov+v/6pe82x9/XWYaRncf/GLfe91snu3+w03hHVjx6Y3DjSRoIhUVfn54R4jRVPHP/RQ/Knjt2xxf+0196uucu/QwfdMaNmli/v557vXquV+4IHuzz1X9WfzXbEizLJcp477H/5Qcpldu9wvuywc44QJ6YtFyUJEqrzFi8PdDCHM4vvii3tP9MVrD/XqhXL77ec+eLD7E0+4r1q1d1/z5++dLv6ss5K/u2NlmTnTvUWLcLfHeDcUKyhwP+eccExPP52eeJQsRKTamDYt3MME3E86qeTawy23hJtMlVUDKSgIyaVRo5BUHn204m5lu3JluNlXeWotkyeHG30dfnjiN/Hats399NNDLeyVV1J/79IoWYhItVJQEGoLrVq5N25ccu0hUZ9+6j5gQDjDHXNMuC1vKj75JNzQqqjGAu7t2oVb/P75z+Ge8InYtcv9ttvC9v37J9+38s037n36uNev7/7OO0kfRpkSTRYaOisiVUphYTgt161bvv24w3PPhdFGGzfCrbfCr38d/+ry1avD1PXPPw9z54Zlxx4bppLfbz+YPj08iq5X6dIFTj89XFV/yinfv/jwu+/CrMKvvBJmFp4wIbVj++orOPnkEN8774T5wypCokNn0/prHxgALANWAKNLWH8zsATIBaYDHaLlPYB/A4ujdRfEey/VLESkJBs3ul9+ue/pF3n77e+X+eyzcIfGY4/dW4PIyXF/8MG994SPtWtX6CN58MFw7/WGDcM2tWu7H3ec+5gx7jNmhI7snj1D5/tvflP+jvfPPw/HcMABob+nIpDpZiigNvAJ0BmoB3wAdClW5lSgUfT8auC56PkPgcOi5wcB64BmZb2fkoWIlOWdd8L9z8H94ovdP/wwnMBPOGFvgujZ0/2++8JJPhnbt4fkcMcdIVnUrr13n02auL/xRsUdxyefuLdp437QQaEfpbwSTRZpa4Yys+OBu9z9x9Hr/45qMveVUr4n8Dt3P7GEdR8AQ919eWnvp2YoEYln+3a49164//6981l16xaamM47L0xxUhE2b4aZM8MEkOefH65cr0gffRSapA44INwwq3Xr1PeV8ek+zGwoMMDdr4xeXwwc6+7XlVL+d8AX7n5PseV9gD8BXd19d7F1o4BRAO3bt++9evXqij8QEck6ixeHfoczzoAjj8x0NKmZMyf0lXTuHBJT8+ap7SfRZJHOWUdKukC9xMxkZiOAHOChYsvbAH8GRhZPFADuPtHdc9w9p1WrVhUQsojUBF27wvXXV99EAaHT/dVXwzTyZ50VJnRMp3TODZUHHBzzuh2wtnghM+sPjAFOcfcdMcv3B/4G3OHu76UxThGRaun002HqVNi6NT33N4mVzmQxFzjMzDoBnwMXAhfFFoj6KZ4kNFdtiFleD3gZeNrdX0hjjCIi1do551TO+6StGcrdC4HrgGnAUuB5d19sZmPNbFBU7CGgMfCCmS0ys9ei5ecDJwOXRcsXmVmPdMUqIiJl00V5IiI1WFXo4BYRkSyhZCEiInEpWYiISFxKFiIiEpeShYiIxKVkISIicSlZiIhIXEoWIiISl5KFiIjEpWQhIiJxKVmIiEhcShYiIhKXkoWIiMSlZCEiInEpWYiISFxKFiIiEpeShYiIxKVkISIicSlZiIhIXEoWIiISl5KFiIjEpWQhIiJxKVmIiEhcShYiIhKXkoWIiMSlZCEiInEpWYiISFxKFiIiEldak4WZDTCzZWa2wsxGl7D+ZjNbYma5ZjbdzDrErHvLzDaZ2V/TGaOIiMSXtmRhZrWBx4CBQBdgmJl1KVZsIZDj7t2AF4EHY9Y9BFycrvhERCRx6axZ9AFWuPtKd98JTAUGxxZw9xnuvjV6+R7QLmbddODbNMYnIiIJSmeyaAusiXmdFy0rzRXAm8m8gZmNMrN5ZjYvPz8/hRBFRCQR6UwWVsIyL7Gg2Qggh9D0lDB3n+juOe6e06pVqxRCFBGRRNRJ477zgINjXrcD1hYvZGb9gTHAKe6+I43xiIhIitJZs5gLHGZmncysHnAh8FpsATPrCTwJDHL3DWmMRUREyiFtycLdC4HrgGnAUuB5d19sZmPNbFBU7CGgMfCCmS0ysz3JxMxmAy8Ap5tZnpn9OF2xiohI2dLZDIW7vwG8UWzZr2Oe9y9j25PSGJqIiCRBV3CLiEhcShYiIhKXkoWIiMSlZCEiInEpWYiISFxKFiIiEpeShYiIxKVkISIicSlZiIhIXEoWIiISl5KFiIjEpWQhIiJxKVmIiEhcShYiIhKXkoWIiMSlZCEiInEpWYiISFwJJQszO8TM6kfP+5nZ9WbWLL2hiYhIVZFozeIlYJeZHQr8AegEPJu2qEREpEpJNFnsdvdC4BzgUXe/CWiTvrBERKQqSTRZFJjZMOBS4K/RsrrpCUlERKqaRJPFSOB44F53/9TMOgHPpC8sERGpSuokUsjdlwDXA5hZc6CJu9+fzsBERKTqSHQ01D/NbH8zOwD4AJhsZo+kNzQREakqEm2Gauru3wDnApPdvTfQP31hiYhIVZJosqhjZm2A89nbwS0iIjVEosliLDAN+MTd55pZZ2B5+sISEZGqJNEO7heAF2JerwR+mq6gRESkakm0g7udmb1sZhvMbL2ZvWRm7dIdnIiIVA2JNkNNBl4DDgLaAq9Hy8pkZgPMbJmZrTCz0SWsv9nMlphZrplNN7MOMesuNbPl0ePSBOMUEZE0SDRZtHL3ye5eGD3+CLQqawMzqw08BgwEugDDzKxLsWILgRx37wa8CDwYbXsAcCdwLNAHuDO6vkNERDIg0WTxpZmNMLPa0WMEsDHONn2AFe6+0t13AlOBwbEF3H2Gu2+NXr4HFDVt/Rj4u7t/5e5fA38HBiQYq4iIVLBEk8XlhGGzXwDrgKGEKUDK0hZYE/M6L1pWmiuAN5PZ1sxGmdk8M5uXn58fJxwREUlVQsnC3T9z90Hu3srdf+DuQwgX6JXFStpViQVDTSUHeCiZbd19orvnuHtOq1ZltoqJiEg5lOdOeTfHWZ8HHBzzuh2wtnghM+sPjAEGufuOZLYVEZHKUZ5kUdKv/1hzgcPMrJOZ1QMuJIyo2rsDs57Ak4REsSFm1TTgR2bWPOrY/lG0TEREMiChi/JKUWKT0p6V7oVmdh3hJF8bmOTui81sLDDP3V8jNDs1Bl4wM4Ci5q6vzOx/CAkHYKy7f1WOWEVEpBzMvfRzvpl9S8lJwYCG7l6eZFOhcnJyfN68eZkOQ0SkWjGz+e6eE69cmSd7d29ScSGJiEh1VZ4+CxERqSGULEREJC4liyRNmQIdO0KtWuHvlCmZjkhEJP2qTAd1dTBlCowaBVujCUpWrw6vAYYPz1xcIiLppppFEsaM2ZsoimzdGpaLiGQzJYskfPZZcstFRLKFkkUS2rdPbrmISLZQskjCvfdCo0b7LmvUKCwXEclmNT5ZJDO6afhwmDgROnQAs/B34kR1botI9qvRo6FSGd00fLiSg4jUPDW6ZqHRTSIiianRyUKjm0REElOjk4VGN4mIJKZGJ4vKGt2kKUJEpLqr0cmiMkY3FXWir14N7ns70ZUwRKQ6KfPmR9VJVb35UceOIUEU16EDrFpV2dGIiOwr0Zsf1eiaRWVQJ7qIZAMlizRTJ7qIZAMlizTTFCEikg2ULNIs1U50jaASkaqkRk/3UVmSnSJEN1kSkapGNYsqKJVpSFQTEZF0Us2iCkp2BJVqIiKSbqpZVEHJjqDShIgikm5KFlVQsiOoUr2WQ01XIpIoJYsqKNkRVKlcy6FpSEQkGZruIwsU77OAUBMpK8FoGhIRgSoy3YeZDTCzZWa2wsxGl7D+ZDNbYGaFZja02LoHzOyj6HFBOuOs7lK5lkPTkIhIMtI2GsrMagOPAWcAecBcM3vN3ZfEFPsMuAy4tdi2ZwK9gB5AfWCmmb3p7t+kK97qLtlrOdq3L7lmoWlIRKQk6axZ9AFWuPtKd98JTAUGxxZw91XungvsLrZtF2Cmuxe6+3fAB8CANMZa42gaEhFJRjqTRVtgTczrvGhZIj4ABppZIzNrCZwKHFy8kJmNMrN5ZjYvPz+/3AHXJJqGRESSkc6L8qyEZQn1prv722Z2DPAvIB/4N1BYQrmJwEQIHdyph1ozaRoSEUlUOmsWeexbG2gHrE10Y3e/1917uPsZhMSzvILjkyTp4j+RmiudyWIucJiZdTKzesCFwGuJbGhmtc2sRfS8G9ANeDttkUpCNIJKpOZKW7Jw90LgOmAasBR43t0Xm9lYMxsEYGbHmFkecB7wpJktjjavC8w2syWEZqYR0f4kg1K9+E99HCLVX1onEnT3N4A3ii37dczzuYTmqeLbbSeMiJIq5N57S774r7QRVOrjEMkemu5DEpbsCCr1cYhkD033IWlTq1aYd6o4M9hd/MoaEcmIKjHdh9RsqfRxgPo5RKoiJQtJm1SuEtdsuCJVk5KFpE0qV4mn2s+h2ohIeqnPQqqUVPo5UpmiXUQC9VlItZRKP0cqtRHVRESSo2QhVUoq/RzJXlmufhGR5ClZSJWSSj9HsrURXf8hkjwlC6lyhg8Pt3bdvTv8jdfvkGxtRHNciSRPyUKqvWRrI7r+QyR5ShaSFZKpjej6D5HkKVlIjVOZ13+IZAslC6mRku0XSbWfQ01Xki2ULEQSkOq9PNR0JdlCyUIkAan0c2jqEskmShYiCUilnyOVpivVRqSq0txQImnSsWM42RfXoUPoJ6mobUTKQ3NDiWRYZUxdAmq2ksqhZCGSJpUxdYmaraSyqBlKpApJdrp1NVtJeSXaDFWnMoLJlIKCAvLy8ti+fXumQ5EENGjQgHbt2lG3bt1Mh5IxRQlhzJjQ9NS+fWi2Kq02onmupLJkdbLIy8ujSZMmdOzYETPLdDhSBndn48aN5OXl0alTp0yHk1HDhyd+06b27UuuWSQyz1WiCUkEsrzPYvv27bRo0UKJohowM1q0aKFaYJIqc54rdaTXbFmdLAAlimpE31XyKmueK3WkS9YnC5FsVxnzXGkiRVGyiFHR1eyNGzfSo0cPevToQevWrWnbtu2e1zt37kxoHyNHjmTZsmVllnnssceYUkE/8fr27cuiRYsqZF9SNaUyz5U60iWrO7iTUXzIYlE1G1Lv+GvRosWeE+9dd91F48aNufXWW/cp4+64O7VqlZy3J0+eHPd9rr322tQClBrp3ntLHp5bVj9Hqh3pkj3SWrMwswFmtszMVpjZ6BLWn2xmC8ys0MyGFlv3oJktNrOlZjbB0tygXZnV7BUrVnDUUUdx1VVX0atXL9atW8eoUaPIycmha9eujB07dk/Zol/6hYWFNGvWjNGjR9O9e3eOP/54NmzYAMAdd9zBo48+uqf86NGj6dOnD4cffjj/+te/APjuu+/46U9/Svfu3Rk2bBg5OTlxaxDPPPMMRx99NEcddRS33347AIWFhVx88cV7lk+YMAGA8ePH06VLF7p3786IESMq/DOTipNKP0cqHempUCd6FVb0y7aiH0Bt4BOgM1AP+ADoUqxMR6Ab8DQwNGb5CcD/RfuoDfwb6FfW+/Xu3duLW7JkyfeWlcZwuXDPAAAPAklEQVTMPXTd7fswS3gXZbrzzjv9oYcecnf35cuXu5n5+++/v2f9xo0b3d29oKDA+/bt64sXL3Z39xNPPNEXLlzoBQUFDvgbb7zh7u433XST33fffe7uPmbMGB8/fvye8r/85S/d3f3VV1/1H//4x+7uft999/k111zj7u6LFi3yWrVq+cKFC78XZ9H7rVmzxjt06OD5+fm+c+dOP/nkk/3111/39957zwcMGLCn/Ndff+3u7q1bt/YdO3bssywVyXxnUrmeeca9Q4fwf6JDh/C6oss3arTv/79GjeJvJ+UDzPMEzunprFn0AVa4+0p33wlMBQYXS1Sr3D0X2F1sWwcaEJJMfaAusD6NsaZ8X+ZUHXLIIRxzzDF7Xv/lL3+hV69e9OrVi6VLl7JkyZLvbdOwYUMGDhwIQO/evVlVyiW655577vfKvPvuu1x44YUAdO/ena5du5YZ35w5czjttNNo2bIldevW5aKLLmLWrFkceuihLFu2jBtuuIFp06bRtGlTALp27cqIESOYMmVKjb6oLpsl05GeyugpdaJXbelMFm2BNTGv86Jlcbn7v4EZwLroMc3dl1Z4hDEqq5pdZL/99tvzfPny5fzmN7/hnXfeITc3lwEDBpR4vUG9evX2PK9duzaFhYUl7rt+/frfK+NJTutSWvkWLVqQm5tL3759mTBhAj/72c8AmDZtGldddRXvv/8+OTk57Nq1K6n3k+ySyolfdyOs2tKZLErqY0jojGVmhwJHAu0ICeY0Mzu5hHKjzGyemc3Lz88vV7CptONWlG+++YYmTZqw//77s27dOqZNm1bh79G3b1+ef/55AD788MMSay6xjjvuOGbMmMHGjRspLCxk6tSpnHLKKeTn5+PunHfeedx9990sWLCAXbt2kZeXx2mnncZDDz1Efn4+W4ufKaRGSeXEr7sRVm3pHA2VBxwc87odsDbBbc8B3nP3LQBm9iZwHDArtpC7TwQmQphIsLwBJzPNQkXq1asXXbp04aijjqJz586ceOKJFf4eP//5z7nkkkvo1q0bvXr14qijjtrThFSSdu3aMXbsWPr164e7c/bZZ3PmmWeyYMECrrjiCtwdM+OBBx6gsLCQiy66iG+//Zbdu3dz22230aRJkwo/Bqk+Uhk9lcoorbJqMJq+pIIl0rGRyoOQiFYCndjbwd21lLJ/ZN8O7guAf0T7qAtMB84u6/3K28Gd7QoKCnzbtm3u7v7xxx97x44dvaCgIMNRfZ++s+yQamd1sp3iqQxMSfY9sh0JdnCnrWbh7oVmdh0wjTCiaZK7LzazsVFwr5nZMcDLQHPgbDO72927Ai8CpwEfEpqu3nL319MVa02wZcsWTj/9dAoLC3F3nnzySerU0WU2kh7Jzp4bu10yNYJkazDpuJ6qpsjq+1ksXbqUI488MkMRSSr0nUkydP+P8tNtVUUk6yU7MEXTlqRO7RAiUq1Vxv0/RDULEalBUr2eStdyKFmISA2SyvVUupYjULJIo379+n3vArtHH32Ua665psztGjduDMDatWsZOnRoiWX69etH8Q794h599NF9Lo77yU9+wqZNmxIJvUx33XUX48aNK/d+RDIh2ft/pDoNSbK1kapee1GySKNhw4YxderUfZZNnTqVYcOGJbT9QQcdxIsvvpjy+xdPFm+88QbNmjVLeX8iNVEqneLJ1kaqQ+2lxnRw33gjVPQ9fXr0gGhm8BINHTqUO+64gx07dlC/fn1WrVrF2rVr6du3L1u2bGHw4MF8/fXXFBQUcM899zB48D7zLLJq1SrOOussPvroI7Zt28bIkSNZsmQJRx55JNu2bdtT7uqrr2bu3Lls27aNoUOHcvfddzNhwgTWrl3LqaeeSsuWLZkxYwYdO3Zk3rx5tGzZkkceeYRJkyYBcOWVV3LjjTeyatUqBg4cSN++ffnXv/5F27ZtefXVV2nYsGGpx7ho0SKuuuoqtm7dyiGHHMKkSZNo3rw5EyZM4IknnqBOnTp06dKFqVOnMnPmTG644QYg3EJ11qxZutJbqrxUOsWTvbK8OlyJrppFGrVo0YI+ffrw1ltvAaFWccEFF2BmNGjQgJdffpkFCxYwY8YMbrnlljIn+3v88cdp1KgRubm5jBkzhvnz5+9Zd++99zJv3jxyc3OZOXMmubm5XH/99Rx00EHMmDGDGTNm7LOv+fPnM3nyZObMmcN7773HU089xcKFC4EwqeG1117L4sWLadasGS+99FKZx3jJJZfwwAMPkJuby9FHH83dd98NwP3338/ChQvJzc3liSeeAGDcuHE89thjLFq0iNmzZ5eZhESqilQ6xZOtjVSHIb01pmZRVg0gnYqaogYPHszUqVP3/Jp3d26//XZmzZpFrVq1+Pzzz1m/fj2tW7cucT+zZs3i+uuvB6Bbt25069Ztz7rnn3+eiRMnUlhYyLp161iyZMk+64t79913Oeecc/bMfHvuuecye/ZsBg0aRKdOnejRowdQ9jToAJs3b2bTpk2ccsopAFx66aWcd955e2IcPnw4Q4YMYciQIQCceOKJ3HzzzQwfPpxzzz2Xdu3aJfIRimRUKlejJ1sbqQ5DelWzSLMhQ4Ywffp0FixYwLZt2+jVqxcAU6ZMIT8/n/nz57No0SIOPPDAEqclj1XSzQI//fRTxo0bx/Tp08nNzeXMM8+Mu5+yajBF05tD2dOgx/O3v/2Na6+9lvnz59O7d28KCwsZPXo0v//979m2bRvHHXcc//nPf1Lat0hlS7ZTPNnaSHUY0qtkkWaNGzemX79+XH755ft0bG/evJkf/OAH1K1blxkzZrC6pJ8VMU4++WSmRP8SPvroI3Jzc4Ewvfl+++1H06ZNWb9+PW+++eaebZo0acK3335b4r5eeeUVtm7dynfffcfLL7/MSSedlPSxNW3alObNmzN79mwA/vznP3PKKaewe/du1qxZw6mnnsqDDz7Ipk2b2LJlC5988glHH300t912Gzk5OUoWkrWSHaJbHYb01phmqEwaNmwY55577j4jo4YPH87ZZ59NTk4OPXr04IgjjihzH1dffTUjR46kW7du9OjRgz59+gDhrnc9e/aka9eu35vefNSoUQwcOJA2bdrs02/Rq1cvLrvssj37uPLKK+nZs2eZTU6l+dOf/rSng7tz585MnjyZXbt2MWLECDZv3oy7c9NNN9GsWTN+9atfMWPGDGrXrk2XLl323PVPJBslOylisuUru1NcEwlKlaLvTCQxtWqFGkVxZqG5LFGaSFBEJIulcmfB8lCyEBGphlLtFE9V1ieLbGlmqwn0XYkkLpVO8fLI6g7uBg0asHHjRlq0aFHisFOpOtydjRs30qBBg0yHIlJtJNspXh5ZnSzatWtHXl4e+fn5mQ5FEtCgQQNdqCdSRWV1sqhbty6dOnXKdBgiItVe1vdZiIhI+SlZiIhIXEoWIiISV9ZcwW1m+UDRBEstgS8zGE4m1eRjh5p9/DX52KFmH395jr2Du7eKVyhrkkUsM5uXyOXr2agmHzvU7OOvyccONfv4K+PY1QwlIiJxKVmIiEhc2ZosJmY6gAyqyccONfv4a/KxQ80+/rQfe1b2WYiISMXK1pqFiIhUICULERGJK6uShZkNMLNlZrbCzEZnOp7KZmarzOxDM1tkZvPib1G9mdkkM9tgZh/FLDvAzP5uZsujv80zGWO6lHLsd5nZ59H3v8jMfpLJGNPFzA42sxlmttTMFpvZDdHyrP/uyzj2tH/3WdNnYWa1gY+BM4A8YC4wzN2XZDSwSmRmq4Acd68RFyaZ2cnAFuBpdz8qWvYg8JW73x/9YGju7rdlMs50KOXY7wK2uPu4TMaWbmbWBmjj7gvMrAkwHxgCXEaWf/dlHPv5pPm7z6aaRR9ghbuvdPedwFRgcIZjkjRy91nAV8UWDwb+FD3/E+E/UtYp5dhrBHdf5+4LouffAkuBttSA776MY0+7bEoWbYE1Ma/zqKQPsQpx4G0zm29mozIdTIYc6O7rIPzHAn6Q4Xgq23Vmlhs1U2VdM0xxZtYR6AnMoYZ998WOHdL83WdTsijpVnjZ0caWuBPdvRcwELg2aqqQmuNx4BCgB7AOeDiz4aSXmTUGXgJudPdvMh1PZSrh2NP+3WdTssgDDo553Q5Ym6FYMsLd10Z/NwAvE5rmapr1UbtuUfvuhgzHU2ncfb2773L33cBTZPH3b2Z1CSfLKe7+v9HiGvHdl3TslfHdZ1OymAscZmadzKwecCHwWoZjqjRmtl/U4YWZ7Qf8CPio7K2y0mvApdHzS4FXMxhLpSo6UUbOIUu/fzMz4A/AUnd/JGZV1n/3pR17ZXz3WTMaCiAaLvYoUBuY5O73ZjikSmNmnQm1CQi3y30224/fzP4C9CNMz7weuBN4BXgeaA98Bpzn7lnXEVzKsfcjNEM4sAr4WVEbfjYxs77AbOBDYHe0+HZC231Wf/dlHPsw0vzdZ1WyEBGR9MimZigREUkTJQsREYlLyUJEROJSshARkbiULEREJC4lC5E4zGxXzGyeiypyRmMz6xg7c6xIVVUn0wGIVAPb3L1HpoMQySTVLERSFN0/5AEzez96HBot72Bm06NJ3aabWfto+YFm9rKZfRA9Toh2VdvMnoruT/C2mTWMyl9vZkui/UzN0GGKAEoWIoloWKwZ6oKYdd+4ex/gd4TZA4ieP+3u3YApwIRo+QRgprt3B3oBi6PlhwGPuXtXYBPw02j5aKBntJ+r0nVwIonQFdwicZjZFndvXMLyVcBp7r4ymtztC3dvYWZfEm5QUxAtX+fuLc0sH2jn7jti9tER+Lu7Hxa9vg2o6+73mNlbhBscvQK84u5b0nyoIqVSzUKkfLyU56WVKcmOmOe72NuXeCbwGNAbmG9m6mOUjFGyECmfC2L+/jt6/i/CrMcAw4F3o+fTgash3AbYzPYvbadmVgs42N1nAL8EmgHfq92IVBb9UhGJr6GZLYp5/Za7Fw2frW9mcwg/vIZFy64HJpnZL4B8YGS0/AZgopldQahBXE24UU1JagPPmFlTwo29xrv7pgo7IpEkqc9CJEVRn0WOu3+Z6VhE0k3NUCIiEpdqFiIiEpdqFiIiEpeShYiIxKVkISIicSlZiIhIXEoWIiIS1/8HTzLhrOOTQYkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1ce31197668>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "acc = history.history['binary_accuracy']\n",
    "val_acc = history.history['val_binary_accuracy']\n",
    "\n",
    "loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    "\n",
    "epochs = range(1, len(acc) + 1)\n",
    "\n",
    "plt.plot(epochs, acc, 'bo', label='Training acc')\n",
    "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Accuracy\")\n",
    "plt.legend()\n",
    "\n",
    "plt.figure()\n",
    "\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.legend()\n",
    "plt.show();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000/1000 [==============================] - 5s 5ms/step\n"
     ]
    }
   ],
   "source": [
    "values = model.evaluate(x=[testSetOne, testSetTwo], y=testLabels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 0.21182597386837004\n"
     ]
    }
   ],
   "source": [
    "print(\"Loss: \" + str(values[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.686\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: \" + str(values[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save the model and tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"QQPairs.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('QQTokenizer.pickle', 'wb') as handle:\n",
    "    pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL, fix_imports=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.0"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}