models.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.4"
    },
    "colab": {
      "name": "keras-cnn-with-fasttext-embeddings.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "GffWHGGdpgWP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#train.csv - id, tweet_text, labels one hot encoded\n",
        "#test.csv - id, tweet_text\n",
        "#test_labels.csv - id, labels one hot encoded -- of test texts"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fZqjgsbhEPrf",
        "colab_type": "code",
        "outputId": "20b296a0-cde4-426a-d482-f4f50b275651",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 120
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mBNliZZPETJ5",
        "colab_type": "code",
        "outputId": "f99c4b34-4c06-4437-e782-67d1d2b97636",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "%cd drive/My Drive/EmotionDetection/Classification_system/"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/EmotionDetection/Classification_system\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "_cell_guid": "29a8d31b-451e-4427-afb3-fd40cd01bc60",
        "_uuid": "887320b9e2d8efaaa27c3f7913debaf53c37ed52",
        "trusted": true,
        "id": "1tYkdoTrpaiS",
        "colab_type": "code",
        "outputId": "da245212-6a6c-4643-fff9-fe5defbfe16e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        }
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import keras\n",
        "from keras import optimizers, Model\n",
        "from keras import backend as K\n",
        "from keras import regularizers\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Activation, Dropout, Flatten, SpatialDropout1D, LSTM, Bidirectional, TimeDistributed, Input\n",
        "from keras.layers import Embedding, Conv1D, MaxPooling1D, GlobalMaxPooling1D \n",
        "from keras.utils import plot_model\n",
        "from keras.preprocessing import sequence\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.callbacks import EarlyStopping\n",
        "\n",
        "from tqdm import tqdm\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import RegexpTokenizer \n",
        "import os, re, csv, math, codecs\n",
        "\n",
        "sns.set_style(\"whitegrid\")\n",
        "#np.random.seed(0)\n",
        "\n",
        "#DATA_PATH = ''\n",
        "#EMBEDDING_DIR = ''\n",
        "\n",
        "MAX_NB_WORDS = 200000\n",
        "tokenizer = RegexpTokenizer(r'\\w+')\n",
        "#stop_words = set(stopwords.words('english'))\n",
        "#stop_words.update(['.', ',', '\"', \"'\", ':', ';', '(', ')', '[', ']', '{', '}'])\n",
        "\n",
        "#from subprocess import check_output\n",
        "#print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n",
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "6d95a418-778d-4683-85f7-33351d82c18d",
        "_uuid": "92b4cf0b1af7f4a423eacc83a1fb43e66400056f",
        "id": "nHnl04ZJpaiW",
        "colab_type": "text"
      },
      "source": [
        "Let's load the data and the embeddings..."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "_cell_guid": "3f6f79a5-ba5b-486d-b212-3332e6c2b6d4",
        "_uuid": "b56846b453212ccc053461ba730f56dacb433c23",
        "trusted": true,
        "id": "e_rDB4-npaiW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#load embeddings\n",
        "#print('loading word embeddings...')\n",
        "#embeddings_index = {}\n",
        "#f = codecs.open('../input/fasttext/wiki.simple.vec', encoding='utf-8')\n",
        "#for line in tqdm(f):\n",
        "#    values = line.rstrip().rsplit(' ')\n",
        "#    word = values[0]\n",
        "#    coefs = np.asarray(values[1:], dtype='float32')\n",
        "#    embeddings_index[word] = coefs\n",
        "#f.close()\n",
        "#print('found %s word vectors' % len(embeddings_index))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QjwUYSIW27IN",
        "colab_type": "code",
        "outputId": "7b4eb52a-fa9b-4a7b-9c60-74729f941cc8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "from gensim.models import FastText\n",
        "embeddings_index = FastText.load(\"../Embeddings/hinglish_plus_english_fasttext_stopwords_present_rarewords_absent.model\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:253: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sAXs-KC49LDr",
        "colab_type": "code",
        "outputId": "843fd1f4-99e1-47d9-dce9-0a452ce833b3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "from gensim.models import Word2Vec\n",
        "embeddings_index = Word2Vec.load(\"../Embeddings/hinglish_plus_english_word2vec_stopwords_present_rarewords_absent.model\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:253: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "_cell_guid": "1ba47ff9-ac70-4a6e-9811-461f6819ca89",
        "_uuid": "eee4b60da112577dbdd4b17b3760bc365e483222",
        "trusted": true,
        "id": "8hRGZDszpaib",
        "colab_type": "code",
        "outputId": "9b5943d6-0b5d-4ede-8204-a742f8d57f0c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 346
        }
      },
      "source": [
        "#load data\n",
        "\n",
        "train_df = pd.read_csv('../Dataset/all_tweets_processed.csv', sep=',', header=0, encoding='ISO-8859-1')\n",
        "test_df = pd.read_csv('../Dataset/all_tweets_processed.csv', sep=',', header=0)\n",
        "test_df = test_df.fillna('_NA_')\n",
        "\n",
        "print(\"num train: \", train_df.shape[0])\n",
        "print(\"num test: \", test_df.shape[0])\n",
        "#print(test_df)\n",
        "#label_names = [\"sarcastic\", \"not_sarcastic\"]\n",
        "label_names = ['anger', 'disgust', 'fear', 'happiness', 'sadness', 'surprise']\n",
        "\n",
        "y_train = train_df[label_names].values\n",
        "y_test = test_df[label_names].values\n",
        "#y_train=y_train[0:4999]\n",
        "#y_test=y_train[:251]\n",
        "#print(len(y_train))\n",
        "#print(len(y_test))\n",
        "#visualize word distribution\n",
        "train_df['doc_len'] = train_df['tweet_text'].astype(str).apply(lambda words: len(words.split(\" \")))\n",
        "#test_df['doc_len'] = test_df['tweet_text'].astype(str).apply(lambda words: len(words.split(\" \")))\n",
        "max_seq_len = np.round(train_df['doc_len'].mean() + train_df['doc_len'].std()).astype(int)\n",
        "#max_seq_len = train_df['doc_len'].max()\n",
        "print(max_seq_len)\n",
        "sns.distplot(train_df['doc_len'], hist=True, kde=True, color='b', label='doc len')\n",
        "plt.axvline(x=max_seq_len, color='k', linestyle='--', label='max len')\n",
        "plt.title('tweet length'); \n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "num train:  149088\n",
            "num test:  149088\n",
            "24\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEXCAYAAACjyo8UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXhTVfrA8W+60tKFliWtUIqssi+lUBwQbalFFoGhdRkGxRmEER1BBZxBxZ8ooE4ZKKgoKDiKigICQpEOtDBFQAZQqIMVRSy20AbZuy8hvz8ODbTpTtK0ue/nefKkubm5eW8I7z1577nn6EwmkwkhhBAOy8neAQghhLAtSfRCCOHgJNELIYSDk0QvhBAOThK9EEI4OEn0Qgjh4CTRC1FL4eHh7Nu3r97fNyMjgy5dulBSUlLv7y0aN0n0osGyR0JdtmwZM2fOrNf3rIy9DijC8UiiF0IIByeJXjRIs2bN4syZM/zlL3+hb9++rFy5kmeffZZVq1YBYDAY6NKlCx999BEAv/76KwMGDODq1asA7Nq1izFjxtC/f38eeOABfvjhB/O2DQYDf/3rXwkLCyM8PJwPPvgAgOTkZN555x2+/PJL+vbty7333lttnFevXmXFihUMGzaMgQMHMn36dC5dugRcL7Vs3LiRO++8k4EDB7J8+XLzawsKCnj22WcJDQ3lnnvuYeXKldxxxx2V7n+pLVu2VLg9ISplEqKBuuuuu0x79+41P163bp1p6tSpJpPJZPriiy9MERERpunTp5uf+8tf/mIymUymY8eOmcLCwkxHjhwxlZSUmD7//HPTXXfdZSosLDQZjUbTuHHjTMuWLTMVFhaafv31V1N4eLgpOTnZZDKZTEuXLjU988wzNY7r/fffN8XExJgyMzNNhYWFphdeeMH01FNPmUwmkyk9Pd3UuXNn03PPPWfKz883paammrp37246ceKEyWQymf7xj3+YJkyYYLp06ZIpMzPTNGrUKNOQIUMq3f/qtidEZaRFLxqNAQMGcPjwYa5evcrBgweZPHky33zzDQAHDx5kwIABAHz66afcf//99O7dG2dnZ8aNG4erqytHjhzhu+++48KFCzzxxBO4ubkRFBTEfffdx7Zt2+oU09q1a3nqqacICAjAzc2NJ554goSEhDInTJ944gmaNGnCbbfdxm233Wb+dfHll18ydepUfH19CQgI4KGHHqrRe1a2PSEq42LvAISoqbZt2+Lh4UFqaiqHDx/m8ccfZ/369Zw8eZKDBw8yceJEAM6cOcOmTZtYs2aN+bXFxcWcPXsWJycnzp49S//+/c3PGY3GMo9r48yZMzz++OM4OV1vMzk5OXH+/Hnz4xYtWpj/9vDwIC8vD4CzZ88SGBhofi4gIKBG71nZ9oSojCR60aiEhoaSkJBAcXExer2e0NBQNm3axOXLl+natSsAgYGB/OUvf+Gxxx6zeP23335LmzZt+Pe//13h9nU6Xa3iCQgIYMGCBYSEhFg8l5GRUeVrW7ZsSVZWFh07dgQgKyurVu8tRE1J6UY0WC1atCA9Pb3MsgEDBrBmzRpzC3zgwIGsWbOGkJAQnJ2dAYiJiWHt2rUcPXoUk8lEXl4eu3fvJicnh169etG0aVNWrFhBQUEBRqORH3/8kZSUFACaN2/O6dOnzSd1q/Pggw+yZMkSTp8+DcCFCxfYuXNnjV57zz338M4773D58mUMBkOZXyCV7b8QdSGJXjRYU6ZMYfny5fTv35/33nsPUC363NxcQkNDAQgJCaGgoKBM6aVnz568/PLLzJs3j9DQUO6++24+//xzAJydnXn77bf54YcfiIiIICwsjOeff56cnBwAhg8fDqgDyLhx46qN8aGHHiI8PJw//elP9O3bl/vuu8980KjO448/TkBAABEREUyaNImoqCjc3Nyq3H8h6kJnMsnEI0I0BB9//DHbtm2zaNkLcbOkRS+EnZw9e9bci+jkyZOsXr2aYcOG2Tss4YDkZKwQdlJcXMyLL75IRkYG3t7ejBw5kj/84Q/2Dks4ICndCCGEg5PSjRBCOLgGV7o5cuQI7u7uVt1mYWGh1bfZ2NT2Mzh+/DgAXbp0sVVI9Urr3wGt7z84/mdQWFhInz59KnyuwSV6d3d384Uv1pKammr1bTY2tf0MSi822r17t40iql9a/w5off/B8T+D1NTUSp+T0o0QQji4BteiFw3D888/b+8QhBBWIoleVEj6cwvhOCTRiwodOXIEoNKTO0JYU3FxMRkZGRQUFNj0PaqqYzcWTZo0oU2bNri6utb4NZLoRYVmzJgBOM7JWNGwlV401q5du1qPIFpT+fn5eHh42GTb9cVkMnH+/HkyMjK49dZba/w6ORkrhLC7goICmjdvbrMk7yh0Oh3Nmzev9S+fGiX65ORkoqKiiIyMZMWKFRbPFxUVMWPGDCIjI4mJiTGPw52RkUGvXr0YM2YMY8aMYe7cubUKTgihHZLka6Yun1O1pRuj0ci8efNYvXo1er2e6OhowsPDzZMlAKxbtw4fHx927NhBfHw8sbGxLFmyBFCzAm3evLnWgQkhhLCOalv0KSkpBAcHExQUhJubGyNHjiQxMbHMOklJSeaxu6Oioti/fz8yhI4QQlx34MABpk6dapf3rrZFbzAYysxlqdfrLSZWMBgM5rkvXVxc8Pb25uLFi4Aq34wdOxYvLy9mzJhR7dychYWFVj8zXlBQcBPb9CMvz/Jj8vQsAS7eVFz1qbafwZQpU4Cqr7ZrTG7uO9D4NfT9Ly4uJj8/36bvYTKZbP4eVSksLMRoNFolhtr2ILJpr5tWrVqxa9cu/Pz8+N///sfjjz9OfHw8Xl5elb6moQ2BcOoU7NljuTwqCoKDazaZc0NQ28/A0S4Vd/TL36vT0Pc/NTXV5j1iqup1k5GRweTJk+nTpw/ffvstPXr0YPz48SxdupQLFy4QGxtLr169SElJYf78+RQWFtKkSRMWLFhA+/btef/99zl+/DgLFy7k+PHjPPPMM6xbt67M+7m7u+Ps7Gye0P3ll1/mp59+oqSkhCeeeIJhw4bx+eefk5SURH5+Punp6QwbNozZs2dbxOvq6mrx71lV4q820ev1+jKTFhsMBvR6vcU6mZmZBAQEUFJSQnZ2Nn5+fuh0OvPUaD169KBt27b88ssv9OzZs7q3FXa2b98+AG6//XY7RyK06M4777RYdt999zFt2jTy8vIYMWKExfOTJk1i0qRJnDt3jujo6DLP1aSb8K+//kpcXBwLFiwgOjqaLVu28Mknn5CYmMjbb7/NW2+9Rfv27fnoo49wcXFh3759LF68mGXLlvHQQw8xceJEduzYwfLly3nppZeqPHC9/fbbhIWFsXDhQq5cuUJMTIz5/1pqaiqbNm3Czc2N4cOHM3HiRHPFpK6qTfQ9e/YkLS2N9PR09Ho98fHxLFq0qMw64eHhbNy4kb59+5KQkEBYWBg6nY4LFy7g6+uLs7Mz6enppKWlERQUdFMBi/oxZ84cQPrRC+1o06aNebTWjh07MmjQIHQ6HV26dDFP/p6dnc2zzz7LqVOn0Ol0FBcXA+Dk5MSrr77Kvffey/33309ISEiV7/XVV1+RlJTEqlWrAFXWyczMBGDQoEF4e3sD0KFDB06fPm37RO/i4sLcuXOZPHkyRqOR8ePH06lTJ+Li4ujRowcRERFER0cza9YsIiMj8fX1ZfHixQAcPHiQpUuX4uLigpOTEy+99BLNmjW7qYCFEI6vqgaGp6dnlc+3aNGiTg2UGydmd3JyMj/W6XQYjUYA4uLiGDhwIG+++SYZGRk89NBD5tekpaXh6enJ2bNna/R+S5cupX379mWWHT16tEwczs7O5ve+GTWq0Q8dOpShQ4eWWTZ9+nTz3+7u7ixdutTidVFRUURFRd1kiPXj4kW4csVyuR3P3QghGpjs7Gxz6Xrjxo1llr/yyiusWbOGl19+me3btzN8+PBKtzN48GDWrFnDCy+8gE6n4/vvv6dbt242i1uujL3myhVISLC8FRXZOzIhREMxefJk/vnPfzJ27FhKSkrMyxcsWMCECRO49dZbmT9/PosWLeL8+fOVbmfatGmUlJRw7733MnLkSOLi4mwad4ObM9YWvQNqss1Tp1RiLy8sDL7+2nK56nVjpQDrQW0/19KTYY5So2/ovU5sraHvf33E5whj3ZSq6POq6jOUQc3qqKREHRxu5OMDfn72icfaSq9sFkI0fpLo6yg317KlHxXlOIlehicWwnFIjV5UaOfOnezcudPeYQghrEBa9KJCr7zyCiAzTQnhCKRFL4QQDk4SvRBCODgp3QghGpzKLmCsKx8faNKk5usvW7YMT09P/vznP9/U+/bt25dvv/32prZhDZLohRANTukFjNYSFVW7RO9oJNGLCr3zzjv2DkGIerV8+XI2bdqEv78/gYGBdO/eHVAXIr344ovk5+fTtm1bFixYgK+vL6dOneLFF1/kwoULODs7ExcXR9u2bSvd/rvvvsuXX35JUVERkZGRPPnkk2RkZPDoo48SEhLCt99+i16v56233qKJlY9KUqMXFerSpYt5JD8hHN3//vc/tm3bxqZNm1i5ciXfffed+bnZs2czc+ZMtmzZQufOnXnjjTcAmDlzJhMmTOCLL75g7dq1tGzZstLtf/XVV5w6dYr169ezefNmjh07xsGDBwE4deoUEyZMID4+Hm9vbxKs+VPmGmnRV+PNN+HAAZg2zd6R1K8tW7YAMHr0aDtHIoTtHTp0iGHDhpmHSAgPDwfUYGXZ2dkMGDAAgHHjxjF9+nRycnIwGAxERkYCamDHquzdu5e9e/cyduxYAPLy8khLSyMwMJA2bdqYhy7o3r27eUhka5JEX4ULF2D1aiguBoMBys234tBK5xyQRC/EzTOZTEyZMoUHHnigzPKMjAyLYYkLCwut/v5SuqnC9u1QOuTboUP2jUUIYTuhoaHs3LmTgoICcnJy2LVrFwDe3t74+Phw6FoC2Lx5M6GhoXh5eREQEGC+eryoqKjKuWAHDx7Mhg0byM3NBdRMfVWNbmlt0qKvxIULsHcvjB0LR46oRD9yZN22VVlXMUcaBE0Ia/LxUT1lrLm9qnTv3p0RI0YwZswY/P39y0x3+tprr5lPxgYFBbFw4UIAXn/9debOnUtcXByurq7ExcVVOoPe4MGD+fnnn80tek9PT/7xj3/g5FQ/bW1J9JUobc3/+c/wr3/B2rVw5gzcckvtt1VZVzFHGgRNCGvy87P+/43qJhF67LHHeOyxxyyWd+3alc8++8xiebt27fjggw+q3OaNfegffvhhHn74YYt1tm7dav77ZvvtV0ZKNxUoKFCt+UGDVGIPCQGdTso3QojGSVr0FTh9Wo0336uXeuzjA507q0Q/erRK+qVMprKPHcWHH35o7xCEEFYiLfoKZGSo+xvLbf37q543P/98fdl//gPPPQc1nAu4UQkKCqq03iiELTSwye4arLp8TpLoK5CeDp6eZWuEoaHQrBmsWaO6W6anw7p1cP48vP22480t++mnn/Lpp5/aOwyhEU2aNOH8+fOS7KthMpk4f/58ra+cldJNBTIyoE2bsiUZDw+YOBGWLYOtW1WvHGdnePBB+PBDdQAYNcp+MVvb8uXLAbj//vvtHInQgjZt2pCRkcFvv/1ms/coLi7G1dXVZtuvL02aNKFNmza1eo0k+nKuXlWJ/o47LJ/r0QNuv131yAGYMAF+9zu4dAm++ALi4+Hxx28+hoq6Y0pXTOHIXF1dufXWW236Hg19gnRbkkRfztmzqjRTWXk6JgaOH4eOHWHwYLXsnnsgMRH27LFOoq+oO2ZEhPTFF0LUjST6ctLT1X1lv4w8PWHuXJXkS7tbOjlBcDAcPWq7uCqajBykL74QonpyMracjAxVew8MrHydJk3Apdwhsl07+OknyMuzaXhCCFFr0qIvJz1dJfnyibw6wcFgNKrhEm6/3Tax1af169fbOwQhhJVIi76c9PTKyzZVaddO3V8bYrrRa9GiBS1atLB3GEIIK5BEf4MrV9StLtcJNWumhjF2lET//vvv8/7779s7DCGEFUiiv0FFV8TWRq9ejjMejiR6IRyH1Ohv8Msv6r4upRtQiX7HDrh8GXx9q1+/pAROnbJcXt0oe0IIURuS6G/w/ffQti00bVq315cOgnb4MFybiaxKlXWZDAur2/sLIURFalS6SU5OJioqisjISFasWGHxfFFRETNmzCAyMpKYmBgySmsg15w5c4a+ffvy3nvvWSdqG8jOhpMnoVu3um+jNNE7SvlGCOEYqk30RqORefPm8e677xIfH8/WrVs5ceJEmXXWrVuHj48PO3bsYNKkScTGxpZ5/tVXX2XIkCHWjdzK9u9Xwx/cTKL384Nbb7U8ITt/ftlpCYUQoj5Vm+hTUlIIDg4mKCgINzc3Ro4cSWJiYpl1kpKSGDduHABRUVHs37/fPArdzp07ad26NZ06dbJB+NazZw+4u0P79je3ndtvh5071aBnpdtduRI2blSDnxmNNx9rfdi2bRvbtm2zdxhCCCuotkZvMBgICAgwP9br9aSkpFisE3jtUlIXFxe8vb25ePEi7u7urFy5klWrVrFq1aoaBVRYWEhqampt9qFaBQUF1W5z9+7OtG1bzLlzF8ssz8vzITPTcpCZipafO+dKTMwVPvnkVp566gJ/+9tZZs1qS/PmHnTrlseePV6cO1fAAw9cRKer3bYrW/fcOVfy8qof8a8mn4Ejk/3X9v6Dtj8Dm56MfeONN3j44YdpWouzm+7u7lYfYa66UetOnlQXSt1/v7P5gFXK0xMCAy3jr2h5ixYQEtKCP/8ZVq9uzuDBzTlwQI2N07q1F76+sHVrE5ycAgkIqN22K1u3RQsIDra8sKn8CJh5eefw9GxR40HQ3nrrLQCmTZtW/cqNgJZHLgTZf3D8z6Cqg1i1iV6v15OVlWV+bDAY0Ov1FutkZmYSEBBASUkJ2dnZ+Pn5cfToURISEoiNjeXKlSs4OTnh7u7OH//4x5vYHev797/V/c3U5280bx588glMmaKGU/jDH9RsVKGhaiz7Eyfghh9JNlF+BMzMzGICA2s+CFrpZMiOkuiF0LJqa/Q9e/YkLS2N9PR0ioqKiI+PJ7xc38Hw8HA2btwIQEJCAmFhYeh0Oj7++GOSkpJISkri4YcfZurUqQ0uyYNKiK1bqytbrSEgAObMUX/PmaMGQQO1fS8vlegrc/o0vPCC6qIphBDWUG2L3sXFhblz5zJ58mSMRiPjx4+nU6dOxMXF0aNHDyIiIoiOjmbWrFlERkbi6+vL4sWL6yN2q9m/X00gYs1JvmfOhL59ITLy+hW3Op0ax76iRF9YqE7WHjqkeucEBkJIiOV6X3yhykx/+pOa9UoIIapToxr90KFDGTp0aJll06dPN//t7u7O0qVLq9zGX//61zqEZ3sGg7pZq2xTytUVhg+3XN6xoxrh8vLlsssPHFDdMu++Gzp1gjffhDNn4JZbrq9TUKCuvC0qgrg4uOGfQAghKqX5sW6++07dWzvRV6ZjR3VfvlWfkqJOrP7+9+qm01mWbw4fVkl++HA1dMKSJSr5CyFEVTSf6Et7inbpUj/v17atau3fmOgLCyE1FXr3Vgm+RQvVqi+f6PfvV3X+sWNV6SYtTbXwbWH37t3s3r3bNhsXQtQrzSX6ixdVa7j0tm8ftGqlui/WB2dndVHWjYk+NVUNcFY6hAJAv36QmanKN6Dq8j/9BIMGqYNBSIg6sVvu2jUhhLCguURf2u2w9Pbf/6oWdFFR/cXQsaNK3Lm56nFKijqxeuPFw/36lS3fbNmiHpcOeObkBD16wK5d6iBhbbGxsRZDWQghGifNJfobGY2q1VzXYYnrqmNH1bPmm2/U+DopKSppOztfX8fXVyX+r76CtWthwwbo2rVsH/hevdRJ3X37rB/j1q1b2bp1q/U3LISod5pO9AaDag23bl2/79u+vRoKec4ciI9XI2feWLYpdeed6mB04ICKs/zQx926qXr/li31ErYQopHSdKI/fVrd13eLvkkT+NvfVF/5rVtVGaZ7d8v1QkIgNhYWL1ZX1vbsWfZ5Dw9Vyimf6HNzVY+ci2WH7RFCaJSmE31Ghkqyth6OoCKtWsEHH6hW+rBhdZ/sJCICjh9XJ2oBioshJkYl+u3brRevEKLx0nSiP31atapd7DTPVpMmcP/9MH583bcREaHuX3pJ1fMnTYIvv1Tz3h46VPdhkT08PPCQS2+FcAiankowI6NsT5fGKCgIHngAPvpI3QAWLlQ9iR59VE2PWL7kUxNffvmldQMVQtiNZhN9bq6qYdd3fd4WPvkEli1TLXonJxg9WvXTb9pUncitS6IXQjgOzSb60pGXyw0/32i1aKGumC3l5qZO5n79dd2GSXj55ZcBeOGFF6wUoRDCXjRbo8/JUffe3vaNw5YGDlQXgh05UvvXJiYmWkwZKYRonDSb6PPy1H19DX1gD+3bQ/PmlpOVCyG0RfOJvq7dGhuD0mESTpxQV+AKIbRJ84nekVv0AO3aqRr9+fPqdExJSdlB3U6dkgurhHB0mj0Zm5en+rE7Ofihrl07dX/6tCu9eqneRl9/XXadiuaRbd68eb3EJ4SwPc0m+txcxy7blAoIUAe006dda/W6DRs22CgiIUR9c/D2bOXy8hy/bAPqF0twcO0TvRDCcUii14B27SAry5Xi4pq/5u9//zt///vfbRaTEKL+aLZ0k5fnOBdLVaddO7h6VUdGRs1fs3//fpvFI4SoX5pt0efmaqdFf+ut6v6XX+wbhxDCPjSZ6E0mbZVu/PzA29tIWpq9IxFC2IMmE31xsepProVeN6Vaty6WRC+ERmmyRl86KbdWWvQArVsX8cMPTbhypWbrt3GEYT2FEIBGE71Wroq9UXBwEQA7d8Itt1S//po1a2wckRCivmiydKPFRB8UVEzr1mrsepPJ3tEIIeqTJhO9Fks3Oh3cdRf8+OP1+WWrMmPGDGbMmGH7wIQQNqfJRK+FkSsrMnAg+PjArl3Vr3vkyBGO1GUgeyFEg6PpRK+lFj2oWad+/3s1EcmFC/aORghRXzSb6HU68PCwdyT17777VI0+OdnekQgh6osmE31urkryjj5EcUVat1aThe/dC0ajvaMRQtQHDaY6bV0VW5EhQ+DKFTh6tPJ1OnfuTOfOnesvKCGEzdQo0ScnJxMVFUVkZCQrVqyweL6oqIgZM2YQGRlJTEwMGddGz0pJSWHMmDGMGTOGe++9lx07dlg3+jrSeqLv3l0Ni7BnT+XrrFixosJ/ayFE41PtBVNGo5F58+axevVq9Ho90dHRhIeH07FjR/M669atw8fHhx07dhAfH09sbCxLliyhU6dObNiwARcXF86ePcuYMWO46667cHGx73VaWpl0pDLOzvC730F8PJw7Z+9ohBC2Vm2LPiUlheDgYIKCgnBzc2PkyJEkJiaWWScpKYlx48YBEBUVxf79+zGZTHh4eJiTemFhITqdzga7UHtab9GDSvQAX31V8fNTpkxhypQp9ReQEMJmqm1aGwwGAgICzI/1ej0pKSkW6wReG9zdxcUFb29vLl68iL+/P0ePHmXOnDmcOXOG119/vdrWfGFhIampqXXZl0oVFBSYt5mX15LsbD+ggMzM6wO/5OX5lHlcl+XnzrmSl/dbBeu2JDPTctaP2my7snVr+p7FxcVkZmaW2U6nTn7s2ePKb79dtNhGaR96a/9b2MuN3wEt0vr+g7Y/A5vXUHr37k18fDw///wzzz77LHfccQfu7u6Vru/u7k7Xrl2tGkNqaqp5m2lpUFAALVs2JTDwev3G05Myj+uyvEULCA5uYbHuqVMVT3JSm21Xtm5N3zMzM5PAwMAy2+nbV10pazK1oGvXstvwvPaTx9r/FvZy43dAi7S+/+D4n0FVB7FqSzd6vZ6srCzzY4PBgF6vt1gnMzMTgJKSErKzs/Hz8yuzTocOHfD09OTHH3+sVfDWlpcHV69K6QbUxOEAP/9s3ziEELZVbaLv2bMnaWlppKenU1RURHx8POHh4WXWCQ8PZ+PGjQAkJCQQFhaGTqcjPT2dkpISAE6fPs3Jkydp3bq1DXaj5i5fVveS6KH0eH3ypH3jEELYVrWlGxcXF+bOncvkyZMxGo2MHz+eTp06ERcXR48ePYiIiCA6OppZs2YRGRmJr68vixcvBuDw4cOsXLkSFxcXnJyc+L//+z/8/f1tvlNVkUR/nZ8fuLpWnOj79OlT/wEJIWyiRjX6oUOHMnTo0DLLpk+fbv7b3d2dpUuXWrxu7NixjB079iZDtK5Ll9S9JHp1ZbBeX3GiX7JkSf0HJISwCc1dGVvaotdyP/obVZbohRCOQ3OJvnQqPWnRK3o9pKdDYWHZ5X/84x/54x//aJ+ghBBWpblELy36svR61QupfM+bjIwM81AWQojGTZOJXqeDKrrya0ppF8vjx+0bhxDCdjSX6C9dUmUbLQ5RXJHSLpaS6IVwXJpLd9nZ2pxwpDIeHtCypSR6IRyZfYeRtIPcXGjSxN5RNCzt21sm+kGDBtknGCGE1Wku0eflSX2+vPbtISGh7LKFCxfaJxghhNVprnSTmyuJvrwOHdRk4TI2vRCOSZOJXko3ZbVvr+5vHG9u/PjxjB8/3j4BCSGsSnOJXko3lkoT/Y11+vPnz3P+/Hn7BCSEsCrNJXop3Vhq0wZcXODECXtHIoSwBU0lepNJSjcVcXGBdu0k0QvhqDTV66aoCEpKpEVfXkkJtG4NqalqZipQs3DZeQ53IYSVaOq/ck6OupcWfVm5uWpYiBMnYPt29bdeH0HHjvaOTAhhDZpK9NnZ6l5a9JZatYL8fJX0vbxg5MgXiIqyd1RCCGvQVI1eWvSVa9lS3Z89a984hBDWp8lELy16S61aqfvSRL906T08/PA99gtICGE1UroRADRvrmrzv/2mHhcX51NQYN+YhBDWockWvZRuLLm6gr+/lG6EcESaTPTSoq9Yy5bXW/RCCMchiV6YtWoliV4IR6TJGr2UbirWsqU6GOblQc+eo+jSxd4RCSGsQVOJPidHnXB0dbV3JA1Tac+b336Du++eKf3ohXAQmivdyHyxlZO+9GmQ7j4AABmZSURBVEI4Jk2lvOxsaNrU3lE0XKWJ/rffYNGiO7n//jvtGo8Qwjo0V7qxZaIvKbk+KNiN8vNt957W5OYGzZpJi14IR6O5RO/pabvt5+bC119bLg8Ls917WlurVpCVJecxhHAkmird2LpF7whuvRV+/RWuXrV3JEIIa9FUopcaffU6dwajsfGUm4QQ1dNc6SYgwN5RNGwdO6peSb6+9zFqlL2jEUJYg+YSvZeXvaOwLmufAG7SBIKDobBwGhMn3lxsQoiGoUaJPjk5mfnz53P16lViYmKYMmVKmeeLioqYPXs2x44do1mzZixevJg2bdqwd+9eFi1aRHFxMa6ursyaNYtBgwbZZEdqIjvbtidj7cEWJ4A7d4YdO/I4fx6Cgx3sAxNCg6qt0RuNRubNm8e7775LfHw8W7du5US5WaTXrVuHj48PO3bsYNKkScTGxgLg5+fH8uXL2bJlC6+++iqzZ8+2zV7UgMkkJ2NrqksXuHp1BBMmjLB3KEIIK6g20aekpBAcHExQUBBubm6MHDmSxMTEMuskJSUxbtw4AKKioti/fz8mk4lu3bqh1+sB6NSpE4WFhRQVFdlgN6pXUKB6kkiir16HDur+yhX7xiGEsI5qSzcGg4GAG85g6vV6UlJSLNYJDAxUG3Rxwdvbm4sXL+Lv729eJyEhgW7duuHm5lbl+xUWFpKamlqrnahOQUEBhw//CHTGZMohMzPbYp28PB8yMy0zW22WW2Mbttp2cXExmZmZNd62m5uJy5ex+r+FvRQUFDjMvtSF1vcftP0Z1MvJ2J9++onY2FhWrVpV7bru7u507drVqu+fmpqKr29nAFq18sLb2/KMrKcnBAZaNvdrs9wa27DVtjMzMwkMDKzxtps21XHxIgQFdXWIE9ipqalW/141Jlrff3D8z6Cqg1i1pRu9Xk9WVpb5scFgMJdjblwnMzMTgJKSErKzs/Hz8wMgKyuLJ554gtdee422bdvWaQesoXQseind1EzpSetyVTohRCNUbaLv2bMnaWlppKenU1RURHx8POHh4WXWCQ8PZ+PGjYAq0YSFhaHT6bhy5QpTpkzhmWeeISQkxDZ7UEOS6GsnPHwSTZpM4to/qxCiEau2dOPi4sLcuXOZPHkyRqOR8ePH06lTJ+Li4ujRowcRERFER0cza9YsIiMj8fX1ZfHixQCsWbOGX3/9lTfffJM333wTgFWrVtG8eXPb7lUFSicdadpUrvqsicGDJ5GTA1u2qL76Lpq64kIIx1Kj/75Dhw5l6NChZZZNnz7d/Le7uztLly61eN20adOYNm3aTYZoHaUtekfrR28rOTnn+N3vYOPGFiQnQ7kfcUKIRkQz7bTSRO8IJxbrwzvvROPrC02a7GbjRkn0QjRmmhnUrLR0Iy36mnNygrvvhk2b1AVnQojGSTOJXk7G1s24cZCRAYcP2zsSIURdaSrROzuDu7u9I2lcRo9Wn9vmzfaORAhRV5pJ9NnZqj6v09k7ksaleXPo2RP++197RyKEqCtNnYyVE7E1d8cdj9G7t/o7JOR6nV4OlEI0Pppp0efkgLe3vaNoPEJD72f06PsB6NcPzp+H9HQ7ByWEqBNNJXpp0dfchQvpnDmjMnu/fmrZN9/YMSAhRJ1pJtGX1uhFzaxePZGnnlJTTPXurU7ISs8bIRonzSR6Kd3UnYcHdO0qLXohGitNJXpp0dddv36S6IVorDST6KV0c3P69YOsLLg2GrUQohHRTKKX0s3NKR1lWur0QjQ+muhHbzJBbq606Gtj2LBnzL1tQJ2Q1elU+WbUKPvFJYSoPU0k+rw8J0wmadHXRu/eoxk27Ppjb2/o3Fnq9EI0Rpoo3Vy5onbz2uyGogayso7z88/HyywLCZFEL0RjpIlEn53tDICvr50DaUQ++mgqc+ZMLbOsXz91dexvv9kpKCFEnWgi0Ze26Js1s3MgjZxcIStE46SJRJ+To1r0kuhvTt++6l4SvRCNiyYSfWmLXko3N6dZM+jQQbpYCtHYaKLXTWmNvlkzyMuzczCNXL9+amz6U6fKLvfxkZPdQjRUGkn011v0kuhrZsSI5+nf33J5v36wbh18/nnZaRmjoiTRC9FQaaJ0k53tjIcHuLnZO5LGo2vXYQwePMxieekVsjI2vRCNh0YSvZOciK2l9PQjHDt2xGJ56QnZ8qUbIUTDpZHSjbMk+lr67LMZ7NgBI0bsLrO8RQto3Vpa9EI0JtKiF7XWvTv8+qu9oxBC1JRGEr2zdK20op49wWCA/Hx7RyKEqAlNJPorV6RFb009eqh7Kd8I0ThoItHn5EiN3ppKhyw+frz6dYUQ9ufwJ2NNJtWil9JN7Ywdu4CBAyt+zt9fXSF79CiMHl2/cQkhas/hW/QFBVBcLKWb2urQ4XZCQm6v9Pk+fVTp5ty5egxKCFEnDp/oL19W95Loa+fnn/dx+PC+Sp/v00fdH7Hsai+EaGBqlOiTk5OJiooiMjKSFStWWDxfVFTEjBkziIyMJCYmhoyMDAAuXrzIxIkT6du3L/PmzbNu5DV06ZK6l9JN7WzaNIfXX59T6fMtW6r+9EeP1mNQQog6qTbRG41G5s2bx7vvvkt8fDxbt27lxIkTZdZZt24dPj4+7Nixg0mTJhEbGwuAu7s706dPZ/bs2baJvgZKE7206K2vTx/46Sc18boQouGqNtGnpKQQHBxMUFAQbm5ujBw5ksTExDLrJCUlMW7cOACioqLYv38/JpMJT09P+vfvj7u7u22irwEp3dhOnz7qZHdKir0jEUJUpdpEbzAYCAgIMD/W6/UYDAaLdQIDAwFwcXHB29ubixcvWjnUupEWve0EBakeODIRiRANW4PrXllYWEhqaqrVtpea2gwI5OzZn4AS8vJakplZbLFeXp4PmZlXbmq5NbZhq20XFxeTmZlZ420XFhZRXKyr8N/ixs/wttu8+frrpvz00wXy8s5arNtQFBQUWPV71dhoff9B259BtYler9eTlZVlfmwwGNDr9RbrZGZmEhAQQElJCdnZ2fjVcXByd3d3unbtWqfXVsTTU92HhnbC01ONunjtx4fFeoGBTW9quTW2YattZ2ZmEhgYWONtT5z4FgMGgKen5b+FTnf9Mxw2DPbtgz17mvPSS80t1m0oUlNTrfq9amy0vv/g+J9BVQexaks3PXv2JC0tjfT0dIqKioiPjyc8PLzMOuHh4WzcuBGAhIQEwsLC0Ol0Nxm2dVy6BC4uJjw87B1J4xIU1Ifg4D4kJGBxKyq6vl7r1hAcDOvX2y9WIUTVqm3Ru7i4MHfuXCZPnozRaGT8+PF06tSJuLg4evToQUREBNHR0cyaNYvIyEh8fX1ZvHix+fXh4eHk5ORQXFzMzp07WbVqFR07drTpTt3o0iXw9jai0zW4KlWDlpq689pflpOPlDdoEKxdq/rUl/avF0I0HDXKfkOHDmXo0KFllk2fPt38t7u7O0uXLq3wtUlJSTcR3s1Tif6qXWNojLZte4WvvoJHH60+0YeGqqkF//UvSfRCNESauDLWx8do7zAcmpeXqtWvWVO2rCOEaBgcPtFfugReXtKit7Vx49S4N7t22TsSIUR5mkj00qK3vSFDVO+dzZvtHYkQojyHT/SXL0uNvj40aQJRUfDFF+pqWSFEw+HwXVFKe92I2pkw4R369IEbLqGoUkkJ/O53sHEjxMer6QYBfHygjpdUCCGsxKFb9MXFkJsrLfq6CAjoQrt2XWq8fm6uupBKp4O3377e5/6K5UW7Qoh65tCJvjTJSIu+9o4e3cLu3Vtq9RovL+jYUYYuFqKhcehEXzqgmZyMrb2dOxfx4YeLav263r0hI0NmnhKiIdFEopfulfWnd291L616IRoOTSR6adHXn1atoE0b+O9/7R2JEKKUQyf60klH5GRs/Ro0CNLSIDPT3pEIIcDBE/310o206OvTgAHg5AT799s7EiEEOHg/+tIWpb+/JPraeuSRD+nXT7XMa8vHB7p3hwMHwCgfvRB259At+pMn1QQZHh5yqWZt+fsHERAQVOfXDxqkflHt3WvFoIQQdeLwif7WW+0dReN08OCnbN/+aZ1f36uXGvtmwwYrBiWEqBOHT/Tt29s7isYpOXk569Ytr/PrXV1VrX7bNulqKYS9OWyiLypSF+5Ioref0aPVODd/+APk59s7GiG0y2ET/a+/wtWrkujtycsLFi2C77+HWbPsHY0Q2uWwif7kSXUvid6+hgyBp5+GN9+EpUtlCGMh7EESvbCpkhKYOlVNNTh9OjzwAPz4I1y8aO/IhNAOh+1Hf/IkuLur7pUyVG7tTZ26npAQOH785raTmwtffw3jx6t/j88+g4MHYe1adbJWCGF7Dt2ib9dOXaEpas/LqwV+fi2stj0nJ7j3XtW6P3NGnaj9+murbV4IUQWHbtFL2abu9u17H4MB9PpJVt1uv36g18Pq1ap+//vfw7Rp6qAMMiOVELbgsIn+l1/g9tvtHUXjtX//+xw7Bo8+Osnq227dGj76CObOhc8/h3XroGVLaNpUHZxvu01d6DZiBHTtavW3F0JzHDLRX7yoLr+XFn3D5esLDz6okvnu3fDbb6qef+YMHD6s/v1mzlS/AKKi1Ng5t92muswWFoKHB7Rtqw4QQoiqOWSilx43jYevL4wZc/1xVBQEB6sB6T77DNasgVdfrbxbZuvWqkdPZCSEhKgDgJR/hCjLoRO9jHPTeAUGqu6Y06fDd9/Be+9BVhY4O6vhFQoLVTLftw8+/hj+9S91wrd1a3WALypS6xQVqZte34Hbb1flvA4d1MHEaLw+qbmHhyodubrae8+FsD5J9KLB8/GBbt3U7UZhYdCnj0rox4+rf/e0NDVfbWGhStouLiqJ//CDjq++qvqCrdIDRbt26jVXr6oDwblzqpTUvLk6QPTooUpK3bqpgdvc3NT1AoWF6mrgtm3VwUOIhsJhE32LFipBiLr561+3ERraOAYkc3dXo2X26qUeh4VZdt3MzDxHdHQgubnw889qiAxXVygoUEM0FBVBXh6cPauec3dX4/O4uanvUatWkJ0Np07Brl1q/cp4eamTyCNGwEMPSQlR2J/DJnr5z3Vz3Nw88fCwdxTW5eWlTureeKHWqVOQkGC5bkUHi9Lle/eqcwhnz0JxsWrlp6WpXw95eXD6tOr19dJL6ta7Nwwfrm63364OHkLUJ4dL9FlZ8NVXMGmSvSNp3HbvfotTpyA4eFq9vm9JiUq+N2poI186O6sJ0Nu0UY8rOyh066aWx8erwd1eew28vSEiQiX9qKjr1w8IYUsOl+gXLVI/q59+2t6RNG6HD3/GTz/Bo4/Wb6IvHTLhRmFh1tl2fR9EgoJg8GDVTfTKFVXy2b5dJf5Nm9Q6HTrAHXeo29Chcl5J2IZDJfpz52D5ctU/u1Mne0cjGhpbHkQqUv7A0qePuj35pLpI7NgxdVuzRl0pDCrxh4RA//7q1q+f6oIqxM2oUaJPTk5m/vz5XL16lZiYGKZMmVLm+aKiImbPns2xY8do1qwZixcvps2137XvvPMO69evx8nJieeff54hQ4ZYfy+uWbxY1Uife85mbyFEjVV0YAF1cAkIULeICPUL9Oef1cnhY8dUl9HPPru+fufOMHCgmoe3Xz91MGjeXHr2iJqrNtEbjUbmzZvH6tWr0ev1REdHEx4eTseOHc3rrFu3Dh8fH3bs2EF8fDyxsbEsWbKEEydOEB8fT3x8PAaDgUceeYSEhAScnZ2tviPZ2bBsGcTEyGXzonFxc1Pf2dJa/5gxkJOjfg24uKjrCL78Ej788PprvLxUh4P27dU1B82bX7/5+6vuoW5u12/p6W64ul5/XP5vGfzPsVWb6FNSUggODiYoKAiAkSNHkpiYWCbRJyUl8cQTTwAQFRXFvHnzMJlMJCYmMnLkSNzc3AgKCiI4OJiUlBT69u1rk50JD4dXXrHJpoWoV6U9hEqT//jxqjR55owaLqJpU/X4xx9hzx417MfVq1VtsUOV7+fiUvbA4Opq+YvhxsdVPVeXx6Uqus6h/LK6Pi4p6YiLi3W3ae1tLFwIjz5qufxm6Uymquf82b59O3v27GH+/PkAbNq0iZSUFObOnWteZ9SoUbz77rsEBAQAMGzYMD777DPeeOMNevfuzZhr17jPmTOHO+64g+HDh1f6fkeOHMHd3f2md0wIIbSksLCQPn36VPhcgzsZW1mgQggh6qbaypxerycrK8v82GAwoNfrLdbJzMwEoKSkhOzsbPz8/Gr0WiGEELZVbaLv2bMnaWlppKenU1RURHx8POHh4WXWCQ8PZ+PGjQAkJCQQFhaGTqcjPDyc+Ph4ioqKSE9PJy0tjV6l16kLIYSoF9WWblxcXJg7dy6TJ0/GaDQyfvx4OnXqRFxcHD169CAiIoLo6GhmzZpFZGQkvr6+LF68GIBOnTpxzz33MGLECJydnZk7d65NetwIIYSoXLUnY4UQQjRu0ntWCCEcnCR6IYRwcA6f6JOTk4mKiiIyMpIVK1bYOxyby8zMZOLEiYwYMYKRI0fyr3/9C4BLly7xyCOPcPfdd/PII49w+fJlO0dqW0ajkbFjxzJ16lQA0tPTiYmJITIykhkzZlBU1YDyDuDKlSs8+eSTDB8+nHvuuYdvv/1WU9+B999/n5EjRzJq1CiefvppCgsLNfcduJFDJ/rS4Rveffdd4uPj2bp1KydOnLB3WDbl7OzM3/72N7Zt28ann37Kxx9/zIkTJ1ixYgWDBg3i3//+N4MGDXL4g94HH3xAhw7XrwaNjY1l0qRJ7NixAx8fH9avX2/H6Gxv/vz5DBkyhO3bt7N582Y6dOigme+AwWDggw8+YMOGDWzduhWj0WgemkVL34EbOXSiv3H4Bjc3N/PwDY6sVatWdO/eHQAvLy/at2+PwWAgMTGRsWPHAjB27Fh27txpzzBtKisri927dxMdHQ2AyWTi66+/JioqCoBx48Y59PcgOzubgwcPmvffzc0NHx8fTX0HjEYjBQUFlJSUUFBQQMuWLTX1HSjPoRO9wWAwD8sA6sIug8Fgx4jqV0ZGBqmpqfTu3Zvz58/TqlUrAFq2bMn58+ftHJ3tLFiwgFmzZuF0baSuixcv4uPjg8u1gU4CAgIc+nuQkZGBv78/f//73xk7dizPPfcceXl5mvkO6PV6/vSnP3HXXXcxePBgvLy86N69u6a+A+U5dKLXstzcXJ588knmzJmDl5dXmed0Oh06Bx3jdteuXfj7+9OjRw97h2I3JSUlfP/99zz44INs2rQJDw8PizKNI38HLl++TGJiIomJiezZs4f8/Hz27Nlj77DsqsGNdWNNWh2Cobi4mCeffJLRo0dz9913A9C8eXPOnj1Lq1atOHv2LP7+/naO0ja++eYbkpKSSE5OprCwkJycHObPn8+VK1coKSnBxcWFrKwsh/4eBAQEEBAQQO/evQEYPnw4K1as0Mx3YN++fbRp08a8f3fffTfffPONpr4D5Tl0i74mwzc4GpPJxHPPPUf79u155JFHzMvDw8PZdG3+uk2bNhEREWGvEG3qmWeeITk5maSkJP75z38SFhbGokWLGDhwIAnXZgHfuHGjQ38PWrZsSUBAACdPngRg//79dOjQQTPfgVtuuYWjR4+Sn5+PyWRi//79dOzYUVPfgfIc/srY//znPyxYsMA8fMNjjz1m75Bs6tChQ0yYMIHOnTuba9RPP/00vXr1YsaMGWRmZnLLLbewZMkSmjVrZudobevAgQOsWrWKd955h/T0dJ566ikuX75M165diY2Nxc3Nzd4h2kxqairPPfccxcXFBAUFsXDhQq5evaqZ78DSpUvZtm0bLi4udO3alfnz52MwGDT1HbiRwyd6IYTQOocu3QghhJBEL4QQDk8SvRBCODhJ9EII4eAk0QshhIOTRC+EEA5OEr3QnGXLlvHee+/d9HYyMjIYNWqUFSISwrYk0QshhINz6LFuhCi1fPlyNm3ahL+/P4GBgXTv3p3U1FRefPFF8vPzadu2LQsWLMDX15dTp07x4osvcuHCBZydnYmLi6Nt27ZVbt9oNBIbG8t///tfioqKmDBhAg888AAHDhzgjTfewM/Pjx9//JHu3bsTGxvrsAOKiYZJWvTC4f3vf/9j27ZtbNq0iZUrV/Ldd98BMHv2bGbOnMmWLVvo3Lkzb7zxBgAzZ85kwoQJfPHFF6xdu5aWLVtW+x7r16/H29ubDRs2sGHDBj777DPS09MB+P7775kzZw7btm0jIyODw4cP225nhaiAtOiFwzt06BDDhg3Dw8MDUAO85efnk52dzYABAwA1EcX06dPJycnBYDAQGRkJgLu7e43eY+/evRw/ftw8aFZ2djanTp3C1dWVXr16medFuO222zh9+jT9+/e39m4KUSlJ9EJYgclk4vnnn2fIkCFllh84cKDMwFnOzs4Yjcb6Dk9onJRuhMMLDQ1l586dFBQUkJOTw65du/Dw8MDHx4dDhw4BsHnzZkJDQ/Hy8iIgIMA8zV5RURH5+fnVvsfgwYP55JNPKC4uBuCXX34hLy/PdjslRC1Ii144vO7duzNixAjGjBmDv78/PXv2BOC1114zn4wtHcoX4PXXX2fu3LnExcXh6upKXFwcQUFBVb5HTEwMp0+f5ve//z0mkwk/Pz/eeustm++bEDUhwxQLIYSDk9KNEEI4OCndCFGN48ePM3v27DLL3NzcWLdunZ0iEqJ2pHQjhBAOTko3Qgjh4CTRCyGEg5NEL4QQDk4SvRBCOLj/B2OHzkIMoCNAAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "dc87384b-710c-4ec4-aec1-48bf8b2439f3",
        "_uuid": "4df404e0692a1d585336e8994b776f5cd1fd0fab",
        "id": "lm1AY3N-paid",
        "colab_type": "text"
      },
      "source": [
        "Let's pre-process the text, tokenize it and pad it to a maximum length (as in the figure above)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "_cell_guid": "19aeb800-f1c8-4206-8bf8-323256ff5c14",
        "_uuid": "c600a3f2135982bf6f9b46cce3a97b1e2e5b912c",
        "scrolled": true,
        "trusted": true,
        "id": "T83c7twMpaie",
        "colab_type": "code",
        "outputId": "420dd204-8d27-4e05-a93a-4d38e01a6c34",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 100
        }
      },
      "source": [
        "processed_docs_train = train_df['tweet_text'].astype(str).tolist()\n",
        "processed_docs_test = test_df['tweet_text'].astype(str).tolist() \n",
        "num_classes = len(label_names)\n",
        "\n",
        "#raw_docs_train = train_df['tweet_text'].tolist()\n",
        "#raw_docs_test = test_df['tweet_text'].tolist() \n",
        "\n",
        "#print(processed_docs_train)\n",
        "\n",
        "#print(\"pre-processing train data...\")\n",
        "#processed_docs_train = []\n",
        "#for doc in tqdm(raw_docs_train):\n",
        "#    tokens = tokenizer.tokenize(str(doc))\n",
        "#    filtered = [word for word in tokens if word not in stop_words]\n",
        "#    filtered = [word for word in tokens]\n",
        "#    processed_docs_train.append(\" \".join(filtered))\n",
        "\n",
        "#print(processed_docs_train)\n",
        "#end for\n",
        "\n",
        "#processed_docs_test = []\n",
        "#for doc in tqdm(raw_docs_test):\n",
        "#    tokens = tokenizer.tokenize(doc)\n",
        "#    #filtered = [word for word in tokens if word not in stop_words]\n",
        "#    filtered = [word for word in tokens]\n",
        "#    processed_docs_test.append(\" \".join(filtered))\n",
        "#end for\n",
        "\n",
        "print(\"tokenizing input data...\")\n",
        "tokenizer = Tokenizer(num_words=MAX_NB_WORDS, lower=True, char_level=False)\n",
        "total_docs=[]\n",
        "total_docs.extend(processed_docs_train)\n",
        "total_docs.extend(processed_docs_test)\n",
        "#tokenizer.fit_on_texts(processed_docs_train + processed_docs_test)  #leaky\n",
        "tokenizer.fit_on_texts(total_docs)\n",
        "word_seq_train = tokenizer.texts_to_sequences(processed_docs_train)\n",
        "#print(word_seq_train)\n",
        "word_seq_test = tokenizer.texts_to_sequences(processed_docs_test)\n",
        "word_index = tokenizer.word_index\n",
        "print(len(word_index))\n",
        "print(\"dictionary size: \", len(word_index))\n",
        "\n",
        "#pad sequences\n",
        "word_seq_train = sequence.pad_sequences(word_seq_train, maxlen=max_seq_len)\n",
        "print(word_seq_train[2])\n",
        "word_seq_test = sequence.pad_sequences(word_seq_test, maxlen=max_seq_len)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tokenizing input data...\n",
            "15036\n",
            "dictionary size:  15036\n",
            "[   0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0 3034 1211   12 1211   63 2702    1]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "3b2c9452-5780-4920-8896-0b9dfa7c562b",
        "_uuid": "e8742cc989f4c71a8c401feef5e54c0e7718865a",
        "id": "dGXCyaL1paik",
        "colab_type": "text"
      },
      "source": [
        "We can now prepare our embedding matrix limiting to a max number of words:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "_cell_guid": "d524dcca-8299-46d0-adae-f12f7c1a3171",
        "_uuid": "08e7abf92ce07c856f5e7d6f9ce371997495910b",
        "trusted": true,
        "id": "163KAE38paih",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#training params\n",
        "batch_size = 64\n",
        "num_epochs = 8 \n",
        "\n",
        "#model parameters\n",
        "num_filters = 128 \n",
        "num_kernels=200\n",
        "stride=1\n",
        "embed_dim = 300\n",
        "weight_decay = 1e-4\n",
        "\n",
        "#parallel layers \n",
        "kernel_size=7\n",
        "kernel_size_p1=3\n",
        "kernel_size_p2=6\n",
        "kernel_size_p3=9\n",
        "kernel_size_p4=12"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "_cell_guid": "18708bcb-0ae5-467f-8725-e2dbb50f989e",
        "_uuid": "630959b52ab4e120581fac11915bf432e40b8b5f",
        "trusted": true,
        "id": "gNgE0oY2paik",
        "colab_type": "code",
        "outputId": "55120022-2298-4a4d-b3db-a7808b15b80d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "#embedding matrix\n",
        "print('preparing embedding matrix...')\n",
        "words_not_found = []\n",
        "nb_words = min(MAX_NB_WORDS, len(word_index))\n",
        "embedding_matrix = np.zeros((nb_words, embed_dim))\n",
        "for word, i in word_index.items():\n",
        "    if i >= nb_words:\n",
        "        continue\n",
        "    #print(word)\n",
        "\n",
        "    if embeddings_index.wv.__contains__(word): \n",
        "\n",
        "        embedding_vector = embeddings_index.wv[word]\n",
        "    \n",
        "        embedding_matrix[i] = embedding_vector\n",
        "    else:\n",
        "        words_not_found.append(word)\n",
        "#print(embedding_matrix)\n",
        "print('number of null word embeddings: %d' % np.sum(np.sum(embedding_matrix, axis=1) == 0))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "preparing embedding matrix...\n",
            "number of null word embeddings: 2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tjASbzxBejTG",
        "colab_type": "code",
        "outputId": "d87ed348-cdf0-4e29-ad8f-46ae496a6e99",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "print(len(embedding_matrix[34]))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "300\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YDILrYH2J3oK",
        "colab_type": "code",
        "outputId": "603b00a3-e85d-44f8-92b3-f3c8c1bebf5c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "print(words_not_found)\n",
        "#print(embeddings_index.wv.__contains__(\"abe\"))\n",
        "#print(embeddings_index.wv['ddlj'])\n",
        "#print(len(words_not_found))\n",
        "#print(len(embedding_matrix))\n",
        "#print(len(word_index))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['d3']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "f1db355a-b0bf-4f58-8565-efb196c5b139",
        "_uuid": "13eccb942d6c0027e6d4f317c9072579c7c747a2",
        "id": "2Yq1FFm8pain",
        "colab_type": "text"
      },
      "source": [
        "It's interesting to look at the words not found in the embeddings:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "_cell_guid": "e6777c80-d2e5-49fb-9d69-75de9f322b6f",
        "_uuid": "fe157163ee51b46f011cd1b5395c36c633c11b18",
        "trusted": true,
        "id": "HZTXmrBwpaio",
        "colab_type": "code",
        "outputId": "e762663c-517b-4b9b-a3f7-5e2ec7a0f37f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "print(\"sample words not found: \", np.random.choice(words_not_found, 10))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "sample words not found:  ['msguy90' 'dinusirvi' 'suvaro' \"gaal'\" 'ranbhoomi' 'pradesiya' 'ujarey'\n",
            " '681819043007836160\\xa0\\x85' 'journojuno' '2semi']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "3382a8db-f082-48c3-82de-898e7a716c5f",
        "_uuid": "5d5b2e211c3b2a320aaa63c39999e913fe471f4e",
        "id": "OFD6Q9c8paig",
        "colab_type": "text"
      },
      "source": [
        "Let's define our training and model parameters:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "86182902-7292-46dc-bedb-6aded3526a00",
        "_uuid": "92fa5491dd28f186b9ff4772a4045b4e5d59df8d",
        "id": "NPAhnVnOpair",
        "colab_type": "text"
      },
      "source": [
        "We can finally define the CNN architecture"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4k79Iohm1-Ky",
        "colab_type": "code",
        "outputId": "410aa4ae-3204-4580-ba32-75efa3241b4d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 502
        }
      },
      "source": [
        "#CNN architecture\n",
        "print(\"training CNN ...\")\n",
        "model = Sequential()\n",
        "model.add(Embedding(nb_words, embed_dim,\n",
        "          weights=[embedding_matrix], input_length=max_seq_len, trainable=False))\n",
        "model.add(Conv1D(num_kernels, kernel_size, activation='relu', padding='same'))\n",
        "model.add(MaxPooling1D(2))\n",
        "model.add(Conv1D(num_kernels, kernel_size, activation='relu', padding='same'))\n",
        "#model.add(GlobalMaxPooling1D())\n",
        "model.add(MaxPooling1D(2))\n",
        "#model.add(Dense(64, activation='relu', kernel_regularizer=regularizers.l2(weight_decay)))\n",
        "model.add(GlobalMaxPooling1D())\n",
        "model.add(Dropout(0.5))\n",
        "model.add(Dense(128, activation='relu'))\n",
        "model.add(Dense(64, activation='relu'))\n",
        "#model.add(Dense(num_classes, activation='sigmoid'))  #multi-label (k-hot encoding)\n",
        "model.add(Dense(num_classes, activation='sigmoid')) \n",
        "\n",
        "adam = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "training CNN ...\n",
            "Model: \"sequential_2\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_4 (Embedding)      (None, 24, 300)           4510800   \n",
            "_________________________________________________________________\n",
            "conv1d_11 (Conv1D)           (None, 24, 200)           420200    \n",
            "_________________________________________________________________\n",
            "max_pooling1d_3 (MaxPooling1 (None, 12, 200)           0         \n",
            "_________________________________________________________________\n",
            "conv1d_12 (Conv1D)           (None, 12, 200)           280200    \n",
            "_________________________________________________________________\n",
            "max_pooling1d_4 (MaxPooling1 (None, 6, 200)            0         \n",
            "_________________________________________________________________\n",
            "global_max_pooling1d_4 (Glob (None, 200)               0         \n",
            "_________________________________________________________________\n",
            "dropout_4 (Dropout)          (None, 200)               0         \n",
            "_________________________________________________________________\n",
            "dense_10 (Dense)             (None, 128)               25728     \n",
            "_________________________________________________________________\n",
            "dense_11 (Dense)             (None, 64)                8256      \n",
            "_________________________________________________________________\n",
            "dense_12 (Dense)             (None, 6)                 390       \n",
            "=================================================================\n",
            "Total params: 5,245,574\n",
            "Trainable params: 734,774\n",
            "Non-trainable params: 4,510,800\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "1e1e03f0-9cf1-4e3d-d420-e8546bcd01b2",
        "id": "hdWy1E3m2Qw6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 702
        }
      },
      "source": [
        "hist = model.fit(word_seq_train, y_train, batch_size=batch_size, epochs=num_epochs, validation_split=0.1, shuffle=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 134179 samples, validate on 14909 samples\n",
            "Epoch 1/20\n",
            "134179/134179 [==============================] - 14s 105us/step - loss: 1.3792 - accuracy: 0.4575 - val_loss: 1.2005 - val_accuracy: 0.5503\n",
            "Epoch 2/20\n",
            "134179/134179 [==============================] - 14s 104us/step - loss: 1.1912 - accuracy: 0.5519 - val_loss: 1.1353 - val_accuracy: 0.5793\n",
            "Epoch 3/20\n",
            "134179/134179 [==============================] - 14s 102us/step - loss: 1.1234 - accuracy: 0.5818 - val_loss: 1.1003 - val_accuracy: 0.5970\n",
            "Epoch 4/20\n",
            "134179/134179 [==============================] - 14s 104us/step - loss: 1.0736 - accuracy: 0.6028 - val_loss: 1.0841 - val_accuracy: 0.6008\n",
            "Epoch 5/20\n",
            "134179/134179 [==============================] - 14s 108us/step - loss: 1.0360 - accuracy: 0.6200 - val_loss: 1.0685 - val_accuracy: 0.6086\n",
            "Epoch 6/20\n",
            "134179/134179 [==============================] - 15s 108us/step - loss: 1.0030 - accuracy: 0.6344 - val_loss: 1.0603 - val_accuracy: 0.6147\n",
            "Epoch 7/20\n",
            "134179/134179 [==============================] - 14s 104us/step - loss: 0.9721 - accuracy: 0.6466 - val_loss: 1.0479 - val_accuracy: 0.6174\n",
            "Epoch 8/20\n",
            "134179/134179 [==============================] - 14s 103us/step - loss: 0.9437 - accuracy: 0.6578 - val_loss: 1.0530 - val_accuracy: 0.6195\n",
            "Epoch 9/20\n",
            "134179/134179 [==============================] - 14s 105us/step - loss: 0.9183 - accuracy: 0.6689 - val_loss: 1.0466 - val_accuracy: 0.6210\n",
            "Epoch 10/20\n",
            "134179/134179 [==============================] - 14s 103us/step - loss: 0.8914 - accuracy: 0.6785 - val_loss: 1.0620 - val_accuracy: 0.6206\n",
            "Epoch 11/20\n",
            "134179/134179 [==============================] - 14s 104us/step - loss: 0.8677 - accuracy: 0.6858 - val_loss: 1.0531 - val_accuracy: 0.6214\n",
            "Epoch 12/20\n",
            "134179/134179 [==============================] - 14s 105us/step - loss: 0.8448 - accuracy: 0.6968 - val_loss: 1.0467 - val_accuracy: 0.6239\n",
            "Epoch 13/20\n",
            "134179/134179 [==============================] - 14s 104us/step - loss: 0.8231 - accuracy: 0.7044 - val_loss: 1.0627 - val_accuracy: 0.6223\n",
            "Epoch 14/20\n",
            "134179/134179 [==============================] - 14s 104us/step - loss: 0.8010 - accuracy: 0.7117 - val_loss: 1.0727 - val_accuracy: 0.6253\n",
            "Epoch 15/20\n",
            "134179/134179 [==============================] - 14s 103us/step - loss: 0.7800 - accuracy: 0.7212 - val_loss: 1.0731 - val_accuracy: 0.6214\n",
            "Epoch 16/20\n",
            "134179/134179 [==============================] - 14s 106us/step - loss: 0.7616 - accuracy: 0.7271 - val_loss: 1.0877 - val_accuracy: 0.6184\n",
            "Epoch 17/20\n",
            "134179/134179 [==============================] - 14s 106us/step - loss: 0.7449 - accuracy: 0.7338 - val_loss: 1.0949 - val_accuracy: 0.6243\n",
            "Epoch 18/20\n",
            "134179/134179 [==============================] - 14s 104us/step - loss: 0.7258 - accuracy: 0.7418 - val_loss: 1.1026 - val_accuracy: 0.6188\n",
            "Epoch 19/20\n",
            "134179/134179 [==============================] - 14s 103us/step - loss: 0.7099 - accuracy: 0.7467 - val_loss: 1.1276 - val_accuracy: 0.6202\n",
            "Epoch 20/20\n",
            "134179/134179 [==============================] - 14s 103us/step - loss: 0.6959 - accuracy: 0.7525 - val_loss: 1.1163 - val_accuracy: 0.6233\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "8faab3d7-9e29-44e8-f5a4-bf76738ff93a",
        "id": "FKbPqLRU22Mr",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 619
        }
      },
      "source": [
        "print(\"training CNN2 ...\")\n",
        "\n",
        "inputt = Input(shape=(max_seq_len,))\n",
        "model = Embedding(nb_words, embed_dim,weights=[embedding_matrix],input_length=max_seq_len, trainable = False)(inputt)\n",
        "tower_1 = Conv1D(num_kernels, kernel_size_p1, padding='same', activation='relu', strides=stride)(model)\n",
        "tower_2 = Conv1D(num_kernels, kernel_size_p2, padding='same', activation='relu', strides=stride)(model)\n",
        "tower_3 = Conv1D(num_kernels, kernel_size_p3, padding='same', activation='relu', strides=stride)(model)\n",
        "tower_4 = Conv1D(num_kernels, kernel_size_p4, padding='same', activation='relu', strides=stride)(model)\n",
        "model = keras.layers.concatenate([tower_1, tower_2, tower_3, tower_4], axis=1)\n",
        "model = GlobalMaxPooling1D()(model)\n",
        "model = Dropout(0.5)(model)\n",
        "#model = Flatten()(model)\n",
        "model = Dense(128, activation='relu')(model)\n",
        "model = Dense(64, activation='relu')(model)\n",
        "output = Dense(num_classes, activation='sigmoid')(model)\n",
        "model = Model(inputt, output)\n",
        "\n",
        "#adam = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "model.summary()\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "training CNN2 ...\n",
            "Model: \"model_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            (None, 24)           0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_1 (Embedding)         (None, 24, 300)      4510800     input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv1d_1 (Conv1D)               (None, 24, 200)      180200      embedding_1[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "conv1d_2 (Conv1D)               (None, 24, 200)      360200      embedding_1[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "conv1d_3 (Conv1D)               (None, 24, 200)      540200      embedding_1[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "conv1d_4 (Conv1D)               (None, 24, 200)      720200      embedding_1[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)     (None, 96, 200)      0           conv1d_1[0][0]                   \n",
            "                                                                 conv1d_2[0][0]                   \n",
            "                                                                 conv1d_3[0][0]                   \n",
            "                                                                 conv1d_4[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "global_max_pooling1d_1 (GlobalM (None, 200)          0           concatenate_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dropout_1 (Dropout)             (None, 200)          0           global_max_pooling1d_1[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 128)          25728       dropout_1[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 64)           8256        dense_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 6)            390         dense_2[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 6,345,974\n",
            "Trainable params: 1,835,174\n",
            "Non-trainable params: 4,510,800\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "94ceef79-7eed-4a9f-be7b-67efcaab7841",
        "id": "dPvTEGZUu6NM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        }
      },
      "source": [
        "hist = model.fit(word_seq_train, y_train, batch_size=batch_size, epochs=num_epochs, validation_split=0.1, shuffle=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 134179 samples, validate on 14909 samples\n",
            "Epoch 1/8\n",
            "134179/134179 [==============================] - 45s 333us/step - loss: 1.7860 - accuracy: 0.2035 - val_loss: 1.7680 - val_accuracy: 0.1826\n",
            "Epoch 2/8\n",
            "134179/134179 [==============================] - 38s 283us/step - loss: 1.5490 - accuracy: 0.3549 - val_loss: 1.3263 - val_accuracy: 0.4834\n",
            "Epoch 3/8\n",
            "134179/134179 [==============================] - 38s 281us/step - loss: 1.3280 - accuracy: 0.4862 - val_loss: 1.2361 - val_accuracy: 0.5348\n",
            "Epoch 4/8\n",
            "134179/134179 [==============================] - 38s 283us/step - loss: 1.2424 - accuracy: 0.5287 - val_loss: 1.1813 - val_accuracy: 0.5621\n",
            "Epoch 5/8\n",
            "134179/134179 [==============================] - 38s 284us/step - loss: 1.1901 - accuracy: 0.5520 - val_loss: 1.1403 - val_accuracy: 0.5806\n",
            "Epoch 6/8\n",
            "134179/134179 [==============================] - 38s 281us/step - loss: 1.1474 - accuracy: 0.5720 - val_loss: 1.1076 - val_accuracy: 0.5892\n",
            "Epoch 7/8\n",
            "134179/134179 [==============================] - 38s 282us/step - loss: 1.1173 - accuracy: 0.5845 - val_loss: 1.0892 - val_accuracy: 0.5972\n",
            "Epoch 8/8\n",
            "134179/134179 [==============================] - 38s 285us/step - loss: 1.0885 - accuracy: 0.5974 - val_loss: 1.0812 - val_accuracy: 0.6064\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "WVpSh0yY3KAv",
        "colab": {}
      },
      "source": [
        "#LSTM Params\n",
        "\n",
        "lstm_out1 = 150\n",
        "bilstm_out1 = 96\n",
        "lstm_out2 = 32\n",
        "dropout = 0.2\n",
        "recurrent_dropout = 0.2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "bde03aa6-5eb0-467f-debe-ea7b5eaadceb",
        "id": "X-9ZGCwY3RwS",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 338
        }
      },
      "source": [
        "#LSTM\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Embedding(nb_words, embed_dim,\n",
        "          weights=[embedding_matrix], input_length=max_seq_len, trainable=False))\n",
        "model.add(LSTM(lstm_out1, dropout_U = 0.2, dropout_W = 0.2))\n",
        "#model.add(MaxPooling1D(2))\n",
        "model.add(Dense(64, activation='relu'))\n",
        "\n",
        "model.add(Dense(num_classes, activation='sigmoid'))\n",
        "model.compile(loss='categorical_crossentropy',\n",
        "                  optimizer='adam',\n",
        "                  metrics=['accuracy'])\n",
        "\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: UserWarning: Update your `LSTM` call to the Keras 2 API: `LSTM(150, dropout=0.2, recurrent_dropout=0.2)`\n",
            "  \"\"\"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_2 (Embedding)      (None, 24, 300)           4510800   \n",
            "_________________________________________________________________\n",
            "lstm_1 (LSTM)                (None, 150)               270600    \n",
            "_________________________________________________________________\n",
            "dense_4 (Dense)              (None, 64)                9664      \n",
            "_________________________________________________________________\n",
            "dense_5 (Dense)              (None, 6)                 390       \n",
            "=================================================================\n",
            "Total params: 4,791,454\n",
            "Trainable params: 280,654\n",
            "Non-trainable params: 4,510,800\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "5b6a5c06-10c0-4faf-95fb-18fe462c7972",
        "id": "ZvIccE0huvAR",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        }
      },
      "source": [
        "hist = model.fit(word_seq_train, y_train, batch_size=batch_size, epochs=num_epochs, validation_split=0.1, shuffle=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 134179 samples, validate on 14909 samples\n",
            "Epoch 1/8\n",
            "134179/134179 [==============================] - 165s 1ms/step - loss: 1.2247 - accuracy: 0.5355 - val_loss: 1.0920 - val_accuracy: 0.5982\n",
            "Epoch 2/8\n",
            "134179/134179 [==============================] - 167s 1ms/step - loss: 1.0764 - accuracy: 0.5982 - val_loss: 1.0434 - val_accuracy: 0.6137\n",
            "Epoch 3/8\n",
            "134179/134179 [==============================] - 165s 1ms/step - loss: 1.0242 - accuracy: 0.6202 - val_loss: 1.0157 - val_accuracy: 0.6251\n",
            "Epoch 4/8\n",
            "134179/134179 [==============================] - 164s 1ms/step - loss: 0.9881 - accuracy: 0.6353 - val_loss: 1.0064 - val_accuracy: 0.6289\n",
            "Epoch 5/8\n",
            "134179/134179 [==============================] - 163s 1ms/step - loss: 0.9609 - accuracy: 0.6456 - val_loss: 0.9890 - val_accuracy: 0.6390\n",
            "Epoch 6/8\n",
            "134179/134179 [==============================] - 164s 1ms/step - loss: 0.9389 - accuracy: 0.6547 - val_loss: 0.9920 - val_accuracy: 0.6399\n",
            "Epoch 7/8\n",
            "134179/134179 [==============================] - 163s 1ms/step - loss: 0.9239 - accuracy: 0.6607 - val_loss: 0.9773 - val_accuracy: 0.6448\n",
            "Epoch 8/8\n",
            "134179/134179 [==============================] - 169s 1ms/step - loss: 0.9069 - accuracy: 0.6680 - val_loss: 0.9847 - val_accuracy: 0.6442\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "ac0613e8-b75f-4d0a-f6fe-18da2b919a9c",
        "id": "ZnCmYX6V7F5P",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 438
        }
      },
      "source": [
        "# BiLSTM\n",
        "\n",
        "# main model\n",
        "inputt = Input(shape=(max_seq_len,))\n",
        "model = Embedding(nb_words, embed_dim,weights=[embedding_matrix],input_length=max_seq_len, trainable = False)(inputt)\n",
        "model =  Bidirectional (LSTM (lstm_out1, return_sequences=True, dropout_U = 0.2, dropout_W = 0.2 ), merge_mode='concat')(model)\n",
        "model = TimeDistributed(Dense(10,activation='relu'))(model)\n",
        "model = Flatten()(model)\n",
        "model = Dense(64,activation='relu')(model)\n",
        "output = Dense(num_classes,activation='sigmoid')(model)\n",
        "model = Model(inputt,output)\n",
        "model.compile(loss='categorical_crossentropy',optimizer='adam', metrics=['accuracy'])\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: UserWarning: Update your `LSTM` call to the Keras 2 API: `LSTM(150, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)`\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_2 (InputLayer)         (None, 24)                0         \n",
            "_________________________________________________________________\n",
            "embedding_3 (Embedding)      (None, 24, 300)           4510800   \n",
            "_________________________________________________________________\n",
            "bidirectional_1 (Bidirection (None, 24, 300)           541200    \n",
            "_________________________________________________________________\n",
            "time_distributed_1 (TimeDist (None, 24, 10)            3010      \n",
            "_________________________________________________________________\n",
            "flatten_1 (Flatten)          (None, 240)               0         \n",
            "_________________________________________________________________\n",
            "dense_7 (Dense)              (None, 64)                15424     \n",
            "_________________________________________________________________\n",
            "dense_8 (Dense)              (None, 6)                 390       \n",
            "=================================================================\n",
            "Total params: 5,070,824\n",
            "Trainable params: 560,024\n",
            "Non-trainable params: 4,510,800\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "798138f4-ae86-4c93-fafa-1fe52a78e21e",
        "id": "zQDSwJ81Hzu3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        }
      },
      "source": [
        "hist = model.fit(word_seq_train, y_train, batch_size=batch_size, epochs=num_epochs, validation_split=0.1, shuffle=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 134179 samples, validate on 14909 samples\n",
            "Epoch 1/8\n",
            "134179/134179 [==============================] - 306s 2ms/step - loss: 1.2002 - accuracy: 0.5453 - val_loss: 1.0537 - val_accuracy: 0.6122\n",
            "Epoch 2/8\n",
            "134179/134179 [==============================] - 304s 2ms/step - loss: 1.0268 - accuracy: 0.6219 - val_loss: 0.9941 - val_accuracy: 0.6383\n",
            "Epoch 3/8\n",
            "134179/134179 [==============================] - 305s 2ms/step - loss: 0.9566 - accuracy: 0.6503 - val_loss: 0.9670 - val_accuracy: 0.6501\n",
            "Epoch 4/8\n",
            "134179/134179 [==============================] - 304s 2ms/step - loss: 0.9097 - accuracy: 0.6695 - val_loss: 0.9504 - val_accuracy: 0.6570\n",
            "Epoch 5/8\n",
            "134179/134179 [==============================] - 301s 2ms/step - loss: 0.8727 - accuracy: 0.6847 - val_loss: 0.9444 - val_accuracy: 0.6625\n",
            "Epoch 6/8\n",
            "134179/134179 [==============================] - 302s 2ms/step - loss: 0.8439 - accuracy: 0.6952 - val_loss: 0.9277 - val_accuracy: 0.6683\n",
            "Epoch 7/8\n",
            "134179/134179 [==============================] - 300s 2ms/step - loss: 0.8204 - accuracy: 0.7044 - val_loss: 0.9263 - val_accuracy: 0.6701\n",
            "Epoch 8/8\n",
            "134179/134179 [==============================] - 304s 2ms/step - loss: 0.7982 - accuracy: 0.7127 - val_loss: 0.9238 - val_accuracy: 0.6750\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lzg_abDUYpeu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Attention BiLSTM\n",
        "\n",
        "from keras import backend as K\n",
        "from keras.engine.topology import Layer\n",
        "from keras import initializers, regularizers, constraints\n",
        "\n",
        "\n",
        "class Attention(Layer):\n",
        "    def __init__(self, step_dim,\n",
        "                 W_regularizer=None, b_regularizer=None,\n",
        "                 W_constraint=None, b_constraint=None,\n",
        "                 bias=True, **kwargs):\n",
        "        self.supports_masking = True\n",
        "        self.init = initializers.get('glorot_uniform')\n",
        "\n",
        "        self.W_regularizer = regularizers.get(W_regularizer)\n",
        "        self.b_regularizer = regularizers.get(b_regularizer)\n",
        "\n",
        "        self.W_constraint = constraints.get(W_constraint)\n",
        "        self.b_constraint = constraints.get(b_constraint)\n",
        "\n",
        "        self.bias = bias\n",
        "        self.step_dim = step_dim\n",
        "        self.features_dim = 0\n",
        "        super(Attention, self).__init__(**kwargs)\n",
        "\n",
        "    def build(self, input_shape):\n",
        "        assert len(input_shape) == 3\n",
        "\n",
        "        self.W = self.add_weight( shape=(input_shape[-1],),\n",
        "                                 initializer=self.init,\n",
        "                                 name='{}_W'.format(self.name),\n",
        "                                 regularizer=self.W_regularizer,\n",
        "                                 constraint=self.W_constraint)\n",
        "        self.features_dim = input_shape[-1]\n",
        "        #print(self.name)\n",
        "        if self.bias:\n",
        "            self.b = self.add_weight(shape=(input_shape[1],),\n",
        "                                     initializer='zero',\n",
        "                                     name='{}_b'.format(self.name),\n",
        "                                     regularizer=self.b_regularizer,\n",
        "                                     constraint=self.b_constraint)\n",
        "        else:\n",
        "            self.b = None\n",
        "\n",
        "        self.built = True\n",
        "\n",
        "    def compute_mask(self, input, input_mask=None):\n",
        "        return None\n",
        "\n",
        "    def call(self, x, mask=None):\n",
        "        features_dim = self.features_dim\n",
        "        step_dim = self.step_dim\n",
        "\n",
        "        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n",
        "                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n",
        "\n",
        "        if self.bias:\n",
        "            eij += self.b\n",
        "\n",
        "        eij = K.tanh(eij)\n",
        "\n",
        "        a = K.exp(eij)\n",
        "\n",
        "        if mask is not None:\n",
        "            a *= K.cast(mask, K.floatx())\n",
        "\n",
        "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
        "\n",
        "        a = K.expand_dims(a)\n",
        "        weighted_input = x * a\n",
        "        return K.sum(weighted_input, axis=1)\n",
        "\n",
        "    def compute_output_shape(self, input_shape):\n",
        "        return input_shape[0],  self.features_dim\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YeGgEApsZHlz",
        "colab_type": "code",
        "outputId": "33657fd9-cae3-4c3c-d95a-1c6cfa72cca2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 385
        }
      },
      "source": [
        "inp = Input(shape=(max_seq_len, ))\n",
        "x = Embedding(nb_words, embed_dim, weights=[embedding_matrix],\n",
        "                  trainable=False)(inp)\n",
        "x = Bidirectional(LSTM(300, return_sequences=True, dropout=0.25,\n",
        "                          recurrent_dropout=0.25))(x)\n",
        "x = Attention(max_seq_len)(x)\n",
        "x = Dense(256, activation=\"relu\")(x)\n",
        "x = Dropout(0.25)(x)\n",
        "x = Dense(num_classes, activation=\"sigmoid\")(x)\n",
        "model = Model(inputs=inp, outputs=x)\n",
        "\n",
        "model.compile(loss='categorical_crossentropy',optimizer='adam', metrics=['accuracy'])\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_3\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_3 (InputLayer)         (None, 24)                0         \n",
            "_________________________________________________________________\n",
            "embedding_4 (Embedding)      (None, 24, 300)           4510800   \n",
            "_________________________________________________________________\n",
            "bidirectional_2 (Bidirection (None, 24, 600)           1442400   \n",
            "_________________________________________________________________\n",
            "attention_1 (Attention)      (None, 600)               624       \n",
            "_________________________________________________________________\n",
            "dense_9 (Dense)              (None, 256)               153856    \n",
            "_________________________________________________________________\n",
            "dropout_2 (Dropout)          (None, 256)               0         \n",
            "_________________________________________________________________\n",
            "dense_10 (Dense)             (None, 6)                 1542      \n",
            "=================================================================\n",
            "Total params: 6,109,222\n",
            "Trainable params: 1,598,422\n",
            "Non-trainable params: 4,510,800\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "117e6604-d4f6-4ae8-e902-0f38df1ccf67",
        "id": "mY8qgojCZxJ9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        }
      },
      "source": [
        "hist = model.fit(word_seq_train, y_train, batch_size=batch_size, epochs=num_epochs, validation_split=0.1, shuffle=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 134179 samples, validate on 14909 samples\n",
            "Epoch 1/8\n",
            "134179/134179 [==============================] - 365s 3ms/step - loss: 1.1726 - accuracy: 0.5582 - val_loss: 1.0160 - val_accuracy: 0.6255\n",
            "Epoch 2/8\n",
            "134179/134179 [==============================] - 363s 3ms/step - loss: 0.9988 - accuracy: 0.6333 - val_loss: 0.9496 - val_accuracy: 0.6569\n",
            "Epoch 3/8\n",
            "134179/134179 [==============================] - 366s 3ms/step - loss: 0.9234 - accuracy: 0.6636 - val_loss: 0.9144 - val_accuracy: 0.6720\n",
            "Epoch 4/8\n",
            "134179/134179 [==============================] - 367s 3ms/step - loss: 0.8695 - accuracy: 0.6846 - val_loss: 0.8943 - val_accuracy: 0.6778\n",
            "Epoch 5/8\n",
            "134179/134179 [==============================] - 366s 3ms/step - loss: 0.8276 - accuracy: 0.7010 - val_loss: 0.8783 - val_accuracy: 0.6865\n",
            "Epoch 6/8\n",
            "134179/134179 [==============================] - 367s 3ms/step - loss: 0.7915 - accuracy: 0.7136 - val_loss: 0.8857 - val_accuracy: 0.6860\n",
            "Epoch 7/8\n",
            "134179/134179 [==============================] - 363s 3ms/step - loss: 0.7618 - accuracy: 0.7230 - val_loss: 0.8806 - val_accuracy: 0.6900\n",
            "Epoch 8/8\n",
            "134179/134179 [==============================] - 366s 3ms/step - loss: 0.7381 - accuracy: 0.7325 - val_loss: 0.8778 - val_accuracy: 0.6910\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "01c86abe-c064-4802-b09e-830bb37bd174",
        "id": "qzTXo_HP7e2e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 385
        }
      },
      "source": [
        "inp = Input(shape=(max_seq_len, ))\n",
        "x = Embedding(nb_words, embed_dim, weights=[embedding_matrix], trainable=False)(inp)\n",
        "x = Bidirectional(LSTM(lstm_out1 , return_sequences=True, dropout=0.25, recurrent_dropout=0.25))(x)\n",
        "x = Attention(max_seq_len)(x)\n",
        "x = Dense(256, activation=\"relu\")(x)\n",
        "x = Dropout(0.25)(x)\n",
        "x = Dense(num_classes, activation=\"sigmoid\")(x)\n",
        "model = Model(inputs=inp, outputs=x)\n",
        "\n",
        "model.compile(loss='categorical_crossentropy',optimizer='adam', metrics=['accuracy'])\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_7\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_8 (InputLayer)         (None, 24)                0         \n",
            "_________________________________________________________________\n",
            "embedding_8 (Embedding)      (None, 24, 300)           4510800   \n",
            "_________________________________________________________________\n",
            "bidirectional_7 (Bidirection (None, 24, 300)           541200    \n",
            "_________________________________________________________________\n",
            "attention_7 (Attention)      (None, 300)               324       \n",
            "_________________________________________________________________\n",
            "dense_13 (Dense)             (None, 256)               77056     \n",
            "_________________________________________________________________\n",
            "dropout_7 (Dropout)          (None, 256)               0         \n",
            "_________________________________________________________________\n",
            "dense_14 (Dense)             (None, 6)                 1542      \n",
            "=================================================================\n",
            "Total params: 5,130,922\n",
            "Trainable params: 620,122\n",
            "Non-trainable params: 4,510,800\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "c41f5062-86c2-4201-d878-bb57f2003485",
        "id": "1DdEyto2uvk8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        }
      },
      "source": [
        "hist = model.fit(word_seq_train, y_train, batch_size=batch_size, epochs=num_epochs, validation_split=0.1, shuffle=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 134179 samples, validate on 14909 samples\n",
            "Epoch 1/8\n",
            "134179/134179 [==============================] - 199s 1ms/step - loss: 1.1853 - accuracy: 0.5511 - val_loss: 1.0387 - val_accuracy: 0.6147\n",
            "Epoch 2/8\n",
            "134179/134179 [==============================] - 201s 1ms/step - loss: 1.0230 - accuracy: 0.6229 - val_loss: 0.9742 - val_accuracy: 0.6432\n",
            "Epoch 3/8\n",
            "134179/134179 [==============================] - 202s 2ms/step - loss: 0.9578 - accuracy: 0.6495 - val_loss: 0.9315 - val_accuracy: 0.6637\n",
            "Epoch 4/8\n",
            "134179/134179 [==============================] - 203s 2ms/step - loss: 0.9125 - accuracy: 0.6671 - val_loss: 0.9211 - val_accuracy: 0.6677\n",
            "Epoch 5/8\n",
            "134179/134179 [==============================] - 202s 2ms/step - loss: 0.8803 - accuracy: 0.6811 - val_loss: 0.9102 - val_accuracy: 0.6702\n",
            "Epoch 6/8\n",
            "134179/134179 [==============================] - 200s 1ms/step - loss: 0.8535 - accuracy: 0.6903 - val_loss: 0.9030 - val_accuracy: 0.6762\n",
            "Epoch 7/8\n",
            "134179/134179 [==============================] - 202s 2ms/step - loss: 0.8324 - accuracy: 0.6993 - val_loss: 0.8941 - val_accuracy: 0.6801\n",
            "Epoch 8/8\n",
            "134179/134179 [==============================] - 201s 2ms/step - loss: 0.8137 - accuracy: 0.7058 - val_loss: 0.8936 - val_accuracy: 0.6787\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LnF9gZEntcKl",
        "colab_type": "code",
        "outputId": "76d8f1c2-3a55-4e2d-fbd4-b00911c1ad05",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "print(y_train)\n",
        "print(len(y_train))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0]\n",
            " [1]\n",
            " [0]\n",
            " ...\n",
            " [0]\n",
            " [0]\n",
            " [0]]\n",
            "4999\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "bcc9c1ac-db54-4356-bce7-0ef39dbaf909",
        "_uuid": "169c30a270b00bf2088afc4268ada3042318adb3",
        "id": "Mdu_nMcApaiu",
        "colab_type": "text"
      },
      "source": [
        "Because of the multi-label loss, we are using k-hot encoding of the output and sigmoid activations. As a result, the loss is binary cross-entropy."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "_cell_guid": "e439dcfb-c97c-4ae5-b6d4-e48b5c44b919",
        "_uuid": "c48a4db47861146bc4e90b3be6cd23106edb1255",
        "trusted": true,
        "id": "iRnZuvWNpaiu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#define callbacks\n",
        "early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.01, patience=4, verbose=1)\n",
        "callbacks_list = [early_stopping]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "_cell_guid": "38c67614-db6d-4d57-80d6-e46404bf4147",
        "_uuid": "c1a947e9bc4acefed432c40ec79fa318921278a2",
        "trusted": true,
        "id": "2miFvICypaiy",
        "colab_type": "code",
        "outputId": "86a8b66a-fbdb-46b1-e2eb-47c4798fc8b6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 183
        }
      },
      "source": [
        "#model training\n",
        "hist = model.fit(word_seq_train, y_train, batch_size=batch_size, epochs=num_epochs, callbacks=callbacks_list, validation_split=0.1, shuffle=True, verbose=2)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-21-70b64faee771>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mhist\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mword_seq_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnum_epochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_split\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'callbacks_list' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "dfd0eaee-5f7b-4e6d-b809-dfe1316a30db",
        "_uuid": "b48d3b7623e2ce8e814ed246595c213f4321af64",
        "id": "s7bnOpv0pai1",
        "colab_type": "text"
      },
      "source": [
        "Let's make predictions on the test data:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "_cell_guid": "5e2c397f-ae3f-4382-891c-89b172fe9e48",
        "_uuid": "25540b75e933d23b7e204ce34df09415392c8474",
        "trusted": true,
        "id": "IfCPM49qpai1",
        "colab_type": "code",
        "outputId": "a7584652-0e29-4d2d-8d98-f728fc8dc5f4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "y_pred = model.predict(word_seq_test)\n",
        "print(y_pred)\n",
        "from sklearn.metrics import classification_report\n",
        "\n",
        "#y_pred = model.predict(word_seq_test, batch_size=64, verbose=1)\n",
        "#y_pred_bool = np.argmax(y_pred, axis=1)\n",
        "#y_final=y_pred_bool.round\n",
        "print(classification_report(y_test, y_pred.round()))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0.74227786]\n",
            " [0.5047133 ]\n",
            " [0.8221329 ]\n",
            " [0.4604183 ]\n",
            " [0.91411513]\n",
            " [0.52935994]\n",
            " [0.9285195 ]\n",
            " [0.699492  ]\n",
            " [0.50055104]\n",
            " [0.77881575]\n",
            " [0.45935008]\n",
            " [0.99589455]\n",
            " [0.714272  ]\n",
            " [0.99890804]\n",
            " [0.95115834]\n",
            " [0.8540589 ]\n",
            " [0.73217684]\n",
            " [0.5370274 ]\n",
            " [0.6116317 ]\n",
            " [0.9704306 ]\n",
            " [0.85503036]\n",
            " [0.83672565]\n",
            " [0.95010334]\n",
            " [0.56763524]\n",
            " [0.9049514 ]\n",
            " [0.595439  ]\n",
            " [0.4202568 ]\n",
            " [0.96245635]\n",
            " [0.9527545 ]\n",
            " [0.92329115]\n",
            " [0.6713236 ]\n",
            " [0.9456618 ]\n",
            " [0.9726731 ]\n",
            " [0.9448101 ]\n",
            " [0.92816687]\n",
            " [0.9555551 ]\n",
            " [0.8508468 ]\n",
            " [0.8108252 ]\n",
            " [0.966388  ]\n",
            " [0.72358817]\n",
            " [0.870922  ]\n",
            " [0.8803662 ]\n",
            " [0.916993  ]\n",
            " [0.4897167 ]\n",
            " [0.6230776 ]\n",
            " [0.9450136 ]\n",
            " [0.54772323]\n",
            " [0.4123038 ]\n",
            " [0.897395  ]\n",
            " [0.7520355 ]\n",
            " [0.73475075]\n",
            " [0.5940251 ]\n",
            " [0.78745985]\n",
            " [0.95681524]\n",
            " [0.9990702 ]\n",
            " [0.45634347]\n",
            " [0.66067195]\n",
            " [0.8372949 ]\n",
            " [0.9591135 ]\n",
            " [0.7314591 ]\n",
            " [0.766696  ]\n",
            " [0.5349784 ]\n",
            " [0.8003424 ]\n",
            " [0.97009665]\n",
            " [0.99233997]\n",
            " [0.8244031 ]\n",
            " [0.87771237]\n",
            " [0.9276703 ]\n",
            " [0.7717381 ]\n",
            " [0.9410118 ]\n",
            " [0.47282058]\n",
            " [0.9737935 ]\n",
            " [0.9753846 ]\n",
            " [0.62053657]\n",
            " [0.8215027 ]\n",
            " [0.8899912 ]\n",
            " [0.95080733]\n",
            " [0.820426  ]\n",
            " [0.89305866]\n",
            " [0.481136  ]\n",
            " [0.45684874]\n",
            " [0.40697888]\n",
            " [0.35650057]\n",
            " [0.35948825]\n",
            " [0.633059  ]\n",
            " [0.3915243 ]\n",
            " [0.57666194]\n",
            " [0.7994752 ]\n",
            " [0.90936375]\n",
            " [0.8249937 ]\n",
            " [0.46319082]\n",
            " [0.99859744]\n",
            " [0.9079212 ]\n",
            " [0.9986525 ]\n",
            " [0.94310516]\n",
            " [0.97357935]\n",
            " [0.913643  ]\n",
            " [0.80092055]\n",
            " [0.67491966]\n",
            " [0.43463588]\n",
            " [0.5436821 ]\n",
            " [0.91777134]\n",
            " [0.31179318]\n",
            " [0.87334573]\n",
            " [0.41838694]\n",
            " [0.9593488 ]\n",
            " [0.9577276 ]\n",
            " [0.9859461 ]\n",
            " [0.6989277 ]\n",
            " [0.96672153]\n",
            " [0.9910868 ]\n",
            " [0.48332423]\n",
            " [0.721962  ]\n",
            " [0.15088503]\n",
            " [0.9062177 ]\n",
            " [0.5246694 ]\n",
            " [0.97888994]\n",
            " [0.7402297 ]\n",
            " [0.9622817 ]\n",
            " [0.99180156]\n",
            " [0.9578579 ]\n",
            " [0.7577247 ]\n",
            " [0.50708914]\n",
            " [0.8378285 ]\n",
            " [0.97335726]\n",
            " [0.21027836]\n",
            " [0.7123154 ]\n",
            " [0.7432666 ]\n",
            " [0.96112615]\n",
            " [0.70152575]\n",
            " [0.45434943]\n",
            " [0.48072693]\n",
            " [0.91412145]\n",
            " [0.57959723]\n",
            " [0.7729785 ]\n",
            " [0.55168545]\n",
            " [0.19854046]\n",
            " [0.5587514 ]\n",
            " [0.5270437 ]\n",
            " [0.9598869 ]\n",
            " [0.94605273]\n",
            " [0.79907656]\n",
            " [0.52405244]\n",
            " [0.7964704 ]\n",
            " [0.57090926]\n",
            " [0.52057266]\n",
            " [0.5168334 ]\n",
            " [0.8191581 ]\n",
            " [0.9319135 ]\n",
            " [0.97721297]\n",
            " [0.7059561 ]\n",
            " [0.71647006]\n",
            " [0.5612078 ]\n",
            " [0.970769  ]\n",
            " [0.9905308 ]\n",
            " [0.8064747 ]\n",
            " [0.8789808 ]\n",
            " [0.9169957 ]\n",
            " [0.8176109 ]\n",
            " [0.5183277 ]\n",
            " [0.8521568 ]\n",
            " [0.9389965 ]\n",
            " [0.70594954]\n",
            " [0.7918134 ]\n",
            " [0.9064908 ]\n",
            " [0.539589  ]\n",
            " [0.6483057 ]\n",
            " [0.61430615]\n",
            " [0.5730697 ]\n",
            " [0.6519397 ]\n",
            " [0.5350784 ]\n",
            " [0.7052703 ]\n",
            " [0.97779435]\n",
            " [0.8043238 ]\n",
            " [0.7542768 ]\n",
            " [0.8700237 ]\n",
            " [0.9086068 ]\n",
            " [0.9370471 ]\n",
            " [0.5842851 ]\n",
            " [0.5164504 ]\n",
            " [0.5460341 ]\n",
            " [0.9315135 ]\n",
            " [0.5051968 ]\n",
            " [0.80824506]\n",
            " [0.9518588 ]\n",
            " [0.9256354 ]\n",
            " [0.9734153 ]\n",
            " [0.9330235 ]\n",
            " [0.8574803 ]\n",
            " [0.9580519 ]\n",
            " [0.9992644 ]\n",
            " [0.5168982 ]\n",
            " [0.82458484]\n",
            " [0.97898793]\n",
            " [0.8007729 ]\n",
            " [0.6164814 ]\n",
            " [0.9721495 ]\n",
            " [0.8006912 ]\n",
            " [0.89718646]\n",
            " [0.9158278 ]\n",
            " [0.9340074 ]\n",
            " [0.7945628 ]\n",
            " [0.95767045]\n",
            " [0.536821  ]\n",
            " [0.58343977]\n",
            " [0.6288666 ]\n",
            " [0.6962182 ]\n",
            " [0.9983413 ]\n",
            " [0.9846473 ]\n",
            " [0.59699106]\n",
            " [0.78497756]\n",
            " [0.5601653 ]\n",
            " [0.7142819 ]\n",
            " [0.53680253]\n",
            " [0.5982459 ]\n",
            " [0.95150876]\n",
            " [0.6513692 ]\n",
            " [0.64788043]\n",
            " [0.82645017]\n",
            " [0.6492197 ]\n",
            " [0.9314587 ]\n",
            " [0.6872775 ]\n",
            " [0.58569175]\n",
            " [0.8924366 ]\n",
            " [0.64594907]\n",
            " [0.93469375]\n",
            " [0.72019774]\n",
            " [0.9519454 ]\n",
            " [0.84732735]\n",
            " [0.84125596]\n",
            " [0.8453936 ]\n",
            " [0.9321068 ]\n",
            " [0.5615602 ]\n",
            " [0.73711634]\n",
            " [0.5122554 ]\n",
            " [0.90542454]\n",
            " [0.6162301 ]\n",
            " [0.80554885]\n",
            " [0.95690376]\n",
            " [0.6867216 ]\n",
            " [0.98236406]\n",
            " [0.91653895]\n",
            " [0.59803087]\n",
            " [0.7188799 ]\n",
            " [0.99461186]\n",
            " [0.9459417 ]\n",
            " [0.6302087 ]\n",
            " [0.8328825 ]\n",
            " [0.98112434]\n",
            " [0.8867212 ]\n",
            " [0.88802487]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       1.00      0.09      0.17       250\n",
            "           1       0.00      1.00      0.01         1\n",
            "\n",
            "    accuracy                           0.10       251\n",
            "   macro avg       0.50      0.55      0.09       251\n",
            "weighted avg       1.00      0.10      0.17       251\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "_cell_guid": "16a97ecf-a5bb-46c5-814f-ab4483afe139",
        "_uuid": "cc103f87ffee97d82a6c18f9f7d360a739a8482c",
        "trusted": true,
        "id": "mG6a7tvnpai5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#create a submission\n",
        "submission_df = pd.DataFrame(columns=['tweet_id'] + label_names)\n",
        "submission_df['tweet_id'] = test_df['tweet_id'].values \n",
        "submission_df[label_names] = y_test \n",
        "submission_df.to_csv(\"./cnn_fasttext_submission.csv\", index=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "34f04d6b-f37f-4917-9128-1a4d2ddb318a",
        "_uuid": "0739cf53451a26d1d8a7800b18c97ed706fb4b96",
        "id": "eKtwAWdQpai8",
        "colab_type": "text"
      },
      "source": [
        "Looking at training and validation loss / accuracy figures below, we can see there is no sign of over-fitting."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "_cell_guid": "89fdc75d-43eb-445c-8588-af0eed779ca3",
        "_uuid": "c70a282f66aa426654d9b02fec4a1f31f59286ab",
        "trusted": true,
        "id": "HleZR1m_pai9",
        "colab_type": "code",
        "outputId": "00788b82-bef4-45e5-d500-def59c7abf00",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        }
      },
      "source": [
        "#generate plots\n",
        "plt.figure()\n",
        "plt.plot(hist.history['loss'], lw=2.0, color='b', label='train')\n",
        "plt.plot(hist.history['val_loss'], lw=2.0, color='r', label='val')\n",
        "plt.title('CNN sentiment')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Cross-Entropy Loss')\n",
        "plt.legend(loc='upper right')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de3zO9fvA8de9w70NYyzuOcyQ03L4NpFjzByGmTkXir4lhBBybliZL5EkkUiFIoc1zCnHUU45/JY2CmFzWDmMOe3k8/vjnWXZds92H3Zv1/PxuB/uw+dwbdi19+l66zRN0xBCCCH+xc7aAQghhMifJEEIIYTIlCQIIYQQmZIEIYQQIlOSIIQQQmRKEoQQQohMSYIQIp/p378/YWFh1g5DCEkQwjZt2LCBrl274uPjQ7Nmzejfvz8///wzAPPmzaNGjRps2rQp/fjU1FRq1KhBXFwcAOPGjaNGjRpERUWlH3P+/Hlq1Khh0a9j3rx5jB49OsN7ixcvpkuXLhaNA9T3ZM6cORa/r8i/JEEIm7N06VJCQ0MZNGgQP/74I7t27aJ3797s2LEj/Rg3NzfmzZtHWlpaltdxc3Pjo48+skTIQtgkSRDCpiQmJvLxxx8THBxM27ZtKVKkCI6Ojvj5+TF27Nj045o1a4ajoyPr16/P8lqdO3fm1KlTHDp0KEf3XrRoES+88AI+Pj74+/uzf/9+AB48eMCiRYto3bo1DRs2ZPjw4SQkJAAQFxdHjRo1CAsLw9fXl4YNG7JgwQIAIiMj+eyzz9i8eTM+Pj506tQJgFdeeYXVq1cDsG7dOl566SVCQ0OpX78+rVq14ujRo6xbt44WLVrQuHHjDN1RycnJzJgxA19fX5o0aUJwcDD3798H4ODBgzRv3pwvvviCxo0b06xZM9auXQvAqlWr2LBhA0uWLMHHx4dBgwbl6HsiCjZJEMKmHDt2jKSkJNq0aZPtcTqdjuHDh/PJJ5+QkpKS6THOzs4MHDgwR90qZ8+eZcWKFaxZs4Zjx46xZMkSypcvD8CyZcvYvn07y5cvZ+/evZQoUYKQkJAM5x85coQtW7bw1VdfMX/+fM6cOUPz5s0ZOHAg7du359ixY1kms6ioKGrUqMHBgwfp2LEjI0eO5JdffuGHH37ggw8+ICQkhDt37gAwa9Ys/vjjD77//nu2bdvGn3/+yfz589OvdfXqVRITE4mMjGTatGmEhIRw8+ZNXnzxRQIDA3n99dc5duwYCxcuNPo9EQWfJAhhUxISEihZsiQODg5Gj23VqhWlSpVK/208My+99BKXL19mz5492V7L3t6e5ORkzpw5Q0pKChUqVKBixYoArFy5krfffhsPDw/0ej1Dhw5l69atpKampp8/dOhQnJ2dqVmzJjVr1uTkyZM5/IqhQoUKdOvWDXt7ezp06MDly5cZMmQIer2eZs2aodfruXDhApqm8d133zFhwgTc3NwoVqwYAwcOJCIiIv1aDg4ODBkyBEdHR1q0aEGRIkX4448/chyLKFyM/y8TIh9xc3Pjxo0bpKam5ihJjBgxgvHjxxMUFJTp53q9nsGDBzN37txsWxJeXl5MmDCBefPmcfr0aZo1a8a4ceMwGAxcunSJIUOGYGf3z+9bdnZ2XLt2Lf31U089lf7cxcWFu3fv5uTLBcDd3T39ubOz82PXc3Jy4s6dO1y/fp179+7RtWvX9M80TePBgwfpr93c3DJ83540FlG4SAtC2BQfHx/0ej3bt2/P0fFNmzbFy8uLb775JstjunbtSmJiItu2bcv2WoGBgXz77bfs2rULnU7HrFmzAPDw8ODzzz/n559/Tn/88ssvGAwGo/HpdLocfR05UbJkSZydnYmIiEiP48iRIxw7dixH55syFlEwSIIQNsXV1ZVhw4YREhLC9u3buXfvHikpKezZs4eZM2dmes6IESNYvHhxltd0cHDgrbfeyvaYs2fPsn//fpKTk9Hr9Tg5OaW3GHr16sVHH33ExYsXAbh+/XqOE5i7uzsXL17M8Ft+btnZ2dGjRw9CQ0PTWy/x8fHs3bs3x7E8nAYsBEiCEDbotddeY9y4cXz66ac0btwYX19fVqxYQevWrTM9/rnnnqNu3brZXrNjx46ULl06y8+Tk5OZPXs2DRs2pFmzZly/fp2RI0cC0LdvX/z8/Hjttdfw8fGhZ8+eGdZXZKddu3YANGzY0CRrH9555x28vLzo2bMn9erV49VXX83xGEP37t05ffo09evXZ/DgwXmORdg+nWwYJIQQIjPSghBCCJEpSRBCCCEyJQlCCCFEpiRBCCGEyFSBWSh3/PhxnJyccn1+UlJSns63JFuKFWwrXluKFWwrXluKFWwr3rzEmpSUxLPPPpvpZwUmQTg5OeHt7Z3r82NiYvJ0viXZUqxgW/HaUqxgW/HaUqxgW/HmJdaYmJgsP5MuJiGEEJmSBCGEECJTkiCEEEJkqsCMQQghRG6kpKQQFxeXvrHSo+9n1z+fn+QkVmdnZypUqICjo2OOrysJQghRqMXFxeHq6kqlSpUyVLS9d+8eLi4uVows54zFqmka165dIy4ujsqVK+f4utLFJIQo1O7fv4+7u3uBLneu0+lwd3d/rJVkjCQIIUShV5CTw0O5+RoLfYK4fh1efhm2bHG1dihCCJGvFPoEce4crFgBI0dWYNo0kOLnQghLunXrFitWrHji89544w1u3bplhoj+UegTRL16MHcu6HQakybBgAGQkmLtqIQQhcWtW7f49ttvH3s/NTU12/M+//xzihcvbq6wAJnFBMCwoPPYafDOOC8WL4a4OPjuO3CVXichhJnNnj2bCxcuEBQUhIODA05OThQvXpw//viDrVu3MnjwYK5cuUJSUhJ9+/blxRdfBMDPz481a9Zw9+5d+vfvT/369Tl27BgGg4FPP/0UZ2fnPMdW6FsQHD0KlSrx3xU92L35Hk89BVu2QPPmcOmStYMTQlhSQADodOpRpIhL+vO8PAICsr/nqFGjqFixIuHh4YwZM4bo6GgmTpzI1q1bAQgNDWXdunWsXbuWZcuWcePGjceuceHCBfr06UNERASurq7p5+aVJIhKlaBcOYoePkzD2T3ZH5lC1apw/Dg0bgy//mrtAIUQhUmdOnXw9PRMf71s2TI6depEz549uXz5MufPn3/snHLlyqUX66tVqxYXL140SSySIEqVgm3bSC1RAjZupOr7r7L/xwc0bgwXLkDTprBrl7WDFEJYQkSEmqiiaXD37r3053l5REQ8WQxFihRJf37w4EF++uknVq1axfr163nmmWdISkp67By9Xp/+3N7enrS0tFx/Dx4lCQKgVi1iP/sMihWDb77hqZBh7Niu0aUL3LwJ/v5qppMQQpha0aJFuXPnTqafJSYmUqJECVxcXDhz5gzHjx+3aGySIP52v25dCA8HJyeYPx+X6cGsXg3Dh6tZTS+/DKGhMg1WCGFaJUuWpF69enTs2JGZM2dm+Kx58+akpqbSvn17Zs+eneXGPuYis5ge5ecHq1ZBt27w/vvYlyzJRx+NpFIlGDkSJk6E8+dh/nxwkO+cEMJEZs+enen7er2exYsXZ/rZzp07AShVqhRr165Nf//11183WVzSgvi3oCD44gv1fNQo+OILRoyA1avB2RkWLVKH3L5t3TCFEMLcJEFkpm9ftXoO4I03YO1aunWDHTvA3R02bYIWLeDyZeuGKYQQ5iQJIivDhsGUKfDgAfTuDT/8QJMmsH8/PP20Wj7RuDFER1s7UCGEMA9JENkJDlaJIjkZunSBAweoVk0liYYN1XhE06awZ4+1AxVCCNOTBJEdnQ7mzFFdTnfuQPv28MsvlC4NO3dC586QkABt20ImpVSEEMKmSYIwxs4OlixRI9MPs8GZMxQpAmvWwFtvqQZG794wY4ZMgxVCFBxmTRCRkZH4+/vTpk0bFi1a9Njn3377LYGBgQQFBdGrVy9Onz6d/tlnn31GmzZt8Pf3Z+/eveYM0zgHB1i5Elq2hCtXoE0buHQJe3s1lv1whtq4cTB4MBgpwiiEELnm4+NjsXuZLUGkpaUREhLC4sWLiYiIYOPGjRkSAEBgYCAbNmwgPDyc/v37M336dABOnz5NREQEERERLF68mKlTp5ps6XiuOTurhXQNGsAff6gkce0aOp1aI7F6tVpjt3Ch6nqSabBCCFtntgQRFRWFl5cXnp6e6PV6AgIC2LFjR4ZjihUrlv783r176Vvi7dixg4CAAPR6PZ6ennh5eREVFWWuUHPO1RU2b4ZnnlHTlzp0gMREALp3V9NgS5VStVd8fVVjQwghsjNr1qwMGwbNmzePTz/9lH79+tGlSxcCAwPZvn27VWIzW4KIj4/Hw8Mj/bXBYCA+Pv6x41asWEHr1q354IMPmDRp0hOdaxXu7rBtm6oCe+iQai78vRF406ZqhlOVKnDkiJoGGxNj3XCFEE/gkXrfLkWK5L3Wdw7qfXfo0IHNmzenv968eTNdunRh/vz5hIWF8dVXXzFjxgw0KwxwWr1gRJ8+fejTpw8bNmxgwYIFzJgxI1fXSUpKIiYPP43v37//ROc7LlhApZdfxmHnThI7diRuzpz0+htffWXP4MGe/PKLC40apfHJJ7HUr38v17HlNVZrs6V4bSlWsK1482usKSkp3Lun/n/q09KwN/H109LSSL6X9f//ypUrc/XqVc6fP8+NGzcoVqwYRYsWZebMmRw9ehSdTkd8fDxxcXE89dRTaJqWHu9Dmb2XmZSUlCf6OzBbgjAYDFx5pI8lPj4eg8GQ5fEBAQFMmTIlV+cCODk5pddDz42YmJgnO9/bW811bdEC1x078J49W5XosFONsv371cym9evt6d+/El99BS+9lOvw8harldlSvLYUK9hWvPk11piYGFxcXNSLLVvS3793794/7+eBPWDsKu3bt2fPnj1cvXqVjh07sn37dm7dukVYWBiOjo74+fmh0+lwcXFJ//NROY3V0dHxsb+D7BKG2bqY6tSpw7lz54iNjSU5OZmIiAj8/PwyHHPu3Ln057t378bLywtQW+lFRESQnJxMbGws586do27duuYKNffq1lV1N4oUga++UqPVfzcDixaFdetgyBA1DbZXL/jgA5kGK4R4XIcOHdi0aRNbt26lXbt2JCYm4u7ujqOjIwcOHDDZBkBPymwtCAcHB4KDg+nfvz9paWl069aNatWqMXfuXGrXrk2rVq1Yvnw5+/fvx8HBgeLFi6d3L1WrVo327dvToUMH7O3tCQ4Oxt7e1A0/E2ncGL7/XvUzzp2rRqmDgwGwt4d586ByZRg9GsaMgXPn1GFSDVYI8VC1atW4c+cOZcqUoUyZMgQGBvLmm28SGBhI7dq1qVKlinUC0wqI6Ohoq56vrV6taXZ2ahOpjz9+7ONVqzRNr1cfBwZq2u3bub9VnmO1MFuK15Zi1TTbije/xppVXHfv3rVwJLmX01gz+1qz+3uRldSm0r27qgUOqn7TsmUZPu7ZE7Zvh5IlYcMGteYuv0zMEkKIzEiCMKXXX4dZs9Tz//5XLax7xAsvqMHrypXh8GHVO3XqlBXiFEKIHJAEYWqjRsGECZCWBi++CLt2Zfi4Rg2VJOrXVwuymzSBffusFKsQAsAqawwsLTdfoyQIc3j/fVWUKSkJOnVSzYVHGAywezd07AjXr0Pr1vDdd9YJVYjCztnZmWvXrhXoJKFpGteuXcPZ2fmJzpO5NOag06npSwkJ8M03qkx4ZKQq0fG3okUhLEwNVyxYoBobFy6oBsjfFUeEEBZQoUIF4uLi+OuvvzK8n5KSgqOjo5WiejI5idXZ2ZkKFSo80XUlQZiLnR18+SXcvKmKM7Vtq/qSKlVKP8TBAebPV2MSY8bAO++oTYg++khNkRVCmJ+joyOVK1d+7P38urAvM+aKVbqYzMnRUZV5bd4cLl5UfUn/quCn06nE8O23oNfDJ59At25w966VYhZCiL9JgjA3FxdYvx7q1YMzZ8DfH27ceOywl16CH34ANzc1+allS/jzTyvEK4QQf5MEYQklSqgaLzVqQFSUWnV9585jhzVvDj/9BF5eqlBs48bw229WiFcIIZAEYTmlS6sy4Z6eap5r165qltO/eHvDgQPw3HNw9qxKEj/+aIV4hRCFniQIS6pYUS2nfpgsXnlFrZf4Fw8PNQ02IEBNg23VCtautXy4QojCTRKEpVWvDlu3QvHiagB74MBMS7wWK6ZqAA4cqBoaPXrAnDlSDVYIYTmSIKzBxwc2blT7XC9Zoua4ZvKT38FBrZGYPl19PHIkjBiRaaNDCCFMThKEtbzwguo3cnBQ9Zv+979MD9PpYNw4WLFCTYP9+GN4++3yD3c5FUIIs5EEYU0dOqiqrzqdqt+0cGGWh/burYYt3Nxg+/bidOyY6UQoIYQwGUkQ1vbSS6ofCVT9pm+/zfLQFi1UxQ5391R27FBLKm7etFCcQohCRxJEfjBw4D8DDX37qtIcWahTB5YtO0+FCmr6a6tWcPWqBWMVQhQakiDyi7FjVc2N1FS1+dDevVkeWqlSMnv3wtNPw5Ej4Ov7WAUPIYTIM0kQ+YVOBzNmQP/+cP++qgV+9GiWh1eqpLqbvL3h11/VmPeFC5YLVwhR8EmCyE90OjVQ3aMH3LoF7dplu+VcuXKwZw88+yycPq2SxOnTFoxXCFGgSYLIb+ztYflyNQL911/Qpk22TYPSpdWmdY0aqcNeeEG1KIQQIq8kQeRHer1aI9GkCcTGqiSRTWlXNzdVCbZlSzUW0aJFtr1TQgiRI5Ig8quiRdVq67p1VUnXdu2yndNarJia/NShA1y7Bn5+qjKsEELkliSI/KxkSbU6rmpVOHYMAgOz3UnIxUVtY9qtm8olbdvCzp0WjFcIUaBIgsjvDAbVf1S+vJr62qMHpKRkebheDytXqkKxd+6oFkU2yyqEECJLZk0QkZGR+Pv706ZNGxYtWvTY50uXLqVDhw4EBgbSr18/Ll68mP6Zt7c3QUFBBAUFMWjQIHOGmf9VqqRaEu7usGkT5SZMyLasq4OD2g570CBVCbZLF1izxmLRCiEKCLMliLS0NEJCQli8eDERERFs3LiR0/+ag+nt7c3atWvZsGED/v7+fPDBB+mfOTs7Ex4eTnh4OAuzqVFUaDzzDGzeDMWKUSIiQtVwyoadHXz6KYwapRocL74IX39toViFEAWC2RJEVFQUXl5eeHp6otfrCQgIYMeOHRmOadSoES4uLgA8++yzXJHlwNlr0ADmz1fPR49WuwllQ6eDDz6AyZPhwQPo1++fsk9CCGGMg7kuHB8fj4eHR/prg8FAVFRUlsevWbOG5s2bp79OSkqia9euODg4MGDAAFq3bp3t/ZKSkoiJicl1vPfv38/T+RZTvz4V6tXD9ehRbrz5JlemTDF6yosvwu3bpZg928DgwXD2bDyvvZZ9cjElm/neYluxgm3Fa0uxgm3Fa65YzZYgnkR4eDgnTpxg+fLl6e/t2rULg8FAbGws/fr1o3r16lSsWDHLazg5OeHt7Z3rGGJiYvJ0viWdmTwZ127dKLl6NSVHjoSGDY2eM2sWVKkCQ4bArFkGihY1MHmyamWYmy19b20pVrCteG0pVrCtePMSa3aJxWgX01dffcXt27fRNI0JEybQpUsX9u3bZ/SmBoMhQ5dRfHw8BoPhseN++uknFi5cyIIFC9Dr9RnOB/D09OT5558nOjra6D0Li+Rq1dTggqbBm2+qAn85MHiwGry2s4OpU7PcyE4IIYAcJIi1a9dSrFgx9u3bx61bt5g5cyazZ882euE6depw7tw5YmNjSU5OJiIiAj8/vwzHREdHExwczIIFC3B3d09//+bNmyQnJwNw/fp1jh49StWqVZ/0ayvY3n0XKlZU6yM+/TTHp/Xrp6bBPtzIbvBgNT4hhBD/ZrSLSfv7V8w9e/YQFBREtWrV0t/L9sIODgQHB9O/f3/S0tLo1q0b1apVY+7cudSuXZtWrVoxc+ZM7t69y/DhwwEoW7YsCxcu5MyZM0yePBmdToemabzxxhuSIP6taFG1/2jnzjBpkioRXq5cjk7t0QOKFFEL6hYuVOslvvhCJQ0hhHjI6I+E2rVr89prrxEXF8eoUaO4ffs2dnY5m/zUokULWrRokeG9h8kA4Msvv8z0vHr16rFhw4Yc3aNQCwpSq6s3bFBdTtnsRvdvAQGwaRN06qRmzN6798++10IIATnoYpo2bRqjRo1izZo1uLi4kJqaSmhoqCViEznx8ceqxsbKlWrF9RPw81Pr74oXVwvpunRRiUIIISAHCeLYsWNUrlyZ4sWLEx4ezoIFC3B1dbVEbCInKlWC4GD1fMgQtdnQE2jSRJUL/3uRNgEBcPu26cMUQtgeowliypQpuLi4cPLkSZYuXUrFihUZO3asJWITOTVypNpa7vffYebMJz69Xj218ZCHh0oWbdtCQoIZ4hRC2BSjCcLBwQGdTsf27dvp06cPffr04c6dO5aITeSUXv/PTKbQUDhz5okvUauWqgVYsSLs36+6n/76y8RxCiFsitEEUbRoUT777DPWr1+Pr68vDx48IDWH8+6FBfn6qhKuSUkwdGiuFjhUraqSxMPq4r6+cOmSySMVQtgIowlizpw56PV6QkNDKV26NFeuXOH111+3RGziSc2apbaX27JF7UiXCxUrQmSkalFER0Pz5nD+vInjFELYBKMJonTp0gQGBpKYmMiuXbtwcnKic+fOlohNPKkyZWD6dPV8xAhITMzVZcqWhd271djEmTNqn+vffjNdmEII22A0QWzatIkePXqwZcsWNm/enP5c5FNvvKGqvl68CDko5JeVp55Su9E93Ba7eXM4ccJ0YQoh8j+jC+UWLlzImjVr0kthXL9+nVdffZV27dqZPTiRC/b2anl0gwYwd66qrVG3bq4uVaKEWicRFAQ7dkCLFur1c8+ZOGYhRL5ktAWhaVqGOklubm45KrUhrKhePbUmIi1NbSuXh2JLRYvCxo1qfcT162p2Uw5qNQohCgCjCaJZs2a8/vrrrFu3jnXr1jFgwABeeOEFS8Qm8uK999TChv37VaGlPHB2hnXrVA2nW7fA3x+2bzdRnEKIfMtoghg7diwvvvgip06d4tSpU7z44ouMGTPGErGJvChRAj78UD0fOxauXs3T5fR6Verp1Vfh7l3o2FGVgBJCFFw5qrrXtm1bxo8fz/jx42nTpg2+vr5mDkuYxEsvQatWqm/IBKvf7e1hyRJVIjwpCbp2hVWrTBCnECJfytWe1DIGYSN0OrXCWq9X3Uw//pjnS9rZwSefqM2GUlOhd29YutQEsQoh8p1cJQidJfapFKZRvfo/rYdBgyAlJc+X1Ongf/+DkBA1/v3aazB/fp4vK4TIZ7Kc5ro0i18LNU3j7t27ZgtImMH48WqzhxMnVHnwUaPyfEmdTm1qV7SoutzQoaoKrNRxFKLgyLIFcefOnUwfd+/epW/fvpaMUeSVi4vqFwKYPFmtfDORkSPVsgudDsaNU0lDeiCFKBiybEEMHTrUknEIc2vfXu0xunatKsORy1pNmRk4ULUk+vWD999XW5jOnq2ShhDCduVqDELYqI8+Uj/J161TuwOZ0Msvw3ffgaMjzJmjhjvS0kx6CyGEhUmCKEwqVICpU9XzoUNNvr9ot24QHq4W1i1apFoUUhleCNtlNEGkya+BBcuwYVCnDvzxh9pcyMTat4fNm1VDZcUK6NkTkpOlr0kIW2Q0QbRt25YZM2Zw+vRpS8QjzM3RERYsUM9nzIBTp0x+C19fVYqjRAkIC4Px48vKwLUQNshogggPD6dy5cpMmjSJnj17smrVKm7Lrva2rWlTeP11tSZi8GCzTDtq1Ejtb+3qCps3l+CDD0x+CyGEmRlNEMWKFaNnz56sXLmS0aNH88knn9CsWTPGjh3LedlqzHbNmAHu7mrTh5UrzXILHx9Ytkw9Hz8etm41y22EEGaSozGIHTt2MGTIEEJDQ3nttdfYvn07LVu2ZMCAAdmeGxkZib+/P23atGHRokWPfb506VI6dOhAYGAg/fr14+LFi+mfhYWF0bZtW9q2bUtYWFguvjSRLXd3lSRALWa4edMstwkKgjff/IsHD1RpqDNnzHIbIYQ5aEb4+flp48eP144cOfLYZ++9916W56WmpmqtWrXSLly4oCUlJWmBgYHa77//nuGY/fv3a3fv3tU0TdNWrFihDR8+XNM0Tbtx44bm5+en3bhxQ0tISND8/Py0hISEbOOMjo429qWY9XxLMlmsaWma1qSJpoGmDR1qmmtm4sSJaC0wUN2mdm1NS0w0263yzJb+HWiabcVrS7Fqmm3Fm5dYszvXaAti/fr1hIaGUq9evcc+mzRpUpbnRUVF4eXlhaenJ3q9noCAAHbs2JHhmEaNGuHi4gLAs88+y5UrVwDYt28fTZs2xc3NjRIlStC0aVP27t37RIlP5ICdnRqwtrdXRf2OHDHbbZYtgxo1VLWP//5XVlsLYQuMJojr168zaNAgGjZsSOPGjXnzzTeJzUGphvj4eDw8PNJfGwwG4uPjszx+zZo1NG/ePFfnijyoW1etrH7wAN5802yr20qUgO+/V4PWa9b807slhMi/jO5JPWrUKHr37s0nf9fyiYiIYOTIkaxevdpkQYSHh3PixAmWL1+e62skJSURExOT6/Pv37+fp/MtydSx2r30ElWWL8fx8GEuh4SQ8NJLJrs2ZIx3+vRiDB3qyYQJGqVKxfLCC3dMeq+8sqV/B2Bb8dpSrGBb8ZorVqMJ4t69e3Tu3Dn9dVBQEEuWLDF6YYPBkN5lBKpVYDAYHjvup59+YuHChSxfvhy9Xp9+7qFDhzKc+/zzz2d7PycnJ7y9vY3GlZWYmJg8nW9JZol1/nzo3p2yc+dSdvBgyOTvKrcejdfbW21uN2WKjrFjK3L4MFStarJb5Zkt/TsA24rXlmIF24o3L7Fml1iMdjE1b96cRYsWERcXx8WLF/n8889p0aIFCQkJJCQkZHlenTp1OHfuHLGxsSQnJxMREYGfn1+GY6KjowkODmbBggW4u7unv9+sWTP27dvHzZs3uXnzJvv27aNZs2Y5+VpFbnXtqpZB37wJo0eb9VbvvqtmNyUkQCrlLqEAACAASURBVOfOkJho1tsJIXLJaAti8+bNAKz811z5iIgIdDrdYwPP6Rd2cCA4OJj+/fuTlpZGt27dqFatGnPnzqV27dq0atWKmTNncvfuXYYPHw5A2bJlWbhwIW5ubgwePJju3bsDMGTIENzc3PL0hQojdDpVErxWLVi+XC2kM9PWsnZ28PXX0LAh/PqrGrRevVqqvwqR7+R6blQ+I9NcTSQkRM1H9fbWtKQkk1wyq3hPntS04sXV7aZNM8mt8syW/h1omm3Fa0uxapptxWu1aa4pKSl8/fXXDBs2jGHDhrF8+XJSTLBtpcinxoxR25TGxKhNHcyoRg1V0E+ng0mTTF6BXAiRR0YTxJQpU/j111/p1asXvXr14tdff2XKlCkWCE1YhZOTWhMB8N57quqrGXXsqCqQaxr07g2//27W2wkhnoDRMYhffvmF9evXp79u3LgxnTp1MmtQwspatYJeveDbb2H4cHjk798cJk6Eo0fVOonOneHAAbVeQghhXUZbEPb29ly4cCH9dWxsLPb29mYNSuQDs2dD8eKwYYPaBciMHg5ae3tDdLTaaOjBA7PeUgiRA0YTxJgxY+jbty+vvPIKL7/8Mv369WPs2LGWiE1YU9myaoNpUJsM3THvgjZXV9WCeLiHhBn2MhJCPKFsu5jS0tI4deoU27Zt4+zZswBUqVIlfUGbKOAGD4Yvv1T9PyEhZq+PUb26GrQODITgYFUuPCDArLcUQmQj2xaEvb09GzduRK/XU7NmTWrWrCnJoTCxt4eFC9U0ow8/VIsWzCwgQI2NPxy0/u03s99SCJEFo11M9erVIyQkhJ9//plff/01/SEKiQYNYNAgSE012+5z/zZhglrYfeuWGrS+dcvstxRCZMLoLKaHdTrmzp2b/p5Op+Prr782X1Qif5k2DdauhchINZrcr59Zb6fTqZ6tkyf/GbReu1YNZgshLMdogggNDcXT0zPDezkp9y0KkJIlYdYs6NtX1WkKDIRSpcx6y4eD1g0aqD/ff1+NSwghLMfo72TDhg177L2HtZNEIfLyy6o209Wrqg/IAqpVU0sxdDqYPFnNuBVCWE6WLYgzZ85w+vRpEhMT2bZtW/r7t2/fJikpySLBiXxEp1MrrOvWhUWLVIW9hg3Nftv27VXrYeJElaMOHVIlOoQQ5pdlC+KPP/5g9+7dJCYmsmvXrvRHdHQ07733niVjFPmFt7fqYtK0fwauLWD8eOjeXQathbC0LFsQrVu3pnXr1hw7dgwfHx9LxiTys3ffVf0+x4+rTYYs0N2o08HSpWrQ+sQJeOUVtZhOBq2FMC+jg9ReXl4sXLiQixcvkvrIb4zTp083a2AinypSBObNg06dVLLo0QPKlTP7bYsVU4PV9eur0lDvvafGJYQQ5mP0d7DBgweTmJhI48aN8fX1TX+IQiwwUG0Jl5gII0da7LZPP/3PoPWUKWYvESVEoZejPanfeecdS8QibMncufDDD7Bqldp9rk0bi9y2XTtVp2n8eNXVdOgQ1KxpkVsLUegYbUH4+vqyZ88eS8QibImX1z8LEwYPhvv3LXbrsWNVz1ZiomrI3LxpsVsLUagYTRBff/01AwcOpG7dutSrVw8fHx/q1atnidhEfvf22/DMM3D6NMycabHb6nTwxRdQp46q1fTyy1IeXAhzMNrFdOzYMUvEIWyRXg8LFkCLFqrfp3dvqFrVIrd+dNB640a1K93UqRa5tRCFRpYtiPBHRgCPHDmS4bPly5ebLyJhW5o3VyU4kpJg6FCLFPN7qEoVWLlSTXcNCVEJQwhhOlkmiC+//DL9+fsPN47529q1a80WkLBBH3yg6jVt3Qpr1lj01m3bwsMZ16+8oor7CSFMI8sEoT3ym6D2r98K//1aFHJlyvzzU3rECDV6bEHvvAMvvgi3b6uV1gkJFr29EAVWlglCp9Nl+jyz10LwxhuqNtOlSxZfwabTwZIlqkzU77/LoLUQppLlIPXZs2cJDAwE4MKFC+nPQcp9i0zY2akB6/r14eOP1SYO//mPxW5ftKgqv9GgAUREqBwlJcOEyJssE8SmTZvyfPHIyEimTZvGgwcP6NGjBwMGDMjw+eHDhwkNDeXUqVN8+OGHtGvXLv0zb29vqlevDkDZsmVZuHBhnuMRZubjowaqP/4Y3nwT9u2zaMGkKlXUuj1/f1UB1sdH7UwnhMidLBNE+fLlH3tv165dtGzZMkcXTktLIyQkhKVLl2IwGOjevTt+fn5UfWQaZNmyZZk+fTpffPHFY+c7OztnmEklbMR778Hq1bB/v1qs0L+/RW/fujXMmKHGJfr1U6XBa9WyaAhCFBhP9Ovdxx9/nONjo6Ki8PLywtPTE71eT0BAADt27MhwTIUKFahZsyZ2Upaz4CheHObMUc/HjlUbDFnYqFHQq9c/g9Y3blg8BCEKBKML5R71JLOX4uPj8fDwSH9tMBiIiorK8flJSUl07doVBwcHBgwYQOvWrY0e/3D/7Ny4f/9+ns63pHwfa506eDZpQrGffiJhwADuv/uuxeMdPVrH0aOVOHXKmaCg23z6aSz29sbPy/ff23+xpXhtKVawrXjNFesTJYiQkBCTB5CVXbt2YTAYiI2NpV+/flSvXp2KFStmebyTkxPe3t65vl9MTEyezrckm4h16VKoUwe3sDDc/f2pOHCgxUPYvFmNme/dW4xvv/Vm2jTj59jE9/YRthSvLcUKthVvXmLNLrEY7dvZvHkzt2/fBmDv3r0MHTqUX3/91ehNDQYDV65cSX8dHx+PwWDISbzp5wN4enry/PPPEy0roGxL9eowbhwAFQcNgldfhUf+PVhC5crw3XdqnDw0FGR9pxBPxmiC+PTTTylWrBg///wzBw4coHv37kyZMsXohevUqcO5c+eIjY0lOTmZiIgI/Pz8chTUzZs3SU5OBuD69escPXo0w+C2sBHvvgsTJ/LA0RG++kqNGM+ZAykpFguhVSu10BvUoPWJExa7tRA2z2iCsP+743bPnj307NkTX19fUnLwH9zBwYHg4GD69+9Phw4daN++PdWqVWPu3Lnpg9VRUVE0b96cLVu2MHnyZAICAgA4c+YM3bp1o1OnTvTr14833nhDEoQtcnCA99/n7Pr10LGj2kx65Eh49ln414QFc3r7bejTB+7ckUFrIZ6E0TEIg8FAcHAwP/74I2+88QbJyck8yOEy1RYtWtCiRYsM7w1/ZA/junXrEhkZ+dh59erVY8OGDTm6h8j/Ury8YMMGtYJtxAhVMKl1a+jeHWbPhmzGlkxBp4NFi+DXX9VW2r16qVByMmgtRGFmtAXx0Ucf0axZM5YsWULx4sVJSEhgzJgxlohNFDQBAaqPJzRU7W29Zo3aDu6998y+4VCRImqltbu7qik4aZJZbydEgWA0Qfz111/4+vpSqVIlDh48yJYtW6hbt64lYhMFkZOT2i/05ElVYe/ePbUz3TPPwPr1Zi0XXqmSGrS2t4f//U+t5xNCZM1ognjrrbews7Pj/PnzBAcHc/nyZUaNGmWJ2ERB5umpNnPYtQtq14Y//lD7h3booLaJMxM/P5g1Sz1/9VX45Rez3UoIm2c0QdjZ2eHg4MC2bdt4+eWXGTt2LH/99ZclYhOFga8vHDsGc+dCiRKwZYtKGOPGqaXQZjB8uKr4eveuGrS+ft0stxHC5hlNEA4ODmzcuJHw8HB8fX0BSE1NNXdcojBxcIBhw1TL4fXXITVVFVSqUQO++cbk3U4PB63r1YOzZ9WgdVqaSW8hRIFgNEFMnz6d48ePM2jQIDw9PYmNjaVTp06WiE0UNmXKwOLFcOCAqtt96ZKan9qiBfzf/5n0Vi4uatD6qadg2zaYMMGklxeiQDCaIKpWrcqYMWOoXr06v/32Gx4eHo+V7RbCpJ5/XiWJJUugdGnYu1f9uj90qEn7gypWVAPV9vYwc6YqFS6E+IfRBHHw4EH8/f0JCQlh6tSp+Pv7c/jwYUvEJgozOzt47TXV7TRsmOoXmj9flfD4/HOT9Qn5+qqlGKBud/Kkk0muK0RBYDRBzJgxgyVLlrB8+XJWrFjBkiVLmP5w/2EhzM3NTQ1gHzumfppfuwYDBqjtTQ8cMMkthg2Dvn3VoPXQoRWkHIcQfzOaIFJSUqhSpUr668qVK+eo1IYQJlWnDuzcqabGVqgAR45A48YmKQKo08HChQ+31NbTsCGsWGGasIWwZUYTRO3atZk4cSIHDx7k4MGDTJo0idq1a1siNiEy0unU4rqTJ9Wosl5vsiKALi6qPFSnTgncvaumwQ4eDElJJoxfCBtjNEFMnTqVqlWrsmzZMpYtW0bVqlWZOnWqJWITInNFi8K0aaq4UkCAyYoAFi0K06dfZuFClXsWLIAXXoALF0wYuxA2JNtifWlpaXTq1IktW7bw3//+11IxCZEzVavCxo2q8t7w4SYpAqjTwcCB8Nxz6jKHD6sJVCtWgL+/Gb4GIfKxbFsQ9vb2VK5cmUuXLlkqHiGe3MMigNOmmawIYP36apijfXs1Lt6+PUydCjksZCxEgWC0i+nWrVsEBATQr18/Bg0alP4QIl9xdlbjEiYsAujurhooD3fanTJFlYq6etW0oQuRXxndD+LR/RuEyPceFgEcOFDNXz1xQhUBbNdOTZetXv2JLmdnpzbGa9gQevdWpcKfe04tsHv+eTN9DULkE1m2IM6fP8+RI0d4/vnnMzzs7Ozw8PCwZIxCPLmWLU1aBLBtWzh6VCWKCxegWTM1iG3G6uRCWF2WCSI0NJRixYo99r6rqyuhoaFmDUoIk3i0COBrr6lpsHkoAlixIkRGqoofKSlqGmzfvmorUyEKoiwTxNWrV6lRo8Zj79eoUYOLFy+aNSghTKpMGVXX6eDBjEUAfX2fuAigXg/z5qn8UqQILF+uWhWnTpkndCGsKcsEkZiYmOVJ9828PaQQZvGwCODixaoIYGRkrosA9uqlpsDWrKmWYzRooCZPCVGQZJkgateuzXfffffY+6tXr6ZWrVpmDUoIs7GzU3tOmKAI4DPPwKFD0LMnJCZCjx5qvZ5UohEFRZazmCZMmMDQoUPZsGFDekI4ceIEKSkpfPLJJxYLUAizeFgEsH9/eOst2LMHBgygUu3a8OWXaqpSDri6qklTTZrA6NGq4sehQ2rv63LlzPslCGFuWbYgnnrqKVauXMmQIUMoX7485cuXZ8iQIaxatYrSpUtbMkYhzKdOHbUv9sqVUL48LidOqK6oESNUsyAHdDq1kHvPHpUUfvwRfHxg927zhi6EuRldKNeoUSNeeeUVXnnlFRo3bmyJmISwrEeKAF7r10+9N3cueHurbedyONupSRM1s9bPD/78E1q1gv/9T1ZfC9tlNEHkRWRkJP7+/rRp04ZFixY99vnhw4fp0qULzzzzDFu2bMnwWVhYGG3btqVt27aEhYWZM0whlGLF+HPsWPj5ZzXqfPEidO2qFtrlsGJfmTJqC9OJE1ViGD8eunSBhAQzxy6EGZgtQaSlpRESEsLixYuJiIhg48aNnD59OsMxZcuWZfr06XTs2DHD+wkJCXzyySd89913rF69mk8++YSbN2+aK1QhMvLxgf371XxWV1fYsEGNSM+eDampRk+3t4f331enubmpSh/PPQfHj1sgdiFMyGwJIioqCi8vLzw9PdHr9QQEBLDjX6WYK1SoQM2aNbGzyxjGvn37aNq0KW5ubpQoUYKmTZuyd+9ec4UqxOPs7dX015Mn1fSkO3fUKHSDBmoUOgc6dlSrr3184OxZtb/R0qVmjlsIEzJbgoiPj89QksNgMBAfH2/2c4UwqXLl1JSkiAjw8lLNgEaNVPLIQau2cmX46Sc1Wer+fbWgu39/VUtQiPzOaLE+W5GUlERMTEyuz79//36ezrckW4oVbCveLGOtXBnd2rU8tWAB7l9+iW7+fFK++4748eNJ9PdXA93ZGDkSKlUqwXvvebBkiR0//XSfjz6Kw9Mzb4smCsT3Np+ypXjNFavZEoTBYODKI3sFx8fHYzAYcnzuoUea8fHx8TxvpHSmk5MT3t7euQsWiImJydP5lmRLsYJtxWs01sWL1QK7gQNxPHCACiNHqs0i5s9XzYVsTJyotq7o3h1iYpx58cWqfP01BAaaMd58xJZiBduKNy+xZpdYzNbFVKdOHc6dO0dsbCzJyclERETg5+eXo3ObNWvGvn37uHnzJjdv3mTfvn00a9bMXKEK8WTq1lWLHRYuVJViN2+GWrVUIUAjy6iffVZNkgoKUjObOnVS21jkYOxbCIszW4JwcHAgODiY/v3706FDB9q3b0+1atWYO3du+mB1VFQUzZs3Z8uWLUyePJmAgAAA3NzcGDx4MN27d6d79+4MGTIENzc3c4UqxJOzs1N7Tpw8qQoz3bunSonXq6cGHbLh5qaWV8yYoS4zfbrazvTPPy0UuxA5pRUQ0dHRVj3fkmwpVk2zrXhzHevWrZpWpYqmqWV1mjZggKZdv270tF27NM1gUKeUK6dp+/ZZKF4rsKVYNc224s1LrNmda9aFckIUGm3bqt3rJk4ER0dYtEiVejWy74Svr1p93ayZqkLu6wsffSQbEYn8QRKEEKbi4qJWyB0/rn7i//mn2nfC3x/+tUj0UWXLws6dMGqUGot4+21V+SOHpaCEMBtJEEKY2jPPqMp9ixdDyZLwww9qu9Np0yA5OdNTHB1h1iy1p4Srq9rzukEDtdeEENYiCUIIc3i478TJk/DKK5CUBJMmqWlMkZFZntatm5rlVLu22qXu+edVL5UQ1iAJQghzKlMGvv4aduyAatUgJgZatFDJ49q1TE+pXl1tfPfKK3D3ruqlGjpU5RghLEkShBCW4OcHUVEwebLa2PqLL9Qg9tdfZzoiXbQofPWVWmqh16t1eM2b57iorBAmIQlCCEtxdoYpU1Si8PWFq1ehXz9o3VptgfovOp1aavHjj6oM1KFDapnFtm0Wj1wUUpIghLC0GjXUtKWvvgJ3d/W8Th2YOjXTfqT69eHIEWjXTvVKtWsHISGyEZEwP0kQQliDTgd9+6qR6NdeU7ObpkxRZTx27XrscHd3VVA2JES9njxZ1XXavNmVbdtU6+K339TM2iwmSgnxxApMNVchbJK7OyxZorqaHpbu8PNTyWPWLHhk/3c7O3j3XWjYEHr3hi1bYMuWCple1sVFlfTIyaNEicffc3Ky1DdA5GeSIITID5o3VwvsPvhALbb7+mvYuFG9/u9/M5QTb9tWrb6ePRtiYm7x4EFxEhLI8Lh3Tz0uX85dOM7OOU8wmT0kwRQMkiCEyC+cnNRaiZdegjffhO3b1XTYh9OZHinn7OmpSnLExFzE27t4hstompoe+++kkZPHzZtw44ba3OjKFfXIjYcJ5tHWSdmyZQgNVSvHhW2QBCFEflO1qpqq9O23qu5GZCT85z8wZoyq9eTiku3pOp2aJlu0KJQv/+S31zTV+shNgklIyC7BuLNqFYwYob4UKdCc/0mCECI/0unUQEP79qqM+KJFqlTHypWwYAG0aWPWWxcpoh7lyj35+ZklmGvXYO7cRHbscGX6dNUgGjdOLQAsUsT0X4MwDZnFJER+VrIkfPYZ7Nun6m+cOaMGIfr0gXy6T/vDBFOunCpL1aSJ2jVv3rw49u9XS0Bu3ICxY1Vj6bPPjO6zJKxEEoQQtqBpUzh6FP73P9XF9M03ULMmpefOzbZSbH7TqJFa9rF1K/j4qEH0QYNUIlm1StZ25DeSIISwFY6O6tfuX39Vq+USEnjqs89UjacWLeDLL+H2bWtHaZROpxpBP/+skkK1airHvfSSWhS4ZYvsh5FfSIIQwtZUrgybNsHu3SQEBan+nMhINR3Ww0MtvNu7N9//lLWzg549Vb5btEh1SR07poZdWraE/futHaGQBCGELdLpoEULLk+frvppFi9Wnf137sDSpWpdRY0aEBoKcXHWjjZbjo7wxhuqFTFzphp22bNHfTmdO8ueGNYkCUIIW1e8uFov8eOPaiX2uHHq1/Hff1fTYr28VJfUqlVq/mk+5eIC77wDZ8/ChAmqYRQerspUvfoqnDtn7QgLH0kQQhQkNWrA9Olw/rzqhurRAxwc1KjwSy+pxDF0qKr+l0+7oNzc1IzeM2dgyBCwt1drBatXh+HDVb0pYRmSIIQoiBwcVGf+d9/BpUvw8cdq2tCNG2pzifr11eK7OXPgr7+sHW2mPDzgk09Uo6hPH7Vf98cfQ5UqqljhrVvWjrDgkwQhREHn7g5vvaWmyR4/rn4Nd3eHX36BkSNVq6JrV9iwQf0UzmeefhqWL1cD2AEBapglJEQlig8/zNe9ZjZPEoQQhcl//qOKOF26BGvWqJ+4Dx5AWBh06gQVKqiBgOhoa0f6mP/8R9Uv3LtXLQu5dg1GjVJdT198kS9zm82TBCFEYaTXQ7du6iduXBzMmKG2QI2PV2XGa9VSq9o++0xV8MtHmjVTSWLjRjWAHRurxujr1IF16/Lt0IpNMmuCiIyMxN/fnzZt2rBo0aLHPk9OTmbEiBG0adOGHj16EPf3dLy4uDjq1q1LUFAQQUFBBAcHmzNMIQq3smVV9bzoaLX44I03wNUVDh5Uy5w9PNQgwPbt+Waps06nGj/Hj6vup8qV1VhFt25qv4ydO60dYcFgtgSRlpZGSEgIixcvJiIigo0bN3L6XyUBVq9eTfHixfnhhx949dVXmTVrVvpnFStWJDw8nPDwcEIebqMlhDAfnU61GhYtUmVYly1Tmxfdv69Ke7Rpo34SBweruaj5gJ2dyl0nT6qxd4MBDh+GVq1UuD//bO0IbZvZEkRUVBReXl54enqi1+sJCAhgx44dGY7ZuXMnXbp0AcDf35/9+/ejSftQCOsrUgRefhl27FDJYPJktZ7iwgV47z01ctyypdrY6M4da0eLXg+DB6upsdOmqaUh27dDgwZqpu+pU9aO0DaZrdx3fHw8Hh4e6a8NBgNRUVGPHVP2791DHBwccHV15caNG4DqZurcuTPFihVjxIgR1K9fP9v7JSUlERMTk+t479+/n6fzLcmWYgXbiteWYgULxvvii9CjB0UOHcItLAzXH37Abvdu2L2btMGDSWzXjoQuXbjn45Nh9ztrxNqlC7RsaceSJe4sX16KNWvsCAvT6NIlgcGDr+LhkbPRbFv6t2C2WDUz2bx5szZhwoT012FhYdrUqVMzHBMQEKBdvnw5/XWrVq20a9euaUlJSdr169c1TdO0X375RWvevLmWmJiY7f2io6PzFG9ez7ckW4pV02wrXluKVdOsGG9CgqYtWqRpjRppmhoXVo/q1TVt+nRNu3jxsVOsEWtcnKYNGKBp9vYqPCcnTRs1StOuXjV+ri39W8hLrNmda7YuJoPBwJVHtpOKj4/HYDA8dszlvzfNTU1NJTExkZIlS6LX6ylZsiQAtWvXpmLFivzxxx/mClUI8aRKlFCD2fv3q8HtMWPUYPZvv8H48WpP1A4d1FTapCSrhVm+vJqIFR2tCgMmJam9vKtUUVt/376NSm0pKWqXo1u31PzZP//ELiFBHZCUlG8G5y3NbF1MderU4dy5c8TGxmIwGIiIiGD27NkZjvHz8yMsLAwfHx+2bt1Ko0aN0Ol0XL9+nRIlSmBvb09sbCznzp3D09PTXKEKIfLC21tNk502TZX0WLoU1q+HzZvVo1Qp6NOHYjVqqMGAlBS1aCEl5Z+HqV//673qKSmsSk1luXsK926lwK0UHN5NxfHdFCDzLqca/37DwUENdjg6qj8fPvLy2lTXSk42y1+t2RKEg4MDwcHB9O/fn7S0NLp160a1atWYO3cutWvXplWrVnTv3p133nmHNm3aUKJECebMmQPA4cOH+fjjj3FwcMDOzo6pU6fiJhvYCpG/OTiouacBAXD1KqxYoZLF//0fzJtHfvgVz/Hvx7+lYg+Ojtg7OaBzdAQHB9KSkrBPS1M/fB8mnHy6Gq/K00+r4oxZjP/klk7TCsa0oZiYGLy9va12viXZUqxgW/HaUqxgI/EeOwZffcXto0cpVrKk+q3XwUH9+fBh7HVuzjHyWnNwZP0mByYEOxB9UvW2166tGkKBgXDy5CPfW01TySE5+Z+E8fB5Pnid4OODW1hYrv56svs3ZLYWhBBCAKpIoI8PsfksmemAoB7Qsata8jF5Mpw4AUFBajlI3bqlqVtXjWOUL6+jXDlHDAZHHIoWtXboj7kcE4M5+lgkQQghCjV7e7XfRK9esHChGrw+cAAOHHjqsWPt7NRYfLlyDxNH5s9LlDB5b49VSIIQQgjAyUkVun3tNVW78PDhv0hOLs2lS3Dxonr8+aeqc3jpUvartIsU+SdZZJVMypZV98zPJEEIIcQjXF2hb19o0OAq3t6lM3yWkqJ2eH00aWT2/PZttYXqv6oLPeapp7JPIuXKqWPsrFRWVRKEEELkkKMjVKyoHtm5dct4Erl8WU32unpVTfTK7p7lymWfRFJSTPt1PiQJQgghTKx4cfWoWTPrY9LS1GZ+2SWRixfVJoDnz6tHVmrWrEx0tOnHPSRBCCGEFdjbqwFvDw947rmsj7t3759xj8wSyMWLUKVKEuBs8hglQQghRD7m4qKK5z79dNbHxMRcQqcrYfJ7y45yQgghMiUJQgghRKYkQQghhMiUJAghhBCZkgQhhBAiU5IghBBCZEoShBBCiExJghBCCJGpArNh0PHjx3HK76URhRAin0lKSuLZZ5/N9LMCkyCEEEKYlnQxCSGEyJQkCCGEEJmSBCGEECJTkiCEEEJkShKEEEKITEmCEEIIkalCv2FQZGQk06ZN48GDB/To0YMBAwZYO6QsjR8/nt27d+Pu7s7GjRutHU62Ll++zJgxY7h27Ro6nY6ePXvSr18/a4eVpaSkJPr06UNycjJpaWn4+/szbNgwa4eVrbS0NLp164bBYOCzzz6zdjjZ8vPzo2jRotjZ2WFvb8+6deusHVKWbt26/Uf7XQAABn1JREFUxaRJk/jtt9/Q6XSEhobi4+Nj7bAydfbsWd5+++3017GxsQwbNoxXX33VNDfQCrHU1FStVatW2oULF7SkpCQtMDBQ+/33360dVpYOHTqknThxQgsICLB2KEbFx8drJ06c0DRN0xITE7W2bdvm6+/tgwcPtNu3b2uapmnJycla9+7dtWPHjlk5qux98cUX2siRI7UBAwZYOxSjWrZsqV27ds3aYeTImDFjtO+++07TNE1LSkrSbt68aeWIciY1NVVr0qSJFhcXZ7JrFuoupqioKLy8vPD09ESv1xMQEMCOHTusHVaWGjRoQIkSpt9W0BzKlClDrVq1AChWrBhVqlQhPj7eylFlTafTUbRoUQBSU1NJTU1FZ+od4E3oypUr7N69m+7du1s7lAIlMTGRw4cPp39f9Xo9xYsXt3JUObN//348PT0pX768ya5ZqBNEfHw8Hh4e6a8NBkO+/iFmq+Li4oiJieE///mPtUPJVlpaGkFBQTRp0oQmTZrk63hDQ0N55513sLOznf/Cr7/+Ol27dmXVqlXWDiVLcXFxlCpVivHjx9O5c2cmTpzI3bt3rR1WjkRERNCxY0eTXtN2/nUJm3Tnzh2GDRvGhAkTKFasmLXDyZa9vT3h4eHs2bOHqKgofvvtN2uHlKldu3ZRqlQpateube1Qcuzbb78lLCyMzz//nBUrVnD48GFrh5Sp1NRUoqOj6dWrF99//z0uLi4sWrTI2mEZlZyczM6dO2nXrp1Jr1uoE4TBYODKlSvpr+Pj4zEYDFaMqGBJSUlh2LBhBAYG0rZtW2uHk2PFixenYcOG7N2719qhZOro0aPs3LkTPz8/Ro4cyYEDBxg9erS1w8rWw/9X7u7utGnThqioKCtHlDkPDw88PDzSW4/t2rUjOjraylEZFxkZSa1atXjqqadMet1CnSDq1KnDuXPniI2NJTk5mYiICPz8/KwdVoGgaRoTJ06kSpUq/Pe//7V2OEZdv36dW7duAXD//n1++uknqlSpYuWoMjdq1CgiIyPZuXMnH374IY0aNWLWrFnWDitLd+/e5fbt2+nPf/zxR6pVq2blqDJXunRpPDw8OHv2LKD69Z9++mkrR2VcREQEAQEBJr9uoZ7m6uDgQHBwMP3790+fMphf/+ECjBw5kkOHDnHjxg2aN2/OW2+9RY8ePawdVqaOHDlCeHg41atXJygoCFDxt2jRwsqRZe7PP/9k3LhxpKWloWka7dq1o2XLltYOq0C4du0aQ4YMAdQ4T8eOHWnevLmVo8rau+++y+jRo0lJScHT05Pp06dbO6Rs3b17l59++omQkBCTX1vKfQshhMhUoe5iEkIIkTVJEEIIITIlCUIIIUSmJEEIIYTIlCQIIYQQmSrU01yFyAlvb2+qV6+e/jogIMBkVX/j4uIYNGhQvq/OKwonSRBCGOHs7Ex4eLi1wxDC4iRBCJFLfn5+tGvXjr179+Lk5MTs2bPx8vIiLi6OCRMmcOPGDUqVKsX06dMpV64cV69eZfLkycTGxgIwZcoUypQpQ1paGpMmTeLYsWMYDAY+/fRTnJ2d+frrr1m5ciX29vZUrVqVOXPmWPkrFoWNjEEIYcT9+/cJCgpKf2zatCn9M1dXVzZs2MDLL79MaGgoAO+//z5dunRhw4YNBAYG8v7776e/36BBA9avX09YWFj6qv3z58/Tp08fIiIicHV1ZevWrQAsWrSI77//ng0bNjB16lQLf9VCSIIQwqiHXUwPHx06dEj/7GF55YCAAI4fPw7AsWPH0t8PCgriyJEjABw4cIDevXsDqnKsq6srABUqVMDb2xuAWrVqcfHiRQBq1KjB6NGjCQ8Px97e3gJfqRAZSYIQwsr0en36c3t7e9LS0gDVgujduzfR0dF0796d1NRUa4UoCilJEELkwebNmwHYtGlT+r7FPj4+REREALBhwwbq168PQOPGjfnmm28AVbQuMTExy+s+ePCAy5cv06hRI0aPHk1iYqLNbFwjCg4ZpBbCiIdjEA+98MIL6fsv3Lx5k8DAQPR6PR9++CGgqoGOHz+eJUuWpA9SA0ycOJF3332XtWvXYmdnx5QpUyhdunSm90xLS+Odd97h9u3baJpG3759bWbrS1FwSDVXIXLJz8+PNWvWUKpUKWuHIoRZSBeTEEKITEkLQgghRKakBSGEECJTkiCEEEJkShKEEEKITEmCEEIIkSlJEEIIITL1/2U7uPJ5vO66AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "_cell_guid": "9f6fccbb-7888-41c1-99e5-eb1d95f19021",
        "_uuid": "dcf22acf97cd6321d449db27fde638f5d71d68dd",
        "trusted": true,
        "id": "VdcggnhrpajA",
        "colab_type": "code",
        "outputId": "b07d3341-56c3-49e9-caf6-a397ff1bdfb1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        }
      },
      "source": [
        "plt.figure()\n",
        "plt.plot(hist.history['accuracy'], lw=2.0, color='b', label='train')\n",
        "plt.plot(hist.history['val_accuracy'], lw=2.0, color='r', label='val')\n",
        "plt.title('CNN sentiment')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.legend(loc='upper left')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deVxUVf/A8c+w44KI2lAuZO6pJYmPG2rihiK5YGlumNqmPlKmlj2lT1j2I8UyrQwznzLXSjJDE7NyKXPFyKRyV1xQQQWRdZjfH0dG0WFRmLkz8H2/XvPizp07935B5DvnnHu+R2c0Go0IIYQQt3DQOgAhhBC2SRKEEEIIsyRBCCGEMEsShBBCCLMkQQghhDBLEoQQQgizJEEIYWPGjh1LdHS01mEIIQlC2Kd169YxcOBAfH198ff3Z+zYsezZsweA+fPn06RJE9avX286Pjc3lyZNmpCYmAjAK6+8QpMmTYiPjzcdc+LECZo0aWLV72P+/PlMnjy5wL5PPvmEAQMGWDUOUD+Td9991+rXFbZLEoSwO0uWLGHWrFk899xz/PLLL/z0008MHTqUzZs3m47x9PRk/vz5GAyGQs/j6enJe++9Z42QhbBLkiCEXUlLS+P9999n+vTp9OzZk0qVKuHs7ExAQAAvv/yy6Th/f3+cnZ359ttvCz1X//79+fvvv9m1a1eJrh0VFUWnTp3w9fWlV69e7NixA4C8vDyioqLo3r07bdu2JSwsjMuXLwOQmJhIkyZNiI6O5tFHH6Vt27Z89NFHAGzdupWPP/6YDRs24Ovry2OPPQbAiBEj+PLLLwFYs2YNQ4YMYdasWfj5+dGtWzf27dvHmjVr6NKlC+3bty/QHZWdnU1ERASPPvooHTp0YPr06WRmZgKwc+dOOnfuzKeffkr79u3x9/fn66+/BmDVqlWsW7eOxYsX4+vry3PPPVein4ko3yRBCLsSFxdHVlYWPXr0KPI4nU5HWFgYCxYsICcnx+wxbm5uPPvssyXqVjl69CjLli3jq6++Ii4ujsWLF1O7dm0Ali5dyg8//MAXX3zBtm3bqFatGuHh4QXev3fvXr7//ns+++wzPvjgA44cOULnzp159tln6d27N3FxcYUms/j4eJo0acLOnTvp27cvkyZN4o8//mDTpk3Mnj2b8PBw0tPTAZgzZw7Hjh3jm2++ITY2lvPnz/PBBx+YznXx4kXS0tLYunUrb731FuHh4Vy5coXBgwcTHBzMmDFjiIuLY+HChcX+TET5JwlC2JXLly9TvXp1nJycij22W7dueHl5mT6NmzNkyBDOnj3Lli1bijyXo6Mj2dnZHDlyhJycHOrUqUO9evUAWLlyJS+++CLe3t64uLgwYcIENm7cSG5urun9EyZMwM3NjaZNm9K0aVP++uuvEn7HUKdOHUJCQnB0dKRPnz6cPXuW8ePH4+Ligr+/Py4uLpw8eRKj0cjq1at59dVX8fT0pEqVKjz77LPExMSYzuXk5MT48eNxdnamS5cuVKpUiWPHjpU4FlGxFP+/TAgb4unpyaVLl8jNzS1RknjhhReYNm0a/fr1M/u6i4sL48aNY968eUW2JHx8fHj11VeZP38+hw8fxt/fn1deeQW9Xs+ZM2cYP348Dg43Pm85ODiQnJxsel6zZk3Ttru7O9euXSvJtwtAjRo1TNtubm63nc/V1ZX09HRSUlLIyMhg4MCBpteMRiN5eXmm556engV+bncai6hYpAUh7Iqvry8uLi788MMPJTq+Y8eO+Pj4sHz58kKPGThwIGlpacTGxhZ5ruDgYFasWMFPP/2ETqdjzpw5AHh7e7No0SL27Nljevzxxx/o9fpi49PpdCX6PkqievXquLm5ERMTY4pj7969xMXFlej9ZRmLKB8kQQi7UrVqVSZOnEh4eDg//PADGRkZ5OTksGXLFt555x2z73nhhRf45JNPCj2nk5MT//73v4s85ujRo+zYsYPs7GxcXFxwdXU1tRiefPJJ3nvvPU6fPg1ASkpKiRNYjRo1OH36dIFP+XfLwcGBxx9/nFmzZplaL0lJSWzbtq3EseTfBiwESIIQdmj06NG88sorfPjhh7Rv355HH32UZcuW0b17d7PHt27dmoceeqjIc/bt25datWoV+np2djaRkZG0bdsWf39/UlJSmDRpEgAjR44kICCA0aNH4+vryxNPPFFgfkVRAgMDAWjbtm2ZzH2YMmUKPj4+PPHEEzzyyCOMGjWqxGMMgwYN4vDhw/j5+TFu3LhSxyLsn04WDBJCCGGOtCCEEEKYJQlCCCGEWZIghBBCmCUJQgghhFnlZqLc/v37cXV1vev3Z2Vller91mRPsYJ9xWtPsYJ9xWtPsYJ9xVuaWLOysmjVqpXZ18pNgnB1daVZs2Z3/f6EhIRSvd+a7ClWsK947SlWsK947SlWsK94SxNrQkJCoa9JF5MQQgizJEEIIYQwSxKEEEIIs8rNGIQ5OTk5JCYmmhZMKe7YovribIm5WN3c3KhTpw7Ozs4aRSWEKG/KdYJITEykatWq3H///cVWqszIyMDd3d1KkZXOrbEajUaSk5NJTEykfv36GkYmhChPynUXU2ZmJjVq1Cj3ZYx1Oh01atQoUUtJCCFKqlwnCKg4Ne4ryvcphLCect3FJIQQ5UFeHly4AKdPw5kz6nHzdv36NVmwoOyvKwnCwlJTU1m3bh3Dhg27o/c9/fTTREZG4uHhYaHIhBBaMxrhypXb/+Dfun3uHNy0xPltatXyZP58KOuOBEkQFpaamsqKFStuSxDFram8aNEiS4cmhLCgjIzC/+DfvJ2RUbLz1awJ992nHrVrF9yuXv0YOl3jMv8eJEFYWGRkJCdPnqRfv344OTnh6uqKh4cHx44dY+PGjYwbN45z586RlZXFyJEjGTx4MAABAQF89dVXXLt2jaeffprWrVsTFxeHXq8nMjLSbu64EqK8yc1Vn+iL++N/6VLJzlelyu1/8G/dvvdeKKrUUkKCoWy+uVtUmAQRFATr1xd1xJ3/we3TB2Jiij7mpZde4tChQ6xdu5adO3fy7LPPsm7dOurWrQvArFmz8PT0JDMzk0GDBtGzZ0+qV69e4BwnTpxg7ty5vPnmm4SFhbF582YGDRp0x/EKoZlTp6i8dSscP651JMXKzYU//oC//komznic5GRMj5QU9Ye/sGU4HYG61x9OjlCjhnp4ed3Yzn9es6b6WqlSEcFcuf4oZoqWk5sbWKBuVIVJELaiZcuWpuQAsHTpUjZt2gTA2bNnOXHixG0Jok6dOqZCXM2bN+fMmTPWC1iI0khOhjffhA8+oF5OjtbRlIgT4Hv9USoG4Pz1h4XVr1FDjWKX8SBEhUkQxX3St9ZEuUo3fVzYuXMnv/76K6tWrcLd3Z0RI0aQlZV123tcXFxM246OjuQWNVolhC3IyoIFC1RyuHwZdDrS//UvKnt5aR3ZbTIy4PgJSDwFudd7aipXhiqVc6hSxRlXN9W94+YGbq7g4goONnZXeXLz5ugtcKt7hUkQWqlcuTLp6elmX0tLS6NatWq4u7tz5MgR9u/fb+XohChjRiOsXg3TpsGxY2pf9+4wZw4nXVxsqnz2vn0QGQmrVoHhemLo2hUmTwb/QPj77wQa21C8RUlJSEBvgfNKgrCw6tWr88gjj9C3b19cXV2pWbOm6bXOnTuzcuVKevfuTf369QtdtEMIu/DLL/DSS7Bzp3revDnMmQO9eqmuDxuodZaXB99/r8L66Se1z9ERhg5VoT/yiLbx2RpJEFYQGRlpdr+LiwuffPKJ2dd+/PFHALy8vPjuu+9M+8eMGUNGSe+LE8IaDh+GV16Br79Wz/V6mDkTnnoKiriV25qysmDZMtViOHhQ7atSBZ55BsLCoF49beOzVbbxryeEsD/JySoRfPgh5OSAuztMmaIeVapoHR2g7jpauBDmz1e3poK6dTQsDJ5+Gjw9tY3P1kmCEELcGTMD0Dz1lEoWtWtrHR0AR4/Ce+/B4sVw7Zra9/DDanzhiSfgpvs+RBEkQQghSqaIAWgefljb2K7buVOFs2aNGm8ANQQyeTJ061b2pSjKO0kQQojiFTcAraG8PFi3ToWzfbva5+wMI0bApEnw0EOahmfXJEEIIQpnwwPQGRnw+edq4PnQIbWvWjV4/nn4979ViQpROpIghBC3s+EB6AsX4IMP1OPiRbXPxwdefBFGj4aqVTUNr1yRBGFjfH19iYuL0zoMUVFlZalbft58U9Wh1unUX93wcM0HoP/+G959Fz77DPIXT/TzU+MLISGaN2jKJfmRCiFuDEC/8sqNgno9esDs2ZoOQBuNavhjzhz49lv1HCA4WCWGTp00HwIp1yRBWNicOXO49957TetBzJ8/H0dHR3bu3Elqaiq5ubmEhYXRvXt3jSMVFZYNDkDn5kJ0tApj1y61z9UVRo5UA89Nm2oSVoVj0TWpt27dSq9evejRowdRUVG3vX769GlCQ0MJDg5mxIgRnMufyQK88847BAUF0bt3b958802MxsIK7JZQUJD6ZS/k4V6pUpGvm30EBRV72T59+rBhwwbT8w0bNjBgwAA++OADoqOj+eyzz4iIiCj99yfEnTp8GAYNAn9/lRy8vWHRIti/HwIDNUkOV6+qHq7GjdV8hV27VEns11+HEycgKkqSgzVZrAVhMBgIDw9nyZIl6PV6Bg0aREBAAA0bNjQdExERQf/+/RkwYAA7duwgMjKS2bNns2/fPvbt28e3334LwNChQ9m1axdt27a1VLgW8+CDD5KcnExSUhKXLl3Cw8ODmjVr8vbbb7N7924cHBxISkri4sWL1KpVS+twRUVw6wB0pUpq8HnyZM0GoM+eVXPvPvroxkI7DRuq1kJoaDFrJgiLsViCiI+Px8fHx7T2QVBQEJs3by6QII4cOcK0adMAaNeuHePHjwdAp9ORnZ1NTk4ORqORnJycAkXu7kox9b4tWe47MDCQjRs3cvHiRfr06cO6detISUlhzZo1ODs7ExAQYLbMtxBlygYHoP/8U92mumwZZGerfR06qFz12GOqkJ7QjsUSRFJSEt7e3qbner2e+Pj4Asc0bdqU2NhYQkND2bRpE+np6Vy6dAlfX1/atm2Lv78/RqOR4cOH06BBgyKvl5WVRcIt1SJzcnJKXNjOaDRarAheQEAA4eHhXL58mcWLFxMbG0u1atXIzc1lx44dnD59mszMTDIyMkoUR2HH5OTk3PYzsAWZmZk2GZc59hQrlDBeo5Gq33/PPXPn4nL6NABXO3Tg/OTJZDVtCqmp6mGlWI1G2LmzEkuW1GDbNtVi0emM9OiRxqhRKfj6qt/tf/6xeEhFsqffBUvFqukg9dSpU5k5cybR0dH4+fmh1+txdHTkxIkTHDlyhC1btgAwevRo9uzZg5+fX6HncnV1va3WfEJCQolbBZZsQbRs2ZLMzEy8vb2pV68eAwcO5Pnnn+eJJ56gRYsWPPDAA7i5ueHu7o5Opys2jsJidXZ2tql6+/kSEhJsMi5z7ClWKEG827erj+O3DEBX6dWLKlYeY4iPTyAurhlz5kD+ndzu7qoR88ILOho29AA8rBpTUezpd6E0sRaVWCyWIPR6fYFB56SkJPR6/W3HLFiwAID09HRiY2Px8PBg9erVPPzww1SuXBmATp06ERcXV2SCsHXr1q0zbXt5ebFq1Sqzx8kcCFEmDh1St6yuWaOee3urcYdRo6w6YeDMGdizR+WnTz9taKqoes89arbz88+rNZqFbbLYb0rLli05fvw4p06dQq/XExMTc9u6CCkpKXh6euLg4EBUVBQhISEA3HfffaxevZrc3FyMRiO7d+8mNDTUUqEKUX7kD0B/8IG6V9SKA9DJySoZ7N6tHnv2qARxgzNNm6o7aocPV0t4CttmsQTh5OTE9OnTGTt2LAaDgZCQEBo1asS8efNo0aIF3bp1Y9euXcydOxedToefnx8zZswAoFevXvz2228EBwej0+no1KkTAQEBlgpVCPtn5QHotDTYu7dgQsgv8HozDw8127lNG3jggZOMHVsPB4veXC/KkkXbml26dKFLly4F9oWFhZm2AwMDCQwMvO19jo6OhIeHl0kMRqMRXQWYainzKCooo1EtqnzrDOg5c8qsjGlGhpoakZ8M9uyBv/66Mas5n7u7WrIzPyG0aaNuVc1PCAkJ6ZIc7Ey5nknt5uZGcnIyNWrUKNdJwmg0kpycjJu02SuW7du5f/x4yL87sEWLGzOg71JODhw4ULCb6MAB1Vt1M2dnlX/yE4GfHzz4oNRDKm/K9T9nnTp1SExM5MKFC8Uem5OTg7OzsxWiKj1zsbq5uVGnTh2NIhJWlZmpxhUWLMAdbgxAP/XUHU0cMBhUAbz8RLB7t2op3Dolx8FB5Z78RNCmjUoOrq5l+l0JG1SuE4SzszP169cv0bEV5ZY2YecOHYLBg9V9os7OXBg7llrvvFPsALTRqJbhvHnMYN8+VdriVo0aFUwGvr5w/YZCUcGU6wQhRLmyfDk8+6z6q/7AA7BqFRcrV6bWLcnBaFR3D93cTbRnD6Sk3H7KevUKjhk88ghUr26l70fYPEkQQti69HSYOBE+/VQ9HzwYPv5YLZ+WkMDFiwW7iXbvhpumIJncc8+NRNCmDbRurRaIE6IwkiCEsGV//qnKmh48qCYOvP8+jB3LX3/rCH8etm5twPXqGQV4et5oGeR/rVNH1k4Qd0YShBC2yGiExYtVyyEjA5o1g1WrMLZoyeLFEBYG164BuFC5suoaujkZNGggyUCUniQIIWxNaqoaa1i5Uj1/6imYP5+UrMo88zh8/bXaPWIEDBp0lKCgB6TqqbAISRBC2JJ9+1SX0pEj6tahhQth+HC2boVhwyAxEapWVbuHDoWEhCxJDsJiZF6jELbAaFTjC+3bq+Tw8MOwbx85g4fz+uvQtatKDu3aqbkKQ4dqHbCoCCRBCKG1lBQYOFANLGRnw7hx8NtvHHNuTOfOqryS0QivvQZbt6o7XIWwBuliEkJLO3bAkCFw8qS6bXXxYggJYflyVQo7NVXdffTFF3BLWTMhLE5aEEJoIS8PIiKgUyeVHP71L4iLI7VHCCNHqvGG1FTVsPj9d0kOQhuSIISwtvPnoU8fVYHVYFBrNWzbxq4L9fH1haVL1TIOixbBV1+Bl5fWAYuKSrqYhLCmn35SI8znzqml1D7/HEOvPrzzDkyfrqqmtmoFK1ZA06ZaBysqOmlBCGENBgPMmAHduqnk0Lkz/P47px/uQ48e8OqrKjlMmgS//SbJQdgGaUEIYWmnT6tBhS1b1PTm6dPh9df55jsnxoxRNzHp9fDZZ6VaykGIMicJQghLWr8eQkPh4kW1bsOyZVxrF8CkCareHkDv3vC//6liekLYEuliEsIScnJg6lQIClLJoWdP+P13fq8RgJ+fSg4uLjBvHsTESHIQtkkShBBl7fhxdfvq7Nlqhbe338a4fgPzVtzDv/4FCQmq9t6uXaoWnxTVE7ZKupiEKEtr1sDo0XDlilqNZ8UKzjfswKhg2LBBHfLccxAZqW5lFcKWSQtCiLKQmQkTJkBIiEoO/fpBXBwb0zrw0EMqOXh5qfzx0UeSHIR9kAQhRGn9848qsvfBB6aBhayV0bz0lheBgZCUBI8+qmZEDxigdbBClJx0MQlRGsuWqbUb0tPVKj2rVvFX5dY82V5VXXV0hJkz1Xi1lOUW9kZaEELcjfR0NdYwfLjaHjIE4959fBLXmtatVXJ44AH45ReYNk2Sg7BP0oIQ4k4dOKAW9UlIUOtEz59PyoAxPDNGV2C1twULwMND21CFKA1JEEKUlNEIn3yi7k3NzIQHH4RVq9iS3ILhrW6s9vbRR2ritBD2TrqYhCiJ1FRVZO+ZZ1RyGD2anF928fqqFqbV3tq2VV1LkhxEeSEtCCGKs3cvDB6slgKtUgUWLuRo+2EM660K6+l0arW36dPB2VnrYIUoO9KCEKIwt64T3aoV7N3Lct0wWrVSyaFOHVXBe+ZMSQ6i/JEEIYQ5KSlq0kJYmKqrNGECqRt3MPLNxgwbBmlpstqbKP8smiC2bt1Kr1696NGjB1FRUbe9fvr0aUJDQwkODmbEiBGcO3fO9NqZM2cYPXo0vXv3pk+fPiQmJloyVCFu+PVX1VpYu1atE/311+waMR/f9m4sXQru7hAVJau9ifLPYgnCYDAQHh7OJ598QkxMDN999x2HDx8ucExERAT9+/dn3bp1jBs3jsjISNNrL7/8MmPGjGHDhg18+eWX1KhRw1KhCqHk5VFj0SK1mM+pU9C2LYa9+3n774F07AhHj6q8sW8fPP20FNkT5Z/FEkR8fDw+Pj7UrVsXFxcXgoKC2Lx5c4Fjjhw5Qrt27QBo166d6fXDhw+Tm5tLx44dAahcuTLu7u6WClVUdEajqrndqRP3vPuuWv1tyhQSV2yjx9P3m1Z7e/FFWe1NVCwWu4spKSkJb29v03O9Xk98fHyBY5o2bUpsbCyhoaFs2rSJ9PR0Ll26xPHjx/Hw8GDChAkkJibSvn17Jk+ejGMR01GzsrJISEi463gzMzNL9X5rsqdYwYbjzcmh2vr1eH36KW6HDqldnp6c/b//Y212H1731XHlCtSokcusWWfo1Cmdo0c1jvkWNvuzNcOeYgX7itdSsWp6m+vUqVOZOXMm0dHR+Pn5odfrcXR0JDc3lz179vDNN99w77338uKLL7JmzRoef/zxQs/l6upKs2bN7jqWhISEUr3fmuwpVrDBeK9ehUWL4N13VVcSQO3a8OKL/PmvR1m4rHWB1d6WLHFCr6+nXbxFsLmfbRHsKVawr3hLE2tRicViCUKv1xcYdE5KSkKv1992zIIFCwBIT08nNjYWDw8PvL29adasGXXr1gWgW7du/P7775YKVVQU58+r21Y//BAuXVL7HnwQpkyBoUP5428XQvpncfSoKsr6zjuyoI+o2Cw2BtGyZUuOHz/OqVOnyM7OJiYmhoCAgALHpKSkkJeXB0BUVBQhISGm96amppKSkgLAzp07adiwoaVCFeXd4cPw/PPg4wNvvaWSg78/fPst/PEHjBpFltGFwEA4etTVtNpbWJgkB1GxWawF4eTkxPTp0xk7diwGg4GQkBAaNWrEvHnzaNGiBd26dWPXrl3MnTsXnU6Hn58fM2bMAMDR0ZGXX36Z0NBQAJo3b15k95IQZu3Zo5oBX38N1z+I0K+fqr3doUOBQ7/4As6cgUaNMtmzx00W9BECC49BdOnShS63zCIKCwszbQcGBhIYGGj2vR07dmTdunWWDE+UR0YjxMaqxPDjj2qfszOMGgWTJ6vFoG+Rl6eWjwYYMyaZSpVqWy9eIWyY1GIS5UNuLqxerRJD/nhV1apqAeiwMDUIXYjvvoO//4a6daF371RAEoQQIAlC2Lv0dPj0U5g7F44fV/u8veGFF9RKb56exZ4iv/Xw4otST0mIm0mCEPbp4kW1Is+CBZCcrPY1bqzuSBo+XC3kUwI7dsD27aqixtixqmy3EEKRBCHsy/HjEBkJixdDRoba17YtvPwyPPbYHa/tmd96GDdO9UgJIW6QBCHsw/79anxh9WpVCgMgKEjdkdSp013dj/rPP/DNN2rOw7//XcbxClEOSIIQtstoVIstRESoO5MAnJxg5Eh1R1LLlqU6fWSkusSIEXDvvWUQrxDljCQIYXsMBlizRrUY9uxR+ypXVst9vvAC1Ct92YukJPjsM7U9eXKpTydEuSQJQtiOjAz1V3vOHLWCG0CtWuo21eefL9PFF+bPh6wsNWwh1VmFME8ShNDepUuqPtL776t6SQANGqiP9qGhaoWeMnT1qrocqCEMIYR5kiCEdk6dUhVVo6LUfAaA1q3VHUkDB97xHUkltXixyknt28P1JUeEEGZIghDWd+CAur90+XI1AxqgZ0+VGLp2tWiFvNxclZNATZkQQhROEoSwDqMRtm1TA88xMWqfgwM8+aTq52nVyiphfPklnDih5tQ99phVLimE3Sq23PePP/5oKsktxB3Ly6PqDz+o6qlduqjk4O4OEyaoMtzLl1stORiNKj8BvPSSxXqwhCg3im1BrF+/nlmzZtGzZ09CQkJo0KCBNeIS5cHp09CzJ3UOHlTPvbzUjLQJE6BmTauHs3mzmm93zz1qKoUQomjFJog5c+Zw9epVvvvuO6ZNm4ZOp2PgwIEEBQVRpUoVa8Qo7FVkJBw8SM699+I8bRqMHq3mM2gkv/UwcWKJSzUJUaGVaEW5KlWq0KtXL/r06cOFCxfYtGkTAwcOZOnSpZaOT9irnBy1Cg+Q+N57quWgYXLYvx82bVIhPP+8ZmEIYVeKbUFs3ryZNWvWcPLkSfr168eXX35JjRo1yMjIICgoiBEjRlgjTmFvvv8eLlyAZs3IbNFC62iYM0d9HTu2TOfbCVGuFZsgYmNjGTVqFG3atCmw393dnbfeestigQk7l1/HYtQozRd2PnECVq5Ug9IvvqhpKELYlWK7mCZMmMBDDz1kep6ZmUni9aL57du3t1xkwn4lJ8O336rbWIcP1zoa3ntPlXcaPBh8fLSORgj7UWyCCAsLQ3fTJ0AHB4cC60oLcZuVK9UYRI8ecN99moZy6RIsWqS2ZWKcEHem2ARhMBhwcXExPXdxcSEnJ8eiQQk797//qa+jRmkZBQAffaSqeHTvbrXpFkKUG8UmCC8vLzZv3mx6/sMPP1C9enWLBiXs2MGDqkR3tWrQr5+moWRmqvp/IEX5hLgbxQ5Sv/HGG0yePJmZM2diNBq59957iYiIsEZswh7lD04/8USZV2G9U0uXqnUfWrVSLQghxJ0pNkHUq1eP1atXk3692mZlDe9lFzYuN1f9VQbNu5fy8m7c2jpliuY3Uglhl0pUrO/nn3/m0KFDZGVlmfZNmDDBYkEJO/XDD3D2LDRqpGppa+jbb9Wa0/XqweOPaxqKEHar2DGI6dOns379er64Pit248aNnDlzxuKBCTuU370UGqr5R/bZs9XXSZPA2VnTUISwW8UmiLi4ON555x08PDyYMGECK1eu5Pjx41YITdiVy5chOlolBo1n1//yC/z6K1SvDmPGaBqKEHat2ATh6uoKqJnTSUlJODs7c+HCBYsHJuzM6tpWSD0AAByfSURBVNVqkeeuXVW/jobyWw/PPw9ST1KIu1fsGETXrl1JTU1lzJgxDBw4EJ1Ox+PSqStudXNpDQ399Zcaf3B1VfUBhRB3r8gEkZeXR/v27fHw8KBXr1507dqVrKwsqlataq34hD345x/Vp1OlilpLWkORkWphoJEjwdtb01CEsHtFdjE5ODgQHh5ueu7i4iLJQdzu88/V10GDNC3pfe6cCkWnUyvGCSFKp9gxiPbt27Nx40aMRuMdn3zr1q306tWLHj16EBUVddvrp0+fJjQ0lODgYEaMGMG5c+cKvH716lU6d+5cIEkJG5OXdyNBaNy9NH8+ZGerCdxNmmgaihDlQrFjECtXrmTJkiU4OTnh4uKC0WhEp9Oxb9++It9nMBgIDw9nyZIl6PV6Bg0aREBAAA0bNjQdExERQf/+/RkwYAA7duwgMjKS2fkjjMB77713W5lxYWN++glOnYL774dOnTQLIy0NPvxQbUtRPiHKRrEJIi4u7q5OHB8fj4+PD3Xr1gUgKCiIzZs3F0gQR44cYdq0aQC0a9eO8ePHm147cOAAycnJdOrUiQMHDtxVDMIK8genR45U5b01snixutO2Y0fo0EGzMIQoV4pNELt37za7v7hP9klJSXjfNEqo1+uJj48vcEzTpk2JjY0lNDSUTZs2kZ6ezqVLl6hWrRoRERHMnj2bX3/9tSTfB1lZWSQkJJToWHMyMzNL9X5rspVYHdLTafTVVzgAhzt0IKeQmCwdb04OREQ0BJwZMuQUCQlX7/pctvKzLSl7iteeYgX7itdSsRabIBYvXmzazsrKIj4+nubNm/N5fr9zKUydOpWZM2cSHR2Nn58fer0eR0dHli9fTufOnQskmOK4urrSrFmzu44lISGhVO+3JpuJdckSyMiATp1o2KtXoYdZOt5ly9QAdZMmMG5c3VI1ZGzmZ1tC9hSvPcUK9hVvaWItKrEUmyAWLlxY4PnZs2eZNWtWsRfV6/UFBp2TkpLQ6/W3HbNgwQIA0tPTiY2NxcPDg7i4OPbu3cuKFStIT08nJyeHSpUqMXny5GKvK6zo5tIaGjEab0yMmzxZ014uIcqdEhXru5m3tzdHjhwp9riWLVty/PhxTp06hV6vJyYmhsjIyALHpKSk4OnpiYODA1FRUYSEhAAUOG7NmjUcOHBAkoOtOXYMtmxRJb01nDi5aRP8/jvo9TaxuqkQ5UqxCWLmzJmmJUfz8vJISEjgwQcfLP7ETk5Mnz6dsWPHYjAYCAkJoVGjRsybN48WLVrQrVs3du3axdy5c9HpdPj5+TFjxozSf0fCOvK7GAcOBA8PzcLIbz2EhYGbm2ZhCFEuFZsgWrRoYdp2dHQkKCiI1q1bl+jkXbp0oUuXLgX23byedWBgIIGBgUWeY+DAgQzUeHauuEVenk2U1ti3T1UYr1wZnntOszCEKLeKTRC9evXC1dUVR0dHQM1vyMjIwF3j1cKEhrZvV11Mdeqo4nwayV8Q6JlnVOVWIUTZKnZIb9SoUWRmZpqeZ2Zm8tRTT1k0KGHj8lsPI0bA9Q8O1nb8uCog6+gIL7ygSQhClHvFJoisrKwCy4xWrlyZjIwMiwYlbFh6uvrLDJrevfTuu2AwwJAhmlcXF6LcKjZBuLu78+eff5qeHzhwADcZDay4oqPh6lVo106zgkfJyfDJJ2pbymoIYTnFjkG8+uqrhIWFcc8992A0Grl48SLvvvuuNWITtsgG5j589BFcuwY9e8LDD2sWhhDlXrEJ4qGHHmLDhg0cO3YMgPr16+Msi/xWTKdOwebNajWewYM1CSEzU1VtBZg6VZMQhKgwiu1iWrZsGRkZGTRu3JjGjRtz7do1li1bZo3YhK1ZulRNXe7XT7Pbhj7/HM6fB19fCAjQJAQhKoxiE8Tq1avxuGkiVLVq1fjyyy8tGpSwQUaj5t1LBsONW1unTFELAwkhLKfYBJGXl1dgsSCDwUBOTo5FgxI26Lff1NKi3t6q818D334Lhw6Bj4+m1T2EqDCKHYPw9/fnhRdeYMiQIYBaQKhz584WD0zYmPzWw/Dh4HTHJbxKzWiEd95R25MmaRKCEBVOsf/NpkyZwqpVq1ixYgUATZo04eLFixYPTNiQzExYuVJta9S99MsvqhHj5QVjxmgSghAVTrFdTA4ODjz88MPUrl2bP/74g99++40GDRpYIzZhK9auhStXoHVruKk2lzXlF+UbN07VXhJCWF6hLYhjx44RExPDd999R/Xq1enTpw8AS5cutVpwwkZoPDidkKDGH1xdYcIETUIQokIqNEH07t0bPz8/Pv74Y3x8fAD43//+Z624hK04exY2bgRnZ3jySU1CyF8eZNQote6DEMI6Cu1iWrBgAbVq1WLkyJG89tpr7Nixo8DdTKKC+OILVd67b1+oWdPqlz97Vk2/0OngpZesfnkhKrRCWxDdu3ene/fuXLt2jc2bN/PZZ5+RkpLCjBkz6NGjB/7+/taMU2jBBuY+vP8+ZGerdYkaNdIkBCEqrGIHqStVqkRwcDALFy5ky5YtPPjggyxatMgasQmt7d0Lf/4JtWrB9TEoa0pLU3WXQIryCaGFO7qbvFq1agwePJjBGtXhEVaW33oYOlSNQVjZokXq5il/f1U8VghhXcW2IEQFlZUFy5erbQ26l3Jy4L331LYU5RNCG5IghHkxMZCSAg89BK1aWf3yq1ap4rFNm0JQkNUvL4RAEoQozM2D01auindzWY0pU8BBfkuF0IT81xO3O38e1q9XCz4PG2b1y8fGwh9/wL33anJ5IcR1kiDE7ZYvh9xc6N1bk5lp+a2HiRPV7GkhhDYkQYjbaTj3Yd8++PFHqFIFnnvO6pcXQtxEEoQo6PffYf9+tWJccLDVL59flO+ZZ8DT0+qXF0LcRBKEKCi/9fDkk1bv3zl2DFavVms9vPCCVS8thDBDEoS4IScH8tcb16B76d13VdmnJ5+EunWtfnkhxC0kQYgbvv9e3cHUrBm0aWPVSycnw+LFalvKaghhGyRBiBs0nPvw4Ydw7RoEBkLLlla9tBCiEJIghJKcDOvWqVlpw4db9dIZGTB/vtqW1oMQtkMShFBWrlR1tXv0gNq1rXrpzz6DCxfUiqZdu1r10kKIIlg0QWzdupVevXrRo0cPoqKibnv99OnThIaGEhwczIgRIzh37hwACQkJDB48mKCgIIKDg1m/fr0lwxSg2dwHg+HGinFTpli9Z0sIUYQ7Kvd9JwwGA+Hh4SxZsgS9Xs+gQYMICAigYcOGpmMiIiLo378/AwYMYMeOHURGRjJ79mzc3NyIiIjg/vvvJykpiZCQEPz9/fHw8LBUuBXbwYOwezd4eED//la99DffwOHDUL8+hIRY9dJCiGJYrAURHx+Pj48PdevWxcXFhaCgIDZv3lzgmCNHjtDueqH/du3amV6vX78+999/PwB6vR4vLy9SUlIsFarIbz0MHgzu7la7rNF4Y2LcpElq/oMQwnZYLEEkJSXh7e1teq7X60lKSipwTNOmTYmNjQVg06ZNpKenc+nSpQLHxMfHk5OTQ7169SwVasVmMKh1p8Hq3Uvbt8POneDlBU89ZdVLCyFKQNPPbFOnTmXmzJlER0fj5+eHXq/H0dHR9Pr58+eZMmUKEREROBRT8zkrK4uEhIS7jiUzM7NU77emsoy18vbt1Dtzhux69ThSvTpY4GdQWLzTp9cBqjJ48AVOnrxY5te9G/b0ewD2Fa89xQr2Fa/FYjVayL59+4yjR482PV+4cKFx4cKFhR5/9epVY6dOnUzP09LSjP379zdu2LChRNc7ePDg3QdbBu+3pjKNdcgQoxGMxpkzy+6ctzAX759/qsu6uRmN589b7NJ3zJ5+D4xG+4rXnmI1Gu0r3tLEWtR7LdbF1LJlS44fP86pU6fIzs4mJiaGgICAAsekpKSQl5cHQFRUFCHXRymzs7MZP348/fr1IzAw0FIhisuX1SgxwIgRVr30nDnq61NPQa1aVr20EKKELNbF5OTkxPTp0xk7diwGg4GQkBAaNWrEvHnzaNGiBd26dWPXrl3MnTsXnU6Hn58fM2bMAGDDhg3s2bOHy5cvEx0dDcD//d//0axZM0uFWzGtXg2ZmRAQAD4+VrvsmTNq2EOnU4PTQgjbZNExiC5dutClS5cC+8LCwkzbgYGBZlsI/fr1o1+/fpYMTYBmcx/ef1/VBQwJgZvuehZC2BiZSV1RHToEv/4KlSvDwIFWu2xqKnz0kdqWshpC2DZJEBVVfuvh8cfV8m1WsmiRShKdO0Pbtla7rBDiLkiCqIjy8mDpUrVtxe6l7Gy15gPA1KlWu6wQ4i5JgqiIfv4ZTp5UA9OdO1vtsitXwunT8OCD0Lu31S4rhLhLkiAqov/9T30NDVXlva3g5rIakydb7bJCiFKQ/6YVTVoafP212h450mqX/f57OHAA7rsPhg612mWFEKUgCaKi+fprtXSbvz80aGC1y+a3HsLCwNXVapcVQpSCJIiKJr97adQoq13ywAE3fvoJqlaFZ5+12mWFEKUkCaIiOXYMtmxRJb0ff9xql/300xqASg7VqlntskKIUpIEUZF8/rn6OmCAWhzICo4ehdjYqjg5qe4lIYT9kARRURiNNxKEFbuX5s6FvDwdw4ZBnTpWu6wQogxIgqgotm9XH+dr11bF+azgyBH49FO1PXmyVS4phChDkiAqivzB6ZEj4aZFmSzh5Ek13tCsGWRkQOfOV2nRwqKXFEJYgKwCXBFcuwZffqm2LVha49QpmDULFi9W1VodHGD4cHjmmTNAY4tdVwhhGdKCqAiio9UEubZtoUmTMj99YiKMH69Kdy9cCLm5ajLcn3+qkk81axrK/JpCCMuTFkRFYKG5D2fOwNtvQ1SUKsSn08GQITB9uupeEkLYN0kQ5d2pU7B5s5q+PHhwmZzy7Fn4v/+Djz+GrCy174knVGJo3rxMLiGEsAGSIMq7L75Qt7g+9hhUr16qU507BxERqhspM1PtGzQIZsxABqGFKIckQZRnRmOZdC8lJcE776iV4DIy1L6BA1VieOihUkcphLBRkiDKs5074Z9/wNsbeva847dfuKASw4cfqhuhAPr1g//+F1q1KttQhRC2RxJEeZa/rOiwYeBU8n/qixdV9dUFC24khuBglRgeeaTswxRC2CZJEOVVZqZawg1KPPchORnmzIH58yE9Xe0LClKJwc/PMmEKIWyXJIjy6ttv4fJl9ZG/ZcsiD01JgchIeP99uHpV7evdWyWGf/3L8qEKIWyTJIjy6uZlRQtx6ZIqpjdvnppHB9Crl0oM7dpZPEIhhI2TBFEenT0LGzeCs7PZ9T0vX4b33oN334XUVLWvRw944w1o397KsQohbJYkiPJo2TLIy1NzH2rWNO2+ckW1FubOVdsA3bqpxNCxo0axCiFsliSI8sbM3IfUVDW+MHeu6lYC6NpVJYZOnTSJUghhByRBlDf79qkqeTVrkubfm/mz1AB0Sop6uUsXlRi6dNE2TCGE7ZMEUd5cn/uwp/FQApu4kJysdnfqpBJD164axiaEsCuSIMqR9EvZ6BYvpxLw9K+jSEaNLbzxhlpETqfTOkIhhD2R9SDKgWvX1AS38ffHUOlaMvG0xL1dK2JjYds2NRAtyUEIcaekBWHHrl1TlVUjIuD8eYhGdS+5PB3KLx/rJCkIIUrFoi2IrVu30qtXL3r06EFUVNRtr58+fZrQ0FCCg4MZMWIE586dM70WHR1Nz5496dmzJ9HR0ZYM0+5kZup47z1o0ABeekklhx6tLvCYYwxGR0eahg+T5CCEKDWLtSAMBgPh4eEsWbIEvV7PoEGDCAgIoGHDhqZjIiIi6N+/PwMGDGDHjh1ERkYye/ZsLl++zIIFC/j666/R6XQMHDiQgIAAqlWrZqlwNWc0qlLaly8XfFy5UvD5pUvwzTcNuHBBva91azXG0OfQcnQv5qriSd7e2n4zQohywWIJIj4+Hh8fH+rWrQtAUFAQmzdvLpAgjhw5wrRp0wBo164d48ePB2D79u107NgRT09PADp27Mi2bdvo27evpcItNaNR1TEq7g98Uftzc0t6NWd8fVVi6Nv3+viC7//USyUszCeEEMWxWIJISkrC+6ZPsnq9nvj4+ALHNG3alNjYWEJDQ9m0aRPp6elcunTJ7HuTkpKKvF5WVhYJCQl3HW96eia//fY3V686kprqQFraja9XrzqQmupIWlr+19v3Xb3qQF5e6fp1XF3zqFo1Dw8PA1WqqK9Vqxpu23fvvel07pyDTgd//QWuf//NA/v3Y/Dw4FCjRhhL8XOwhMzMzFL921iTPcUK9hWvPcUK9hWvpWLVdJB66tSpzJw5k+joaPz8/NDr9Tg6Ot7VuVxdXWnWrNkdv+/kSTU34OjRu7psAZUrg6fnjUe1agWfF7WvWjVwc3NADQsV/c+SkJBQ8Hv95BMAHIcNo6kNruRzW7w2zJ5iBfuK155iBfuKtzSxFpVYLJYg9Hp9gUHnpKQk9Hr9bccsWLAAgPT0dGJjY/Hw8ECv17Nr164C7/2XhepOZ2aqBXIAPDxK/gf91n0eHqo2ntXl5Kh1p6FUy4oKIcStLJYgWrZsyfHjxzl16hR6vZ6YmBgiIyMLHJOSkoKnpycODg5ERUUREhICgL+/P3PnzuXK9Ypy27dvZ9KkSRaJs3FjtVDOX38l0KKFfXxaKGDjRnUbU9Om0KaN1tEIIcoRiyUIJycnpk+fztixYzEYDISEhNCoUSPmzZtHixYt6NatG7t27WLu3LnodDr8/PyYMWMGAJ6enowbN45BgwYBMH78eNOAtWVihbvs2dJe/rKioaEyG04IUaYsOgbRpUsXutxSFS4sLMy0HRgYSGBgoNn3Dho0yJQgRCFSUtTKcQ4OMGKE1tEIIcoZKbVhz1auhOxs6N4datfWOhohRDkjCcKe3dy9JIQQZUwShL1KSIBdu9TtU/37ax2NEKIckgRhr/JbD088AZUqaRuLEKJckgRhjwwGWLpUbUv3khDCQiRB2KHKO3bAmTOqnGvHjlqHI4QopyRB2KFqa9eqDZn7IISwIEkQ9ubKFar+8IPalrkPQggLkhXlAK5dwzElBdMiC7ZsxQocsrJUhcH779c6GiFEOSYJIiEB/PxofO2a1pHcGRmcFkJYmCSIqlWhfn1yz5zByU4KMl2rV49KUoZECGFhkiDq1IEDBzhkR7XfTyQk0KxyZa3DEEKUczJILYQQwixJEEIIIcySBCGEEMIsSRBCCCHMkgQhhBDCLEkQQgghzJIEIYQQwixJEEIIIczSGY1Go9ZBlIX9+/fj6uqqdRhCCGFXsrKyaNWqldnXyk2CEEIIUbaki0kIIYRZkiCEEEKYJQlCCCGEWZIghBBCmCUJQgghhFmSIIQQQphV4RcM2rp1K2+99RZ5eXk8/vjjPPPMM1qHVKhp06bx888/U6NGDb777jutwynS2bNnmTp1KsnJyeh0Op544glCbXiZ1KysLIYNG0Z2djYGg4FevXoxceJErcMqksFgICQkBL1ez8cff6x1OEUKCAigcuXKODg44OjoyJo1a7QOqVCpqam89tpr/PPPP+h0OmbNmoWvr6/WYZl19OhRXnzxRdPzU6dOMXHiREaNGlU2FzBWYLm5ucZu3boZT548aczKyjIGBwcbDx06pHVYhdq1a5fxwIEDxqCgIK1DKVZSUpLxwIEDRqPRaExLSzP27NnTpn+2eXl5xqtXrxqNRqMxOzvbOGjQIGNcXJzGURXt008/NU6aNMn4zDPPaB1Ksbp27WpMTk7WOowSmTp1qnH16tVGo9FozMrKMl65ckXjiEomNzfX2KFDB2NiYmKZnbNCdzHFx8fj4+ND3bp1cXFxISgoiM2bN2sdVqHatGlDtWrVtA6jRO655x6aN28OQJUqVXjggQdISkrSOKrC6XQ6Kl9fxjU3N5fc3Fx0Op3GURXu3Llz/PzzzwyStcnLVFpaGrt37zb9XF1cXPDw8NA4qpLZsWMHdevWpXbt2mV2zgqdIJKSkvD29jY91+v1Nv1HzF4lJiaSkJDAww8/rHUoRTIYDPTr148OHTrQoUMHm4531qxZTJkyBQcH+/kvPGbMGAYOHMiqVau0DqVQiYmJeHl5MW3aNPr3789//vMfrl27pnVYJRITE0Pfvn3L9Jz289sl7FJ6ejoTJ07k1VdfpUqVKlqHUyRHR0fWrl3Lli1biI+P559//tE6JLN++uknvLy8aNGihdahlNiKFSuIjo5m0aJFLFu2jN27d2sdklm5ubkcPHiQJ598km+++QZ3d3eioqK0DqtY2dnZ/PjjjwQGBpbpeSt0gtDr9Zw7d870PCkpCb1er2FE5UtOTg4TJ04kODiYnj17ah1OiXl4eNC2bVu2bdumdShm7du3jx9//JGAgAAmTZrEb7/9xuTJk7UOq0j5/69q1KhBjx49iI+P1zgi87y9vfH29ja1HgMDAzl48KDGURVv69atNG/enJo1a5bpeSt0gmjZsiXHjx/n1KlTZGdnExMTQ0BAgNZhlQtGo5H//Oc/PPDAAzz11FNah1OslJQUUlNTAcjMzOTXX3/lgQce0Dgq81566SW2bt3Kjz/+yNy5c2nXrh1z5szROqxCXbt2jatXr5q2f/nlFxo1aqRxVObVqlULb29vjh49Cqh+/QYNGmgcVfFiYmIICgoq8/NW6NtcnZycmD59OmPHjjXdMmirv7gAkyZNYteuXVy6dInOnTvz73//m8cff1zrsMzau3cva9eupXHjxvTr1w9Q8Xfp0kXjyMw7f/48r7zyCgaDAaPRSGBgIF27dtU6rHIhOTmZ8ePHA2qcp2/fvnTu3FnjqAr3+uuvM3nyZHJycqhbty5vv/221iEV6dq1a/z666+Eh4eX+bml3LcQQgizKnQXkxBCiMJJghBCCGGWJAghhBBmSYIQQghhliQIIYQQZlXo21yFKIlmzZrRuHFj0/OgoKAyq/qbmJjIc889Z/PVeUXFJAlCiGK4ubmxdu1arcMQwuokQQhxlwICAggMDGTbtm24uroSGRmJj48PiYmJvPrqq1y6dAkvLy/efvtt7rvvPi5evMiMGTM4deoUAP/973+55557MBgMvPbaa8TFxaHX6/nwww9xc3Pj888/Z+XKlTg6OtKwYUPeffddjb9jUdHIGIQQxcjMzKRfv36mx/r1602vVa1alXXr1jF8+HBmzZoFwJtvvsmAAQNYt24dwcHBvPnmm6b9bdq04dtvvyU6Oto0a//EiRMMGzaMmJgYqlatysaNGwGIiorim2++Yd26dbzxxhtW/q6FkAQhRLHyu5jyH3369DG9ll9eOSgoiP379wMQFxdn2t+vXz/27t0LwG+//cbQoUMBVTm2atWqANSpU4dmzZoB0Lx5c06fPg1AkyZNmDx5MmvXrsXR0dEK36kQBUmCEEJjLi4upm1HR0cMBgOgWhBDhw7l4MGDDBo0iNzcXK1CFBWUJAghSmHDhg0ArF+/3rRusa+vLzExMQCsW7cOPz8/ANq3b8/y5csBVbQuLS2t0PPm5eVx9uxZ2rVrx+TJk0lLS7ObhWtE+SGD1EIUI38MIl+nTp1M6y9cuXKF4OBgXFxcmDt3LqCqgU6bNo3FixebBqkB/vOf//D666/z9ddf4+DgwH//+19q1apl9poGg4EpU6Zw9epVjEYjI0eOtJulL0X5IdVchbhLAQEBfPXVV3h5eWkdihAWIV1MQgghzJIWhBBCCLOkBSGEEMIsSRBCCCHMkgQhhBDCLEkQQgghzJIEIYQQwqz/BwCYschrtD6RAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "febbb068-edfb-4157-8bd8-38d3ae698e92",
        "_uuid": "1bf8b52609c0538f619ac4d6efea04b6491a4673",
        "id": "WjBwLxlopajD",
        "colab_type": "text"
      },
      "source": [
        "**References:**\n",
        "\n",
        "[1] P. Bojanowski, E. Grave, A. Joulin, T. Mikolov, \"Enriching Word Vectors with Subword Information\", arXiv, 2016  \n",
        "[2] FastText Embeddings: https://github.com/facebookresearch/fastText/blob/master/pretrained-vectors.md  \n",
        "[3] F. Chollet, \"Deep Learning with Python\", Manning Publications, 2017  "
      ]
    }
  ]
}