NER_Project_Transformer.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/RashaAlshawi/NER-using-Deep-learning/blob/main/NER_Project_Transformer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SQLvm96MKKgt"
      },
      "source": [
        "### ` Intelligently Predicting Viral Spillover Risks from Bats  and other Wild Mammals `     \n",
        "🦇"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ue-Kl0oPZtfr"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import os\n",
        "import sys\n",
        "import matplotlib.pyplot as plt\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.metrics import precision_recall_fscore_support,accuracy_score\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UiCshzV-BuKU",
        "outputId": "0a868af5-1b65-4374-bfe4-72f3154cc808"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#virus ds \n",
        "data=pd.read_csv('/content/drive/MyDrive/viruses/viruses_dataset.csv')"
      ],
      "metadata": {
        "id": "7ZdaWptAHhjg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data=data.fillna(method='ffill')"
      ],
      "metadata": {
        "id": "hApOx6TEKw3v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# change the label of the first column\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "data['sentence_id']=LabelEncoder().fit_transform(data['sentence_id'])"
      ],
      "metadata": {
        "id": "AVQ3ChQ3K7If"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# label info \n",
        "labels = data.label.unique()\n",
        "labels"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wc6vtV0bK99q",
        "outputId": "616a9a8a-89ed-4e41-f9ac-a84c8a0549f6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['O', 'B-VIRUS', 'I-VIRUS', 'B-HOST', 'I-HOST', 'B-DETECTIONMETHOD',\n",
              "       'I-DETECTIONMETHOD'], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "label_to_ID = {}\n",
        "for i, label in enumerate(labels):\n",
        "    label_to_ID[label] = i\n",
        "label_to_ID"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NSrpVeCTLE2S",
        "outputId": "a2da929f-0fdf-4879-bb2e-4dc19800dcff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'B-DETECTIONMETHOD': 5,\n",
              " 'B-HOST': 3,\n",
              " 'B-VIRUS': 1,\n",
              " 'I-DETECTIONMETHOD': 6,\n",
              " 'I-HOST': 4,\n",
              " 'I-VIRUS': 2,\n",
              " 'O': 0}"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# for prediction #####\n",
        "id2tag = {}\n",
        "for i, j in label_to_ID.items():\n",
        "    id2tag[j] = i"
      ],
      "metadata": {
        "id": "Gz6Kff6wLHcT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#id2word, id2tag\n",
        "id2tag"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F92NBn2OU5ZK",
        "outputId": "3ea53556-3585-4f38-fe95-7a5a7cd5adf5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: 'O',\n",
              " 1: 'B-VIRUS',\n",
              " 2: 'I-VIRUS',\n",
              " 3: 'B-HOST',\n",
              " 4: 'I-HOST',\n",
              " 5: 'B-DETECTIONMETHOD',\n",
              " 6: 'I-DETECTIONMETHOD'}"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# map the IOB labels into numbers\n",
        "data['labels_ids']=(data['label'].map(label_to_ID))"
      ],
      "metadata": {
        "id": "7NG5kcDLLOhy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qpXQtTcWjUUC",
        "outputId": "2bb48c81-f468-4b2d-f6b9-cf1c842b0b61"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n"
          ]
        }
      ],
      "source": [
        "#Group each sentence in one list \n",
        "pre_data=data.groupby(\n",
        "    ['sentence_id'],as_index=False)['label','labels_ids'].agg(lambda x:list(x)) # agg query apply on the groupby"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Group each sentence in one list \n",
        "pre_data1=data.groupby(['sentence_id'],as_index=False)['word'].agg(lambda x:' '.join(x)) # agg query apply on the groupby"
      ],
      "metadata": {
        "id": "x6vLfvgCEEkU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pre_data['words']=pre_data1['word']"
      ],
      "metadata": {
        "id": "XiC-molQEHvt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pre_data.head(1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81
        },
        "id": "5O5x7ZrdLU46",
        "outputId": "ce45ab4f-d96f-45ed-e1de-7c536adf87ea"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   sentence_id                                              label  \\\n",
              "0            0  [O, O, O, O, O, O, O, O, O, B-VIRUS, O, O, O, ...   \n",
              "\n",
              "                                          labels_ids  \\\n",
              "0  [1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, ...   \n",
              "\n",
              "                                               words  \n",
              "0  To determine whether transplacental transmissi...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0d5fb3e0-6ec7-4125-a756-4d6af4d0b196\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence_id</th>\n",
              "      <th>label</th>\n",
              "      <th>labels_ids</th>\n",
              "      <th>words</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>[O, O, O, O, O, O, O, O, O, B-VIRUS, O, O, O, ...</td>\n",
              "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, ...</td>\n",
              "      <td>To determine whether transplacental transmissi...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0d5fb3e0-6ec7-4125-a756-4d6af4d0b196')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0d5fb3e0-6ec7-4125-a756-4d6af4d0b196 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0d5fb3e0-6ec7-4125-a756-4d6af4d0b196');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ynEjWjvxkpzP"
      },
      "outputs": [],
      "source": [
        "print(pre_data['labels'][0],end='')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZVfPnmEwCNA5",
        "outputId": "8570316e-c790-46d9-9305-23720dc1fd80"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "406\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "406"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ],
      "source": [
        "print(len(np.unique(pre_data['labels_ids'])))\n",
        "len(np.unique(pre_data['label']))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ssx_6C8ECKyU"
      },
      "outputs": [],
      "source": [
        "def num_labels (labels, data):\n",
        "    label_count = {}\n",
        "    for label in labels:\n",
        "        len_label = len(data[data['label'] == label])\n",
        "        label_count[label] = len_label\n",
        "    \n",
        "    return label_count"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "labels_count = num_labels(labels, data)\n",
        "labels_count"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YpQ3h8VCFv_H",
        "outputId": "64af1059-a0fc-47f7-9814-c713c210b745"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'B-DETECTIONMETHOD': 839,\n",
              " 'B-HOST': 2102,\n",
              " 'B-VIRUS': 2883,\n",
              " 'I-DETECTIONMETHOD': 157,\n",
              " 'I-HOST': 671,\n",
              " 'I-VIRUS': 1414,\n",
              " 'O': 88674}"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 516
        },
        "id": "msydiHzPELno",
        "outputId": "6345c3f4-cf98-42ec-866f-2ccfe30ac070"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 720x432 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmkAAAHzCAYAAAB7ZIpqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZhkdXn3//dHFseojAhuj4ADYpDFh23colHcAibiHhU1ivqTK65xIY/GDdSYGCUa3PKEGCWaKEEeXEYRF2QxEaOAmCCLIgIOQVkdQERZ7t8f54wUTXdP9XRVn1PV79d19dVV51tV5+57BubT55zv96SqkCRJUr/coesCJEmSdHuGNEmSpB4ypEmSJPWQIU2SJKmHDGmSJEk9ZEiTJEnqIUOapHkluTDJhQt8z6FJKsk+46lqNJIc2da5amDbqnbbkR3WVUlOmrGt8572oTfScmJIk3qs/QdxQ1/7LHIfJyVZ0IKJSQ5s933gYvY9jZLs0/bm0K5r2RizBURJ3di06wIkDeXt84xdOOZ9P24j3vMh4Cjg4hHXshQuAXYG1nVYw87A9R3ufy596I20bBjSpAlQVYd2uO8fb8R7rgCuGEM5Y1dVNwLndlxDp/ufSx96Iy0nnu6UpsjgdUtJnpnkO0muT3JVkqOS3Hfgtava05yPbp8PnkI9aeB1t7kmrR37ePv04zPet2pmHbPU+MD2WrCfJvlNkp8n+VSSnWZ57b2SHJbkvCS/TPKL9vGRSXZYQF8en+Sb7WdcleRzSR44x2tnve5qmFra95zYvuWQ2U5LD54qTrJfe7p53eAp5w2dckzywiTfS/KrJJcl+ViSe8/yujmvJ5z5Z7S+rnb40TNqP3S+3rRj90ny4Xafv0lyeZJjk+w9y2sHe/CYtgfXJrkmyZeS7DzXzy4tJx5Jk6bTy4EnA18ATgYeCjwb2D3JHlX1a+AXNKdRDwTux21PqV44z2cf2b73KcDngTMHxn4xX1FJ9gOOBTYD1gDnA9sATwf+KMljquqM9rW/A/wHcH/ga+3r09b6FOAY4IL59td+zjOBfwN+036/FHgkcCrwXxt6/wJr+Vz7lhfS9P2kgY+5cMbHPhPYD/gy8H/bzxrGa4E/aH+W49uf5UXAPkkeWlWXD/k5M51J83fgEOAimj/n9U6a741Jtgf+HfhfwDeATwPbAn9M8+f6jKr64ixvfRJN/9b3YBfgD4EHJ9mlPSIrLVuGNGkCzHMR+g1V9e5Ztu8HPLiq/nvgMz4FHEDzj+LRVfUL4ND2SMr9hj2lWlVHJqH9nM9V1ZFD/gxb0vzjfT3wqKo6e2BsN+DbwEeBvdrNj6MJRX9XVa+d8VmbA3ccYp93Af4BuAX4/ao6bWDs/cBrhql92Fqq6nNJfkET0k7aQE//EPjDqjp+yBrWeyLw0Kr63kAN63+WdwMvWeDnAVBVZwJnJjkEuHCBp9j/L01Ae0tVvWugro8ApwD/nOR+VXXdjPc9Fdi3qk4YeM9fA28EXgy8Z2N+FmlaeLpTmgyHzPH1xjle/4HBgNb6x/b7Q8ZS4Ya9ALgbcMhgQAOoqrNo6tszyS4z3vermR9UVb+pqmuH2OdTgLsDnxoMaK1DWfgF8IupZabPb0RAA/jkYEBrHUrzszw3yQbD6ygl2YbmyN7FzAhVVfUtmmB+d5qjpTMdNRjQWke037v6eyr1hkfSpAlQVVngW2YGEoCftt+3XGQ5G+vh7ffd5zgy+Lvt952Bs2lOF14CvDHJXsBxNKccz6yqm4fc5/qjcifPHKiqdUnOpL0mbwNGUctM39nI923oZ9mZ256CHrc92+/fbCcWzPQN4Pnt6z4xY6yPf0+l3jCkSdNptmvDbmq/b7KUhQzYqv3+0g287i4AVXVNkofRXCf1ZGDfdvyK9jTaX84RCgatbL//fI7xn23g/Yywlo3a9yw29LOsnGN8XNbv79I5xtdvv9ssY7f7e1pVN7Wn07v6eyr1hiFN0lJZf2px96oa6oL9qloLvCTNv9q7AI8FXgG8jeZyjbcOuc97zTF+uxmRY6zldh+5wNevt6GfZfAU7i3A5nO8frbQtDHW72+uXt5nxuskDclr0iTdDJBkIUcu1p/iW8h7vt1+//0FvAeAavygqj4IPKHd/NQh3npG+/12pzSTrAT2GEMtG9ObhZjvZ7kBOGdg6GrgXkk2m+VzVs/x+bewsNrXXx/3yCSz/eL/mPb7GbOMSZqHIU3Sle337cb8no/TnN46JMntLgpPcocMrKuWZNcksx01Wr9tmBX5P08TVJ6bZGYoOZQhTw0usJaN6c1C/EmSPWdsO5TmZ/l0u7zKet+hOWPyosEXp7md1yPm+PwraZbPGEp7hPFrwCpmzJZN8lDguTR/Bp8d9jMlNTzdKU2AeZbggGYZjMVcKH4CzXpWxyY5jmYG40VV9cl53nMqTTB5TZKtuPV6qA9W1ayntarqynbNss8C305yAvADmtN+29JMLNgKWNG+5QnAe5OcCvwQuIxmTbWn0Bztee+GfrCqui7JQTRrin0zyeA6abvRLA/xqA19zgJrOY9mksFzktxIs+ZY0czKvGiIfW3Il4H/SHL0wM/ySJp12GbO9v0gTUD7+ySPo7kofw+aXn+RZp2ymU5oa19Dc/TrRuCUqjplnpr+lGYixXuT/AHNhID166TdArxoI2fASsuaIU2aDIfMM3Yhi5vN91GahVSfA/wfmv8vnAzMGdKq6uokz2jrOhC4czv0L8xz7VFVnZDkfwMH01x8//s0i8z+D80swP838PKv0ByNehRNGNqCJpR8DXhfu7zDBlXVMe0iuocAzwJ+TRPOHk4TaoYJaUPXUlU3J3kazZplfwzclWbh23+nCWyL9X6aoPsamgWKr6NZePZNVXXZ4Aur6uwkjwf+CtifZvLIN2l+9qcze0j7M5pQ+TiatdzuQDNhYs6QVlUXtEcq39K+Zx/gGprFdt9VVd/duB9VWt5StbHXrkqSJGlcvCZNkiSphwxpkiRJPWRIkyRJ6iFDmiRJUg8Z0iRJknpo6pbg2HrrrWvVqlVdlyFJkrRBp59++hVVdY/ZxqYupK1atYrTTjut6zIkSZI2KMmc6yd6ulOSJKmHDGmSJEk9ZEiTJEnqoV5fk5ZkH+CdNDdhPqqqTuq0IEmSNDI33ngja9eu5YYbbui6lLFbsWIF22yzDZttttnQ71nykJbkYzQ39b2sqnYb2L4fcDiwCfDRqno3zU1+rwNWAGuXulZJkjQ+a9eu5a53vSurVq0iSdfljE1VceWVV7J27Vq23377od/XxenOI4H9Bjck2QT4MPBEYBfggCS7AN+sqicCbwDevsR1SpKkMbrhhhvYaqutpjqgASRhq622WvARwyUPaVV1CnDVjM0PAc6vqguq6jfAUcBTquqWdvxq4I5LWKYkSVoC0x7Q1tuYn7MvEwfuC/x04Pla4L5Jnp7kH4BPAh+a681JDkpyWpLTLr/88jGXKkmSpsVd7nKXeccvvPBCdtttt3lfM9OBBx7IMcccs5iygJ5PHKiqY4Fjh3jdEcARAKtXr65x1yVJkkYvhx020s+rgw8e6ecttb4cSbsE2Hbg+TbtNkmSpLG77rrreNzjHsdee+3Fgx70ID7/+c//duymm27iec97HjvvvDPPfOYzuf766wE4/fTTefSjH83ee+/Nvvvuy6WXXjrSmvoS0r4LPCDJ9kk2B54DfKHjmiRJ0jKxYsUKPvvZz3LGGWdw4okn8vrXv56q5uTceeedx8tf/nLOOecctthiCz7ykY9w44038qpXvYpjjjmG008/nRe/+MW8+c1vHmlNXSzB8WlgH2DrJGuBQ6rqn5K8EvgKzRIcH6uqHyx1bZIkaXmqKt70pjdxyimncIc73IFLLrmEn//85wBsu+22POIRjwDg+c9/Ph/4wAfYb7/9OOuss3jCE54AwM0338x97nOfkda05CGtqg6YY/txwHEb+7lJ9gf233HHHTf2IyRJ0jL1r//6r1x++eWcfvrpbLbZZqxateq3S2bMnJmZhKpi11135dRTTx1bTb2eOLAQVbUGWLN69eqXjntfo76wcaEm/UJISZL6Zt26ddzznvdks80248QTT+Siiy767djFF1/MqaeeysMf/nA+9alP8chHPpKddtqJyy+//Lfbb7zxRn74wx+y6667jqymvlyTJkmS1JnnPe95nHbaaTzoQQ/iE5/4BA984AN/O7bTTjvx4Q9/mJ133pmrr76al73sZWy++eYcc8wxvOENb2D33Xdnjz324Fvf+tZIa5qaI2mSJGmydXGm6LrrrgNg6623nvPU5bnnnjvr9j322INTTjnldtuPPPLIkdTmkTRJkqQeMqRJkiT10NSEtCT7Jzli3bp1XZciSZK0aFMT0qpqTVUdtHLlyq5LkSRJQ1q/YOy025ifc2pCmiRJmiwrVqzgyiuvnPqgVlVceeWVrFixYkHvc3anJEnqxDbbbMPatWu5/PLLuy5l7FasWME222yzoPcY0iRJUic222wztt9++67L6C1Pd0qSJPXQ1IQ0Z3dKkqRpMjUhzdmdkiRpmkxNSJMkSZomhjRJkqQeMqRJkiT1kCFNkiSphwxpkiRJPTQ1Ic0lOCRJ0jSZmpDmEhySJGmaTE1IkyRJmiaGNEmSpB4ypEmSJPWQIU2SJKmHDGmSJEk9ZEiTJEnqIUOaJElSD01NSHMxW0mSNE2mJqS5mK0kSZomUxPSJEmSpokhTZIkqYcMaZIkST1kSJMkSeohQ5okSVIPGdIkSZJ6yJAmSZLUQ4Y0SZKkHpqakOYdByRJ0jSZmpDmHQckSdI0mZqQJkmSNE0MaZIkST1kSJMkSeohQ5okSVIPGdIkSZJ6yJAmSZLUQ4Y0SZKkHjKkSZIk9ZAhTZIkqYcMaZIkST1kSJMkSeohQ5okSVIPTU1IS7J/kiPWrVvXdSmSJEmLNjUhrarWVNVBK1eu7LoUSZKkRZuakCZJkjRNDGmSJEk9ZEiTJEnqIUOaJElSDxnSJEmSesiQJkmS1EOGNEmSpB4ypEmSJPWQIU2SJKmHDGmSJEk9ZEiTJEnqIUOaJElSDxnSJEmSesiQJkmS1EOGNEmSpB4ypEmSJPWQIU2SJKmHpiakJdk/yRHr1q3ruhRJkqRFm5qQVlVrquqglStXdl2KJEnSog0V0pL8XpInDTzfKsmnk/x3ksOSbDK+EiVJkpafYY+kvRvYe+D5e4E/BH4IvAx404jrkiRJWtaGDWk7A6cBJNkMeCbw2qp6BvBm4LnjKU+SJGl5Gjak3QW4pn38EODOwBfb52cA2424LkmSpGVt2JB2CbB7+/iJwFlVdVn7fEvg+lEXJkmStJxtOuTrPg38VZJ9aK5FO2RgbC/gRyOuS5IkaVkbNqQdCtwAPIxmEsH7B8Z2Bz4z2rIkSZKWt6FCWlXdDLxrjrGnjrQiSZIkDX0kDYAk/xt4FLAV8A9V9bMkOwI/r6prx1GgJEnScjRUSEtyR+BfgKcDAQpYA/wMeA/NemlvHFONkiRJy86wszvfBTwe+BPgXjRBbb0vA/uOuC5JkqRlbdjTnQcAb6mqT81yC6ifAKtGWpUkSdIyN+yRtK2Ac+b5jDuOphxJkiTB8CHtJ8DD5xh7CHDeaMqRJEkSDB/SPgG8McnzgM3abZXkMcBrgY+NozhJkqTlatiQ9h7gS8Angavbbf8OfB04vqo+OIbaJEmSlq2FLGb7nCQfppnJeU/gSpqAdvIY65MkSVqWFrSYbVV9E/jmmGqRJElSa9jTnZIkSVpCc4a0JDcneUj7+Jb2+VxfNy1dyZIkSdNvvtOd7wDWDjyu8ZcjSZIkmCekVdXbBx4fuiTVSJIkCRjymrQkH0uy/Rxj90viOmmSJEkjNOzEgQOBe8wxtjXwwpFUI0mSJGBhszvnuibt3sCvRlCLJEmSWnNek5bkacDTBja9PckVM152J+D3gdPHUJskSdKyNd/szu1oAhg0R9H2AH494zW/Br4F/MXoS2skuTNwMnBoVX1xXPuRJEnqk/lmdx4OHA6Q5CfAU6vq+4vdYTvJ4EnAZVW128D2/dr9bQJ8tKre3Q69ATh6sfuVJEmaJENdk1ZV248ioLWOBPYb3JBkE+DDwBOBXYADkuyS5AnA2cBlI9q3JEnSRJjvmrRHAWdU1XXt43lV1SnD7LCqTkmyasbmhwDnV9UF7b6PAp4C3AW4M01w+1WS46rqlmH2I0mSNMnmuybtJOBhwHfax3PN7kw7tski6rgv8NOB52uBh1bVKwGSHAhcMVdAS3IQcBDAdtttt4gyJEmS+mG+kPYYmlONAI+lw9tCVdWRGxg/AjgCYPXq1d6+SpIkTbz5Jg6cPPD4pDHXcQmw7cDzbdptkiRJy9Kwt4W6IMnuc4ztluSCRdbxXeABSbZPsjnwHOALi/xMSZKkiTXsHQdWAXecY2wFcL9hd5jk08CpwE5J1iZ5SVXdBLwS+ApwDnB0Vf1g2M+UJEmaNvNdkzbTXNd6rQZ+MfSHVB0wx/bjgOMWUM9tJNkf2H/HHXfc2I+QJEnqjTmPpCV5bZKLk1xME9DWrH8+8HU5zfpmxy9VwXOpqjVVddDKlSu7LkWSJGnR5juSdgFwQvv4hcBpwOUzXvNrmhmgHx19aZIkScvXfLM7Pw98HiAJwDuq6idLVJckSdKyNtQ1aVX1onEXIkmSpFsNPXEgyQ7As4DtaGZ0DqqqeskoC1soJw5IkqRpMlRIS/JU4GiaiQaX0VyLNqjzVf6rag2wZvXq1S/tuhZJkqTFGvZI2jtp7t/5vKqaOXlAkiRJIzZsSNsBeL0BTZIkaWkMe8eBc4GtxlmIJEmSbjVsSPs/wJvayQOSJEkas2FPdx5KcyTtnCQ/Aq6aMV5V9ehRFiZJkrScDRvSbgbOG2chi+USHJIkaZoMu5jtPmOuY9FcgkOSJE2TYa9JkyRJ0hKaM6QleV+SbWdse3qSlTO2/W6SL4yrQEmSpOVoviNpfwbcZ/2TJJsAnwHuP+N1WwJ/NPrSJEmSlq/5QlqG3CZJkqQR85o0SZKkHpqakJZk/yRHrFu3rutSJEmSFm1DIa2G3Na5qlpTVQetXLlywy+WJEnquQ2tk3ZEkmtnbPunJNcNPL/riGuSJEla9uYLaadw+6NmJ8/yul+0r5UkSdKIzBnSJuEuA5IkSdNqaiYOSJIkTRNDmiRJUg8Z0iRJknpoakKa66RJkqRpMjUhzXXSJEnSNJmakCZJkjRNDGmSJEk9ZEiTJEnqIUOaJElSDxnSJEmSemjOkJbk2CQ7to9fkGSrpStLkiRpeZvvSNpTgLu3jz8O3H/85UiSJAnmD2k/Bx7ePg5Q4y9HkiRJMH9IOxp4f5KbaQLat5PcPMfXTUtTriRJ0vKw6TxjrwX+A9gFOAQ4ErhkCWraKEn2B/bfcccduy5FkiRp0eYMaVVVwGcAkhwIHF5V31+iuhasqtYAa1avXv3SrmuRJElarPmOpP1WVW0/7kIkSZJ0q6HXSUtynySHJflukh+339+T5N7jLFCSJGk5GiqkJfld4PvAq4HrgO+03/8MODPJA8ZWoSRJ0jI01OlO4G+AdcBDqurC9RuT3A/4ajv+9JFXJ0mStEwNe7rzMcBbBwMaQFVdBBzajkuSJGlEhg1pmwPXzjF2bTsuSZKkERk2pJ0JvCrJbV6fJMDL23FJkiSNyLDXpL0D+CJwTpJ/Ay4F7g38MfAA4I/GU54kSdLyNOw6accneRLwl8CbufVenqcDT6qqr46vREmSpOVn2CNpVNXxwPFJfgfYEri6qq4fW2WSJEnL2NAhbb02mBnOJEmSxmjoOw70XZL9kxyxbt26rkuRJElatKkJaVW1pqoOWrlyZdelSJIkLdqCT3dKOeywTvdfBx/c6f4lSVoKU3MkTZIkaZpsMKQl2TzJGUn+YCkKkiRJ0hAhrap+A2wP3DT+ciRJkgTDn+78GuCRNEmSpCUy7MSBDwL/kmRT4HM0t4WqwRdU1QUjrk2SJGnZGjakndx+fx3w2jles8niy5EkSRIMH9JeNNYqJEmSdBvD3mD9n8ddiCRJkm61oHXSktwhyW5JHp3kzuMqSpIkabkbOqQleQXwM+C/gG8AO7XbP5fk1eMpT5IkaXkaKqQleSlwOM3MzmcBGRj+JvCM0ZcmSZK0fA17JO11wN9W1UHAZ2eMnUt7VE2SJEmjMWxI2x74yhxjvwTuNppyJEmSBMOHtCuAVXOM7QRcMpJqJEmSBAy/TtoXgbclOQm4qN1WSbamWdz2c2OoTZJ6I4cd1un+6+CDO92/pKU37JG0twC/Bs4Cvk5zS6gPAOcANwPvGEt1kiRJy9RQIa2qrgBWA38NbAb8mOYo3IeAh1fVurFVOKQk+yc5Yt26zkuRJElatGFPd1JV1wLvbL96p6rWAGtWr1790q5rkSRJWqyhQxpAki2A3YD7AmuBs9rwJkmSpBEaOqQleRvweuAu3LqY7bVJ3ltVfzmO4iRJkparoUJakrcDbwU+ChwF/By4F3AA8PYkm1bVoeMqUpIkabkZ9kjaS2nuOPDnA9t+AHwjyTrgIODQEdcmSZK0bA27BMdK5r7jwPHtuCRJkkZk2JD2n8CD5xh7cDsuSZKkEZnzdGeSwQD3auCzSW4CPsOt16Q9C3gx8JRxFilJkrTczHdN2k00dxZYL8C72y9mbP+vDXyWJEmSFmC+YPUObhvSJEmStETmDGkuqSFJktSdYScOSJIkaQkt5I4DOwPPBLYFVswYrqp64SgLkyRJWs6GvePAC4CP0Vyjdhnwmxkv8do1SZKkERr2SNpbgc8DL6mqX4yxHkmSJDF8SLs38KcGNKlbOeywTvdfBx/c6f4laTkZduLAfwA7j7MQSZIk3WrYI2mvBI5NciXwVeDqmS+oqltGWZgkSdJyNmxIWwt8D/iXOcZrAZ8lSZKkDRg2WP0j8Gzgc8C53H52pyRJkkZo2JD2FODPq+rwcRYjSZKkxrATB34JnD3OQiRJknSrYUPax4HnjrMQSZIk3WrY050XAQck+RpwPLPP7vzYKAuTJElazoYNaX/ffr8f8LhZxovmtlGSJEkagWFD2vZjrUKSJEm3MVRIq6qLxl2IJEmSbjXsxAFJkiQtoaGOpCX5Cc11Z3Oqqh1GUtFt97sz8GfA1sAJVfX3G3iLJEnSVBj2SNrJs3ydBWzRfsZJw+4wyceSXJbkrBnb90tyXpLzk7wRoKrOqao/BZ4FPGLYfUiSJE26Ya9JO3C27UnuRrMkx9cXsM8jgQ8Bnxj4nE2ADwNPoLlP6HeTfKGqzk7yZOBlwCcXsA9JkqSJtqhr0qrqF8B7gbct4D2nAFfN2PwQ4PyquqCqfgMcRXMrKqrqC1X1ROB5i6lVkiRpkgy7BMd8bgC2WeRn3Bf46cDztcBDk+wDPB24I3DcXG9OchBwEMB22223yFIkSZK6t9EhLcmmwG7AocAPRlXQoKo6iSGud6uqI4AjAFavXj3vBAdJkqRJMOzszluYe3bnNcAfLbKOS4BtB55v026TJElaloY9kvYObh/SbqC5p+eXq2rdIuv4LvCAJNvThLPn4A3dJUnSMjbs7M5DR7XDJJ8G9gG2TrIWOKSq/inJK4GvAJsAH6uqsZxClSRJmgSjmDiwIFV1wBzbj2OeyQEbkmR/YP8dd9xxYz9CkiSpN+YMaUmGXlYDoKresfhyNl5VrQHWrF69+qVd1iFJkjQK8x1JO3SI9w9ep9ZpSJMkSZom84W0zTbw3j2AvwT2BX40sookSdJv5bDDOtt3HXxwZ/vWPHccqKqbZ/sCdgD+BfhPYBeaRWR3WZpyJUmSloehbwuVZNskH6VZuPaxwMHAA6rqo21461SS/ZMcsW7dYlcDkSRJ6t4GQ1qSeyQ5HPgh8Ayaa892qKq/a++z2QtVtaaqDlq5cmXXpUiSJC3afLM7VwJvAF4FBDgc+JuqunqJapMkSVq25ps48BNgJfBVmgkClwJbJtlythdX1QWjL0+SJGl5mi+k3a39vi/wB0N81iaLL0eSJEkwf0h70ZJVIUmSpNuYM6RV1T8vZSGL5W2hJEnSNBl6CY6+c3anJEmaJkt+g3VJ0mTpcsV7cNV7LV9TcyRNkiRpmhjSJEmSesiQJkmS1EOGNEmSpB4ypEmSJPXQ1IS0JPsnOWLdunVdlyJJkrRoUxPSXCdNkiRNk6kJaZIkSdPEkCZJktRDhjRJkqQeMqRJkiT1kCFNkiSphwxpkiRJPTQ1Ic110iRJ0jSZmpDmOmmSJGmaTE1IkyRJmiaGNEmSpB4ypEmSJPWQIU2SJKmHDGmSJEk9ZEiTJEnqIUOaJElSDxnSJEmSesiQJkmS1ENTE9K8LZQkSZomUxPSvC2UJEmaJlMT0iRJkqaJIU2SJKmHDGmSJEk9ZEiTJEnqIUOaJElSDxnSJEmSesiQJkmS1EOGNEmSpB4ypEmSJPWQIU2SJKmHDGmSJEk9NDUhzRusS5KkaTI1Ic0brEuSpGkyNSFNkiRpmhjSJEmSesiQJkmS1EOGNEmSpB4ypEmSJPWQIU2SJKmHDGmSJEk9ZEiTJEnqIUOaJElSDxnSJEmSesiQJkmS1EOGNEmSpB4ypEmSJPWQIU2SJKmHDGmSJEk9ZEiTJEnqIUOaJElSD01NSEuyf5Ij1q1b13UpkiRJizY1Ia2q1lTVQStXruy6FEmSpEWbmpAmSZI0TQxpkiRJPWRIkyRJ6iFDmiRJUg8Z0iRJknrIkCZJktRDhjRJkqQeMqRJkiT1kCFNkiSphzbtugBJkqTZ5LDDOt1/HXxwp/v3SJokSVIPGdIkSZJ6yJAmSZLUQ4Y0SZKkHjKkSZIk9ZAhTZIkqYcMaZIkST1kSJMkSeohQ5okSVIPGdIkSZJ6yJAmSZLUQ4Y0SZKkHjKkSZIk9ZAhTZIkqYdSVV3XMFJJLgcuGvNutgauGPM+lht7Olr2c/Ts6ejZ09Gyn6O3FD29X1XdY7aBqQtpSyHJaVW1uus6pok9HS37OXr2dPTs6WjZz9Hruqee7pQkSeohQ5okSVIPGdI2zhFdFzCF7Olo2c/Rs6ejZ09Hy36OXqc99Zo0SZKkHvJImiRJUg8Z0iRJknpo01FjJ6UAABGNSURBVK4LmBRJVgA7tk/Pr6obuqxH0vgk2bSqbuq6jmliT6WF80jaBiTZNMl7gLXAPwOfAH6a5D1JNuu2usmU5H5JVg48f0ySw5O8LsnmXdY2ieznWHyn6wKmkD2VFsiQtmHvBe4ObF9Ve1fVXsD9gbsBh3Va2eQ6GrgzQJI9gM8AFwO7Ax/psK5JZT9HL10XMIXs6YgleWGSM5L8sv06LckLuq5rkvWtp57u3LAnAb9bA9Ngq+qaJC8DzgX+rLPKJtedqup/2sfPBz5WVX+b5A7AmR3WNans5+jdI8nr5hqsqvctZTFTwp6OUJIXAq8BXgecQROC9wLem6Sq6pNd1jeJ+thTQ9qGVc2yTklV3ZzE9Us2zuBv1I8F/gKgqm5J/GV7I9jP0dsEuAse/RklezpaLwOeVlUXDmz7RpJnAEcBhrSF611PDWkbdnaSF1TVJwY3Jnk+zZE0Ldw3khwNXApsCXwDIMl9gN90WdiEsp+jd2lVvaPrIqaMPR2tLWaECQCq6sIkW3RQzzToXU8NaRv2CuDYJC8GTm+3rQbuBDyts6om22uAZwP3AR5ZVTe22+8NvLmzqiaX/Rw9j/aMnj0drV9t5Jjm1rueeseBISV5LLBr+/Tsqjqhy3okjU+S7WiO/NzYPt8J+EPgoqo6ttPiJpQ9Ha0k1wPnzzYE7FBVd17ikiZeH3tqSNOSS3ItMPgXr4ArgBOBN1TVlZ0UNqHs5+glOQV4SVX9KMmONMtH/CuwC/CdqvqLTgucQPZ0tJLcb77xqrpoqWqZFn3sqSFNvZBkS+BA4Peq6o87Lmfi2c/FSfLfVfWg9vE7gbtX1SvadedOXz+m4dnT8UiyPbc9y3NBl/VMgz711HXS1AtVdXVVvZ9mDTotkv1ctMHfXh8LfA2gqn4D3NJJRZPPno5Qki3aCUMnAC9uv76e5DNOHNg4feypEwfUG+0dHPw7OSL2c1H+K8lhwCU0t4P7KkCSu3Va1WSzp6P1AeBs4DlVdQtAmjV33gp8CHBR24XrXU893akll+Tps2zekmaG4r87TX9h7OfoJbkTzULV96FZHPj77fbfA+7vQqELZ09HK8mPquoBCx3T3PrYU0OallySj8/YVMCVwElV9aUOSppo9nN8kqygOeoDcH5V3dBlPdPAno7GBgLF+VW142xjmlsfe2pIU68kuXNV/bLrOqaF/dw4STYF/gp4Ec19UANsC3wcePPAWnQakj0drST/DPwYeOfgXXGSvJXmVoZ/0llxE6qPPXXigDqR5L5JVrczu0hyzyR/Bfyo49Imkv0cufcCd6dZG2nvqtqLZhLG3YDDOq1sctnT0XoV8CDg/CT/r/36MbB7O6aF611PPZKmJZfkNTQr4Z8P3BH4CPA3wCeA91TVpR2WN3Hs5+gl+RHNb841Y/smwLle77Nw9nQ8ktyfZq05aJaL+HGX9UyDPvXUmV/qwkHATlV1VbsK+Q+BR1TV6Rt4n2ZnP0evZoaJduPNSfzNduPY0xFKstfA00va7yvXb6+qM5a+qsnWx54a0tSFG6rqKoCqujjJeQaKRbGfo3d2khdU1ScGNyZ5PnBuRzVNOns6Wn878Hhv4DRuvT9q0axFp4XpXU893akll+Qy4KiBTc8ZfF5Vr17yoiaY/Ry9JPcFjqW5qfL6wLsauBPwtKq6ZK73anb2dHySfK+q9uy6jmnSl556JE1d+PMZzz3qszj2c8TawPDQJI/l1tvDHFdVJ3RY1kSzp2Pl0ZbR60VPPZImSTMkuft84+tPL2t49nR8kpzRzpbViPSlpx5J05JLsoZ5fkupqicvYTkTz36Oxek0PQ3NCvn/025Pu32HjuqaZPZ0hJJ8kFv/u98myQcGx73MYeH62FNDmrrgmkijZT9HrKq2X/+4L9emTDp7OnKnDTz2EofR6F1PDWnqwkOAo6rqp10XMiXs53h5Tcjo2dPF26mq3tR1EVOmdz31jgPqwv8CvpXkm0lenuQeXRc04eyntPzs13UBU6h3PXXigDqRJMCjaJaLeCrwfeDTwLFVdW2XtU0i+zlaSV438PR1wPsGx6vqfWhB7OloJfk+sA+3ruN1G07EWLg+9tSQps61t4V5PPBumsPNv9NxSRPNfi5ekkPmG6+qty9VLdPCno5Wkl/TrIo/W6CoqnIixgL1saeGNHUqyYNojv48G7gC+HRVHd5tVZPLfkrLg5MvRq+PPXXigJZckgfQBInnADfTrI7/B1V1QaeFTSj7uTT6sm7SNLGn0vwMaerC8TTXSz27qs7qupgpYD+XxqzXqWhR7OnGm/MIeZJNq+qmpSxmSvSup87u1JKrqvtX1VsGA0WSM7qsaZLZzyXzpa4LmEL2dOP9f+sfJPnkjLHvLHEt06J3PTWkqS/8jXq07Ofo/V07i1YjUlVv6bqGCXbngce7zhjz7+nG6V1PDWnqC3+jHi37uQhJHpbkpCTHJtkzyVnAWcDPk/RuLaVJkOTaJNfM8nVtkmu6rm8CzTfrzxmBG6d3PfWaNPXF3yVJOd14JDxCsWgfAt4ErAS+ATyxqr6d5IE01/8d32Vxk6iq7tp1DVPmbkmeRnOw5W5Jnt5uD83fWy1c73rqEhxackkeRrOG11XAO4FPAlvT/IfxgqryH8AFSHIts/+WF5q1fbZY4pImXpIzq2qP9vE5VbXzwFjvpulr+Uny8fnGq+pFS1XLtOhjTz2Spi54lGKEPEIxFrcMPP7VjDF/s1XnDGGj18eeeiRNS86jFOq7JDcDv6Q5Gnkn4Pr1Q8CKqtqsq9okuN1ttm7H22wtXB976pE0dcGjFOq1qtqk6xqkDTgMOBP4MvBrnNE5Cr3rqUfStOQ8SiFJi5Nkd+AAYD/gdJpLRU5w8tXG62NPDWmSJE2wJL9HEy4eD7yhqr7QcUkTry89dZ00SZImVJJ7AHsCDwLWApd1W9Hk61NPvSZNkqQJk+TFwLOAFcAxwLOqyoC2CH3sqac7JUmaMEluobkLxkXtptv8Y15VT17yoiZcH3vqkTRJkibPY7ouYAr1rqceSZMkSeohj6RJkjRhkpzI3OtKVlU9binrmQZ97KlH0iRJmjBJ9p5l88OA/wNcVlUPXuKSJl4fe2pIkyRpgiV5NPBWmlmJ76qqL3dc0sTrS0893SlJ0gRKsi/wFppbGL2rqk7suKSJ17eeeiRNkqQJk+S7wD2A9wKnzhyvqjOWvKgJ18eeGtIkSZowSU5i/ovcH7uE5UyFPvbUkCZJktRDXpMmSdIESnJP4BXAru2mHwAf7vpWRpOsbz31BuuSJE2YJI8Avts+/UT7BfCddkwL1MeeerpTkqQJk+TbwMuq6nsztu8B/ENVPbSbyiZXH3vqkTRJkibPFjPDBEBVnQnctYN6pkHvempIkyRp8iTJlrNsvDv+276xetdT/yAlSZo87we+muTRSe7afu0DfLkd08L1rqdekyZJ0gRK8iSa+0oOzkR8b1Wt6a6qyda3nhrSJEmSesh10iRJmjBJPsjcq+NTVa9ewnKmQh97akiTJGnynNZ1AVOodz01pEmSNHl2qqo3dV3ElOldT53dKUnS5Nmv6wKmUO966pE0SZImzybtml6ZbbCqrlrieqZB73rq7E5JkiZMkl8DlzB7oKiq2mGJS5p4feypR9IkSZo8Z1fVnl0XMWV611OvSZMkSeohQ5okSZPnH5PcY+bGJPdIsqKLgqZA73pqSJMkafLsAfz+LNsfiffu3Fi966kTByRJmjBJTq+qvecY+0FV7TrbmObWx556JE2SpMnzO/OM+W/7xuldT/2DlCRp8lyW5CEzNyZ5MHB5B/VMg9711NOdkiRNmDZMHA0cCZzebl4NvAB4TlX9Z0elTaw+9tSQJknSBEpyT+AVwG7tph8AH6qqy7qrarL1raeGNEmSJkySLarqmjnGtquqi5e6pknXx556TZokSZPnpPUPkpwwY+xzS1vK1Dhp/YO+9NSQJknS5Bm8v+Td5xnT8HrXU0OaJEmTp+Z4PNtzDad3PfUG65IkTZ57JnkdzRGe9Y9pn9/u1kYaSu966sQBSZImTJJD5huvqrcvVS3Too899UiaJEmT58qq+lDXRUyZ3vXUa9IkSZo8L+66gCnUu54a0iRJknrIa9IkSZowSW4Crp9tCKiq2mKJS5p4feyp16RJkjR5/ruq9uy6iCnTu556ulOSJKmHDGmSJE2ez3RdwBTqXU8NaZIkTZ491j9I8jeDA0m+uvTlTIXe9dSQJknS5Nlx4PETZox5x4GN07ueGtIkSZouLtswet67U5IkDeV3kuxJc7DlTu3jtF936rSyydW7nrpOmiRJEybJifONV9VjlqqWadHHnhrSJEmSesjTnZIkTaAkWwHPBR7YbjoH+FRVXdVdVZOtbz114oAkSRMmyc7AWcDewA+BHwEPBs5K8sD53qvZ9bGnnu6UJGnCJDkGOLqqjp6x/RnAc6vqGd1UNrn62FNDmiRJEybJeVW100LHNLc+9tTTnZIkTZ5fbuSY5ta7njpxQJKkyXPPJK+bZXvwjgMbq3c9NaRJkjR5/hG46xxjH13KQqZI73rqNWmSJEk95DVpkiRNgSRndF3DtOm6p4Y0SZKmQ7ouYAp12lNDmiRJ0+FLXRcwhTrtqdekSZI0BZJsDVxZ/sM+NTySJknShEnysCQnJTk2yZ5JzqK5pdHPk+zXdX2TKMm1Sa6Z5evaJNd0UpOBW5KkyZLkNOBNwErgCOCJVfXt9h6Tn66qPTstUCPhkTRJkibPplX11ar6DPCzqvo2QFWd23FdGiFDmiRJk+eWgce/mjHmKbIp4elOSZImTJKbae4nGeBOwPXrh4AVVbVZV7VpdAxpkiRJPeTpTkmSpB4ypEmSJPWQIU3SREpyYJKa4+sXC/ysVUkOTbLDLGMXJjly4Pk+7WvvMMtnVJIDN/ZnkqRBm3ZdgCQt0h8Da2dsu2mBn7EKOAT4d+CCGWNPAwYXstynfe1fctsZdpcCDwd+vMB9S9KsDGmSJt2ZVXX+uD68qr435Ot+DXx7XHVIWn483Slpag2cEn1Ykn9tb/HyP0k+kGRF+5p9gBPbt3xt4JTpPu34b093JjmU5igawI3rX9uOzXq6M8mjk5zQ3lrml0m+kmS3Ga/ZN8m3kqxLcl2S85K8bSxNkTQxDGmSJt0mSTad8TXz/22fpDkN+XTg74FXAH/Rjp3RPgd4Nc0py4e322f6KPBP7eNHDrx2Vkn+CDgBuA54PvBc4K7AN5Ns275mB+ALwE+AZwNPBt4H3Hmon17S1PJ0p6RJN9ttcL4EPGng+aeqav0RsK8neShwAHBIVV2T5Ox27Jz1t9eZTVWtTbL++rf/rKoNXft2OHByVT1l/YYkJ9Jc9/Z64DXAXsDmwMuqav21b9/YwOdKWgYMaZIm3dO4/cSBmbM7vzTj+X8Djx9bRUCSBwD3B/4qyeD/a68HTgUe1T4/E7gROCrJx4BTquqycdYmaTJ4ulPSpDurqk6b8TVzIsFVM57/GrjjmOu6Z/v9n2hC2ODXk4CtANpa96X5//EngZ8l+XaSR4+5Pkk955E0SRqPK9vvfwF8fZbx36x/UFUnAicmuSPwCOAdwJeSrKqqK8ZeqaReMqRJUnNkDZobVS/ktdfO87rzgAuBXavq3cMU0S7j8Y0kdwE+D2wPGNKkZcqQJmnS7ZFk61m2n7aAz/ghzQK4L05yFU0QO6+qZgth6ycZvD7Jl4Gbq+p2+6qqSvIK4PNJNgeOpglc9wJ+D7i4qt6X5E9prk87DvgpsDXN0bf/Ac5awM8gacoY0iRNus/Msf0ew35AVV2Z5JXAG4CTgU2AxwAnzfLyLwIfAV4OvA1I+zXb5x6X5FHAm2mW77gT8DOaRW//rX3Z94EnAn9Ncx3bVTR3PnheVf1q2J9B0vRJVXVdgyRJkmZwdqckSVIPGdIkSZJ6yJAmSZLUQ4Y0SZKkHjKkSZIk9ZAhTZIkqYcMaZIkST1kSJMkSeohQ5okSVIP/f+DzPHQRz4GFQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "plt.figure(figsize = (10, 6))\n",
        "plt.hist(data.label, log = True, label = 'label', color = 'darkcyan', bins = 16)\n",
        "plt.xlabel('Entities', fontsize = 16)\n",
        "plt.ylabel('Number of  Entities', fontsize = 16)\n",
        "plt.title(\"Entities distribution \", fontsize = 20)\n",
        "plt.grid(alpha=0.0)\n",
        "plt.legend()\n",
        "plt.xticks(fontsize=10)\n",
        "plt.yticks(fontsize=10)\n",
        "plt.xticks(rotation=90)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "J34m3XDZUTT2"
      },
      "outputs": [],
      "source": [
        "X=pre_data['words']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8c4UOlxBBwrO"
      },
      "outputs": [],
      "source": [
        "y=pre_data['labels_ids']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Wle7mZKCAZw_",
        "outputId": "fc2edca5-555c-491f-9b1e-76d68be8b99b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-VIRUS', 'I-VIRUS', 'I-VIRUS', 'O', 'O', 'O', 'O', 'O', 'O', 'B-HOST', 'I-HOST', 'B-HOST', 'I-HOST', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-DETECTIONMETHOD', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-DETECTIONMETHOD', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-DETECTIONMETHOD', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-HOST', 'I-HOST', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-DETECTIONMETHOD', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']"
          ]
        }
      ],
      "source": [
        "print(pre_data['label'][4],end='')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s_ki9AcLXyb0",
        "outputId": "48d3f9d5-5339-4c93-e3a5-3896511a1611"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(303, 303)"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ],
      "source": [
        "len(X[100].split()),len(y[100])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-_uFOqxEYLDQ",
        "outputId": "a49b2022-2fb1-4795-ecbf-0eb9a06e0f44"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "sampele from x: To determine whether transplacental transmission could explain overwintering of bluetongue virus in the United Kingdom we studied calves born to dams naturally infected during pregnancy Approximately were infected transplacentally some had compromised health In all infected calves viral load decreased after birth no evidence of persistent infection was found\n",
            "Its label: [1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n"
          ]
        }
      ],
      "source": [
        "print('sampele from x:',X[0])\n",
        "print('Its label:',y[0])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B1hTEE_RHqa8"
      },
      "source": [
        "## Tokenization and padding"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G9a5QbVWHqIr",
        "outputId": "ee7d5bf6-84b5-4b68-a717-2498f786dbf9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of unique words in the dataset: 9629\n"
          ]
        }
      ],
      "source": [
        "# 1. for the trainig data\n",
        "max_word=data.word.nunique()\n",
        "maxlen = len((max((X), key=lambda x: len(x.split()))).split())\n",
        "print(\"Number of unique words in the dataset: {}\".format(max_word))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "52hxNVRWf1JF",
        "outputId": "4c838ea8-32ec-490b-c90f-d4a28cab3f81"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "384"
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ],
      "source": [
        "maxlen"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CZJSemuCEJhy"
      },
      "outputs": [],
      "source": [
        "tokenizer=Tokenizer(num_words=max_word,oov_token='<OOV>') # using the maximum number in our data to aviod lossing any. \n",
        "tokenizer.fit_on_texts(X)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wpwngun5FrzS"
      },
      "outputs": [],
      "source": [
        "seq=tokenizer.texts_to_sequences(X)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yCBNblLKCCp5"
      },
      "outputs": [],
      "source": [
        "# will need it later for prediction \n",
        "word2id = tokenizer.word_index"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "word2id"
      ],
      "metadata": {
        "id": "2RKc0uR8VA01"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fhRWbbdoGFVh"
      },
      "outputs": [],
      "source": [
        "id2word = {}\n",
        "for key, value in word2id.items():\n",
        "    id2word[value] = key"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "id2word"
      ],
      "metadata": {
        "id": "-bDdhNDnEzgZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "n9ojRmVxeIyd"
      },
      "outputs": [],
      "source": [
        "X_p=pad_sequences(seq,padding='post',maxlen=maxlen)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jTNlk3G6gkK5"
      },
      "source": [
        " preprocess labels by assigning a unique identifier for each one of them.\n",
        "\n",
        "labels have different lenghts, to solve this \n",
        "\n",
        "We can assign a new tag for the zeros that we used in padding\n",
        "We can use the O tag for them."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sOfc-TLoebCB"
      },
      "outputs": [],
      "source": [
        "y_p=pad_sequences(y,padding='post',maxlen=maxlen)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Before Padding:\",len(y[0]),len(y[8]))\n",
        "print(\"After Padding:\",len(y_p[0]),len(y_p[8]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fm718tPFG0pl",
        "outputId": "09a549e1-a384-4bb7-ec2b-325a205e864f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Before Padding: 49 135\n",
            "After Padding: 432 432\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RaMrh6Z6VqlU",
        "outputId": "b5ecb87d-aa5e-4f92-b9f7-e1421d7c13f9"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((524, 432), (524, 432))"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ],
      "source": [
        "X_p.shape,y_p.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mqsvSlQIVJ6s"
      },
      "outputs": [],
      "source": [
        "#Split the data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "J-yxLWNWVJUn"
      },
      "outputs": [],
      "source": [
        "# 85% train and validation,    15% for test\n",
        "samples_train=int(len(X)*0.80)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.utils import shuffle\n",
        "X_s,y_s=shuffle(X_p,y_p,random_state=0)"
      ],
      "metadata": {
        "id": "mgRv2dHqI7QK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v2uWiDASHz-x",
        "outputId": "c097067d-285f-470b-c55c-3a1913af4975"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(((450, 432), (450, 432)), ((74, 432), (74, 432)))"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ],
      "source": [
        "X_train=X_p[:samples_train]\n",
        "y_train=y_p[:samples_train]\n",
        "X_test=X_p[samples_train:]\n",
        "y_test=y_p[samples_train:]\n",
        "(X_train.shape,y_train.shape),(X_test.shape,y_test.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CkC1-nQulhOH"
      },
      "outputs": [],
      "source": [
        "embedding_dim = 150\n",
        "max_words = 36000\n",
        "num_tags=len(labels)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Models   \n",
        "1. LSTM\n",
        "2. Transformer "
      ],
      "metadata": {
        "id": "xwNrx0DxSab2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "## 1.  basic LSTM model\n",
        "model = tf.keras.models.Sequential([\n",
        "    tf.keras.layers.Embedding(max_words, embedding_dim, input_length=maxlen),\n",
        "    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(units=100, activation='tanh', return_sequences=True)),\n",
        "    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(units=100, activation='tanh', return_sequences=True)),\n",
        "    tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(num_tags, activation='softmax'))\n",
        "])"
      ],
      "metadata": {
        "id": "r8UnY_ySSKTX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iaTFdX-9cnlQ"
      },
      "outputs": [],
      "source": [
        "#2. Transformer\n",
        "class TransformerBlock(layers.Layer):\n",
        "    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):\n",
        "        super(TransformerBlock, self).__init__()\n",
        "        self.att = keras.layers.MultiHeadAttention(\n",
        "            num_heads=num_heads, key_dim=embed_dim\n",
        "        )\n",
        "        self.ffn = keras.Sequential(\n",
        "            [\n",
        "                keras.layers.Dense(ff_dim, activation=\"relu\"),\n",
        "                keras.layers.Dense(embed_dim),\n",
        "            ]\n",
        "        )\n",
        "        self.layernorm1 = keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "        self.layernorm2 = keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "        self.dropout1 = keras.layers.Dropout(rate)\n",
        "        self.dropout2 = keras.layers.Dropout(rate)\n",
        "\n",
        "    def call(self, inputs, training=True):\n",
        "        attn_output = self.att(inputs, inputs)\n",
        "        attn_output = self.dropout1(attn_output, training=training)\n",
        "        out1 = self.layernorm1(inputs + attn_output)\n",
        "        ffn_output = self.ffn(out1)\n",
        "        ffn_output = self.dropout2(ffn_output, training=training)\n",
        "        return self.layernorm2(out1 + ffn_output)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DlnBIy94cyTj"
      },
      "outputs": [],
      "source": [
        "class TokenAndPositionEmbedding(layers.Layer):\n",
        "    def __init__(self, maxlen, vocab_size, embed_dim):\n",
        "        super(TokenAndPositionEmbedding, self).__init__()\n",
        "        self.token_emb = keras.layers.Embedding(\n",
        "            input_dim=vocab_size, output_dim=embed_dim\n",
        "        )\n",
        "        self.pos_emb = keras.layers.Embedding(input_dim=maxlen, output_dim=embed_dim)\n",
        "\n",
        "    def call(self, inputs):\n",
        "        maxlen = tf.shape(inputs)[-1]\n",
        "        positions = tf.range(start=0, limit=maxlen, delta=1)\n",
        "        position_embeddings = self.pos_emb(positions)\n",
        "        token_embeddings = self.token_emb(inputs)\n",
        "        return token_embeddings + position_embeddings\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def ner_transformer(num_tags,vocab_size,maxlen=128, embed_dim=32, num_heads=2, units=32,dropout_rate=0.1):\n",
        "\n",
        "  inputs=keras.layers.Input(shape=(maxlen,),dtype='int32')\n",
        "  embedding= TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)(inputs)\n",
        "  transformer_block = TransformerBlock(embed_dim, num_heads, units)(embedding)\n",
        "  x = layers.Dropout(dropout_rate)(transformer_block)\n",
        "  x=layers.Dense(units, activation=\"relu\")(x)\n",
        "  x = layers.Dropout(dropout_rate)(x)\n",
        "  y = layers.Dense(num_tags, activation=\"softmax\")(x)\n",
        "  model=keras.Model(inputs=[inputs],outputs=[y])\n",
        "  return model\n"
      ],
      "metadata": {
        "id": "mh03HOlMMQV6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = ner_transformer(num_tags, max_words, maxlen,embed_dim=embedding_dim, num_heads=4, units=64)"
      ],
      "metadata": {
        "id": "aW35bALHNwvR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.build(input_shape=[None,maxlen])"
      ],
      "metadata": {
        "id": "JHHUVvHE1IYC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.summary()"
      ],
      "metadata": {
        "id": "1-o-GN9R1ErZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7150fb4a-da6b-496f-877c-655087929ca8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_1 (InputLayer)        [(None, 432)]             0         \n",
            "                                                                 \n",
            " token_and_position_embeddin  (None, 432, 150)         5464800   \n",
            " g (TokenAndPositionEmbeddin                                     \n",
            " g)                                                              \n",
            "                                                                 \n",
            " transformer_block (Transfor  (None, 432, 150)         381964    \n",
            " merBlock)                                                       \n",
            "                                                                 \n",
            " dropout_2 (Dropout)         (None, 432, 150)          0         \n",
            "                                                                 \n",
            " dense_2 (Dense)             (None, 432, 64)           9664      \n",
            "                                                                 \n",
            " dropout_3 (Dropout)         (None, 432, 64)           0         \n",
            "                                                                 \n",
            " dense_3 (Dense)             (None, 432, 7)            455       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 5,856,883\n",
            "Trainable params: 5,856,883\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P3YJoh3BlmYn"
      },
      "outputs": [],
      "source": [
        "optimizer = tf.keras.optimizers.Adam(learning_rate=0.0001)\n",
        "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy']) #metric"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%tensorboard --logdir logs"
      ],
      "metadata": {
        "id": "39RKrqRf3RVb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "history = model.fit(X_train,y_train,validation_split=0.2,epochs=20,batch_size=15)"
      ],
      "metadata": {
        "id": "FKAyzqPlD2vW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "585e0721-e4eb-4bf8-f694-395f72205819"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/20\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/dispatch.py:1082: UserWarning: \"`sparse_categorical_crossentropy` received `from_logits=True`, but the `output` argument was produced by a sigmoid or softmax activation and thus does not represent logits. Was this intended?\"\n",
            "  return dispatch_target(*args, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 2s 35ms/step - loss: 0.6036 - accuracy: 0.8439 - val_loss: 0.2377 - val_accuracy: 0.9570\n",
            "Epoch 2/20\n",
            "24/24 [==============================] - 1s 22ms/step - loss: 0.2112 - accuracy: 0.9650 - val_loss: 0.2182 - val_accuracy: 0.9574\n",
            "Epoch 3/20\n",
            "24/24 [==============================] - 1s 21ms/step - loss: 0.1888 - accuracy: 0.9650 - val_loss: 0.1978 - val_accuracy: 0.9573\n",
            "Epoch 4/20\n",
            "24/24 [==============================] - 1s 21ms/step - loss: 0.1738 - accuracy: 0.9650 - val_loss: 0.1868 - val_accuracy: 0.9575\n",
            "Epoch 5/20\n",
            "24/24 [==============================] - 1s 22ms/step - loss: 0.1604 - accuracy: 0.9654 - val_loss: 0.1754 - val_accuracy: 0.9577\n",
            "Epoch 6/20\n",
            "24/24 [==============================] - 1s 21ms/step - loss: 0.1475 - accuracy: 0.9659 - val_loss: 0.1654 - val_accuracy: 0.9585\n",
            "Epoch 7/20\n",
            "24/24 [==============================] - 1s 22ms/step - loss: 0.1357 - accuracy: 0.9666 - val_loss: 0.1557 - val_accuracy: 0.9597\n",
            "Epoch 8/20\n",
            "24/24 [==============================] - 1s 22ms/step - loss: 0.1242 - accuracy: 0.9678 - val_loss: 0.1468 - val_accuracy: 0.9608\n",
            "Epoch 9/20\n",
            "24/24 [==============================] - 1s 22ms/step - loss: 0.1139 - accuracy: 0.9692 - val_loss: 0.1393 - val_accuracy: 0.9624\n",
            "Epoch 10/20\n",
            "24/24 [==============================] - 1s 21ms/step - loss: 0.1034 - accuracy: 0.9707 - val_loss: 0.1320 - val_accuracy: 0.9642\n",
            "Epoch 11/20\n",
            "24/24 [==============================] - 1s 21ms/step - loss: 0.0952 - accuracy: 0.9725 - val_loss: 0.1269 - val_accuracy: 0.9655\n",
            "Epoch 12/20\n",
            "24/24 [==============================] - 1s 22ms/step - loss: 0.0871 - accuracy: 0.9742 - val_loss: 0.1216 - val_accuracy: 0.9669\n",
            "Epoch 13/20\n",
            "24/24 [==============================] - 1s 22ms/step - loss: 0.0794 - accuracy: 0.9759 - val_loss: 0.1173 - val_accuracy: 0.9681\n",
            "Epoch 14/20\n",
            "24/24 [==============================] - 1s 22ms/step - loss: 0.0734 - accuracy: 0.9773 - val_loss: 0.1145 - val_accuracy: 0.9688\n",
            "Epoch 15/20\n",
            "24/24 [==============================] - 1s 22ms/step - loss: 0.0688 - accuracy: 0.9785 - val_loss: 0.1121 - val_accuracy: 0.9693\n",
            "Epoch 16/20\n",
            "24/24 [==============================] - 1s 21ms/step - loss: 0.0643 - accuracy: 0.9800 - val_loss: 0.1105 - val_accuracy: 0.9700\n",
            "Epoch 17/20\n",
            "24/24 [==============================] - 1s 23ms/step - loss: 0.0609 - accuracy: 0.9805 - val_loss: 0.1091 - val_accuracy: 0.9706\n",
            "Epoch 18/20\n",
            "24/24 [==============================] - 1s 21ms/step - loss: 0.0569 - accuracy: 0.9819 - val_loss: 0.1069 - val_accuracy: 0.9713\n",
            "Epoch 19/20\n",
            "24/24 [==============================] - 1s 22ms/step - loss: 0.0541 - accuracy: 0.9827 - val_loss: 0.1061 - val_accuracy: 0.9717\n",
            "Epoch 20/20\n",
            "24/24 [==============================] - 1s 22ms/step - loss: 0.0516 - accuracy: 0.9836 - val_loss: 0.1062 - val_accuracy: 0.9718\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "id": "JlbWpEidHWpc",
        "outputId": "bf77214a-5a7d-4bc0-ba15-b34abad26081"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f1e100caf10>"
            ]
          },
          "metadata": {},
          "execution_count": 47
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZxcdZ3/+9en9u7qrt6XpLMjEIQYlgAqQgIIIiIo/mJAxithe6CyCDMqAgpXcRfH8V4GRX+CYWAAcfCHiDpiwkCuoCQQAgQISYCkO530vlR3V9f2vX+cU9VV3dVLkuqurqrP8/Gox9m+dc63T3e/z7e+dRYxxqCUUir/OXJdAaWUUtmhga6UUgVCA10ppQqEBrpSShUIDXSllCoQrlxtuLa21ixatChXm1dKqby0efPmDmNMXaZlOQv0RYsWsWnTplxtXiml8pKIvDveMu1yUUqpAqGBrpRSBUIDXSmlCoQGulJKFQgNdKWUKhAa6EopVSA00JVSqkDkXaC/8E4X3//TG+htf5VSKl3eBfrLe3q4++md9A5Fcl0VpZSaVSYNdBH5lYi0icir4ywXEfmpiOwQka0icnz2qzmiPuADoK1/eDo3o5RSeWcqLfT7gHMmWP5R4HD7dRVw96FXa3z15V4A2vo00JVSKtWkgW6MeQbomqDIBcA6Y3keqBSROdmq4GjJQO8PTdcmlFIqL2WjD70J2JMy3WzPG0NErhKRTSKyqb29/aA2pl0uSimV2Yx+KWqMuccYs8IYs6KuLuPdHydV5nXh9zi1y0UppUbJRqC3APNTpufZ86ZNfcCnXS5KKTVKNu6H/jhwjYg8BJwM9BpjWrOw3nHVlXu1y0WpfGQMxKMQj4GJp7xi1rLE9ETL4zFrOh4dWVdyfPR0bPwyJpY+zDgvOmqbqcNM8+MTlEtZ9qEb4L3nZ333ThroIvKfwCqgVkSagdsAt/W7MT8DngTOBXYAg8DarNdylPpyL6/t7ZvuzSg1uxgDsQjEwuOEVOq8yPhBFotYy2ORlPWNmk6WCUMsOn6ZWHjU+Kh58QzzZz0BhxPEmTJ0jJp2gTgyzHfa80e/3wniHlnm8k5LzScNdGPMxZMsN8AXs1ajKagv97Ghr20mN6nyWVoQjg6yaErwRNMDKG1ZJH08Hhs7niw/ejqaeTy5/UQAZgrOlFCMR2d2vzlc4HCD0wNOlzV0uO1xLzgTyzxWQHnL08s6Pell7HEjLkAAB8YIGKwhDozBniZ9Oi6AJKcRBwYHVq+xjJp2YOz1g9jzAZzWfCP2e7CWGZLrwAgGY81LbCwet65MT8wzcTDGnjeqbKb5qcviBmLgN4vwTcOvLGePoDsU9QEvA+EYA8NR/N4D/BGMgb69dkvBjOxwE7en45NMT/WWA3Jg9TogKXWesN5x+49qop9l1NDE7U1kWJY2TPk4HB/1cTntY3I8fbvJj6DR9I+6yemo9bE1bTrlPYmyY1p/o1qC8VGhna09n7YbJH1aXBhcIG4MVivOmnZZwZGYxoHBLoMTY1wYvBicGOMA48QYwRiH/RKIizUvLiPbjosVHvbuJQ4mboeGscaxp00cK5gS8xLBaEgLq+Sft0mUi0M8EWr2uDGYeMyeH8WYcLJMMvxiMbtuKfPs8an/DxWuxttvw3fkkVlfb34GevJc9GEWH2igv/hr+P3101ArBSnBkhp28fRxK9BcVlglA86JMYmAs1pbJm63tozV4jJxB5hE68tplcEJxp1snVktMLu1ZuzWWVysRpiRkRZhzKS84phoHBONWcNYLGU6iokkhhFMJHoQgRSzX1m6XYXDgbhc4HIhDgfidFrjTic4nYj9SixPLBOny1rutt/jECTRbZAYF0mfHj0uYjdmneB0WOsXu0xy3IE47G6LtPkp406HtQ6HA3Fa3RAZh4myzkRZq0vDKmPVRxyJ8cR2R49LejnE/plSlpP4uUbKg1jrkNSXw140an7i/aPni9WwGz3f4c1Rl8tsM7xrF/Ne/Adn7n6Trt90UFnltf/xIhCNjvwTZpqOxTBvPoUZaoKGY5I7O9GalsR02lDGlBuZto36Bx9z47BMAZBoFTGqVWTPS4wnP/6lriexLHW9cTOybTMyzyTLp7wn5WXi40zHEx8d4yPjduvK+tgYw8SsL4GsAIxlsfUVt18HIfFP73SOhJnDkQy6xHzxeBC3e+RV6saRGE9d5nGnlcPtRlwuOxwdKcHmHAkepzN92pFh6HIjbpe1LpcLRk2PP8/+eZTKIO8CPbhhA+U/+hH/AvAi7BuvYGorJvlywkAX4vVDqCe9/Hihm4kxY0P9gKdBGHskT7aAUg8mGZfbrRexWxuMtKwQSbYIRlohiffY63I60lpkVkvEMeF4aitOXE5w2i0/l/WljzXPORJ2TlfKvJFl4rSD1uVGXE7r9+Qc+R0lwyy5zGkFqtM58jt1OpOtuMS6cThGDspKFaG8C/SKCy/EfOBUPn73c3z+zCO5+INL0lowE7Zi3tkI930MLnoQln5s5iuvlFLTKO8C3VVVRU1lJZ0VO2nxVOCur5/6m3eut04hWvSh6augUkrlSF52xokIdWUHcXHRzg0w70TwVUxPxZRSKofyMtDBOnXxgC7/H+yCvS/BYadPX6WUUiqH8jfQy70HdoOut/8HMHDYGdNWJ6WUyqU8DnTfgXW57NwA3gqYO60PVFJKqZzJ40D30jsUIRSJTV7YGCvQF59qXZaslFIFKH8DPWBdadU+lVZ61y7o3a3950qpgpa/gV5+AE8u2rneGi7RQFdKFa78DfRkC30KZ7rs3ACVC6F6yTTXSimlcid/A32qLfRYBN5+xupu0cvClVIFLG8DvcbvwemQyU9dbNkM4X49XVEpVfDyNtAdDqG2zDP5xUU711s3l1p82sxUTCmlciRvAx2meC76zg3WueclVTNTKaWUypE8D/RJrhYd6oGWTXq6olKqKOR3oAcmuUHXO89aD2jQ/nOlVBHI60CvK/fROTBMNDbO0212bgBPmXWHRaWUKnB5Hej15V6MgY5gOHOBneth0anW08aVUqrA5X2gA5nPdOl6G7rf1v5zpVTRyO9AD9gXF2X6YnTXBmuol/srpYpEfgd6soWeIdB3boDAPKg9fIZrpZRSuZHXgV5bNk6XSzxmPdDisFV6ub9SqmjkdaB7XA6q/Z6xLfS9L0GoV09XVEoVlbwOdBjn4qKd6wGBxatyUSWllMqJvA/0unLv2Fvo7twAc5aDvyY3lVJKqRzI+0Afcz+X4X5o/oeerqiUKjr5H+gBL+39w8TjxprxzkaIR7X/XClVdPI/0Mu9ROOG7kH7atGd68FdCvNPzm3FlFJqhk0p0EXkHBF5U0R2iMhNGZYvEJENIvKSiGwVkXOzX9XMxjy5aOcGWHgKuLwzVQWllJoVJg10EXECdwEfBd4LXCwi7x1V7FbgEWPMccBFwL9nu6LjaQikXFzUswc639L+c6VUUZpKC/0kYIcxZpcxJgw8BFwwqowBAvZ4BbA3e1WcWLKF3hfSy/2VUkXNNYUyTcCelOlmYHQH9e3Af4vItYAf+HCmFYnIVcBVAAsWLDjQumZUn9pC79gAZY1Qf1RW1q2UUvkkW1+KXgzcZ4yZB5wL3C8iY9ZtjLnHGLPCGLOirq4uKxv2uZ2U+1x09A3Brqet7ha93F8pVYSmEugtwPyU6Xn2vFSXA48AGGOeA3xAbTYqOBX15V48Ha/CUJeerqiUKlpTCfQXgMNFZLGIeLC+9Hx8VJndwJkAInIUVqC3Z7OiE6kv97Gg+3lrYsmqmdqsUkrNKpMGujEmClwD/Bl4HetsltdE5Jsicr5d7J+BK0XkZeA/gUuNMWa6Kj1afcDLUYOboWEZlNXP1GaVUmpWmcqXohhjngSeHDXvGynj24BTslu1qWsqjXNMbBtmyRfQ3nOlVLHK+ytFAZbFXsMjMQbnr8x1VZRSKmcKItCPCL7AsHGzr3J5rquilFI5UxCBPqfjOf4eX8r+wYL4cZRS6qDkfwL27aW09y02xo/J/GxRpZQqElP6UnRW2/U0ABvjy6gd/aALpZQqIvkf6Ds3YPx1vB1bNPZRdEopVUTyu8slHoddG5Alq6grL9UuF6VUUcvvQG97DQba4bAzrIdFa5eLUqqI5Xeg71xvDZesoj7g1Ra6Uqqo5Xmgb4C6oyAwl/pyH+3ah66UKmL5G+iRIXj3b8mnE9WVe+kfjjIUjuW4YkoplRv5G+i7n4PYcPJ2uQ2BxLNFtR9dKVWc8jfQd64HpwcWfhCw7okOaD+6Uqpo5XGgPw3zTwaPH0h5FJ32oyulilR+BnqwDfa/kuw/h5SHRWuXi1KqSOVnoNuX+7NkJNCrSt24naJdLkqpopWfl/7vXA8l1TBn5Ha5IkJdmVe7XJQ6SJFIhObmZkIh/ZQ7G/h8PubNm4fb7Z7ye/Iv0I2xzj9fshIczrRFdQGfdrkodZCam5spLy9n0aJFiOizv3LJGENnZyfNzc0sXrx4yu/Lvy6X9jcguC95umKq+nIv7drlotRBCYVC1NTUaJjPAiJCTU3NAX9ayr9AT17uf/qYRfXlXvb3aQtdqYOlYT57HMzvIv8CfckqOPvbUDl/zKL6ch/dgxHC0fiMV0sppXIt//rQG462XhkkzkVvDw7TVFkyk7VSSmVBWVkZwWAw19XIW/nXQp9A8mpR7XZRShWh/GuhT2Dk4iL9YlSpQ/F///41tu3ty+o63zs3wG0fz/zpejRjDF/5ylf44x//iIhw6623smbNGlpbW1mzZg19fX1Eo1HuvvtuPvjBD3L55ZezadMmRITLLruMG264Iat1zxeFFegBvZ+LUoXgv/7rv9iyZQsvv/wyHR0dnHjiiZx22mk8+OCDfOQjH+GWW24hFosxODjIli1baGlp4dVXXwWgp6cnx7XPnYIK9Bq/BxFo1y4XpQ7JVFvS02Xjxo1cfPHFOJ1OGhoaWLlyJS+88AInnngil112GZFIhE984hMce+yxLFmyhF27dnHttdfysY99jLPPPjundc+lgupDdzkd1Pj1yUVKFarTTjuNZ555hqamJi699FLWrVtHVVUVL7/8MqtWreJnP/sZV1xxRa6rmTMFFeiA/WxRDXSl8tmpp57Kww8/TCwWo729nWeeeYaTTjqJd999l4aGBq688kquuOIKXnzxRTo6OojH43zqU5/ijjvu4MUXX8x19XOmoLpcAPvZotrlolQ+++QnP8lzzz3H8uXLERF+8IMf0NjYyK9//Wt++MMf4na7KSsrY926dbS0tLB27Vricev6k+9+97s5rn3uiDEmJxtesWKF2bRpU9bX+5VHX+bpN9v5xy0fzvq6lSpkr7/+OkcddVSuq6FSZPqdiMhmY8yKTOULsMvFR0dwmFg8NwcqpZTKlYIL9IaAl7iBzgHtR1dKFZcpBbqInCMib4rIDhG5aZwynxaRbSLymog8mN1qTl1d4uIivS+6UqrITPqlqIg4gbuAs4Bm4AURedwYsy2lzOHA14BTjDHdIlI/XRWeTPJ+Lnqmi1KqyEylhX4SsMMYs8sYEwYeAi4YVeZK4C5jTDeAMaYtu9WcuuT9XPRMF6VUkZlKoDcBe1Kmm+15qY4AjhCR/09EnheRczKtSESuEpFNIrKpvb394Go8ibrkDbq0ha6UKi7Z+lLUBRwOrAIuBn4hIpWjCxlj7jHGrDDGrKirq8vSptN5XU4qS916cZFSquhMJdBbgNSnScyz56VqBh43xkSMMW8D27ECPiesq0W1y0UplVk0Gs11FabFVK4UfQE4XEQWYwX5RcBnRpX5HVbL/F4RqcXqgtmVzYoeiPpyH/u1y0Wpg/fHm2DfK9ldZ+My+Oj3Ji32iU98gj179hAKhbj++uu56qqr+NOf/sTNN99MLBajtraWv/71rwSDQa699trkbXNvu+02PvWpT6U9JOPRRx/liSee4L777uPSSy/F5/Px0ksvccopp3DRRRdx/fXXEwqFKCkp4d577+XII48kFovx1a9+lT/96U84HA6uvPJKjj76aH7605/yu9/9DoC//OUv/Pu//zuPPfZYdvfRIZo00I0xURG5Bvgz4AR+ZYx5TUS+CWwyxjxuLztbRLYBMeDLxpjO6az4ROrLvbzdMZCrzSulDsGvfvUrqqurGRoa4sQTT+SCCy7gyiuv5JlnnmHx4sV0dXUB8K1vfYuKigpeecU68HR3d0+67ubmZv72t7/hdDrp6+vj2WefxeVy8dRTT3HzzTfz29/+lnvuuYd33nmHLVu24HK56Orqoqqqii984Qu0t7dTV1fHvffey2WXXTat++FgTOleLsaYJ4EnR837Rsq4AW60XzlXF/DS3j+MMUYfeqvUwZhCS3q6/PSnP022fPfs2cM999zDaaedxuLFiwGorq4G4KmnnuKhhx5Kvq+qqmrSda9evRqn0wlAb28vn/vc53jrrbcQESKRSHK9V199NS6XK217n/3sZ/mP//gP1q5dy3PPPce6deuy9BNnT8HdnAusLpdwLE7PYIQqvyfX1VFKTdHTTz/NU089xXPPPUdpaSmrVq3i2GOP5Y033pjyOlIbcaFQ+ndpfr8/Of71r3+d008/nccee4x33nmHVatWTbjetWvX8vGPfxyfz8fq1auTgT+bFNyl/5B6Lrr2oyuVT3p7e6mqqqK0tJQ33niD559/nlAoxDPPPMPbb78NkOxyOeuss7jrrruS7010uTQ0NPD6668Tj8cn7OPu7e2lqck6A/u+++5Lzj/rrLP4+c9/nvziNLG9uXPnMnfuXO644w7Wrl2bvR86iwo80PVMF6XyyTnnnEM0GuWoo47ipptu4v3vfz91dXXcc889XHjhhSxfvpw1a9YAcOutt9Ld3c0xxxzD8uXL2bBhAwDf+973OO+88/jgBz/InDlzxt3WV77yFb72ta9x3HHHpZ31csUVV7BgwQLe9773sXz5ch58cOROJpdccgnz58+ftXelLLjb5wK83THA6T96mjtXL+dTJ8yblm0oVWj09rmTu+aaazjuuOO4/PLLZ2R7B3r73NnXCZQF2uWilMq2E044Ab/fz5133pnrqoyrIAPd73Xh9zi1y0UplTWbN2/OdRUmVZB96AD1AZ+20JVSRaVgA72u3Eu7Xi2qlCoiBRvoej8XpVSxKdhAb9AuF6VUkSnYQK8v9zIYjhEcLsy7qilV7MrKysZd9s4773DMMcfMYG1mh8IN9EDiQRfa7aKUKg4FedoiWPdzAetc9CV14x/JlVJjff8f3+eNrqnfP2UqllYv5asnfXXc5TfddBPz58/ni1/8IgC33347LpeLDRs20N3dTSQS4Y477uCCC0Y/AXNioVCIz3/+82zatAmXy8WPf/xjTj/9dF577TXWrl1LOBwmHo/z29/+lrlz5/LpT3+a5uZmYrEYX//615NXpuaDAg50vbhIqXyyZs0avvSlLyUD/ZFHHuHPf/4z1113HYFAgI6ODt7//vdz/vnnH9BdVO+66y5EhFdeeYU33niDs88+m+3bt/Ozn/2M66+/nksuuYRwOEwsFuPJJ59k7ty5/OEPfwCs+73kkwIOdLuFrl0uSh2wiVrS0+W4446jra2NvXv30t7eTlVVFY2Njdxwww0888wzOBwOWlpa2L9/P42NjVNe78aNG7n22msBWLp0KQsXLmT79u184AMf4Nvf/jbNzc1ceOGFHH744Sxbtox//ud/5qtf/SrnnXcep5566nT9uNOiYPvQAyUuPC4H7dpCVypvrF69mkcffZSHH36YNWvW8MADD9De3s7mzZvZsmULDQ0NY26Je7A+85nP8Pjjj1NSUsK5557L+vXrOeKII3jxxRdZtmwZt956K9/85jezsq2ZUrAtdBGhvtzLfm2hK5U31qxZw5VXXklHRwf/8z//wyOPPEJ9fT1ut5sNGzbw7rvvHvA6Tz31VB544AHOOOMMtm/fzu7duznyyCPZtWsXS5Ys4brrrmP37t1s3bqVpUuXUl1dzT/90z9RWVnJL3/5y2n4KadPwQY6JC4u0ha6Uvni6KOPpr+/n6amJubMmcMll1zCxz/+cZYtW8aKFStYunTpAa/zC1/4Ap///OdZtmwZLpeL++67D6/XyyOPPML999+P2+2msbGRm2++mRdeeIEvf/nLOBwO3G43d9999zT8lNOnIG+fm3D1/ZvZ0R7kqRtXTut2lCoEevvc2edAb59bsH3oYJ2Lrl+KKqWKRcF3ufSFooQiMXxuZ66ro5TKsldeeYXPfvazafO8Xi9///vfc1Sj3CrwQLdOXWzvH2Z+dWmOa6OUyrZly5axZcuWXFdj1ijoLpe6gD5bVClVPAo60JNXi+p90ZVSRaDAA33kfi5KKVXoCjrQa/wenA7RLhelVFEo6EB3OITaMo92uShVgCa6H3qxKuhAB6vbRbtclFLTJRqdPQ/RKejTFgEaAl5aerTLRakDse8732H49ezeD9171FIab7553OXZvB96MBjkggsuyPi+devW8aMf/QgR4X3vex/3338/+/fv5+qrr2bXrl0A3H333cydO5fzzjuPV199FYAf/ehHBINBbr/9dlatWsWxxx7Lxo0bufjiizniiCO44447CIfD1NTU8MADD9DQ0EAwGOTaa69l06ZNiAi33XYbvb29bN26lZ/85CcA/OIXv2Dbtm3867/+6yHtXyiCQK8r97FlT0+uq6GUmkQ274fu8/l47LHHxrxv27Zt3HHHHfztb3+jtraWrq4uAK677jpWrlzJY489RiwWIxgM0t3dPeE2wuEwiduXdHd38/zzzyMi/PKXv+QHP/gBd955J9/61reoqKjglVdeSZZzu918+9vf5oc//CFut5t7772Xn//854e6+4AiCPT6ci+dA2GisTguZ8H3MCmVFRO1pKdLNu+Hbozh5ptvHvO+9evXs3r1amprawGorq4GYP369axbtw4Ap9NJRUXFpIGe+iSj5uZm1qxZQ2trK+FwmMWLFwPw1FNP8dBDDyXLVVVVAXDGGWfwxBNPcNRRRxGJRFi2bNkB7q3MCj/QA16MgY5gmMYKX66ro5SaQOJ+6Pv27RtzP3S3282iRYumdD/0g31fKpfLRTweT06Pfr/f70+OX3vttdx4442cf/75PP3009x+++0TrvuKK67gO9/5DkuXLmXt2rUHVK+JTKnJKiLniMibIrJDRG6aoNynRMSISMY7geXCyLno2o+u1Gy3Zs0aHnroIR599FFWr15Nb2/vQd0Pfbz3nXHGGfzmN7+hs7MTINnlcuaZZyZvlRuLxejt7aWhoYG2tjY6OzsZHh7miSeemHB7TU1NAPz6179Ozj/rrLO46667ktOJVv/JJ5/Mnj17ePDBB7n44oununsmNWmgi4gTuAv4KPBe4GIReW+GcuXA9cCsuiuOXi2qVP7IdD/0TZs2sWzZMtatWzfl+6GP976jjz6aW265hZUrV7J8+XJuvPFGAP7t3/6NDRs2sGzZMk444QS2bduG2+3mG9/4BieddBJnnXXWhNu+/fbbWb16NSeccEKyOwfg1ltvpbu7m2OOOYbly5ezYcOG5LJPf/rTnHLKKclumGyY9H7oIvIB4HZjzEfs6a8BGGO+O6rcT4C/AF8G/sUYM+HNzmfifugArb1DfOC76/nOJ5fxmZMXTPv2lMpXej/0mXXeeedxww03cOaZZ45bZjruh94E7EmZbrbnpW7geGC+MeYPE61IRK4SkU0isqm9vX0Kmz50tWVeRNBH0SmlZoWenh6OOOIISkpKJgzzg3HIX4qKiAP4MXDpZGWNMfcA94DVQj/UbU+F2+mgutSjFxcpVYDy8X7olZWVbN++fVrWPZVAbwHmp0zPs+cllAPHAE/b54Y2Ao+LyPmTdbvMlLpyL+36pahSkzLGTHqO92xSyPdDP5jHg06ly+UF4HARWSwiHuAi4PGUjfYaY2qNMYuMMYuA54FZE+YA9QG9/F+pyfh8Pjo7Ow8qSFR2GWPo7OzE5zuwU60nbaEbY6Iicg3wZ8AJ/MoY85qIfBPYZIx5fOI15F59uZft+/pzXQ2lZrV58+bR3NzMTH2/pSbm8/mYN2/eAb1nSn3oxpgngSdHzfvGOGVXHVANZkB9uZeO4DDxuMHhyJ+Pk0rNJLfbnbzCUeWnorgWvr7cSzRu6BoM57oqSik1bYoj0AP21aJ6cZFSqoAVR6CX68OilVKFr0gCXZ8tqpQqfMUR6AGrhd6uga6UKmBFEeg+t5Nyn4s2vfxfKVXAiiLQwepH1y4XpVQhK5pAb9CrRZVSBa5oAt1qoWuXi1KqcBVPoAd8tPUN630qlFIFq3gCvdzLcDROXyia66oopdS0KJpArytPnLqo3S5KqcJUNIGevLhIL/9XShWo4gl0++Ki/dpCV0oVqOIJ9MT9XLSFrpQqUEUT6GVeFyVup56LrpQqWEUT6CJCfUCvFlVKFa6iCXSwLy7S+7kopQpUkQW6T++4qJQqWEUV6HV6gy6lVAErqkCvD3gJDkcZDOvVokqpwlNcga4XFymlCliRBXri2aIa6EqpwlNcgR7Qh0UrpQpX3gV6MBxkIDJwUO/VLhelVCFz5boCB+p3O37HnZvv5IT6E/hQ04c4dd6pLKlYgohM+t6qUjdup2iXi1KqIOVdoK9oXMFn3/tZNrZs5M7Nd3Ln5juZ459jhXvTqZw852RK3aUZ3ysi1Jf7tMtFKVWQ8i7Ql1YvZWn1Um484Ub2DexjY8tGnm1+lj/s+gO/2f4b3A43JzSckAz4xRWL01rvdeVevbhIKVWQJFePZFuxYoXZtGlT1tYXiUV4qe0lnm15lo0tG9nRswOAprKmZLif2HgiX/rPbbzbOcifbzgta9tWSqmZIiKbjTErMi4rlEAfrTXYmgz351ufZyg6hNvhptKxlM62Jfz+8qtYEFgwbdtXSqnpUJSBniocC/Ni24tsbN7I/9n+V3qizQDML1vIGQtWsXL+So6tPxa3wz0j9VFKqYNV9IGe6vXWPq79zVO8O7QZV9nruP1vYyRKmbucU5s+xGnzT+NDcz9Epa9yxuumlFKTOeRAF5FzgH8DnMAvjTHfG7X8RuAKIAq0A5cZY96daJ25CvSEHWNJpawAABM+SURBVG1Bnti6l8e3vs3uoS24yt6gpGI7UenDgYNj649l5fyVrJy3csqnRSql1HQ7pEAXESewHTgLaAZeAC42xmxLKXM68HdjzKCIfB5YZYxZM9F6cx3oCcYY3tzfzxMvt/L7rc3sGXgLT/kbBGreYkh2A9YXqyvnWeG+onEFHqcnx7VWShWrQw30DwC3G2M+Yk9/DcAY891xyh8H/L/GmFMmWu9sCfRUxhhe29vH77fu5YmXW9kb3Ic38CZ1DTsJyutETZhSVyknNZ7EYZWHsTCwkMUVi1kYWEiVryrX1VdKFYGJAn0q56E3AXtSppuBkycofznwx3EqchVwFcCCBbPvDBMR4ZimCo5pquCmc5ayZU8PT2w9lj9sbaW7v4+SwNsEmt7m1fYdPNvyLDETS763wlvBwsBCFgUWsSiwyBqvWMSC8gX4XL4c/lRKqWKR1QuLROSfgBXAykzLjTH3APeA1ULP5razTUQ4bkEVxy2o4pZzj2LTu908sfVwnnxlHx3BYSBGdUWQhY0DVFf04vJ1Mmj28Xzr8zy+8/GR9SDM8c9hYWBhMuQXBhYyr2wec8vmaveNUiprphLoLcD8lOl59rw0IvJh4BZgpTGmoC7FdDiEkxZXc9Liam77+NG82tLL1uYetjb38kpLLxu29xO3D08NAS8rmnzMrx+gItCDeNppCzXzbu+7PLHrCYKRYHK9gtDgb6CprIl5ZfNoKreG88rnMa9sHrUltfplrFJqyqbSh+7C+lL0TKwgfwH4jDHmtZQyxwGPAucYY96ayoZnYx/6wRoKx9jW2svLe6yA39rcw66OARK7tqmyhPfNq+CYpgCL6w3+sm56o/tp6W+hOdhMc38zzcFm2gbb0tbrdXppKmuyAr98XnKYCH2/25+Dn1YplUvZOG3xXOAnWKct/soY820R+SawyRjzuIg8BSwDWu237DbGnD/ROgsp0DPpD0V4bW9fWkv+3c7B5PK6ci/vqSvj8IYy3lNfxnvqylhQ4yYsnbQEW2gJtiSDPjGe2roHqPJWZQz6prImGv2NuBx5d6sepdQk9MKiWaJnMMyrLX28ureXHW1BdrQF2dkWpH945BmnAZ/LCnj7dXh9Oe+pL2NuhY9gtJ/m/mb2BPckW/eJYWuwlagZWY9TnMzxz0nvxrFDv6msiUpvpXbnKJWHNNBnMWMM+/uG7YDvZ0d7MBn2HcFwspzP7WBJrd2irytjSV0ZS+r8LK7143M7icaj7B8c242TmO4KdaVt1+f00ehvpMHfwBz/HBr9jTSWNtLob0xOj3cbYqVU7mig56mewXAy3He0BXnLHrb0DCXLiMDcihKW1Pk5rK6Mw+r8ybBvDPiSrfDByGAy6FuCLewb2Jf2ah9qx5D+t1DuKU8GfTL07VdDaQP1pfV6SqZSM0wDvcAMhqO83THArnb71RFkZ3uQXe0DDIZHzo0v9ThZXGsFfDLoa/0sqfNT6knvX4/EI7QPttM60JoM+daBVvYP7GffoDXdM9wzpi4BT4D60noaShto8Fshn5y2Q1+7d5TKnkO9sEjNMqUeF0fPreDouRVp8xPdN7vag+zsGGBnW5BdHQO8tLubJ7buJfXY3VSZ0qq3v5Q9rL6a4+vnjBu+Q9GhZNi3DbbRNtjG/sH97B/cT9tgG292v0nnUOeYlr7H4aGutC4t5OtK66gpqaG2pJa6kjpqS2oJeAIa/EodAm2hF4lQJMY7nQPsbBuwW/NBdrZb46mt+nKfi8PqrC9kE10476kvY0F1KS7n5M8Uj8QjdA51poX+6OBvG2xjODb2UgW3w01tSe2Er7oS60CgF2SpYqVdLmpc8bhhX1+Ine3WGTc72oPJ0E99mLbbKSys8dtfyPpZWFPK/OpS5leVMqfCN6WwTzDGEIwE6RjqmPTVHeoe0+IHq3+/xldDta+ampIaanw11tAeT52vX+6qQqKBrg5KXyjCrvYB6/RKO/B3tgd5t3OQaHzk78blEJqqSlhQPRLyC6pHXhWlB//gkEg8Qneom/ahdjqHOukY6qB9sJ3OUCedQ510hbqS433hvozrKHGVjAS+r4bqkmqqfdaryltFla8qOV3pq9QHnahZTQNdZVU0Fqe1N8SerkF2dw2yp3uQ3V1D1njXIF0D4bTy5T5XWsDPt18LqktpqizB45p6634ikVjECnc74DuHOjMGf1eoi57hHuImnnE95Z7yZNhX+6qTgV/ls8K/yluF3+2n1F1Kqas0Oe5xePQ7ADXtNNDVjAoOR0fC3n7tTob/EOHoSJCKwJyALxnw89NCv4S6Mu+0hGQsHqMv3Ed3qJvOUCfdoW66Q910DXclx1OX9Qz3pN1dMxOnOCl1lVLiLqHUVZoM/FJ3KX6XFfolrhJK3aUEPIGRl9calnvKCXgC+N1+PTCocWmgq1kjHje09Q9brfrOkRZ+IvT396V/WepzO5JdOPOTXTolzK20XlWl7hkJv7iJ0x/uT7buByODDEQGGIwOMhgZTA6HokNj5g1EBqz5iXnRwXE/HYB1YEiEezLovekHgMTy0cMyT5l2GRU4PW1RzRoOh9BY4aOxwseJi6rHLA9FYjR3D1kt+5TQ3901yPO7OhkIp7eSvS4HcytLaAz4mFPpY25FSdpwTkUJAZ/rkEPfIQ4qvBVUeCsmLzyJuIkzGBmkL9xnvYb7Mo/br/7hfloHWpPLU2/xkEmJq2TMASE1+Ms95ficPjxODx6nB6/TmxymjnucHjyO9OVOh/OQf341fTTQ1aziczuT97EZzRhD92CEPV2DtPYOsbcnZA17Q7T2DPHczk7294WIj/rQ6fc4mVNZwpyKkaBvqiyhqaqEeZWlNFb4staPPxUOcVDmKaPMU8Zc5h7Qe40xDEWH6A/30x/utwI/wzD11TbYxo6eHcnpTGcNTZVLXLid7mTYux3WuMvhwuMcmXY73NZypxu3w50+3z6Q+Jw+vE4vPpcPn8sed/rwurzJZV6XlxJnSdo8PaiMTwNd5Q0RodrvodrvYfn8yoxlorE47cHhZNi39oTYaw9be4d4Y18/HcHhtIusRKCh3GcFfFVJMuybKhPTpZR4ZkeIiIjVN+8upcHfcMDvT3w6GI4NE46FGY4Np40n58XT56WOh2IhIrEIkbj9ikUIx8NE4hHCMWsYDAfTpsPx8Ei5WIRQLHTQ+8DlcOF2uHGKE6fDiVOcuBwuXOJKm04sd4nLmraXOR3OtINM6oHJ7XAnD0KpB6bRBy2wDq4GQ9zEiRMHQ3I8dVnaOAZjDMfXH897qt5z0Ptg3H2T9TUqlUMup4M5FSXMqSgBMj/nNRyN09o7REv3EM09QzR3W+MtPYO8uLubP2xtTTstE6DG70mG/NzKEurKvdSWeakp81BnD2v83hlt6R+M5KcDxn4CmknGGMLxMKFoyDqARIcJxazxxLxQNDR2XizEcHSYmIkRjUeJxqPETCw5HTMxYvHY2OXxGFETJRKLEI1HkwejxAEn9cAUjoUP6VPMVHz9/V/XQFcqGzwuBwtr/CysyfyAkFjcsL8vREtPIujt0O8Z4s39/Wx4s41QJPOXmgGfi9pyL7V+L7XlVsgngr+2zEutPWwI+GZNqz8XRCTZZz/bGGOImVjGsE8MBUFEEASHOEaGqfNSxh1iHegT49P1cBoNdKVGcTokeRbNiYvGLjfGMBiO0REcpiMYpiM4TGdyODLvzX39dA500jMYybidihI3cyp8NAR86cMKazgnUEKg5NC/0FUHRkSS3TT5Jv9qrFSOiQh+rwu/1zVuKz9VJBanayCcPAC09w+zvy/Evt4Q++zhtta+MX37YJ22OaeihIaA1x76aAx4qS7zUlXqpqrUQ6U9LPU4NfyLnAa6UtPM7XTQELBa4BOJxOK09Q+zr3eIfb3DtPYOsb8vRGuvFfovvNPF/r4QkVjm/l2P00GVPz3kK0s9VPtHxqtK3WnDihI3ToceBAqFBrpSs4Tb6bDOsKksGbdMPG7oHgzbrwjdAynjg2F6BiLJ5W+1Bemxl8VGn8uZIuBzJUO+otRDZYk7fdzvprLEOkhU2vMCeiCYlTTQlcojDodQU+alpmzqXyYaY+gLRZPh3j0YpjdxABiM0Ds0Mt4zFOHdzgF6BiP0hSJjuoASRCDgcydb+pUp3T+VJR7rIJA8OCQOBm7KvPqdwHTSQFeqwIkIFSVuKkrcLKyZ+vticUPfkBXyPcnAD9M9MDKve9AadgatxyX2DkbSHno+mtspVJR4CPis7yDKvC7KfC7K7WFZ6jBtuTttvs/t0ANDBhroSqmMnA6hyu+hyu8Bpn6aXSQWt1v+I91C6QeACMHhKMFQJHkjt4FwlGAoSn8oOuYagPHq5vc4Kfe58XudlNlfUpf7XPg96QeFTPNLPE5K3E5KPU58bideV2EcIDTQlVJZ5XY6qCv3Uld+4OeYG2MYjsbtwI9aw5Txfnt8wJ7fP2q8tTdkTYeiBMPRcbuMRnMIlLidVtDbYZ+cTg5dlHgclHpc+NxO/B7rgFDicdlDJ/6U8VKPk1J72n0AD4A5FBroSqlZQ0Twua1Wc+0BfE+QSTxuGIrEGBh1IOgfjhKKxBgMxxgKxxiKpAwT4ynzO4JhBsNRQpE4Q5FYcvxAuJ1ifyJwUep18qUPH8H5yw/sPj5ToYGulCpIDsfI9QL1WV534mAxGLYC3hpaB4DU6cFw1JoXiTE4bM+PxKg6hKd4TUQDXSmlDlDqwQJmz+0LZvedhJRSSk2ZBrpSShUIDXSllCoQGuhKKVUgNNCVUqpAaKArpVSB0EBXSqkCoYGulFIFQsxUb3aQ7Q2LtAPvHuTba4GOLFYn27R+h0brd+hmex21fgdvoTGmLtOCnAX6oRCRTcaYFbmux3i0fodG63foZnsdtX7TQ7tclFKqQGigK6VUgcjXQL8n1xWYhNbv0Gj9Dt1sr6PWbxrkZR+6UkqpsfK1ha6UUmoUDXSllCoQszrQReQcEXlTRHaIyE0ZlntF5GF7+d9FZNEM1m2+iGwQkW0i8pqIXJ+hzCoR6RWRLfbrGzNVP3v774jIK/a2N2VYLiLyU3v/bRWR42ewbkem7JctItInIl8aVWbG95+I/EpE2kTk1ZR51SLyFxF5yx5WjfPez9ll3hKRz81Q3X4oIm/Yv7/HRKRynPdO+LcwzXW8XURaUn6P547z3gn/36exfg+n1O0dEdkyzntnZB8eEmPMrHwBTmAnsATwAC8D7x1V5gvAz+zxi4CHZ7B+c4Dj7fFyYHuG+q0CnsjhPnwHqJ1g+bnAHwEB3g/8PYe/631YF0zkdP8BpwHHA6+mzPsBcJM9fhPw/QzvqwZ22cMqe7xqBup2NuCyx7+fqW5T+VuY5jreDvzLFP4GJvx/n676jVp+J/CNXO7DQ3nN5hb6ScAOY8wuY0wYeAi4YFSZC4Bf2+OPAmeKiMxE5YwxrcaYF+3xfuB1oGkmtp1FFwDrjOV5oFJE5uSgHmcCO40xB3vlcNYYY54BukbNTv07+zXwiQxv/QjwF2NMlzGmG/gLcM50180Y89/GmKg9+TwwL5vbPFDj7L+pmMr/+yGbqH52dnwa+M9sb3emzOZAbwL2pEw3MzYwk2XsP+peoGZGapfC7uo5Dvh7hsUfEJGXReSPInL0jFYMDPDfIrJZRK7KsHwq+3gmXMT4/0S53H8JDcaYVnt8H9CQocxs2JeXYX3iymSyv4Xpdo3dLfSrcbqsZsP+OxXYb4x5a5zlud6Hk5rNgZ4XRKQM+C3wJWNM36jFL2J1IywH/h/gdzNcvQ8ZY44HPgp8UUROm+HtT0pEPMD5wG8yLM71/hvDWJ+9Z925viJyCxAFHhinSC7/Fu4GDgOOBVqxujVmo4uZuHU+6/+fZnOgtwDzU6bn2fMylhERF1ABdM5I7axturHC/AFjzH+NXm6M6TPGBO3xJwG3iNTOVP2MMS32sA14DOtjbaqp7OPp9lHgRWPM/tELcr3/UuxPdEXZw7YMZXK2L0XkUuA84BL7gDPGFP4Wpo0xZr8xJmaMiQO/GGfbOf1btPPjQuDh8crkch9O1WwO9BeAw0Vksd2Kuwh4fFSZx4HE2QT/C1g/3h90ttn9bf8beN0Y8+NxyjQm+vRF5CSs/T0jBxwR8YtIeWIc68uzV0cVexz4v+yzXd4P9KZ0LcyUcVtFudx/o6T+nX0O+D8ZyvwZOFtEquwuhbPtedNKRM4BvgKcb4wZHKfMVP4WprOOqd/LfHKcbU/l/306fRh4wxjTnGlhrvfhlOX6W9mJXlhnYWzH+vb7FnveN7H+eAF8WB/VdwD/AJbMYN0+hPXReyuwxX6dC1wNXG2XuQZ4Desb++eBD85g/ZbY233ZrkNi/6XWT4C77P37CrBihn+/fqyArkiZl9P9h3VwaQUiWP24l2N9L/NX4C3gKaDaLrsC+GXKey+z/xZ3AGtnqG47sPqeE3+DibO+5gJPTvS3MIP7737772srVkjPGV1He3rM//tM1M+ef1/i7y6lbE724aG89NJ/pZQqELO5y0UppdQB0EBXSqkCoYGulFIFQgNdKaUKhAa6UkoVCA10pZQqEBroSilVIP5/lc9uxcjm5isAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "pd.DataFrame(history.history).plot()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J3IaQOruGZNU"
      },
      "source": [
        "Perform the same steps on the test data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fuphSL6AIUsj"
      },
      "source": [
        "## Evaluation and Prediction"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ljtOLcupHLX0",
        "outputId": "c73909d3-ea60-4fa9-a6e2-6d2eb13cb8d7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r1/3 [=========>....................] - ETA: 0s - loss: 0.0613 - accuracy: 0.9823"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/dispatch.py:1082: UserWarning: \"`sparse_categorical_crossentropy` received `from_logits=True`, but the `output` argument was produced by a sigmoid or softmax activation and thus does not represent logits. Was this intended?\"\n",
            "  return dispatch_target(*args, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r3/3 [==============================] - 0s 15ms/step - loss: 0.0625 - accuracy: 0.9827\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.062450848519802094, 0.9826701879501343]"
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ],
      "source": [
        "model.evaluate(X_test,y_test)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "p=model.predict([X_test])"
      ],
      "metadata": {
        "id": "L4kehy99Qp12"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y_predict=np.zeros((y_test.shape[0],y_test.shape[1]),dtype=np.int32)\n",
        "for i in range(len(y_test)):\n",
        "  y_predict[i,:]=np.argmax(p[i], axis=1)"
      ],
      "metadata": {
        "id": "yA2YtvVnQ2HJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def compute_metric(y_test,y_predict):\n",
        "  P=0\n",
        "  R=0\n",
        "  F=0\n",
        "  for i in range(y_test.shape[0]):\n",
        "      precision, recall, f1, _ = precision_recall_fscore_support(y_test[i], y_predict[i], average='weighted', labels=np.unique(y_test))\n",
        "      P+=precision\n",
        "      R+=recall\n",
        "      F+=f1\n",
        "  return {\n",
        "      'Precision':P/y_predict.shape[0],\n",
        "      'Recall':R/y_predict.shape[0],\n",
        "      'F1-Score':F/y_predict.shape[0]}\n",
        "\n"
      ],
      "metadata": {
        "id": "sk3sX5R3Q5cA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "compute_metric(y_test,y_predict)"
      ],
      "metadata": {
        "id": "8yqYlRZhQ7jC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "{'F1-Score': 0.9795434714157091,\n",
        " 'Precision': 0.9793462923964108,\n",
        " 'Recall': 0.9826701701701706}"
      ],
      "metadata": {
        "id": "FcnzraoZRIe0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Results"
      ],
      "metadata": {
        "id": "8D42dAcCRLZ6"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mQSR8leTIlpd"
      },
      "outputs": [],
      "source": [
        "def ner_prediction(model, x_test, id2word, id2tag):\n",
        "    \n",
        "    #reshape to tf shape:\n",
        "    x_test = x_test.reshape((1, maxlen))\n",
        "\n",
        "    # return x_test to its words form\n",
        "    sentence = x_test[x_test > 0]\n",
        "    word_list = []\n",
        "    for word in list(sentence):\n",
        "        word_list.append(id2word[word])\n",
        "    orginal_sententce = ' '.join(word_list)\n",
        "    \n",
        "    len_orginal_sententce = len(word_list)\n",
        "    \n",
        "    # make prediction\n",
        "    prediction = model.predict(x_test)\n",
        "    prediction = np.argmax(prediction[0], axis=1)\n",
        "    \n",
        "    # return the prediction to its orginal form\n",
        "    prediction = list(prediction)[ : len_orginal_sententce] \n",
        "    \n",
        "    pred_tag_list = []\n",
        "    for tag_id in prediction:\n",
        "        pred_tag_list.append(id2tag[tag_id])\n",
        "    \n",
        "    return orginal_sententce,  pred_tag_list,prediction\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WLQ_LhRKORFm"
      },
      "outputs": [],
      "source": [
        "orginal_sententce,  pred_tag_list,prediction = ner_prediction(model=model,\n",
        "                                                    x_test=X_test[1],    #8\n",
        "                                                    id2word=id2word,\n",
        "                                                    id2tag=id2tag)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### To do:    \n",
        "1. build a function to high the entities\n",
        "2."
      ],
      "metadata": {
        "id": "J_oMjrwH1HcV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install colorama"
      ],
      "metadata": {
        "id": "uxAOYHWCYiI_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from colorama import Fore, Back, Style"
      ],
      "metadata": {
        "id": "4IzFTnJrYTJl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(Back.GREEN + 'and with a green background')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NxQY1wRjZMtG",
        "outputId": "2f4b2069-81a8-478b-ac74-5d927cda58d6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[42mand with a green background\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#on_red, on_green, on_yellow, on_blue, on_magenta, on_cyan, on_white.\n",
        "\n",
        "print((colored('foot and mouth','white','on_cyan',attrs=['bold'])) + colored(' B-VIRUS','yellow','on_cyan') )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3jdsVTWjXQoT",
        "outputId": "4e08fd30-4ef3-4d89-d50e-b95a37e9adfa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m\u001b[46m\u001b[37mfoot and mouth\u001b[0m\u001b[46m\u001b[33m B-VIRUS\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# edit this later \" render function\"\n",
        "from termcolor import colored\n",
        "t=orginal_sententce.lower().split()\n",
        "k=17\n",
        "for i,j in enumerate(pred_tag_list):\n",
        "  if j != 'O':   \n",
        "    x=t[i]\n",
        "    sys.stdout.write((colored(x,'magenta','on_white',attrs=['bold'])) +colored('('+j+')','cyan','on_white') +' ') \n",
        "  else: sys.stdout.write(t[i]+' ')   # join them \n",
        "  if i in range(11, len(t),13) :\n",
        "     sys.stdout.write('\\n')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GwVl6JEJLzGp",
        "outputId": "0c08d163-3219-4020-af44-35430fb711cc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m\u001b[47m\u001b[35mhuman\u001b[0m\u001b[47m\u001b[36m(B-HOST)\u001b[0m \u001b[1m\u001b[47m\u001b[35mantibodies\u001b[0m\u001b[47m\u001b[36m(B-DETECTIONMETHOD)\u001b[0m protect against aerosolized eastern \u001b[1m\u001b[47m\u001b[35mequine\u001b[0m\u001b[47m\u001b[36m(B-VIRUS)\u001b[0m encephalitis virus infection eastern \u001b[1m\u001b[47m\u001b[35mequine\u001b[0m\u001b[47m\u001b[36m(B-VIRUS)\u001b[0m \n",
            "\u001b[1m\u001b[47m\u001b[35mencephalitis\u001b[0m\u001b[47m\u001b[36m(I-VIRUS)\u001b[0m virus eeev is one of the most virulent viruses endemic to north \n",
            "america no licensed vaccines or antiviral therapeutics are available to combat this infection \n",
            "which has recently shown an increase in \u001b[1m\u001b[47m\u001b[35mhuman\u001b[0m\u001b[47m\u001b[36m(B-HOST)\u001b[0m cases here we characterize \u001b[1m\u001b[47m\u001b[35mhuman\u001b[0m\u001b[47m\u001b[36m(B-HOST)\u001b[0m \n",
            "monoclonal \u001b[1m\u001b[47m\u001b[35mantibodies\u001b[0m\u001b[47m\u001b[36m(B-DETECTIONMETHOD)\u001b[0m mabs \u001b[1m\u001b[47m\u001b[35misolated\u001b[0m\u001b[47m\u001b[36m(B-DETECTIONMETHOD)\u001b[0m from a survivor of natural eeev infection with potent \n",
            "pm inhibitory activity of eeev cryo electron microscopy reconstructions of two highly neutralizing \n",
            "mabs eeev and eeev were solved in complex with chimeric \u001b[1m\u001b[47m\u001b[35msindbis\u001b[0m\u001b[47m\u001b[36m(B-VIRUS)\u001b[0m eeev virions \n",
            "to and respectively the mabs recognize two distinct antigenic sites that are critical \n",
            "for inhibiting viral entry into cells eeev and eeev protect against disease following \n",
            "stringent lethal aerosol challenge of \u001b[1m\u001b[47m\u001b[35mmice\u001b[0m\u001b[47m\u001b[36m(B-HOST)\u001b[0m with highly pathogenic eeev these studies provide \n",
            "insight into the molecular basis for the neutralizing \u001b[1m\u001b[47m\u001b[35mhuman\u001b[0m\u001b[47m\u001b[36m(B-HOST)\u001b[0m \u001b[1m\u001b[47m\u001b[35mantibody\u001b[0m\u001b[47m\u001b[36m(B-DETECTIONMETHOD)\u001b[0m response against eeev \n",
            "and can facilitate development of vaccines and candidate \u001b[1m\u001b[47m\u001b[35mantibody\u001b[0m\u001b[47m\u001b[36m(B-DETECTIONMETHOD)\u001b[0m therapeutics keywords eastern equine \n",
            "aerosol encephalitis \u001b[1m\u001b[47m\u001b[35mhuman\u001b[0m\u001b[47m\u001b[36m(B-HOST)\u001b[0m \u001b[1m\u001b[47m\u001b[35mantibodies\u001b[0m\u001b[47m\u001b[36m(B-DETECTIONMETHOD)\u001b[0m monoclonal neutralizing prophylaxis therapy "
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPmMWIGz0jvxr3lR1yAXTpL",
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}