02_example_amazon_reviews_seq2seq_colab_with_attetionMay26.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GvEmSE6vTdTS"
   },
   "source": [
    "<h1>TEXT SUMMARIZTION WITH AMAZON-REVIEWS</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 411
    },
    "colab_type": "code",
    "id": "O6YEZIHOYXqA",
    "outputId": "39884855-f3ce-4b8d-bcb1-abe79a0688f7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow==1.15 in /usr/local/lib/python3.6/dist-packages (1.15.0)\n",
      "Requirement already satisfied: gast==0.2.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (0.2.2)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.29.0)\n",
      "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (3.10.0)\n",
      "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (0.8.1)\n",
      "Requirement already satisfied: tensorflow-estimator==1.15.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.15.1)\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.1.2)\n",
      "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (0.34.2)\n",
      "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.12.1)\n",
      "Requirement already satisfied: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.0.8)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.1.0)\n",
      "Requirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.18.4)\n",
      "Requirement already satisfied: tensorboard<1.16.0,>=1.15.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.15.0)\n",
      "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.12.0)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (3.2.1)\n",
      "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (0.9.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (0.2.0)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow==1.15) (46.3.0)\n",
      "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.8->tensorflow==1.15) (2.10.0)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15) (1.0.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15) (3.2.2)\n",
      "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard<1.16.0,>=1.15.0->tensorflow==1.15) (1.6.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<1.16.0,>=1.15.0->tensorflow==1.15) (3.1.0)\n"
     ]
    }
   ],
   "source": [
    "from distutils.version import LooseVersion\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.layers.core import Dense\n",
    "\n",
    "\n",
    "# Check TensorFlow Version\n",
    "assert LooseVersion(tf.__version__) >= LooseVersion('1.1'), 'Please use TensorFlow version 1.1 or newer'\n",
    "!python3.6 -m pip install tensorflow==1.15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "Nu3dP2ypavXH",
    "outputId": "0669edf9-6e53-475c-cbc5-85e9fca6bd08"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.15.0\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['AttentionCellWrapper',\n",
       " 'BasicLSTMCell',\n",
       " 'BasicRNNCell',\n",
       " 'BidirectionalGridLSTMCell',\n",
       " 'CompiledWrapper',\n",
       " 'Conv1DLSTMCell',\n",
       " 'Conv2DLSTMCell',\n",
       " 'Conv3DLSTMCell',\n",
       " 'ConvLSTMCell',\n",
       " 'CoupledInputForgetGateLSTMCell',\n",
       " 'DeviceWrapper',\n",
       " 'DropoutWrapper',\n",
       " 'EmbeddingWrapper',\n",
       " 'FusedRNNCell',\n",
       " 'FusedRNNCellAdaptor',\n",
       " 'GLSTMCell',\n",
       " 'GRUBlockCell',\n",
       " 'GRUBlockCellV2',\n",
       " 'GRUCell',\n",
       " 'GridLSTMCell',\n",
       " 'HighwayWrapper',\n",
       " 'IndRNNCell',\n",
       " 'IndyGRUCell',\n",
       " 'IndyLSTMCell',\n",
       " 'InputProjectionWrapper',\n",
       " 'IntersectionRNNCell',\n",
       " 'LSTMBlockCell',\n",
       " 'LSTMBlockFusedCell',\n",
       " 'LSTMBlockWrapper',\n",
       " 'LSTMCell',\n",
       " 'LSTMStateTuple',\n",
       " 'LayerNormBasicLSTMCell',\n",
       " 'LayerRNNCell',\n",
       " 'MultiRNNCell',\n",
       " 'NASCell',\n",
       " 'OutputProjectionWrapper',\n",
       " 'PhasedLSTMCell',\n",
       " 'RNNCell',\n",
       " 'ResidualWrapper',\n",
       " 'SRUCell',\n",
       " 'TimeFreqLSTMCell',\n",
       " 'TimeReversedFusedRNN',\n",
       " 'UGRNNCell',\n",
       " '__builtins__',\n",
       " '__cached__',\n",
       " '__doc__',\n",
       " '__file__',\n",
       " '__loader__',\n",
       " '__name__',\n",
       " '__package__',\n",
       " '__path__',\n",
       " '__spec__',\n",
       " 'best_effort_input_batch_size',\n",
       " 'stack_bidirectional_dynamic_rnn',\n",
       " 'stack_bidirectional_rnn',\n",
       " 'static_bidirectional_rnn',\n",
       " 'static_rnn',\n",
       " 'static_state_saving_rnn',\n",
       " 'transpose_batch_time']"
      ]
     },
     "execution_count": 2,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "print(tf.__version__)\n",
    "dir(tf.contrib.rnn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "3aee62b9-47ce-e416-5816-8df7126fe690",
    "colab": {},
    "colab_type": "code",
    "id": "FQ7ytEbpTdTV"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import gzip\n",
    "import codecs\n",
    "import re\n",
    "import time\n",
    "from tensorflow.python.ops.rnn_cell_impl import _zero_state_tensors\n",
    "from tensorflow.python.layers.core import Dense\n",
    "from tensorflow.contrib.rnn import GRUCell, DropoutWrapper\n",
    "from tensorflow.contrib.seq2seq import TrainingHelper, GreedyEmbeddingHelper,BasicDecoder, dynamic_decode, \\\n",
    " BahdanauAttention, AttentionWrapper,sequence_loss\n",
    "import helper\n",
    "\n",
    "TOKEN_GO = '<GO>'\n",
    "TOKEN_EOS = '<EOS>'\n",
    "TOKEN_PAD = '<PAD>'\n",
    "TOKEN_UNK = '<UNK>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 82
    },
    "colab_type": "code",
    "id": "flWSMJRkztR4",
    "outputId": "8d573caa-bffa-4dc4-eba2-ebf591f2be7d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords \n",
    "from nltk.tokenize import word_tokenize \n",
    "from numpy import linalg as LA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "O9dyYsyuVIAg",
    "outputId": "04c7a08b-0a6e-4e52-84e7-becbef54f95f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/gdrive/; to attempt to forcibly remount, call drive.mount(\"/content/gdrive/\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "vLDd9ViBV8aX",
    "outputId": "00f86901-80cf-4343-b7cc-7f0233e40196"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rm: cannot remove '/content/gdrive/My': No such file or directory\n",
      "rm: cannot remove 'Drive/best*': No such file or directory\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.listdir('/content/gdrive/My Drive')\n",
    "! rm /content/gdrive/My Drive/best*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "654175ff-07c7-455d-1b8f-d702cee211c4",
    "colab": {},
    "colab_type": "code",
    "id": "gRkJ05iHTdTh"
   },
   "outputs": [],
   "source": [
    "#titledata=[]\n",
    "#artdata=[]\n",
    "#with gzip.open('/content/gdrive/My Drive/news.txt.gz') as artfile:\n",
    "#    for li in artfile:\n",
    "#        artdata.append(li.decode())\n",
    "#with gzip.open('/content/gdrive/My Drive/summary.txt.gz') as titlefile:\n",
    "#    for li in titlefile:\n",
    "#        titledata.append(li.decode())\n",
    "#news = pd.DataFrame({'Text':artdata,'Summary':titledata})\n",
    "#news = news.sample(frac=0.5)\n",
    "#news['Text_len'] = news.Text.apply(lambda x: len(x.split()))\n",
    "#news['Summary_len'] = news.Summary.apply(lambda x: len(x.split()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "F9pKvHHzGNv7"
   },
   "outputs": [],
   "source": [
    "data_new=pd.read_csv('/content/gdrive/My Drive/data_new.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 191
    },
    "colab_type": "code",
    "id": "5fY2Sfu7Gs2H",
    "outputId": "c20e20b3-096c-4fa5-fb56-d01f05517988"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Text_len</th>\n",
       "      <th>Summary_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bought several vitality canned dog food produc...</td>\n",
       "      <td>good quality dog food</td>\n",
       "      <td>23</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>product arrived labeled jumbo salted peanuts p...</td>\n",
       "      <td>not as advertised</td>\n",
       "      <td>18</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>confection around centuries light pillowy citr...</td>\n",
       "      <td>delight says it all</td>\n",
       "      <td>39</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>looking secret ingredient robitussin believe f...</td>\n",
       "      <td>cough medicine</td>\n",
       "      <td>17</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>great taffy great price wide assortment yummy ...</td>\n",
       "      <td>great taffy</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text  ... Summary_len\n",
       "0  bought several vitality canned dog food produc...  ...           4\n",
       "1  product arrived labeled jumbo salted peanuts p...  ...           3\n",
       "2  confection around centuries light pillowy citr...  ...           4\n",
       "3  looking secret ingredient robitussin believe f...  ...           2\n",
       "4  great taffy great price wide assortment yummy ...  ...           2\n",
       "\n",
       "[5 rows x 4 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news=data_new[['cleaned_text','cleaned_summary']]\n",
    "news=news.rename(columns={'cleaned_text':'Text','cleaned_summary':'Summary'})\n",
    "news['Text_len'] = news.Text.apply(lambda x: len(x.split()))\n",
    "news['Summary_len'] = news.Summary.apply(lambda x: len(x.split()))\n",
    "news.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 151
    },
    "colab_type": "code",
    "id": "Rxa-JA_QTdT8",
    "outputId": "e0c71f44-2848-42a6-b657-4eef7c8e441f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['bought several vitality canned dog food products found good quality product looks like stew processed meat smells better labrador finicky appreciates product better'\n",
      " 'product arrived labeled jumbo salted peanuts peanuts actually small sized unsalted sure error vendor intended represent product jumbo'\n",
      " 'confection around centuries light pillowy citrus gelatin nuts case filberts cut tiny squares liberally coated powdered sugar tiny mouthful heaven chewy flavorful highly recommend yummy treat familiar story lewis lion witch wardrobe treat seduces edmund selling brother sisters witch'\n",
      " 'looking secret ingredient robitussin believe found got addition root beer extract ordered made cherry soda flavor medicinal'\n",
      " 'great taffy great price wide assortment yummy taffy delivery quick taffy lover deal']\n",
      "['good quality dog food ' 'not as advertised ' 'delight says it all '\n",
      " 'cough medicine ' 'great taffy ']\n"
     ]
    }
   ],
   "source": [
    "print(news['Text'].head(5).values)\n",
    "print(news['Summary'].head(5).values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "48f10124-4c1b-1f09-512d-352c068de1b4",
    "colab": {},
    "colab_type": "code",
    "id": "ZHaGwlh2TdUI"
   },
   "outputs": [],
   "source": [
    "news_summaries = []\n",
    "for summary in news.Summary:\n",
    "    news_summaries.append(summary)\n",
    "news_texts = []\n",
    "for text in news.Text:\n",
    "    news_texts.append(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "3f980247-3c32-240d-7d3d-7b0c3c6c13e5",
    "colab": {},
    "colab_type": "code",
    "id": "yHNDse2lTdUT"
   },
   "outputs": [],
   "source": [
    "def count_words(words_dict, text):\n",
    "    for sentence in text:\n",
    "        for word in sentence.split():\n",
    "            if word not in words_dict:\n",
    "                words_dict[word] = 1\n",
    "            else:\n",
    "                words_dict[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "_cell_guid": "3e9ce130-88f4-8779-5b5f-86f2a23ab347",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "_P7WgumjTdUe",
    "outputId": "9361aca1-37a4-4b61-c148-ec5f26cc0925"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total words in Vocabulary: 56955\n"
     ]
    }
   ],
   "source": [
    "word_counts_dict = {}\n",
    "count_words(word_counts_dict, news_summaries)\n",
    "count_words(word_counts_dict, news_texts)\n",
    "            \n",
    "print(\"Total words in Vocabulary:\", len(word_counts_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QY8Sk4lWpYLC"
   },
   "outputs": [],
   "source": [
    "if 'glove.6B.50d.txt' not in os.listdir('/content/'):\n",
    "  !unzip /content/gdrive/My\\ Drive/glove6b50dtxt.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BGdGuM_NTdUm"
   },
   "outputs": [],
   "source": [
    "def build_word_vector_matrix(vector_file):\n",
    "    embedding_index = {}\n",
    "    with codecs.open(vector_file, 'r', 'utf-8') as f:\n",
    "        for i, line in enumerate(f):\n",
    "            sr = line.split()\n",
    "            if(len(sr)<26):\n",
    "                continue\n",
    "            word = sr[0]\n",
    "            embedding = np.asarray(sr[1:], dtype='float32')\n",
    "            embedding_index[word] = embedding\n",
    "    return embedding_index\n",
    "# Replace the path here to point to the glove.6B.50d.txt vectors file on your system\n",
    "embeddings_index = build_word_vector_matrix('/content/glove.6B.50d.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "0be42a13-70b2-e9cc-7468-1247b01f109c",
    "colab": {},
    "colab_type": "code",
    "id": "VdDgF69WTdUw"
   },
   "outputs": [],
   "source": [
    "word2int = {} \n",
    "count_threshold = 20\n",
    "value = 0\n",
    "for word, count in word_counts_dict.items():\n",
    "    if count >= count_threshold or word in embeddings_index:\n",
    "        word2int[word] = value\n",
    "        value += 1\n",
    "\n",
    "\n",
    "special_codes = [TOKEN_UNK,TOKEN_PAD,TOKEN_EOS,TOKEN_GO]   \n",
    "\n",
    "for code in special_codes:\n",
    "    word2int[code] = len(word2int)\n",
    "\n",
    "int2word = {}\n",
    "for word, value in word2int.items():\n",
    "    int2word[value] = word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "_cell_guid": "4401990d-4baf-3f30-becc-3a6149716b56",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "f4KKWkrTTdU3",
    "outputId": "0eb9eb2d-a60a-4665-9ecb-71d58c7e2577"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of word embeddings:  36805\n"
     ]
    }
   ],
   "source": [
    "embedding_dim = 50\n",
    "nwords = len(word2int)\n",
    "\n",
    "word_emb_matrix = np.zeros((nwords, embedding_dim), dtype=np.float32)\n",
    "for word, i in word2int.items():\n",
    "    if word in embeddings_index:\n",
    "        word_emb_matrix[i] = embeddings_index[word]\n",
    "    else:\n",
    "        new_embedding = np.array(np.random.uniform(-1.0, 1.0, embedding_dim))\n",
    "        word_emb_matrix[i] = new_embedding\n",
    "print(\"Length of word embeddings: \", len(word_emb_matrix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "25cfd0e3-ae3d-8728-1c82-1a61bb06aa0e",
    "colab": {},
    "colab_type": "code",
    "id": "x_n1GMkcTdU_"
   },
   "outputs": [],
   "source": [
    "def convert_sentence_to_ids(text, eos=False):\n",
    "    wordints = []\n",
    "    word_count = 0\n",
    "    for sentence in text:\n",
    "        sentence2ints = []\n",
    "        for word in sentence.split():\n",
    "            word_count += 1\n",
    "            if word in word2int:\n",
    "                sentence2ints.append(word2int[word])\n",
    "            else:\n",
    "                sentence2ints.append(word2int[TOKEN_UNK])\n",
    "        if eos:\n",
    "            sentence2ints.append(word2int[TOKEN_EOS])\n",
    "        wordints.append(sentence2ints)\n",
    "    return wordints, word_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "360cfdf4-ad4c-0316-56d3-70b6206e75e4",
    "colab": {},
    "colab_type": "code",
    "id": "umsLAGX2TdVJ"
   },
   "outputs": [],
   "source": [
    "id_summaries, word_count = convert_sentence_to_ids(news_summaries)\n",
    "id_texts, word_count = convert_sentence_to_ids(news_texts, eos=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "2a0ae7cd-a845-23dc-3563-ad133e2f02b4",
    "colab": {},
    "colab_type": "code",
    "id": "mPyYg9N3TdVV"
   },
   "outputs": [],
   "source": [
    "def unknown_tokens(sentence):\n",
    "    unk_token_count = 0\n",
    "    for word in sentence:\n",
    "        if word == word2int[TOKEN_UNK]:\n",
    "            unk_token_count += 1\n",
    "    return unk_token_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "50d631a2-fb5a-bb0d-6155-9cd15e70835b",
    "colab": {},
    "colab_type": "code",
    "id": "93RbMaQCTdVc"
   },
   "outputs": [],
   "source": [
    "news_summaries_filtered = []\n",
    "news_texts_filtered = []\n",
    "max_text_length = int(news.Text_len.mean() + news.Text_len.std())\n",
    "max_summary_length = int(int(news.Summary_len.mean() + news.Summary_len.std()))\n",
    "min_length = 4\n",
    "unknown_token_text_limit = 10\n",
    "unknown_token_summary_limit = 4\n",
    "\n",
    "for count,text in enumerate(id_texts):\n",
    "    unknown_token_text = unknown_tokens(id_texts[count])\n",
    "    unknown_token_summary = unknown_tokens(id_summaries[count])\n",
    "    text_len = len(id_texts[count])\n",
    "    summary_len = len(id_summaries[count])\n",
    "    if((unknown_token_text>unknown_token_text_limit) or (unknown_token_summary>unknown_token_summary_limit)):\n",
    "        continue\n",
    "    if(text_len<min_length or summary_len<min_length or text_len>max_text_length or summary_len>max_summary_length):\n",
    "        continue\n",
    "    news_summaries_filtered.append(id_summaries[count])\n",
    "    news_texts_filtered.append(id_texts[count])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "34d28c5f-8016-6b36-664e-3d5ee3db745d",
    "colab": {},
    "colab_type": "code",
    "id": "xuL2rU6mTdVk"
   },
   "outputs": [],
   "source": [
    "def model_inputs():\n",
    "    inputs_data = tf.placeholder(tf.int32, [None, None], name='input_data')\n",
    "    targets = tf.placeholder(tf.int32, [None, None], name='targets')\n",
    "    learning_rate = tf.placeholder(tf.float32, name='learning_rate')\n",
    "    dropout_probs = tf.placeholder(tf.float32, name='dropout_probs')\n",
    "    summary_len = tf.placeholder(tf.int32, (None,), name='summary_len')\n",
    "    max_summary_len = tf.reduce_max(summary_len, name='max_summary_len')\n",
    "    text_len = tf.placeholder(tf.int32, (None,), name='text_len')\n",
    "    return inputs_data, targets, learning_rate, dropout_probs, summary_len, max_summary_len, text_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "9c9b6087-3c28-478d-d311-4213e1c59654",
    "colab": {},
    "colab_type": "code",
    "id": "ANIGHjdQTdVr"
   },
   "outputs": [],
   "source": [
    "def process_encoding_input(target_data, word2int, batch_size):\n",
    "    ending = tf.strided_slice(target_data, [0, 0], [batch_size, -1], [1, 1])\n",
    "    decoding_input = tf.concat([tf.fill([batch_size, 1], word2int[TOKEN_GO]), ending], 1)\n",
    "    return decoding_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "d675562b-a9e0-df71-6979-a052fb78dcbc",
    "colab": {},
    "colab_type": "code",
    "id": "4-nmhhyfTdVz"
   },
   "outputs": [],
   "source": [
    "def get_cell(csize,dprob):\n",
    "    rnc = GRUCell(csize)\n",
    "    rnc = DropoutWrapper(rnc, input_keep_prob = dprob)\n",
    "    return rnc\n",
    "\n",
    "def encoding_layer(csize, len_s, nl, rinp, dprob):\n",
    "    for l in range(nl):\n",
    "        with tf.variable_scope('encoding_l_{}'.format(l)):\n",
    "            rnn_frnt = get_cell(csize,dprob)\n",
    "            rnn_bkwd = get_cell(csize,dprob)\n",
    "            eop, est = tf.nn.bidirectional_dynamic_rnn(rnn_frnt, rnn_bkwd, \n",
    "                                                                    rinp,\n",
    "                                                                    len_s,\n",
    "                                                                    dtype=tf.float32)\n",
    "    eop = tf.concat(eop,nl)\n",
    "    return eop, est"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "524d0246-ddae-b485-5ea4-11ad476447f4",
    "colab": {},
    "colab_type": "code",
    "id": "OBSH-yfcTdV-"
   },
   "outputs": [],
   "source": [
    "def trng_dec_layer(dec_emb_inp, summ_len, cell_dec, st_init, lyr_op, \n",
    "                            v_size, max_summ_len):\n",
    "    helper = TrainingHelper(inputs=dec_emb_inp,sequence_length=summ_len, time_major=False)\n",
    "    dec = BasicDecoder(cell_dec,helper,st_init,lyr_op) \n",
    "    logits, _, _ = dynamic_decode(dec,output_time_major=False,impute_finished=True, \n",
    "                                  maximum_iterations=max_summ_len)\n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "6044b206-7f27-5304-4896-06d388af0949",
    "colab": {},
    "colab_type": "code",
    "id": "ExxIWv2YTdWK"
   },
   "outputs": [],
   "source": [
    "def infr_dec_layer(embeddings, start_token, end_token, decoding_cell, initial_state, op_layer,\n",
    "                             max_summary_len, batch_size):\n",
    "    \n",
    "    start_tokens = tf.tile(tf.constant([start_token], dtype=tf.int32), [batch_size], name='start_tokens')\n",
    "    inf_helper = GreedyEmbeddingHelper(embeddings,start_tokens,end_token)\n",
    "    inf_decoder = BasicDecoder(decoding_cell,inf_helper,initial_state,op_layer)       \n",
    "    inf_logits, _, _ = dynamic_decode(inf_decoder,output_time_major=False,impute_finished=True,\n",
    "                                                            maximum_iterations=max_summary_len)\n",
    "    return inf_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "4b50746f-8f78-0253-9178-56c62e4ac1bf",
    "colab": {},
    "colab_type": "code",
    "id": "IcMgwE8oTdWd"
   },
   "outputs": [],
   "source": [
    "def decoding_layer(dec_emb_op, embs, enc_op, enc_st, v_size, txt_len, \n",
    "                   summ_len,mx_summ_len, rnsize, word2int, dprob, batch_size, nlyrs):\n",
    "    \n",
    "    for l in range(nlyrs):\n",
    "        with tf.variable_scope('dec_rnn_layer_{}'.format(l)):\n",
    "            gru = tf.contrib.rnn.GRUCell(rnn_len)\n",
    "            cell_dec = tf.contrib.rnn.DropoutWrapper(gru,input_keep_prob = dprob)\n",
    "            \n",
    "    out_l = Dense(v_size, kernel_initializer = tf.truncated_normal_initializer(mean = 0.0, stddev=0.1))\n",
    "    \n",
    "    attention = BahdanauAttention(rnsize, enc_op,txt_len,\n",
    "                                                  normalize=False,\n",
    "                                                  name='BahdanauAttention')\n",
    "    cell_dec =  AttentionWrapper(cell_dec,attention,rnn_len)\n",
    "    attn_zstate = cell_dec.zero_state(batch_size , tf.float32 )\n",
    "    attn_zstate = attn_zstate.clone(cell_state = enc_st[0])\n",
    "    with tf.variable_scope(\"decoding_layer\"):\n",
    "        tr_dec_op = trng_dec_layer(dec_emb_op, \n",
    "                                                  summ_len, \n",
    "                                                  cell_dec, \n",
    "                                                  attn_zstate,\n",
    "                                                  out_l,\n",
    "                                                  v_size, \n",
    "                                                  mx_summ_len)\n",
    "    with tf.variable_scope(\"decoding_layer\", reuse=True):\n",
    "        inf_dec_op = infr_dec_layer(embs,  \n",
    "                                                    word2int[TOKEN_GO], \n",
    "                                                    word2int[TOKEN_EOS],\n",
    "                                                    cell_dec, \n",
    "                                                    attn_zstate, \n",
    "                                                    out_l,\n",
    "                                                    mx_summ_len,\n",
    "                                                    batch_size)\n",
    "\n",
    "    return tr_dec_op, inf_dec_op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "19ddcf22-4f6a-d531-071a-021b42b643e3",
    "colab": {},
    "colab_type": "code",
    "id": "pYVCMj7aTdWl"
   },
   "outputs": [],
   "source": [
    "def seq2seq_model(data_inp, data_summ_tgt, dprob, len_txt, len_summ, max_len_summ, \n",
    "                  v_size, rnsize, nlyrs, word2int, batch_size):\n",
    "    \n",
    "    inp_emb = word_emb_matrix\n",
    "    word_embs = tf.Variable(inp_emb, name=\"word_embs\")\n",
    "    inp_enc_emb = tf.nn.embedding_lookup(word_embs, data_inp)\n",
    "    op_enc, st_enc = encoding_layer(rnsize, len_txt, nlyrs, inp_enc_emb, dprob)\n",
    "    \n",
    "    inp_dec = process_encoding_input(data_summ_tgt, word2int, batch_size)\n",
    "    inp_dec_emb = tf.nn.embedding_lookup(inp_emb, inp_dec)\n",
    "    \n",
    "    op_tr, op_inf  = decoding_layer(inp_dec_emb, \n",
    "                                                        inp_emb,\n",
    "                                                        op_enc,\n",
    "                                                        st_enc, \n",
    "                                                        v_size, \n",
    "                                                        len_txt, \n",
    "                                                        len_summ, \n",
    "                                                        max_len_summ,\n",
    "                                                        rnsize, \n",
    "                                                        word2int, \n",
    "                                                        dprob, \n",
    "                                                        batch_size,\n",
    "                                                        nlyrs)\n",
    "    \n",
    "    return op_tr, op_inf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "725e92bf-2309-1a78-c771-641a42b440c6",
    "colab": {},
    "colab_type": "code",
    "id": "t6c1ac3qTdWs"
   },
   "outputs": [],
   "source": [
    "def pad_sentences(sentences_batch):\n",
    "\n",
    "    max_sentence = max([len(sentence) for sentence in sentences_batch])\n",
    "    return [sentence + [word2int[TOKEN_PAD]] * (max_sentence - len(sentence)) for sentence in sentences_batch]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "47e4f70a-6377-68dd-c06c-eed674b2bb3f",
    "colab": {},
    "colab_type": "code",
    "id": "usG7exzOTdWy"
   },
   "outputs": [],
   "source": [
    "def get_batches(summaries, texts, batch_size):\n",
    "    for batch_idx in range(0, len(texts)//batch_size):\n",
    "        start_idx = batch_idx * batch_size\n",
    "        summaries_batch = summaries[start_idx:start_idx + batch_size]\n",
    "        texts_batch = texts[start_idx:start_idx + batch_size]\n",
    "        pad_summaries_batch = np.array(pad_sentences(summaries_batch))\n",
    "        pad_texts_batch = np.array(pad_sentences(texts_batch))\n",
    "\n",
    "        pad_summaries_lens = []\n",
    "        for summary in pad_summaries_batch:\n",
    "            pad_summaries_lens.append(len(summary))\n",
    "        \n",
    "        pad_texts_lens = []\n",
    "        for text in pad_texts_batch:\n",
    "            pad_texts_lens.append(len(text))\n",
    "        \n",
    "        yield pad_summaries_batch, pad_texts_batch, pad_summaries_lens, pad_texts_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7SIkV3M9jYKl"
   },
   "outputs": [],
   "source": [
    "def train_on_batch(session, X, X_seq_len, Y, Y_seq_len, learning_rate, dropout_keep_probability):\n",
    "    feed_dict = {\n",
    "            data_inp: X,\n",
    "            tgts:Y,\n",
    "            lrt:learning_rate,\n",
    "            len_summ:Y_seq_len,\n",
    "            len_txt: X_seq_len,\n",
    "            dprobs: dropout_keep_probability}\n",
    "\n",
    "    _,pred,loss,summary = session.run([\n",
    "            train_op,\n",
    "            tr_cost,\n",
    "            inf_op,\n",
    "            merged_summary_op], feed_dict=feed_dict)\n",
    "    return _,pred, loss, summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6ejZsb2VqY7A"
   },
   "outputs": [],
   "source": [
    "def predict_for_batch(session, X, X_seq_len,Y, Y_seq_len,dropout_keep_probability):\n",
    "\n",
    "    feed_dict={data_inp:X, len_txt:X_seq_len,tgts:Y, len_summ: Y_seq_len,dprobs: dropout_keep_probability}\n",
    "\n",
    "    pred = session.run([inf_op], feed_dict=feed_dict)[0]\n",
    "\n",
    "    return pred\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xCltZfqbQpR1"
   },
   "outputs": [],
   "source": [
    "\n",
    "def predict_for_batch_with_loss(session, X, X_seq_len, Y, Y_seq_len,learning_rate,dropout_keep_probability):\n",
    "    \n",
    "    feed_dict={data_inp:X, len_txt:X_seq_len, tgts:Y, len_summ: Y_seq_len,lrt:learning_rate,dprobs: dropout_keep_probability}\n",
    "    \n",
    "    _,loss,pred = session.run([train_op,tr_cost,infer_predictions], feed_dict=feed_dict)\n",
    "    \n",
    "    return pred, loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "77299c4b-a3cf-785b-981a-42a1bb3a2033",
    "colab": {},
    "colab_type": "code",
    "id": "HGM95d9MTdW5"
   },
   "outputs": [],
   "source": [
    "batch_size =64\n",
    "rnn_len =128\n",
    "n_layers = 2\n",
    "lr = 0.005\n",
    "dr_prob =0.75\n",
    "! rm /content/gdrive/My\\ Drive/best*\n",
    "logs_path='/content/gdrive/My Drive/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "_cell_guid": "68781626-8bf4-0a23-4bb2-f24a5762fa1e",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 443
    },
    "colab_type": "code",
    "id": "AlrcY77-TdXF",
    "outputId": "926dc723-f19b-421c-fa87-af989b9fb190"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-24-08a5f0b64492>:2: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From <ipython-input-24-08a5f0b64492>:14: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn.py:464: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn_cell_impl.py:559: Layer.add_variable (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.add_weight` method instead.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn_cell_impl.py:565: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn_cell_impl.py:575: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn.py:244: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/rmsprop.py:119: calling Ones.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "Graph created.\n"
     ]
    }
   ],
   "source": [
    "train_graph = tf.Graph()\n",
    "with train_graph.as_default():\n",
    "    \n",
    "    data_inp, tgts, lrt, dprobs, len_summ, max_len_summ, len_txt = model_inputs()\n",
    "\n",
    "    tr_opp, inf_opp = seq2seq_model(tf.reverse(data_inp, [-1]),\n",
    "                                                      tgts, \n",
    "                                                      dprobs,   \n",
    "                                                      len_txt,\n",
    "                                                      len_summ,\n",
    "                                                      max_len_summ,\n",
    "                                                      len(word2int)+1,\n",
    "                                                      rnn_len, \n",
    "                                                      n_layers, \n",
    "                                                      word2int,\n",
    "                                                      batch_size)\n",
    "    \n",
    "    tr_op = tf.identity(tr_opp.rnn_output, 'tr_op')\n",
    "    inf_op = tf.identity(inf_opp.sample_id, name='predictions')\n",
    "    \n",
    "    seq_masks = tf.sequence_mask(len_summ, max_len_summ, dtype=tf.float32, name='masks')\n",
    "\n",
    "    with tf.name_scope(\"optimizer\"):\n",
    "        tr_cost = sequence_loss(tr_op,tgts,seq_masks)\n",
    "        optzr = tf.train.RMSPropOptimizer(lr)\n",
    "        grds = optzr.compute_gradients(tr_cost)\n",
    "        capped_grds = [(tf.clip_by_value(grd, -5., 5.), var) for grd, var in grds \n",
    "                        if grd is not None]\n",
    "        train_op = optzr.apply_gradients(capped_grds)\n",
    "    # Get predictions for evaluation.\n",
    "    train_predictions = tr_opp.sample_id\n",
    "    infer_predictions = inf_opp.sample_id\n",
    "    tf.summary.scalar(\"cost\", tr_cost)\n",
    "print(\"Graph created.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "989d2498-df4e-31e8-3133-56df524b8a28",
    "colab": {},
    "colab_type": "code",
    "id": "Tg98B19nTdXS"
   },
   "outputs": [],
   "source": [
    "def text_to_seq(text):\n",
    "    return [word2int.get(word, word2int[TOKEN_UNK]) for word in text.split()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "s4afDgp2zidm"
   },
   "outputs": [],
   "source": [
    "# Program to measure similarity between  \n",
    "# two sentences using cosine similarity. \n",
    "def mean_absolute_error_texts(v1,v2):\n",
    "    meanerrorl2=[]\n",
    "    NN=len(v1)\n",
    "    N2=np.sqrt(NN)\n",
    "    for X,Y in zip(v1,v2):\n",
    "        # tokenization \n",
    "        X_list = word_tokenize(X)  \n",
    "        Y_list = word_tokenize(Y) \n",
    "  \n",
    "        # sw contains the list of stopwords \n",
    "        sw = stopwords.words('english')  \n",
    "        l1 =[];l2 =[] \n",
    "  \n",
    "        # remove stop words from string \n",
    "        X_set = {w for w in X if not w in sw}  \n",
    "        Y_set = {w for w in Y if not w in sw} \n",
    "  \n",
    "        # form a set containing keywords of both strings  \n",
    "        rvector = X_set.union(Y_set)  \n",
    "        for w in rvector: \n",
    "            if w in X_set:\n",
    "               l1.append(1) # create a vector \n",
    "            else:\n",
    "               l1.append(0) \n",
    "            if w in Y_set:\n",
    "               l2.append(1) \n",
    "            else:\n",
    "               l2.append(0) \n",
    "        c = 0\n",
    "        # cosine formula  \n",
    "        for i in range(len(rvector)): \n",
    "                      c+= l1[i]*l2[i] \n",
    "        den=float((sum(l1)*sum(l2))**0.5)\n",
    "        try: \n",
    "           cosine = c /den\n",
    "        except:\n",
    "           cosine=0.0 \n",
    "        meanerrorl2.append(cosine/N2) \n",
    "  \n",
    "    return LA.norm(meanerrorl2,2)*100.0\n",
    "      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "IXy1o-6INrQ1",
    "outputId": "7944e07d-243e-4f93-b5b5-043c23d16513"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27125"
      ]
     },
     "execution_count": 38,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(news_summaries_filtered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "Lc3b_vCaNrtm",
    "outputId": "06cb1a40-5180-4312-a5fe-b50f74667d27"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27125"
      ]
     },
     "execution_count": 39,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(news_texts_filtered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xALwgTyFOtLL"
   },
   "outputs": [],
   "source": [
    "epochs =25\n",
    "min_learning_rate = 0.0006\n",
    "display_step = 400\n",
    "display_plot = display_step\n",
    "early_stop_cnt = 0 \n",
    "early_stop_cnt_max = 3 \n",
    "per_epoch = 3 \n",
    "\n",
    "\n",
    "update_loss = 0 \n",
    "batch_loss = 0\n",
    "summary_update_loss = [] \n",
    "\n",
    "news_summaries_train = news_summaries_filtered[0:21700]\n",
    "news_texts_train = news_texts_filtered[0:21700]\n",
    "\n",
    "news_summaries_test =news_summaries_filtered[21700:]\n",
    "news_texts_test=news_texts_filtered[21700:]\n",
    "\n",
    "pad = word2int[TOKEN_PAD] \n",
    "eos = word2int[TOKEN_EOS]  \n",
    "unk = word2int[TOKEN_UNK]\n",
    "tokens_id=[eos,pad,unk]\n",
    "\n",
    "update_check = (len(news_texts_train)//batch_size//per_epoch)-1\n",
    "checkpoint = logs_path + 'best_so_far_model.ckpt' \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "_cell_guid": "6368ba0d-4083-e182-ca38-5356b307e09f",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "9hbE8nF6TdXM",
    "outputId": "022fb485-2296-40a3-c9e9-e363a5edc260"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start training... \n",
      "\n",
      "Average loss: 8.629\n",
      "Saving model\n",
      "Average loss: 5.939\n",
      "Saving model\n",
      "Average loss: 5.31\n",
      "Saving model\n",
      " \n",
      "Test: epoch 1 loss: 5.244236\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 70, 17, 25]\n",
      "  Response Words: great for the best\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 70, 17, 25]\n",
      "  Response Words: great for the best\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 70, 17, 25]\n",
      "  Response Words: great for the best\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Average loss: 5.113\n",
      "Saving model\n",
      "Average loss: 4.911\n",
      "Saving model\n",
      "Average loss: 4.778\n",
      "Saving model\n",
      " \n",
      "Test: epoch 2 loss: 4.856505\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [25, 258, 538, 165]\n",
      "  Response Words: best snack bar ever\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 70, 17, 115]\n",
      "  Response Words: great for the go\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 70, 17, 115]\n",
      "  Response Words: great for the go\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Average loss: 4.725\n",
      "Saving model\n",
      "Average loss: 4.618\n",
      "Saving model\n",
      "Average loss: 4.531\n",
      "Saving model\n",
      " \n",
      "Test: epoch 3 loss: 4.616913\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 44, 13]\n",
      "  Response Words: great snack and great\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 70, 945, 44]\n",
      "  Response Words: great for quick and\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 44, 13]\n",
      "  Response Words: great snack and great\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Average loss: 4.502\n",
      "Saving model\n",
      "Average loss: 4.436\n",
      "Saving model\n",
      "Average loss: 4.382\n",
      "Saving model\n",
      " \n",
      "Test: epoch 4 loss: 4.4535375\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 44, 13]\n",
      "  Response Words: great snack and great\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [60, 44, 60, 44, 60]\n",
      "  Response Words: delicious and delicious and delicious\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 44, 24]\n",
      "  Response Words: great snack and healthy\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Average loss: 4.359\n",
      "Saving model\n",
      "Average loss: 4.316\n",
      "Saving model\n",
      "Average loss: 4.264\n",
      "Saving model\n",
      " \n",
      "Test: epoch 5 loss: 4.2681923\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 44, 81, 82]\n",
      "  Response Words: great snack and gluten free\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 1017, 70, 17, 115]\n",
      "  Response Words: great bars for the go\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 1017, 1017]\n",
      "  Response Words: great snack bars bars\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Average loss: 4.259\n",
      "Saving model\n",
      "Average loss: 4.211\n",
      "Saving model\n",
      "Average loss: 4.182\n",
      "Saving model\n",
      " \n",
      "Test: epoch 6 loss: 4.211289\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 70, 258]\n",
      "  Response Words: great snack for snack\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [60, 44, 24, 44, 48]\n",
      "  Response Words: delicious and healthy and yummy\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 70, 17, 115]\n",
      "  Response Words: great snack for the go\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Average loss: 4.177\n",
      "Saving model\n",
      "Average loss: 4.144\n",
      "Saving model\n",
      "Average loss: 4.12\n",
      "Saving model\n",
      " \n",
      "Test: epoch 7 loss: 4.1056275\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 44, 81, 82]\n",
      "  Response Words: great snack and gluten free\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [60, 44, 24, 538]\n",
      "  Response Words: delicious and healthy bar\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 1017, 538, 1017]\n",
      "  Response Words: great snack bars bar bars\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Average loss: 4.12\n",
      "No Improvement.\n",
      "Average loss: 4.098\n",
      "Saving model\n",
      "Average loss: 4.079\n",
      "Saving model\n",
      " \n",
      "Test: epoch 8 loss: 4.075004\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 81, 82, 258]\n",
      "  Response Words: great gluten free snack\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 1017, 1017, 1017]\n",
      "  Response Words: great bars bars bars\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 1017, 1017]\n",
      "  Response Words: great snack bars bars\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Average loss: 4.089\n",
      "No Improvement.\n",
      "Average loss: 4.059\n",
      "Saving model\n",
      "Average loss: 4.055\n",
      "Saving model\n",
      " \n",
      "Test: epoch 9 loss: 4.055135\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 70, 81, 82, 258]\n",
      "  Response Words: great snack for gluten free snack\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 70, 17, 115]\n",
      "  Response Words: great for the go\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 70, 258]\n",
      "  Response Words: great snack for snack\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Average loss: 4.064\n",
      "No Improvement.\n",
      "Average loss: 4.061\n",
      "No Improvement.\n",
      "Average loss: 4.038\n",
      "Saving model\n",
      " \n",
      "Test: epoch 10 loss: 3.9903595\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 81, 82, 258]\n",
      "  Response Words: great gluten free snack\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 1017, 70, 17, 115]\n",
      "  Response Words: great bars for the go\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 1017, 1017]\n",
      "  Response Words: great snack bars bars\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Average loss: 4.038\n",
      "No Improvement.\n",
      "Average loss: 4.038\n",
      "Saving model\n",
      "Average loss: 4.011\n",
      "Saving model\n",
      " \n",
      "Test: epoch 11 loss: 3.9808743\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 81, 82, 258]\n",
      "  Response Words: great gluten free snack\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 61, 70, 17, 115]\n",
      "  Response Words: great product for the go\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [48, 48, 44, 48]\n",
      "  Response Words: yummy yummy and yummy\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Average loss: 4.032\n",
      "No Improvement.\n",
      "Average loss: 4.006\n",
      "Saving model\n",
      "Average loss: 4.002\n",
      "Saving model\n",
      " \n",
      "Test: epoch 12 loss: 3.9606946\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 81, 82, 258]\n",
      "  Response Words: great gluten free snack\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 70, 81, 82]\n",
      "  Response Words: great for gluten free\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 70, 64]\n",
      "  Response Words: great snack for these\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Average loss: 4.024\n",
      "No Improvement.\n",
      "Average loss: 4.001\n",
      "Saving model\n",
      "Average loss: 3.999\n",
      "Saving model\n",
      " \n",
      "Test: epoch 13 loss: 3.902139\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 70, 81, 82]\n",
      "  Response Words: great snack for gluten free\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 70, 1032, 538]\n",
      "  Response Words: great for protein bar\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 70, 538]\n",
      "  Response Words: great snack for bar\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Average loss: 4.014\n",
      "No Improvement.\n",
      "Average loss: 3.995\n",
      "Saving model\n",
      "Average loss: 3.989\n",
      "Saving model\n",
      " \n",
      "Test: epoch 14 loss: 3.9783185\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 81, 82, 258]\n",
      "  Response Words: great gluten free snack\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [64, 39, 17, 369, 744]\n",
      "  Response Words: these are the real thing\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [64, 39, 48, 48, 1017]\n",
      "  Response Words: these are yummy yummy bars\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Average loss: 4.002\n",
      "No Improvement.\n",
      "Average loss: 3.989\n",
      "No Improvement.\n",
      "Average loss: 3.973\n",
      "Saving model\n",
      " \n",
      "Test: epoch 15 loss: 3.9253328\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 81, 82, 258]\n",
      "  Response Words: great gluten free snack\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [60, 44, 24, 258]\n",
      "  Response Words: delicious and healthy snack\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 538, 1017]\n",
      "  Response Words: great snack bar bars\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Average loss: 3.983\n",
      "No Improvement.\n",
      "Average loss: 3.975\n",
      "No Improvement.\n",
      "Average loss: 3.958\n",
      "Saving model\n",
      " \n",
      "Test: epoch 16 loss: 3.8239086\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 70, 247]\n",
      "  Response Words: great snack for me\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [60, 44, 60, 44, 448, 55]\n",
      "  Response Words: delicious and delicious and too sweet\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 538, 538]\n",
      "  Response Words: great snack bar bar\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Average loss: 3.986\n",
      "No Improvement.\n",
      "Average loss: 3.965\n",
      "No Improvement.\n",
      "Average loss: 3.959\n",
      "No Improvement.\n",
      " \n",
      "Test: epoch 17 loss: 3.8752964\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [25, 258, 405, 453]\n",
      "  Response Words: best snack out there\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [64, 39, 20, 1017]\n",
      "  Response Words: these are wonderful bars\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [48, 48, 258, 1017]\n",
      "  Response Words: yummy yummy snack bars\n",
      " Ground Truth: these are the best\n",
      "\n",
      "Stopping Training.\n"
     ]
    }
   ],
   "source": [
    "invalid_number_prediction_counts = []\n",
    "all_model_predictions = []\n",
    "all_ground_truth = []\n",
    "history=[]\n",
    "print('Start training... \\n')\n",
    "\n",
    "with tf.Session(graph=train_graph) as sess:\n",
    "    tf_summary_writer = tf.summary.FileWriter(logs_path, graph=train_graph)\n",
    "    merged_summary_op = tf.summary.merge_all()\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for epoch_i in range(epochs):\n",
    "        update_loss = 0\n",
    "        batch_loss = 0\n",
    "        train_loss =0\n",
    "        for batch_i, (summaries_batch, texts_batch, summaries_len, texts_len) in enumerate(\n",
    "                get_batches(news_summaries_train, news_texts_train, batch_size)):\n",
    "            before = time.time()\n",
    "            _,loss,predictions,summary=train_on_batch(sess,texts_batch, texts_len, summaries_batch, summaries_len, lr, dr_prob)\n",
    "            #_,loss,summary = sess.run(\n",
    "            #    [train_op, tr_cost,merged_summary_op],\n",
    "            #    {data_inp: texts_batch,\n",
    "            #     tgts: summaries_batch,\n",
    "            #     lrt: lr,\n",
    "            #     len_summ: summaries_len,\n",
    "            #     len_txt: texts_len,\n",
    "            #     dprobs: dr_prob})\n",
    "            \n",
    "            batch_loss += loss\n",
    "            update_loss += loss\n",
    "            after = time.time()\n",
    "            batch_time = after - before\n",
    "            tf_summary_writer.add_summary(summary, epoch_i * batch_size + batch_i)\n",
    "            \n",
    "            if batch_i % display_step == 0 and batch_i > 0:\n",
    "                print('** Epoch {:>3}/{} Batch {:>4}/{} - Batch Loss: {:>6.3f}, seconds: {:>4.2f}'\n",
    "                      .format(epoch_i+1,\n",
    "                              epochs, \n",
    "                              batch_i, \n",
    "                              len(news_texts_filtered) // batch_size, \n",
    "                              batch_loss / display_step, \n",
    "                              batch_time*display_step))\n",
    "                if ((epoch_i/5==0) and (epoch_i>0)):\n",
    "                         history=pd.DataFrame(history, columns=['epochs','loss','val_loss'])\n",
    "                         from matplotlib import pyplot \n",
    "                         pyplot.plot(history['epochs'],history['loss'], label='train') \n",
    "                         pyplot.plot(history['epochs'],history['val_loss'], label='test') \n",
    "                         pyplot.legend() \n",
    "                         pyplot.show() \n",
    "                batch_loss = 0\n",
    "            \n",
    "            if batch_i % update_check == 0 and batch_i > 0:\n",
    "                print(\"Average loss:\", round(update_loss/update_check,3))\n",
    "                train_loss=round(update_loss/update_check,3)\n",
    "                summary_update_loss.append(update_loss)\n",
    "                \n",
    "                if update_loss <= min(summary_update_loss):\n",
    "                    print('Saving model') \n",
    "                    early_stop_cnt = 0\n",
    "                    saver = tf.train.Saver() \n",
    "                    saver.save(sess, checkpoint)\n",
    "\n",
    "                else:\n",
    "                    print(\"No Improvement.\")\n",
    "                    early_stop_cnt += 1\n",
    "                    if early_stop_cnt == early_stop_cnt_max:\n",
    "                        break\n",
    "                update_loss = 0\n",
    "            \n",
    "\n",
    "        summaries_test_batch, texts_test_batch, summaries_test_len, texts_test_len =\\\n",
    "         next(get_batches(news_summaries_test, news_texts_test, batch_size))\n",
    "    \n",
    "        predictions,test_loss = predict_for_batch_with_loss(sess,texts_test_batch, texts_test_len,summaries_test_batch,summaries_test_len,lr,1.0)\n",
    "        history.append([epoch_i+1,train_loss,test_loss])\n",
    "        #print('history:',history[-1])\n",
    "        print(\" \")  \n",
    "        print('Test: epoch', epoch_i+1, 'loss:', test_loss,)\n",
    "        print(\" \")\n",
    "        for text, summary, result_logits  in list(zip(texts_test_batch, summaries_test_batch, predictions))[-3:]: \n",
    "                     print('\\nText')\n",
    "                     print('  Word Ids:    {}'.format([i for i in text]))\n",
    "                     print('  Input Words: {}'.format(\" \".join( int2word[i] for i in text if i not in tokens_id)))\n",
    "\n",
    "                     print('\\nSummary')\n",
    "                     print('  Word Ids:       {}'.format([i for i in result_logits if i not in tokens_id]))\n",
    "                     print('  Response Words: {}'.format(\" \".join(int2word[i] for i in result_logits if i not in tokens_id )))\n",
    "                     print(' Ground Truth: {}'.format(\" \".join(int2word[i] for i in summary if i not in tokens_id)))\n",
    "                     print('')\n",
    "        model_predictions = []\n",
    "        ground_truth = []\n",
    "        invalid_number_prediction_count = 0\n",
    "\n",
    "        ind=0\n",
    "        for summaries_test_batch, texts_test_batch, summaries_test_len, texts_test_len\\\n",
    "         in get_batches(news_summaries_test,news_texts_test, batch_size=batch_size):\n",
    "                 \n",
    "  \n",
    "                 #summaries_test_len=[np.random.randint(5,8)]\n",
    "                 pred=predict_for_batch(sess, texts_test_batch, texts_test_len,summaries_test_batch,summaries_test_len,1.0)   \n",
    "                 \n",
    "                 for y,p in list(zip(summaries_test_batch,pred)): \n",
    "                     Y=''.join(int2word[i] for i in y if i not in tokens_id)\n",
    "                     O=''.join(int2word[i] for i in p if i not in tokens_id)\n",
    "                     \n",
    "                     if mean_absolute_error_texts([Y],[O])<1e-15 : \n",
    "                             invalid_number_prediction_count += 1\n",
    "                     lmin=min(len(Y),len(O))+1\n",
    "                     OO=O[:lmin]\n",
    "                     YY=Y[:lmin] \n",
    "                     model_predictions.append(OO)\n",
    "                     ground_truth.append(YY)\n",
    "        all_model_predictions.append(model_predictions)\n",
    "        all_ground_truth.append(ground_truth)\n",
    "        invalid_number_prediction_counts.append(invalid_number_prediction_count)\n",
    "        \n",
    "        if early_stop_cnt == early_stop_cnt_max:\n",
    "            print(\"Stopping Training.\")\n",
    "            break        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 265
    },
    "colab_type": "code",
    "id": "wErDv9CAuhdy",
    "outputId": "c5d90c88-82ac-4c24-e24c-4a2001a57ba8"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3yV5fnH8c+V5GRBBmSQhIBh76FEQUAFUQFFtFURLdaNq9bWVe2wjp+ttf1Zf9aqFVfrRsRRQQUVVJRhQDbIkJUwwsoigyTn+v3xHCCEJCTkJE9ycr1fr7zOeO4850Lhmzv3cz/3LaqKMcaY5i/I7QKMMcb4hwW6McYECAt0Y4wJEBboxhgTICzQjTEmQIS49cHx8fGalpbm1scbY0yztHjx4j2qmlDVMdcCPS0tjYyMDLc+3hhjmiUR2VLdMRtyMcaYAGGBbowxAcIC3RhjAoRrY+jGGHMiSktLyczMpLi42O1SGlR4eDipqal4PJ5af48FujGmWcnMzCQqKoq0tDRExO1yGoSqsnfvXjIzM+nUqVOtv8+GXIwxzUpxcTFxcXEBG+YAIkJcXFydfwuxQDfGNDuBHOaHnMifsdkF+rpd+fzPR6spLi13uxRjjGlSml2gZ+4v5IV5m8jYvN/tUowxLVBOTg7PPPNMnb/v/PPPJycnpwEqOqLZBfrgTnF4goWvN+x2uxRjTAtUXaCXlZXV+H0zZ84kNja2ocoCmuEsl1ZhIZzcsQ3fbNjjdinGmBbovvvuY+PGjQwcOBCPx0N4eDht2rRh7dq1rFu3josvvpht27ZRXFzMHXfcweTJk4Ejy50UFBQwduxYhg8fzrfffkv79u354IMPiIiIqHdtzS7QAc7oGs8Tn61j34GDtG0V6nY5xhiXPPTfVazenufXc/ZOieaPF/ap9vhjjz3GypUrWbp0KXPnzuWCCy5g5cqVh6cXvvTSS7Rt25aioiJOPfVULrnkEuLi4o46x/r163nzzTeZMmUKEyZM4N1332XSpEn1rr3ZDbkADOsWjyrWSzfGuO600047aq74U089xYABAxgyZAjbtm1j/fr1x3xPp06dGDhwIACDBg1i8+bNfqmlWfbQ+7ePISo8hHnr93DhgBS3yzHGuKSmnnRjadWq1eHnc+fO5bPPPmP+/PlERkYyYsSIKueSh4WFHX4eHBxMUVGRX2pplj30kOAghnaJY96GPaiq2+UYY1qQqKgo8vPzqzyWm5tLmzZtiIyMZO3atSxYsKBRa2uWPXSA4d0S+HTVLjbtOUDnhNZul2OMaSHi4uIYNmwYffv2JSIignbt2h0+NmbMGJ577jl69epFjx49GDJkSKPW1vwCfeMXMOsBzrrwbcAZR7dAN8Y0pjfeeKPK98PCwvj444+rPHZonDw+Pp6VK1cefv/uu+/2W13Nb8glog3sWkGHHZ/QPjaCr9fbhVFjjIHmGOjJAyGhF7LsLc7oFs/8jXspK/e6XZUxxriu+QW6CAyYCJmLGJ2UT35JGcsyc92uyhhjXFerQBeRzSKyQkSWisgxOzuLyM9EZLmvzbciMsD/pVbQ/3KQIIYUfIYIzLNhF2OMqVMPfaSqDlTV9CqObQLOUtV+wCPA836prjrRydB5BBGrp9EvOcpuMDLGGPw05KKq36rqoeUPFwCp/jhvjQZcAblbmdhuG0u27qegpOaFcYwxJtDVNtAVmCUii0Vk8nHaXg9UOW9HRCaLSIaIZOzeXc/VEnuOg9DWjDr4BWVeZeGPe+t3PmOMqYUTXT4X4Mknn6SwsNDPFR1R20AfrqqnAGOB20TkzKoaichInED/TVXHVfV5VU1X1fSEhIQTKviw0EjofTGJ2z4hNuSgTV80xjSKphzotbqxSFWzfI/ZIvIecBrwVcU2ItIfeAEYq6qN010eeAWy9DVuSlzDuxvaNspHGmNatorL55577rkkJiYydepUSkpK+MlPfsJDDz3EgQMHmDBhApmZmZSXl/OHP/yBXbt2sX37dkaOHEl8fDxz5szxe23HDXQRaQUEqWq+7/l5wMOV2nQEpgNXqeo6v1dZnY5DIaYjF+qX/CV7ADtzi0mKCW+0jzfGuOzj+2DnCv+eM6kfjH2s2sMVl8+dNWsW06ZNY9GiRagq48eP56uvvmL37t2kpKQwY8YMwFnjJSYmhieeeII5c+YQHx/v35p9ajPk0g6YJyLLgEXADFX9RERuFpGbfW0eAOKAZ6qb2tgggoJgwETa719IO/Yxz2a7GGMa0axZs5g1axYnn3wyp5xyCmvXrmX9+vX069eP2bNn85vf/Iavv/6amJiYRqnnuD10Vf0ROGZeuao+V+H5DcAN/i2tlgZMRL56nCsjFjBvfV8uHdTwE2yMMU1EDT3pxqCq3H///dx0003HHFuyZAkzZ87k97//PaNGjeKBBx5o8Hqa352ilcV1gQ6DmeCZx7z1tpyuMaZhVVw+d/To0bz00ksUFBQAkJWVRXZ2Ntu3bycyMpJJkyZxzz33sGTJkmO+tyE0v9UWqzJgIskf/ZqkkrWs3TmYXsnRbldkjAlQFZfPHTt2LFdeeSWnn346AK1bt+a1115jw4YN3HPPPQQFBeHxeHj22WcBmDx5MmPGjCElJaVBLoqKWz3a9PR0zcjw01B70X70bz14pWQEZec9xo1ndvbPeY0xTc6aNWvo1auX22U0iqr+rCKyuJo79gNgyAUgog3SYyw/8cxn/vodbldjjDGuCIxABxhwBbGaR9jmLygpK3e7GmOMaXSBE+hdR1ESFsd4vmTxlv3Hb2+MabZawuSHE/kzBk6gB3uQfpcyKmgJGWs2ul2NMaaBhIeHs3fv3oAOdVVl7969hIfX7UbJwJjl4hM6aBJk/AvPmvdg3GC3yzHGNIDU1FQyMzOp9wJ/TVx4eDipqXW7ryagAp2kfuyO7MqQ/NnkFD5MbGSo2xUZY/zM4/HQqVMnt8tokgJnyAVAhOLeEzg5aAPLln7ndjXGGNOoAivQgaThV1Gugnfpm26XYowxjSrgAt0Tm8LqyHR67/4YvF63yzHGmEYTcIEOsK/rJbTTPexaPtvtUowxptEEZKB3GHopeRpBwaLX3C7FGGMaTUAGeqekeOYED6P9jllQUuB2OcYY0ygCMtBFhKyOFxGuxXjX/NftcowxplEEZKADtB9wNlu9CRywYRdjTAtRq0AXkc0isqK67eXE8ZSIbBCR5SJyiv9LrZth3RKY7j2D1tu/gdxMt8sxxpgGV5ce+khVHVjNOrxjgW6+r8nAs/4orj7iW4exvO1YBIXlU90uxxhjGpy/hlwuAv6jjgVArIgk++ncJ6xrz3585+2Bd+kbEMAL+RhjDNQ+0BWYJSKLRWRyFcfbA9sqvM70vXcUEZksIhkiktEYC+sM7xrPu+VnELR3PWxf0uCfZ4wxbqptoA9X1VNwhlZuE5EzT+TDVPV5VU1X1fSEhIQTOUWdnNapLbNkKGUSCrYUgDEmwNUq0FU1y/eYDbwHnFapSRbQocLrVN97rgr3BNMrrT3zQgbDymlQdtDtkowxpsEcN9BFpJWIRB16DpwHrKzU7EPg577ZLkOAXFVtEpt7Du+awCsHToei/bB+ltvlGGNMg6lND70dME9ElgGLgBmq+omI3CwiN/vazAR+BDYAU4BbG6TaE3BGt3i+9vajOCweltmwizEmcB13gwtV/REYUMX7z1V4rsBt/i3NP3onRxMdGc78VqMYuW46HNgLreLcLssYY/wuYO8UPSQoSBjaNZ4peaeBtxRWvut2ScYY0yACPtABzugaz7cFyRTH9bZhF2NMwGoRgT68WzwAS9uOdeaj7/7B5YqMMcb/WkSgp7aJpFN8K94sGgwSbL10Y0xAahGBDjCsaxyztyreLqOctV285W6XZIwxftViAn141wQKD5azMeVCyMuCTV+5XZIxxvhViwn007vEESQw4+BACIuBZW+5XZIxxvhViwn0mAgPAzrEMndjPvT9Caz5EEry3S7LGGP8psUEOjjTF5dn5lDQ4zIoLQTbns4YE0BaVKAP6xqPV2FeSRdo08lmuxhjAkqLCvSTO7YhMjSYeRv3wIArYNPXkLPt+N9ojDHNQIsK9NCQIIZ0jmPe+j0w4HJAYfnbbpdljDF+0aICHZxdjDbvLWSbJsJJw5xhF9uezhgTAFpcoJ/hWwZg3gbfsMveDbDhc5erMsaY+mtxgd41sTXtosOcYZf+E6BtF/jkN1BW4nZpxhhTLy0u0EWEYV3j+WbjHrxBoTD2caeXPv+fbpdmjDH10uICHZxhl5zCUlZtz4Nu50DPcfDVXyE30+3SjDHmhLXIQB/W1RlH/3rDbueN0X8C9cKnv3OxKmOMqZ9aB7qIBIvI9yLyURXHOorIHN/x5SJyvn/L9K/EqHB6JkU54+gAbU6C4XfC6vfhx7mu1maMMSeqLj30O4A11Rz7PTBVVU8GJgLP1LewhjasazwZm/dTdNC3jO6wO6BNGsy8F8oOulqbMcaciFoFuoikAhcAL1TTRIFo3/MYYHv9S2tYw7vFc7Dcy3eb9zlveMJhzF9gzw+w8Lmav9kYY5qg2vbQnwTuBbzVHH8QmCQimcBM4PaqGonIZBHJEJGM3bt317VWvxrcqS2hwUHOfPRDeoyB7mPgy79A3g73ijPGmBNw3EAXkXFAtqourqHZFcArqpoKnA+8KiLHnFtVn1fVdFVNT0hIOOGi/SEyNIRTTorl6/V7jj4w5s9QXgqzfu9OYcYYc4Jq00MfBowXkc3AW8DZIvJapTbXA1MBVHU+EA7E+7HOBnFGtwTW7MhjV17xkTfbdnbG01dOg83z3CvOGGPq6LiBrqr3q2qqqqbhXPD8QlUnVWq2FRgFICK9cALd3TGVWhjbN4ngIOGpz9cffWD4ryGmI8y8x+mtG2NMM3DC89BF5GERGe97eRdwo4gsA94ErlFt+itedU5ozc9PP4k3F21l9fa8IwdCI52hl+zVsGiKewUaY0wdiFu5m56erhkZGa58dkW5haWM+NscureL4q3JQxAR54AqvH4pbFsEv8iAqHbuFmqMMYCILFbV9KqOtcg7RSuKifRw13k9WLhpHx+v3HnkgIizzktZMcx+wL0CjTGmllp8oANccVpHeiZF8eiMNRSXlh85ENcFht4Oy9+CLfPdK9AYY2rBAh0IDhL+eGEfsnKKmPLVj0cfPOMuiE6FmXdDeZk7BRpjTC1YoPuc3iWO8/sl8czcjezILTpyILQVjH4Udq2EjJfcK9AYY47DAr2C+8f2wqvKYx+vPfpA74ug8wj44n+goMnPxjTGtFAW6BV0aBvJTWd25oOl28k4tMYL+C6Q/hVKC+GzB12rzxhjamKBXsnNI7qQFB3OQ/9djddbYUpnQnc4/VZY+pozldEYY5oYC/RKIkNDuP/8nqzIymXakko7GJ15D0QlOxdIveVVn8AYY1xigV6F8QNSGHRSGx7/5Afyiyvc+h8WBef9D+xYBotfca0+Y4ypigV6FUSEP17Ym70HSnj6iw1HH+x7CaSdAZ8/DAf2ulOgMcZUwQK9Gv1TY7lsUCovfbOJTXsOHDkgAuf/FQ4WwOcPuVegMcZUYoFeg7tH9yAsJJhHZ6w++kBiLxh8Myz5D2TVtEy8McY0Hgv0GiRGhXP72V35bE02X66rNP/8rN9A60SYcTd4q9vIyRhjGo8F+nFcMyyNtLhIHvloNaXlFYI7PBrOfQS2L4HvX3WvQGOM8bFAP46wkGD+MK43G7ILeHX+lqMP9p8AHYc6NxsV7qvy+40xprFYoNfC2T0TObN7An//bB17C0qOHDh0gbQ411kWwBhjXGSBXgsiwgPjelF4sJwnZq87+mBSXzjtRmfhru1L3SnQGGOwQK+1rolRVW9XBzDifmgV77uD1C6QGmPcUetAF5FgEfleRD6q5vgEEVktIqtE5A3/ldh0/GpUd2IiPDz031UctXVfRCyc+zBkfgdzHnWvQGNMi1aXHvodwJqqDohIN+B+YJiq9gF+5YfampyYSA93j65iuzqAAVfAKT+Hr/9mywIYY1xRq0AXkVTgAuCFaprcCPxTVfcDqGq2f8preiaeWs12dSJwwRPQ9Rz46E5YP9u9Io0xLVJte+hPAvcC1Q0Qdwe6i8g3IrJARMZU1UhEJotIhohk7N7dPDeKqHG7umAPXPYKtOsDU6+2i6TGmEZ13EAXkXFAtqrWdI97CNANGAFcAUwRkdjKjVT1eVVNV9X0hISEEyzZfdVuVwfOioxXToXItvDGBMjZ6k6RxpgWpzY99GHAeBHZDLwFnC0ir1Vqkwl8qKqlqroJWIcT8AGr2u3qAKKT4WfvQGkxvHYpFO1v/AKNMS3OcQNdVe9X1VRVTQMmAl+o6qRKzd7H6Z0jIvE4QzCVxiMCS7Xb1R2S2Asmvgb7foS3r4KykmPbGGOMH53wPHQReVhExvtefgrsFZHVwBzgHlUN+MXCq92u7pBOZ8LFz8Dmr+GD20CraGOMMX5Sp0BX1bmqOs73/AFV/dD3XFX1TlXtrar9VPWthii2qalxu7pD+k+As/8AK96BLx5p3AKNMS2K3SlaT+MHpJBe1XZ1FZ1xF5xyNXz9v5DxcuMWaIxpMSzQ68nZrq5P1dvVHWl0ZI76jLtg3azGLdIY0yJYoPtBv9SYw9vVLd2WU3Wj4JAjc9TfucbmqBtj/M4C3U/uHdOT5JgIrn5p0bGLdx1ic9SNMQ3IAt1P4luH8foNg4kMDeaqFxeyITu/6oY2R90Y00As0P2oQ9tI3rhxCEFBwpVTFrJ5z4GqG9ocdWNMA7BA97NO8a14/YbBlJZ7+dkLC8nKKaqmoc1RN8b4lwV6A+jeLopXrx9MXnEpP5uygOy84qob9p8Aox6wOerGGL+wQG8gfdvH8O/rTmN3fgk/e2Hh0XuRVjT8Thh0jc1RN8bUmwV6AzqlYxtevOZUtu4r5KoXF5FbWMWNRyJw/v9Ct/Nsjroxpl4s0BvYkM5xPP/zdDZkF3D1y4soKCk7tlFwCFz6srPh9DvXwPbvG71OY0zzZ4HeCM7qnsDTV57MiqxcrnvlO4oOlh/bKKx1hTnql8P+LY1fqDGmWbNAbyTn9UniycsHkrF5H5NfzTh6+7pDopLgZ9OcOeqvXwZF1dx1aowxVbBAb0QXDkjhL5f05+v1e/jFG0soLa9iR7/EnjDxddi3Ed6/1aYzGmNqzQK9kV2W3oFHLurDZ2uy+dXbSymvch31M+DcR+CHGfDN/zV+kcaYZinE7QJaoqtOT6O41MujM9cQHhLMXy/tT1CQHN1oyC2wbSF8/hC0H+SEvDHG1MB66C658czO3Hlud95dkskfPliJVh5aEYGLnoa2XWDadZC3w51CjTHNRq0DXUSCReR7EfmohjaXiIiKSLp/ygtst5/dlVtGdOH1hVt5dMaaY0M9LAoufxUOFsC0a6G8mg00jDGGuvXQ7wDWVHdQRKJ8bRbWt6iWQkS4d3QPrhmaxgvzNvH32euObZTYCy58CrbOh88ebPQajTHNR60CXURSgQuAF2po9gjwF6CahUtMVZwdj3oz8dQOPPXFBp6ZW8WuR/0vg1NvhPlPw+oPGr9IY0yzUNse+pPAvUAV8+xARE4BOqjqjJpOIiKTRSRDRDJ2795dt0oDmIjw6E/6cfHAFB7/5Ade/mbTsY1GPwrt0+H922BPNVvdGWNatOMGuoiMA7JVdXE1x4OAJ4C7jncuVX1eVdNVNT0hIaHOxQay4CDhb5cNYGzfJB7672reXFRpN6OQMGcLu2APTL0KDlaz1roxpsWqTQ99GDBeRDYDbwFni8hrFY5HAX2Bub42Q4AP7cJo3YUEB/F/E09mZI8EfvveCp7/auPRF0pjO8AlL0D2Gvjo13bTkTHmKMcNdFW9X1VTVTUNmAh8oaqTKhzPVdV4VU3ztVkAjFfVjIYqOpCFhgTx7KRBnN83mT/NXMtd7yw7epmArqNgxP2w/G3IeMm9Qo0xTc4Jz0MXkYdFZLw/izGOcE8wT195Mr8+pzvTl2RxxZQFZOdXuNZ85j3Q9Vz45D7IqnIkzBjTAskxc58bSXp6umZkWCf+eD5esYM7py4jNtLDlJ+n07d9jHOgcB/860zn+U1fOas0GmMCnogsVtUqh7TtTtEmbmy/ZKbdcjoCXPrct8xY7rtjNLItTPg3FOyC6TeCt8oJSMaYFsQCvRnokxLDB78YTp+UGG57YwlPzF6H16vOGi9jHoMNn8FXf3W7TGOMyyzQm4mEqDDeuHEwlw1K5anP13PbG0soPFgG6ddB/4kw98+w4XO3yzTGuMgCvRkJCwnm8Uv78/sLevHpqp1c8ux8MnOKYNzfnSUC3r0Bcra5XaYxxiUW6M2MiHDDGZ158ZpTydxXyMX//IaM7cUw4VVn8a53roayErfLNMa4wAK9mRrZI5H3bhtG67AQrpiygKmbw+DiZ5xpjJ/+zu3yjDEusEBvxromtub924YxuFMc905bziM/dsU75Bfw3RRY/o7b5RljGpkFejMXGxnKK9eeyjVD03hx3iauz7qAstQh8N9fOksEGGNaDAv0ABASHMSD4/vw55/24+uNOUzcfxNlIa3g7augJN/t8owxjcQCPYBccVpHXr9hMD8WR3Fj0W3ovh/hg1/YIl7GtBAW6AFmcOc4PrhtGNtjBvF46QRY/T664Bm3yzLGNAIL9ADUoW0k7946lA3drufT8nS8n/6Bsg/ugO1L3S7NGNOALNADVOuwEP51VTprh/yFaWVnUPb9G/D8Wei/zoKMl21s3ZgAZKsttgCLNu3jfz9cSPfsT7ghYi4nlW0GTyvodwkMugZSTgERt8s0xtRCTastWqC3EOVe5d3FmTz+yVpSC1dzf7sFnFYwBykrgqR+TrD3uwzCY9wu1RhTAwt0c1h+cSn/nLORl+ZtIia4iMe7/8BZ+TMI2rUCPJHQ56dOuKemW6/dmCbIAt0cY+veQv40cw2frNpJ+5hwHhtazvDcj5AV06D0ACT2cYK9/wSIiHW7XGOMjwW6qda3G/fwyEdrWLMjj9PS2vLg6I703jsLFr8CO5ZCSAT0+QkMuho6DLZeuzEu80ugi0gwkAFkqeq4SsfuBG4AyoDdwHWquqWm81mgNx3lXuXt77bxv7N+YF/hQS4blMrdo3uQmL8WlvzbWRfmYD4k9IRB10L6tRAS5nbZxrRI/gr0O4F0ILqKQB8JLFTVQhG5BRihqpfXdD4L9KYnr7iUp7/YwMvfbCIsJJjbRnbl2mFphHuLYNV0p9eetRiS+sNlr0BcF7dLNqbFqfeeoiKSClwAvFDVcVWdo6qFvpcLgNQTKdS4Kzrcw2/P78WsX5/F6V3i+Msnazn371/yyfp89OSr4MYvYOKbkLPV2aDaVnQ0pkmp7Y1FTwL3ArXZifh64OOqDojIZBHJEJGM3bt31/KjTWPrFN+KKT9P5/UbBhPpCeHm15ZwxZQFrN6eBz3Ph1u+gXZ9YfoN8MFtcPCA2yUbY6hFoIvIOCBbVRfXou0knGGZKncsVtXnVTVdVdMTEhLqXKxpXMO6xjPjl8N55OK+/LAznwv+8TX3T1/OtvK2cM0MOONu+P51eH4k7FrtdrnGtHjHHUMXkT8DV+Fc8AwHooHpqjqpUrtzgH8AZ6lq9vE+2MbQm5fcolKe+nw9//52M15VzunVjuuGd2KwLkemT4aSPBjzmDPV0WbCGNNg/DZtUURGAHdXcVH0ZGAaMEZV19fmXBbozdOO3CJenb+FNxdtZX9hKb2So7l5UGvGbXyQ4E1znSmOF/6f3XFqTAOp90XRak76sIiM9738K9AaeEdElorIhyd6XtO0JcdEcO+Ynsy/fxSP/bQfXq9yx0fbOW3LrXzV8TZ09YfOBdOs447QGWP8zG4sMvWiqsz/cS8vzdvM52t3cWrQOp6LeIZY736Czn0IhtxqQzDG+FFNPfSQxi7GBBYRYWiXeIZ2iWfL3gP8Z34nLvyuI3/0/pPzPv0tO5fOIm7SC3ii7CK4MQ3NeujG7wpKyng3Yxu5X/6Tm0peJleimdf/Mc4672LiWtsdpsbUh63lYlzh9SqLF8ylwxe3klC6g394L2VH/1u5ZnhXeiVHu12eMc2SBbpxV0k+edN+QfT695mvffllyS106dyFa4Z2YmTPBMJCgt2u0JhmwwLduE8Vlr6OzribYgnnt9zOe/k9iQwNZmiXeEb0SGBEjwRS20S6XakxTZoFumk6stfCtGshezVbe97AOzKG9zYFkZlTDEC3xNaM6JHAyB6JpKe1JTTEtr01piILdNO0lBbBJ/c5qzcC2iqBwvgBrAnuzmd5HXhnRyJ7yyNoFRrM0K7xjOyRyIgeCaTERrhbtzFNgAW6aZp2roRtCyBzMWRlwJ51hw8VRHVmXUh3vsjvyNwDHVmrHenSro1vaCaR9LQ2eIKt925aHgt00zwU5cD2JUcCPjMDCvcAUBYUxsaQLnxTlMaS8i6s9/QgrUtPRvZsx4geiSTFhLtcvDGNwwLdNE+qztrrWRmHQ153LEPKnPH2fcSwpLwzS71d2Rfbl4j2vUlM6UT3pBi6tWtN+9gIxO5SNQHGAt0EjvJS2LUSMjPQrMUc3PIdYTkbDh8uUQ9bNZEtmsj2oGSKok4iuG1nolK6k3RSN7oltyU5JtyC3jRbFugmsBXlwI5lsG8jxbs2ULhrA7LvR1oVbiPUW3y4WZkGkaXxZEkSuZEdKI9JIzSxC21Te5LapTft2sZa0JsmzwLdtEyqULAL9m2iYOc6cjLXUbp7A568LcQWbaO1FhzVfBdt2eNJ4UDrNLZ3uADSziSlTSTt20TQLiqMELsIa5oAC3RjqlK4j5ysdWRvWc2BnRvQvT8SUbCV9qWbieYAK71pPF92ATO9g9EgD0nR4aTEhpMSG0H72AjnsY3zvH1sBK3CbK070/As0I2pi9JiSr5/E/n2aUJzNnAgPIlF7S5nVthoNuYHsT2niJ25xZR5j/63ExPhORz27WPDfWEfSfd2remc0JrgIBvOMfVngW7MifB6Yf0s+PYfsGUehEU7W+wNvpnyqBSy84vZnlNE5v4itucUk5VTyPYc572s/UXkl5QdPlW4J4geSdH0TvFQnk8AAA8QSURBVI6mT0o0vVOi6ZUUTUSorWNj6sYC3Zj6yloM3z4Nq98HCYK+l8Dpv4Dk/tV+S15xKdv2FbJ2Rz6rd+Sxenseq7bnklfsBH2QQKf4VvROiXFCPtkJ+nhbYtjUwC+BLiLBQAaQVcWeomHAf4BBwF7gclXdXNP5LNBNs7R/Cyx4Fpb8B0oPQOcRMPR26DKqVjszqSpZOUW+cM87HPRZOUWH27SLDjsc7n1SYuidHE3HtpEE2ZCNwX+BfieQDkRXEei3Av1V9WYRmQj8RFUvr+l8FuimWSvaDxkvw8J/QcFOSOwDQ38BfS+FkNA6ny63sJRVO3JZXSHk12cXUO4bp28dFkLPpCgSo8OIDvcQHeEhOjzE9+ghOiLk8PsxvvfCPUE2DTMA1TvQRSQV+DfwKHBnFYH+KfCgqs4XkRBgJ5CgNZzcAt0EhLISWDHNGWffvQaikmHwTTDoWoiIrdepi0vL2ZBdwKrtTtCv2ZnP/gMHySsuJa+ojKLS8hq/3xMs1YZ/VLiH8JAgQoKDCAkWQoODCAkSPCFBeIKc9zzBQXh8jyEVnwcdOnbkvahw55x24bfh+SPQpwF/BqKAu6sI9JXAGFXN9L3eCAxW1T2V2k0GJgN07Nhx0JYtW07gj2NME6QKGz6Hb5+CTV9CaGs45ecw+GZoc1KDfGRJWTn5xWXkFZWSd/jRCfu84lJyi0qrOHbkdUmZ16/1iEB0uIfYSA+xER5iIkOJjXBex/h+c4it8J7zfigxER5bJrkO6rVJtIiMA7JVdbGIjKhPIar6PPA8OD30+pzLmCZFBLqd43ztWOZcQF34L2e8PaKN01sPj63bY1h0jePyYSHBhLUOPuGLqKpKuVcp8yoHy72UlStl5d4jz71eDpY5j6XlSqnv/dJyr/Pc6zw/WOYlv7iMHN8PkJzCg+QUlZJT6FwUzik8SG5RKd4a/sW3Cg0mNtIJ98ToMJJjwkmOiTjyGBtOckw4kaE2178mtfmvMwwYLyLnA+FAtIi8pqqTKrTJAjoAmb4hlxici6PGtDzJA+CSKXDOH2HZm5C3A4pznCUKinOcC6uHXmsNwyYSBOExRwI+og3EdoS2nY98tekEoSe2y5OIEBIshARDuKdhp096vUp+SRm5haXkFB0kp9D5DSKnqJTcQt/rwmKS9i1ibW4cs7Ni2VNw8JjzxER4fCEfTnJsBCkx4STFOI/Jsc4PgIb+szRldZq26OuhVzXkchvQr8JF0Z+q6oSazmVj6KbFU4WDBUeC/niPhfuc1ScL9xx9nqhkX8B38j12OfI6LMqdP1tdqMK6T+HzhyB7tfNex9Mp7Xs5O1PHklXsYUduETtyi9mRU3zkeW4x+w4cG/ptIj2He/ftYsJJina+Kj6PjghptheM6zXkUsNJHwYyVPVD4EXgVRHZAOwDJp7oeY1pMUScwA2LwvkFt5aKcmD/Jtj3o+/L93z9bGftmopaJR7doz8U+nFdnN6/27bMh88edDY6adsZLn4O8nfAsjfxzPwVHULuo0PPC2DAlTBgJAQd3fsuLi1nZ24x23OL2JFTzM4858auHbnFZOUUsWTrfvYXlh7zseGeINpFh9POF/BJMeEkRoWR5Av9Q8ea29i+3VhkTCApyT8S8JUDP3/70W1PGgbp10Gv8Sc01bJedq2Czx+GdZ9A63Zw1m+ci8jBHue4KmQtgWVvOLOIinOgdRL0nwADr4TEXrX+qOLScnbnl7Azr5iducXsynO+duaVsCvX+SGwM6+Yg1VcJI5rFUpidDhJ0WGEhgThVac0VcWrilfB68tQryper/OoHGrje61Hv750UCrXDut0Qv/p7E5RYwwcLIT9m2HfRidQl74BOVugVQKcfJWzrEEDzcg5bP9mmPNnWP62c9F3+K+caZ6hrar/nrISJ/iXveUsxeAtg+SBTrD3vRRaxdW7LFUlt6j0qNDfmVvCrvxiduUWsyu/mNIyRcS59hAkEOR7FBGk0utDx+Xw49HfM6ZvMpcOSj2hWi3QjTHH8nph4+fw3Yuw/lOn69ntPDj1euh6zjHDG/VSsBu++itkvOScd/BNMOxXENm27udZOc35YbRzOQSFQLfRMPAK57Gxf9NwgQW6MaZmOdtgyb+dJQ0KdkFMRxh0tTMM0jrxxM9bnAfzn3amcZYVw8mTYMR9EJ1S/5oP/Zax4h2n5oi20O9SGHAFpJxcq6UYmiMLdGNM7ZSXwtqPnF775q8hyAO9LnR67ScNq31IlpU45/j6b1C4F3pfDGf/HuK7NUDNZfDjHCfc186A8hJI6AkDJjrDSBFt/P+ZLrJAN8bU3e51sPhlWPo6FOc6IZl+HfS/vPplDbzlzvj4nD9B7jZn8bJRD0D7QY1Tc1GOsyLm0jedmTNRKfDTf0GnMxvn8xuBBbox5sQdLIRV053x76zF4Il0lg8+9XpnaAOc8fcfZjozV3avdS5anvMgdBnpXt1Zi2H6ZNi70VkR8+w/BMQYuwW6McY/tn/vBPuKaVBaCCmnQL/LYNV7kLkI4ro6wdn7oqYxhn3wAHz6O+c3jaT+cMkLkNDD7arqxQLdGONfxbmw7G3IeNHpkUclOxc7B06C4Ca43sraGfDh7c5vG6P/B9Kvbxo/cE6ABboxpmGowp51zhozngi3q6lZ/k54/1Znqmb3MTD+aWid4HZVdVZToDev+1qNMU2LiDOE0dTDHCAqCX42Dcb8BTbOgWdPd5ZLCCAW6MaYliMoCIbcDJPnOHfIvn4pzLwHSouO/7315S2HH7+E92+DldMb5COa4GCXMcY0sHZ94MY5zsJgC5+FTV85F0yT+vn/s3audKZyrpjmrKcTGlWntWjqwgLdGNMyecJh7GPOpiTv3wpTzoZRf4Qhtzo9+frIzXLuYF0+FbJXOUsUdD0XRj8KPcY22BCVBboxpmXreg7c8i18+EuY9TvYMBsufrbuyxMU58LqD53e+OZ5gELqaXD+36DPT/2yiNjxWKAbY0yreJj4Oix+BT79LTw7FMb/w1n2oCZlB2HDZ06I//Cxs+xA2y4w4n5nXZm4Lo1S/iEW6MYYA86MnfRrIW04vHsDvD3JWZxs9J8hrPWRdqqwbZET4qumQ9F+iIx31o3pfzm0P8W1Oe4W6MYYU1F8N7h+Nsz9M8z7uzN8cskLzvrty6fCiqnOuu4hEdDzAifEu4w8sjmHiyzQjTGmspBQZ5PvrqNg+k0wZRSgzsbdnc6Cs+6DXuOa3J6txw10EQkHvgLCfO2nqeofK7XpCPwbiAWCgftUdab/yzXGmEaUNhxumQffPAWRcc64eFSS21VVqzY99BLgbFUtEBEPME9EPlbVBRXa/B6YqqrPikhvYCaQ5v9yjTGmkUW0cXrrzcBxA12dxV4KfC89vq/KC8AoEO17HgNU2o3WGGNMQ6vV7HkRCRaRpUA2MFtVF1Zq8iAwSUQycXrnt1dznskikiEiGbt3765H2cYYYyqrVaCrarmqDgRSgdNEpG+lJlcAr6hqKnA+8KqIHHNuVX1eVdNVNT0hofmtcmaMMU1Zne5vVdUcYA4wptKh64GpvjbzgXAg3h8FGmOMqZ3jBrqIJIhIrO95BHAusLZSs63AKF+bXjiBbmMqxhjTiGozyyUZ+LeIBOP8AJiqqh+JyMNAhqp+CNwFTBGRX+NcIL1G3do5wxhjWqjazHJZDpxcxfsPVHi+Ghjm39KMMcbUhW1wYYwxAcK1PUVFZDewxZUPP1o8sMftIqrQVOuCplub1VU3VlfdNJW6TlLVKqcJuhboTYWIZFS34aqbmmpd0HRrs7rqxuqqm6ZaV0U25GKMMQHCAt0YYwKEBTo873YB1WiqdUHTrc3qqhurq26aal2HtfgxdGOMCRTWQzfGmABhgW6MMQGixQa6iHQQkTkislpEVonIHW7XVJFvyeLvReQjt2s5RERiRWSaiKwVkTUicrrbNQGIyK99/w9Xisibvl223KjjJRHJFpGVFd5rKyKzRWS977FNE6nrr77/j8tF5L1D6zU1hdoqHLtLRFREGn2hv+rqEpHbff/dVonI441d1/G02EAHyoC7VLU3MAS4zbfbUlNxB7DG7SIq+T/gE1XtCQygCdQnIu2BXwLpqtoXZwvEiS6V8wrHrkR6H/C5qnYDPve9bmyvcGxds4G+qtofWAfc39hF+bzCsbUhIh2A83AW/nPDK1SqS0RGAhcBA1S1D/A3F+qqUYsNdFXdoapLfM/zccKpvbtVOUQkFbgAeMHtWg4RkRjgTOBFAFU96FtOuSkIASJEJASIxKUds1T1K2BfpbcvwtlvF9/jxY1aFFXXpaqzVLXM93IBzl4Hja6a/2YAfwfu5djd0RpFNXXdAjymqiW+NtmNXthxtNhAr0hE0nAWIKu8E5NbnsT5y+x1u5AKOuEsifyybyjoBRFp5XZRqpqF01PaCuwAclV1lrtVHaWdqu7wPd8JtHOzmGpcB3zsdhGHiMhFQJaqLnO7lkq6A2eIyEIR+VJETnW7oMpafKCLSGvgXeBXqprXBOoZB2Sr6mK3a6kkBDgFeFZVTwYO4M7wwVF8Y9IX4fzASQFaicgkd6uqmm9J6SY1T1hEfocz/Pi627UAiEgk8FvggeO1dUEI0BZniPYeYKqIiLslHa1FB7qIeHDC/HVVne52PT7DgPEishl4CzhbRF5ztyQAMoHMCvvJTsMJeLedA2xS1d2qWgpMB4a6XFNFu0QkGcD32GR+TReRa4BxwM+a0P4FXXB+OC/z/RtIBZaISJKrVTkygenqWITzG3ST2pmtxQa67yfri8AaVX3C7XoOUdX7VTVVVdNwLu59oaqu9zhVdSewTUR6+N4aBax2saRDtgJDRCTS9/90FE3gYm0FHwJX+55fDXzgYi2HicgYnGG98apa6HY9h6jqClVNVNU037+BTOAU398/t70PjAQQke5AKE1j9cXDWmyg4/SEr8LpAS/1fZ3vdlFN3O3A6yKyHBgI/MnlevD9xjANWAKswPk77cot2iLyJjAf6CEimSJyPfAYcK6IrMf5beKxJlLX00AUMNv3d/+5xq6rhtpcV01dLwGdfVMZ3wKubkK/2QB2678xxgSMltxDN8aYgGKBbowxAcIC3RhjAoQFujHGBAgLdGOMCRAW6MYYEyAs0I0xJkD8Py6CDJyf1ysCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "history=pd.DataFrame(history, columns=['epochs','loss','val_loss'])\n",
    "from matplotlib import pyplot \n",
    "pyplot.plot(history['epochs'],history['loss'], label='train') \n",
    "pyplot.plot(history['epochs'],history['val_loss'], label='test') \n",
    "pyplot.legend() \n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 536
    },
    "colab_type": "code",
    "id": "MwDoSYgYMLcE",
    "outputId": "ba09c67d-d73d-4b4f-e0ef-71f983ff00d9"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epochs</th>\n",
       "      <th>loss</th>\n",
       "      <th>val_loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5.310</td>\n",
       "      <td>5.244236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>4.778</td>\n",
       "      <td>4.856505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>4.531</td>\n",
       "      <td>4.616913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>4.382</td>\n",
       "      <td>4.453537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>4.264</td>\n",
       "      <td>4.268192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>4.182</td>\n",
       "      <td>4.211289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>4.120</td>\n",
       "      <td>4.105628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>4.079</td>\n",
       "      <td>4.075004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>4.055</td>\n",
       "      <td>4.055135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>4.038</td>\n",
       "      <td>3.990360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>4.011</td>\n",
       "      <td>3.980874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>4.002</td>\n",
       "      <td>3.960695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>3.999</td>\n",
       "      <td>3.902139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>3.989</td>\n",
       "      <td>3.978318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>3.973</td>\n",
       "      <td>3.925333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16</td>\n",
       "      <td>3.958</td>\n",
       "      <td>3.823909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>17</td>\n",
       "      <td>3.959</td>\n",
       "      <td>3.875296</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    epochs   loss  val_loss\n",
       "0        1  5.310  5.244236\n",
       "1        2  4.778  4.856505\n",
       "2        3  4.531  4.616913\n",
       "3        4  4.382  4.453537\n",
       "4        5  4.264  4.268192\n",
       "5        6  4.182  4.211289\n",
       "6        7  4.120  4.105628\n",
       "7        8  4.079  4.075004\n",
       "8        9  4.055  4.055135\n",
       "9       10  4.038  3.990360\n",
       "10      11  4.011  3.980874\n",
       "11      12  4.002  3.960695\n",
       "12      13  3.999  3.902139\n",
       "13      14  3.989  3.978318\n",
       "14      15  3.973  3.925333\n",
       "15      16  3.958  3.823909\n",
       "16      17  3.959  3.875296"
      ]
     },
     "execution_count": 43,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5cUmIlXWM7Uw"
   },
   "outputs": [],
   "source": [
    "#Evaluate results¶\n",
    "#Because our task is simple and the output is straight-forward, we will use MAE metric to evaluate the trained model during the epochs. Compute the value of the metric for the output from each epoch.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 293
    },
    "colab_type": "code",
    "id": "r5ZnQYTwLpXn",
    "outputId": "16156fa1-c6e1-455f-e075-8e2c81e70386"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1, MAE: 57.898399, Invalid numbers: 63\n",
      "Epoch: 2, MAE: 58.339805, Invalid numbers: 127\n",
      "Epoch: 3, MAE: 57.279348, Invalid numbers: 165\n",
      "Epoch: 4, MAE: 58.127807, Invalid numbers: 104\n",
      "Epoch: 5, MAE: 58.862261, Invalid numbers: 82\n",
      "Epoch: 6, MAE: 58.971174, Invalid numbers: 69\n",
      "Epoch: 7, MAE: 59.096178, Invalid numbers: 81\n",
      "Epoch: 8, MAE: 59.566813, Invalid numbers: 60\n",
      "Epoch: 9, MAE: 59.393306, Invalid numbers: 74\n",
      "Epoch: 10, MAE: 59.582851, Invalid numbers: 50\n",
      "Epoch: 11, MAE: 59.891960, Invalid numbers: 47\n",
      "Epoch: 12, MAE: 59.829067, Invalid numbers: 50\n",
      "Epoch: 13, MAE: 59.680285, Invalid numbers: 52\n",
      "Epoch: 14, MAE: 59.614254, Invalid numbers: 55\n",
      "Epoch: 15, MAE: 59.790584, Invalid numbers: 49\n",
      "Epoch: 16, MAE: 59.844391, Invalid numbers: 46\n",
      "Epoch: 17, MAE: 59.562348, Invalid numbers: 44\n"
     ]
    }
   ],
   "source": [
    "for i, (gts, predictions, invalid_number_prediction_count) in enumerate(zip(all_ground_truth,\n",
    "                                                                            all_model_predictions,\n",
    "                                                                            invalid_number_prediction_counts), 1):\n",
    "       \n",
    "\n",
    "          mae=mean_absolute_error_texts(gts, predictions)\n",
    "          print(\"Epoch: %i, MAE: %f, Invalid numbers: %i\" % (i, mae, invalid_number_prediction_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ifAvzROshiqL"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f76d6e5da20>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxU1dnA8d8zk4SENSwBAmGXHULAiFBQEVwQEeQtKCgW1Na6Wyttta8Va7VVq9b1rdVqwWoRtYpaEMUFUSsqoLLJpqCEJQSUPSHLnPePc2cyCVkmyczcSeb5fj7zmZl779z7JIT7zLnn3OeIMQallFIKwON2AEoppWKHJgWllFIBmhSUUkoFaFJQSikVoElBKaVUQILbAdRFmzZtTNeuXd0OQyml6pWVK1fuNcakVbSuXieFrl27smLFCrfDUEqpekVEvq1snV4+UkopFaBJQSmlVIAmBaWUUgH1uk9BqfqmqKiInJwcCgoK3A5FxYHk5GQyMjJITEwM+TOaFJSKopycHJo1a0bXrl0REbfDUQ2YMYZ9+/aRk5NDt27dQv6cXj5SKooKCgpo3bq1JgQVcSJC69ata9wq1aSgVJRpQlDRUpu/NU0KsaSoAFbOAV+J25EopeKUJoVY8uW/4PUb4JulbkeiGjARYfr06YH3xcXFpKWlMX78eAByc3MZP348gwYNol+/fowbNw6Abdu2kZKSQlZWVuDxzDPPVHiMTz/9lFNPPZXevXszePBgfvrTn3L06FEAFixYQGZmJn379mXgwIEsWLAg8Lnly5dz8sknk5WVRd++fbn99tsBmDNnDtdeey0At99+O40bN2bPnj2BzzVt2jTw2uv1lonx7rvvPi4+n8/H9ddfz4ABAxg4cCAnnXQSW7duBexNsXv37g3pd1U+rvvuu6/C38fevXtJTEzk8ccfL7O8a9euDBw4kMzMTE477TS+/bb0nrKmTZuyZs2awM/RqlUrunXrRlZWFmeccQZ9+vRhzZo1ge3//Oc/8/Of/7zC49eEdjTHkg2L7PPu1XDCGHdjUQ1WkyZNWLt2Lfn5+aSkpLBkyRI6duwYWH/bbbdx5plncsMNNwCwevXqwLoePXrwxRdfVLn/3NxcpkyZwvPPP8/w4cMBeOmllzh06BCbN29m1qxZLFmyhG7durF161bOPPNMunfvTmZmJjNmzOCFF15g0KBBlJSUsHHjxgqP0aZNG+6//37uueee49alpKRUG+P8+fPZuXMnq1evxuPxkJOTQ5MmTWr8uwrViy++yLBhw5g3bx5XXnllmXXvvfcebdq0Yfbs2dx55508+eSTgXUDBw4M/CwzZ85k/PjxTJ48GYDFixdz9dVXs2zZMnbu3Mnjjz8elgoP2lKIFccOw9b37etdq6veVqk6GjduHAsXLgRg3rx5TJs2LbBu165dZGRkBN5nZmbWaN+PPfYYM2bMCCQEgMmTJ9OuXTvuu+8+fvvb3wZGw3Tr1o1bbrmFP//5zwDs2bOH9PR0wH7j79evX4XHuOyyy5g/fz7ff/99jWLz27VrF+np6Xg89hSYkZFBy5YtK9y2qt9VqObNm8f999/Pjh07yMnJqXCb4cOHs2PHjpD3OXbsWNLT03nmmWe48cYbuf322yv9GWpCWwqx4ut3oaQQmra3LQXV4P3+9XWs33kwrPvs16E5s8/rX+12U6dO5Y477mD8+PGsXr2ayy67jA8++ACAa665hgsvvJBHH32UM844g0svvZQOHToA8PXXX5OVlRXYzyOPPMIpp5xSZt9r165lxowZFR533bp1zJo1q8yy7OxsHnvsMQBuvPFGevfuzahRoxg7diwzZswgOTn5uP00bdqUyy67jIceeojf//73Zdbl5+eXifGWW27hwgsvLLPNBRdcwMiRI/nggw8YM2YM06dPZ/DgwTX+XYVi+/bt7Nq1i6FDh3LBBRcwf/58brrppuO2W7x4Meeff37I+wV48MEHGTp0KD179uSSSy6p0Wcroy2FWLFxESSnwpCfwL6vbctBqQjJzMxk27ZtzJs3L9Bn4Hf22WfzzTff8LOf/YwNGzYwePBg8vLygNLLR/5H+YRQV7fddhsrVqzgrLPO4l//+hdjx46tdNvrr7+euXPncujQoTLL/ZeP/I/yCQFsy2Djxo386U9/wuPxMGbMGN55550Kj1PV7yoU8+fP54ILLgBsgpk3b16Z9aeffjodO3bkjTfeqHErpEOHDowePZqrrrqqxnFVJmItBRF5GhgP7DHGDAhafh1wDVACLDTG/NpZfgtwubP8emPMm5GKLeaUFMOmN6HX2dBxCGAgdy10HuZ2ZCqCQvlGH0kTJkxg1qxZLF26lH379pVZ16pVKy666CIuuugixo8fz7JlyzjxxBMr3M8rr7wS+Lb+97//nf79+7Ny5UomTpx43Lb9+vVj5cqVDBo0KLBs5cqV9O9f+rvo0aMHV111FT/72c9IS0s7Lja/1NRULrrookAroyqffPJJoBP2jjvuYMKECTRq1IhzzjmHc845h3bt2rFgwQLGjKm4L6+q31V15s2bx+7du3nuuecA2LlzJ5s3b6Znz56A7VNITU3l4osvZvbs2TzwwAM12r/H4wlcBguHSF4+mgM8CgSGJ4jI6cBEYJAx5piItHWW9wOmAv2BDsDbItLLGBMfYzO3fwL530PvcdDeuX67a7UmBRVRl112GampqQwcOJClS5cGlr/77rsMGzaMxo0bc+jQIb7++ms6d+5c6X4mTZrEpEmTAu87derE0KFDOffcczn55JMBePnllxkxYgSzZs1iypQpjB49mq5du7Jt2zb++Mc/8tJLLwGwcOFCxo0bh4iwefNmvF4vqamplR77l7/8JSeddBLFxcVV/qwnn3xymc7nVatW0b59ezp06IDP52P16tVV9p1U9ruqzqZNmzh8+HCZvoLZs2czb948brvttsCyhIQEHnzwQQYOHMitt95Kq1atQj5GuEXs8pExZhlQvhfoKuBuY8wxZxv/mLKJwPPGmGPGmK3AFmBopGKLORsXgTfJjjhq3gEat4bdX7odlWrgMjIyuP76649bvnLlSrKzs8nMzGT48OH89Kc/5aSTTgJK+xT8j4cffvi4z7dr147nn3+eWbNm0bt3b/r27cubb75Js2bNyMrK4p577uG8886jT58+nHfeedx7772BPoB//vOf9O7dm6ysLC655BKee+45vF5vpT9DmzZtmDRpEseOHQss8/cp+B8333zzcZ/bs2cP5513HgMGDCAzM5OEhITA0NKa/K7Ku/POO8nIyAg85s2bVyZhAvz4xz8+7hISQHp6OtOmTQup5RNJYoyJ3M5FugL/8V8+EpEvgFeBsUABMMsY85mIPAosN8Y862z3FPCGMealCvZ5BXAFQOfOnU8MHtdbLxkDjwyBVt1h+r/tsmcmQv4P8PNl7samwu6rr76ib9++boeh4khFf3MistIYk13R9tHuaE4AWgHDgF8BL0gN78M2xjxhjMk2xmSnpVU4m1z9sncTfP8N9D6ndFn7TNjzFZQUuReXUiouRTsp5AAvG+tTwAe0AXYAnYK2y3CWNXwb7PhnegUlhfRBdnhq3gZ3YlJKxa1oJ4UFwOkAItILSAL2Aq8BU0WkkYh0A3oCn0Y5NndsfAPSs6BF0F2SwZ3NSikVRRFLCiIyD/gY6C0iOSJyOfA00F1E1gLPAzOcVsM64AVgPbAYuCYuRh4d3gM5n0Gfc8sub90DEhvrTWxKqaiL2JBUY0xld2FMr2ihMeYu4K5IxROTNi0GTNn+BACPF9oN0JaCUirq9I5mN218A1p0tgmgvPRM2L0GfL7ox6WUiluaFNxSeBS+fs+2EioagNU+EwoPwQ9box+batCCy0yHy6hRowIVOseNG8f+/fuP26aq0tKRsHTp0kCJaxU6TQpu+WYpFOcff+nIr/1A+7x7TcXrlYpRixYtqvIu5PqipKThd2tWRJOCWzYuhEYtoOvIite37Qfi1c5mFTFLly5l1KhRTJ48mT59+nDxxRdjjGHx4sVMmTKlzHb+b9xXXXUV2dnZ9O/fn9mzZ1e43+BJau666y569erFyJEjK50bYebMmVx//fX86Ec/onv37oGSF+W/6V977bXMmTMncIxbbrmFrKwssrOzWbVqFWeffTY9evQoM5HNwYMHOffcc+nduzdXXnklPudy7FtvvcXw4cMZMmQIU6ZM4fDhw4H9/uY3v2HIkCG8+OKLPPzww/Tr14/MzEymTp1am19zvaOls93gK4GNi6HnGeBNrHibxGRI66OdzQ3ZGzeHvyXYfiCcc/xMY5X5/PPPWbduHR06dGDEiBF89NFHnHHGGVxxxRUcOXKEJk2aMH/+/MAJ8a677qJVq1aUlJQwZsyYKmsGrVy5kueff54vvviC4uJihgwZUmlRvV27dvHhhx+yYcMGJkyYEJhIpiqdO3fmiy++4MYbb2TmzJl89NFHFBQUMGDAgMBENp9++inr16+nS5cujB07lpdffplRo0Zx55138vbbb9OkSRPuueceHnjggUAtotatW7Nq1SrAViHdunUrjRo1qvCSWEOkLQU35KyAo3ttAbyqpGdqS0FF1NChQ8nIyMDj8ZCVlcW2bdtISEhg7NixvP766xQXF7Nw4cJAxdMXXniBIUOGMHjwYNatW8f69esr3fcHH3zApEmTaNy4Mc2bN2fChAmVbnv++efj8Xjo168fubm5IcXu39/AgQM5+eSTadasGWlpaWVO4EOHDqV79+54vV6mTZvGhx9+yPLly1m/fj0jRowgKyuLuXPnlpkGM7jUdmZmJhdffDHPPvssCQnx8R06Pn7KWLNxEXgS4IQzqt6ufSZ8OQ8O5UKzdtGJTUVPDb7RR0qjRo0Cr71eb6Da6NSpU3n00Udp1aoV2dnZNGvWjK1bt3Lffffx2Wef0bJlS2bOnElBQUHY4/DXY0tISAhc7gGOO5b/Mx6Pp8znPR5P4OcoX0VHRDDGcOaZZ1ZYlA4oMy3nwoULWbZsGa+//jp33XUXa9asafDJQVsKbti4CLqMgJRqOuPSnWa5thZUlJ122mmsWrWKJ598MnDp6ODBgzRp0oQWLVqQm5vLG2+8UeU+Tj31VBYsWEB+fj6HDh3i9ddfr1EMXbp0Yf369Rw7doz9+/dXOglOVT799FO2bt2Kz+dj/vz5jBw5kmHDhvHRRx+xZcsWAI4cOcKmTZuO+6zP52P79u2cfvrp3HPPPRw4cCDQ99CQNeyUF4v2brFF8E76afXb+kcg7foSep4Z2biUCuL1ehk/fjxz5sxh7ty5AAwaNIjBgwfTp08fOnXqxIgRI6rcx5AhQ7jwwgsZNGgQbdu2DZTfDlWnTp244IILGDBgAN26dat0usyqnHTSSVx77bVs2bKF008/nUmTJuHxeJgzZw7Tpk0LlNy+88476dWrV5nPlpSUMH36dA4cOIAxhuuvv75BjKqqTkRLZ0dadna28Y+Nrjf++wi8dSv8Yg2kVj5xScBDg2xtpAvmRj42FXFaOltFW6yXzlYbFkG7gaElBLCtBb18pJSKEk0K0XRkH2xfXvkNaxVpP8jOt1BwMHJxKaWUQ5NCNG1+E4wP+lQzFDWYv7M5d21kYlJRV58v2ar6pTZ/a5oUomnjImiWbvsIQqVzKzQoycnJ7Nu3TxODijhjDPv27SM5OblGn9PRR9FSVABb3oVBF1ZcAK8yzdpDkzTtV2ggMjIyyMnJIS8vz+1QVBxITk4mIyOjRp/RpBAtW5dB0RHofW712wYTsa0FbSk0CImJiXTr1s3tMJSqlF4+ipaNCyGpKXQ7peafTc+EvK+g+Fj441JKqSCaFKLB57MF8E4YAwmNqt++vPaZ4CuGvA3hj00ppYJoUoiGXZ/D4d3VF8CrjHY2K6WiRJNCNGxYZOdG6HlW7T7fqru99KSdzUqpCNOkEA0b34DOw6Fxq9p93uOx8zhrS0EpFWGaFCLth22wZ13NblirSHqmvYEtqJSwUkqFmyaFSNvolBeuSWmLirTPhMLDtuSFUkpFiCaFSNuw0E6r2ap73fYTmFvhy7rHpJRSlYhYUhCRp0Vkj4gcV7RHRG4SESMibZz3IiIPi8gWEVktIkMiFVdU5f8A3/639qOOgqX1BU+i9isopSIqki2FOcDY8gtFpBNwFvBd0OJzgJ7O4wrgrxGMK3o2LwFTEp6kkJBkWxzhnuhdKaWCRCwpGGOWAd9XsOovwK+B4IpgE4FnjLUcSBWR9EjFFjUbF0GTttDxxPDsLz3TDkvVYmpKqQiJap+CiEwEdhhjyl8Y7whsD3qf4yyraB9XiMgKEVkR00XFio/B5reh91g7pDQc2mfCkTw4tDs8+1NKqXKilhREpDHwW+C2uuzHGPOEMSbbGJOdlpYWnuAiYduHUHio5gXwqhLobNZ+BaVUZESzpdAD6AZ8KSLbgAxglYi0B3YAnYK2zXCW1V8b34CEFOh+Wvj22W6AfdbOZqVUhEQtKRhj1hhj2hpjuhpjumIvEQ0xxuwGXgN+4oxCGgYcMMbsilZsYWeMTQo9RkNiSvj2m9zcDm3VYalKqQiJ5JDUecDHQG8RyRGRy6vYfBHwDbAFeBK4OlJxRcXu1XAwp+53MVdE51ZQSkVQxCbZMcZMq2Z916DXBrgmUrFE3YZFgEDPs8O/7/RMWL8A8vdDSmr496+Uimt6R3MkbFwEnU6GphHoCPeX0c497p5ApZSqM00K4bZ/u718VNdaR5XRuRWUUhGkSSHcNi22z33COBQ1WLN20LSdDktVSkWEJoVw27AQWp8AbXpG7hja2ayUihBNCuFUcMDetBapS0d+6Zl2vuaigsgeRykVdzQphNOWd8BXFN67mCvSPtMW2tuzPrLHUUrFHU0K4bRxETRuDZ2GRvY4Wu5CKRUhmhTCpaQINr8FvcaCxxvZY6V2hUbNtV9BKRV2mhTC5dv/2j6FSPcngK262m6Azq2glAo7TQrhsvEN8Day9Y6iIT3T3sDmK4nO8ZRScUGTQjgYAxsXQvdRkNQkOsdsnwlFR2Hf19E5nlIqLmhSCIc962H/d5EpgFcZ7WxWSkWAJoVw2LDIPvc6bkrqyEnrA94k2KVltJVS4aNJIRw2LrLzMDdrH71jehOhbV9tKSilwkqTQl0d3AU7V0HvKF468vOXuzAm+sdWSjVImhTqatsH9rnnWdE/dvogyP8eDtbvmUuVUrFDk0Jd7V5jr+237Rv9Y7cfWBqDUkqFgSaFuspdB2m97TX+aGs3ABC9s1kpFTaaFOoqdy20G+jOsRs1hdY9tLNZKRU2mhTq4nAeHM6F9gPci0HnVlBKhZEmhbrwz5Pcrr97MaRnwoHv4Oj37sWglGowNCnURSApuNxSAO1sVkqFhSaFushdB03bQ5M27sXQXstdKKXCR5NCXexe625/AkDTNGiWrv0KSqmwCCkpiEhjEfmdiDzpvO8pIuOr+czTIrJHRNYGLfuziGwQkdUi8oqIpAatu0VEtojIRhE5u7Y/UNQUF9p5kt3sT/Brn6mXj5RSYRFqS+EfwDFguPN+B3BnNZ+ZA5SvELcEGGCMyQQ2AbcAiEg/YCrQ3/nM/4lIhKcvq6N9m+18zG4NRw2Wngl7N0FRvtuRKKXquVCTQg9jzL1AEYAx5iggVX3AGLMM+L7csreMMcXO2+VAhvN6IvC8MeaYMWYrsAWI8ETHdbTbaQC5ffkIbEvBlEDuercjUUrVc6EmhUIRSQEMgIj0wLYc6uIy4A3ndUdge9C6HGfZcUTkChFZISIr8vLy6hhCHeSuteUtWp/gXgx+gbkVtIy2UqpuQk0Ks4HFQCcReQ54B/h1bQ8qIv8LFAPP1fSzxpgnjDHZxpjstLS02oZQd7lrnTkNXChvUV5qF0huoZ3NSqk6SwhlI2PMEhFZBQzDXja6wRiztzYHFJGZwHhgjDGBms87gE5Bm2U4y2JX7jroMcbtKCwRp7NZk4JSqm5CHX00CSg2xiw0xvwHKBaR82t6MBEZi21hTHD6JfxeA6aKSCMR6Qb0BD6t6f6jJhbKW5TXPtMmqpLi6rdVSqlKhHz5yBhzwP/GGLMfe0mpUiIyD/gY6C0iOSJyOfAo0AxYIiJfiMjjzv7WAS8A67GXqa4xxpTU+KeJllxn+GcsDEf1az8QigvsqCillKqlkC4fUXHyqPKzxphpFSx+qort7wLuCjEed+Wus8+xMBzVLz2o3IUbczsopRqEUFsKK0TkARHp4TweAFZGMrCYtnutvYu4SWu3IynVphd4G8EuHYGklKq9UJPCdUAhMN95HAOuiVRQMS93XWxdOgI7CqpdP+1sVkrVSaijj44AN0c4lvrBX97ihBgZeRSsfSasfxWMsSOSlFKqhkIdfdRLRJ4QkbdE5F3/I9LBxaRAeYsYGnnkl54JBfvhwPbqt1VKqQqE2tH8IvA48HcgdkcFRUMslbcor/0g+7xrNaR2djcWpVS9FGpSKDbG/DWikdQXuWtip7xFee36g3hsv0LfKovYKqVUhULtaH5dRK4WkXQRaeV/RDSyWJW7LnbKW5SX1NgmKy13oZSqpVBbCjOc518FLTNA9/CGUw/sXgs9z3Q7isq1z4TvlrsdhVKqngp19FG3SAdSLxzeA0f2xN5w1GDpmbD2JTj6PTSOz8acUqr2Qm0pICIDgH5Asn+ZMeaZSAQVs3KdTuZYHHnk55+zedeX0ON0d2NRStU7oQ5JnQ084jxOB+4FJkQwrtgUKG8Rw0kh3RmBpDexKaVqIdSO5snAGGC3MeZSYBDQImJRxapYLG9RXuNW0DxDO5uVUrUSalLIN8b4sCWzmwN7KDv/QXzIXRvb/Ql+6Tq3glKqdmpSEC8VeBJbCG8Vtix2/CguhLyNsX3pyK/9QNi7GQqPuB2JUqqeCXX00dXOy8dFZDHQ3BgTX19F926y5S3ax1C57Mq0zwSM7QPpNNTtaJRS9UhNRh9lAl39nxGRE4wxL0cortgT6GSuJ5ePwF5C0qSglKqBkJKCiDwNZALrAJ+z2ABxlBT85S16uh1J9Vp0guRU7WxWStVYqC2FYcaYfhGNJNYFyluE3Lhyj4h2NiulaiXUjuaPRSS+k8LutfWjP8GvfSbkroeSIrcjUUrVI6F+7X0Gmxh2Y2ddE8AYYzIjFlksqQ/lLcpLHwQlx+ww2g6D3Y5GKVVPhJoUngIuAdZQ2qcQP+pDeYvyup8OiU3go4dhyj/cjkYpVU+EevkozxjzmjFmqzHmW/8jopHFkt31MCk0TYNhV8G6l20dJKWUCkGoSeFzEfmXiEwTkf/xPyIaWSzJXRf75S0qMuJ6SGkJ79zhdiRKqXoi1KSQgu1LOAs4z3lUObWXiDwtIntEZG3QslYiskRENjvPLZ3lIiIPi8gWEVktIkNq9+NESO7a+tVK8EtuASNvhC1vw7YP3Y5GKVUPVJsURMQL7DPGXFrucVk1H50DjC237GbgHWNMT+Ad5z3AOUBP53EFEDtTfwbKW9SjTuZgQ6+wrZy3fw/GuB2NUirGVZsUjDElwIia7tgYswz4vtziicBc5/Vc4Pyg5c8YazmQKiLpNT1mRNSn8hYVSUyB034DOZ/CpsVuR6OUinGhXj76QkReE5FL6tin0M4Ys8t5vRto57zuCGwP2i7HWXYcEblCRFaIyIq8vLxahFBD9XHkUXmDp0OrHrZvwVfidjRKqRgWalJIBvYBowmxT6E6xhiDLZVR0889YYzJNsZkp6Wl1SWE0OSuBW8jaH1C5I8VKd5EGP2/sGc9rHnJ7WiUUjEs1Cqpl4bpeLkikm6M2eVcHtrjLN9B2fkZMpxl7tu9FtrWk/IWVek3Cdo/CO/dBf0nQUKS2xEppWJQqNNxZojIK85ooj0i8m8RyajF8V4DZjivZwCvBi3/iTMKaRhwIOgyk7ty19XvS0d+Hg+MmQ37v4VVc6vfXikVl0K9fPQP7Im7g/N43VlWKRGZh52Ip7eI5IjI5cDdwJkishk4w3kPsAj4BtiCncjn6gp2GX2B8hYNICkAnDAGuoyE9+/VCXiUUhUK9ZpImjEmOAnMEZFfVPUBY8y0SlaNqWBbA1wTYizRs3uNfa6vw1HLE4EzZsNTZ8Lyv8Kps9yOSCkVY0JtKewTkeki4nUe07Edzw2bf2Kd+joctSKdhkLvcbYm0tHyI4aVUvEu1KRwGXABdhjpLmAyEK7O59iVuxaadYDGrdyOJLxG/w6OHYSPHnQ7EqVUjKkyKYjIPc7LocaYCcaYNGNMW2PM+caY76IQn7ty1zWcS0fB2vWDzAvhk7/BwZ1uR6OUiiHVtRTGiYgAt0QjmJjiL2/RvoF0Mpd3+i32Rrb373U7EqVUDKkuKSwGfgAyReSgiBwKfo5CfO7xl7doKCOPymvZFbIvhVXPwL6v3Y5GKRUjqkwKxphfGWNSgYXGmObGmGbBz1GK0R0NobxFdU79FSQ0sje0KaUUoVdJbdgJoCK719T/8hbVadoWhl0Na/8Nu1a7HY1SKgaEWiXVJyItohBP7Mhd1zDKW1TnR9dBcqpOxKOUAkK/ee0wsEZElgCBW2GNMddHJKpYkLsWep7tdhSRl5IKp/wSltwG2z6CrjWukq6UakBCvU/hZeB3wDJgZdCjYTq8B47kNczhqBXxT8Tzjk7Eo1S8C7VK6lwRSQE6G2M2Rjgm9/nLWzTU4ajl+Sfi+c8vYNOb0Lv8hHlKqXgRapXU84AvsENUEZEsEXktkoG5yl/eoiGPPCpv8HRo1d2ZiMfndjRKKZeEevnodmAosB/AGPMF0D1CMbmvoZa3qIo3EUbfCnvWwVqdiEepeBVqUigyxhwot6zhfp3cvTZ++hOC9Ztki/+9e6e9o1spFXdCTQrrROQiwCsiPUXkEeC/EYzLPcWFsLcBl7eoiscDY27XiXiUimOhJoXrgP7AMeBfwAGgyvkU6q29G8FXHF/9CcFOGANdRuhEPErFqeqqpCY7k+ncC3wHDDfGnGSMudUYUxCVCKMtHjuZg4nYaTuP7IFPHnc7GqVUlFXXUpgLZANrgHOA+yIekdviobxFdTqfDL3OgQ8f0ol4lIoz1SWFfsaY6caYv2En1jk1CjG5K17KW1RnjIenmm4AABtpSURBVH8inofcjkQpFUXVJYUi/wtjTHGEY4kNuWuhXQOafrO22vWHzAvsJaSDu9yORikVJdUlhUHO/AkHReQQ5eZViEaAUXUoN77KW1RnlDMRzzKdiEepeFHdfApeZ/4E/xwKCQ16PgX/HArxOBy1Iq26wYkzdSIepeJIqENS40M8TKxTU6f+CrxJ8N4f3Y5EKRUFmhSC5a6Lv/IW1WnWDoZdZUtfbP3A7WiUUhHmSlIQkRtFZJ2IrBWRec79EN1E5BMR2SIi80UkKeqB7V6rl44qMuIX0KY3PH9x6X0cscAYyNuo5b6VCqOoJwUR6QhcD2QbYwYAXmAqcA/wF2PMCcAPwOVRDcxf3kI7mY+X3Bym/xuSGsOzk+FAjtsR2UTwzh3w2FD49Am3o1GqwXDr8lECkCIiCUBjYBcwGvCX55wLnB/ViOK9vEV1UjvZxFB4GP75P+7f1Pb+vfDhA9CoOSy9G/L3uxuPUg1E1JOCMWYH9s7o77DJ4AB2Frf9QfdC5AAdK/q8iFwhIitEZEVeXl74AtutnczVatcfpv4LftgK86ZBUb47cXz4F1j6Rxh0Ecx4HfJ/sAlCKVVnblw+aglMBLoBHYAmQMhTfRljnjDGZBtjstPS0sIXWO5aLW8Rim6nwP88Ads/gX//1N7HEE0f/x+8fTsM+DFMfBQ6ZMGgabD8cfjh2+jGolQD5MblozOArcaYPGNMEXb+5xFAqnM5CSAD2BHVqHLXQtu+Wt4iFP0nwdi7YcN/YNGs6HX0fvZ3ePMW6HseTPobeLx2+ehbbSG/d/8QnTiUasDcSArfAcNEpLGICDAGWA+8h62vBDADeDWqUeWu00tHNTHsSjsqacXTsCwKdRJX/RMW3gS9xsKPn7Yzxfm16AjDr4U1L8KOlZGPRakGzI0+hU+wHcqrsNVXPcATwG+AX4rIFqA18FTUgvKXt9DhqDVzxu2QORXeu9OetCNl9Qvw2nXQYzRMmQsJFYxWHvkLaJIGb/1Oh6gqVQeuXCsxxswGZpdb/A12HujoC9zJrMNRa0TEXtc/sgdevwGatoVeZ4f3GOtegVd+Dl1HwoXPQWJyxds1amZrNS38JWxcBH3ODW8cSsUJvaMZtLxFXXgT4YJn7NzOL86EnBXh2/eGhbYzO2MoTHve3idRlSEzoE0vWHIblBRVva1SqkKaFMAOR9XyFrXXqBlc/KJtKTw3BfZuqfs+Ny+BF2ZA+iC770ZNq/+MNwHO/APs2wIr59Q9BqXikCYFsJ3M2p9QN03bwvSXQTzw7CTbT1Nb3yyF+dPtaLDp/7Z3VIeq19nQ9RRY+icoOFD7GJSKU5oUio855S00KdRZ6x5w8QtwZC88NxkKajHlxrf/tTfGteoBlyyAlJY1+7wInHUnHN1nb3Jzww/f2nIgu1a7c3yl6kCTwt5NTnkL7WQOi44nwgX/hD3r7bf94sLQP7v9M3v5qUUG/GQBNGlduxg6ZNlRUR//H+zfXrt91NaxQzBvKmxZYm+yU6qe0aTgL2/RXqfgDJueZ8CER2Dr+/Dq1eDzVf+ZnZ/Dsz+2w0p/8pq9HFUXbtzQ5iuxHeN5G+0Ndl+/o/dNqHpHk4K/vEWrHm5H0rBkXQRjbrM3lL19W9Xb7l4D/5wEKS1sLaPm6XU/fmonGHY1rJ5vE040vPN72LQYzrkHzv8rJKfCsvujc2ylwkSTgpa3iJyRv4ShV8B/H4GPH6t4mz0b4JnzIbGxTQipncJ4/BuhcZvo3ND2xb/go4cg+3IY+jM7ImvY1bBxYWlrVKl6IL6TgjE6sU4kidgaSf0mwpu/hTUvlV2/dws8M8HWMJrxOrTsGt7jJzeHUTfDtg/sN/hI+e4Te/Net1NtK8Hv5CsgqRl8EIUyIEqFSXwnhcN74OheHXkUSR4vTHoCuoyAV66Eb963y7/fCnPPs9fhf/KaHbkUCSfOhNY9nRvaiqvdvMb2fwfPX2Q7x6fMLVuTKaWlbTWsWwB5m8J/bKUiIL6TQu4a+6xJIbISk+08DG162ik9N75hWwjF+fCTV6Ftn8gd25sIZ/7ejjJbNTe8+z522A6fLSmCafMrvvlx+DWQmKLzPah6I86TgjPfsA5HjbyUVLj4JXtJZ95UyD8Al7wSnUt3vcfZlsrSP9Xu3omK+Hzw8hV26O2Uf0Bar4q3a9IGsi+zRf2+3xqeYysVQfGdFHavheYdtbxFtLToaO96PuEMe6dyh8HROa4InPUHWwn3o4fCs89377CdyGf/CU4YU/W2w68FT4J7N9MpVQPxnRRy12orIdra9rEJodNJ0T1uxxNh4BT4+FE4UMf5m7583p7gT5wJJ/+8+u2bp8OQS+wIpQM5dTu2UhEWv0mh+Ji9zqz9CfFjtDM09d07a7+P7Z/ZuR26ngLj7rOtkFCMuAEw8NHDtT+2UlEQv0khb6Mtb6HDUeNHyy52xrgv58GuL2v++f3b7Uij5h1tufDgkUbVSe1s55JeNbduxQKVirD4TQqBTmZNCnFl5C/tUNG3bq3ZDW2FR+D5aVBcABdVMtKo2mPfCCWF9hKWUjEqjpPCWkhI1vIW8SYl1d7QtnWZnbMhFP6RRrnrYPI/IK137Y7dugcMmAyfPQVH9tVuH0pFWHwnhbQ+Wt4iHp14qf0ysOR3od3Q9t5dsOE/cNZdtthfXZxyExQdhU/+Wrf9KBUh8ZkUtLxFfEtIsje05W2Az/9Z9barX7RlKob8BIZdVfdjt+0D/SbAJ3+D/P11359SYRafSeFwrpa3iHd9xkPn4fDeH+0cCBXJWQGvXgNdRsK4+0MfaVSdU2bBsYPw2ZPh2Z9SYRSfSSHXqVqpSSF++WdoO7Kn4mGiB3KckUbpdqRRQlL4jp2eCb3G2kmAjh0O336VCoP4TAqNWsCAH+uNa/EuI9v+Hfz3ETi4s3R54RFb06jwKEx7vvYzwFXllFmQ/z2seDr8+1aqDuIzKXQ6CSY/reUtlJ0IyJTAu3fZ9z4fLLjKtiYnP23n2oiETidB91E2IRXlR+YYStWCK0lBRFJF5CUR2SAiX4nIcBFpJSJLRGSz81zDGduVqoWWXW2pii+eszPAvX83rH8VzvwD9Dorssc+9df28tWqajq7lYoit1oKDwGLjTF9gEHAV8DNwDvGmJ7AO857pSLvlJvs/Qvzp8P798Dg6bbkdaR1HQGdfwQfPQjFhZE/nlIhiPogfRFpAZwKzAQwxhQChSIyERjlbDYXWAr8JtrxqYbNGENRiSG/qIT8whLn2UuzgdfR6dM7OND2JDb1/x2e7/aT4BG8ziMh8OzB6xW8ErTcW7rev1xEMMbgM1BU4qPEZyj2GefZeV9iSMy6lvavXcSuD/7Bvl5TA+uLS0yZzxwr9lFY4qOw2D6K/K9LfHZdBcsLi+264GVFJT6MAY8AIngEBPCIIAKC8ywVLTt+e4/z83r8vwuxr70iZX5PHhESvOJsj7Pcg9dDYPvA8YKO439dGqvzXFFcQct8xlDigxKfr/T37vxOS0zw+6D1znPpa7vOGPt3Y7Cj2X3OawwYTOkyQ2Cb0u1LP2eABI+Q6BUSvR6SEjwkeT0k+h8JQqPAa0/QNlK6jddDI2ddp1YpdGndJOz/R8REeu7a8gcUyQKeANZjWwkrgRuAHcaYVGcbAX7wvy/3+SuAKwA6d+584rfffhut0OOSMYbDx4o5fKyY4hKDzznR+YzB57OvS3wm8J+ixJjj1vlPjsHrSnyl+/F/3v8f2ee8L6lk2xL/sZx1BUUlgZP80aISCpyT/dHCksC6o4V2+dGiEkp8x//NJ1DMj70f8GZJNvtpVuffm0eggsNU9BtmQdLvaMlhRhfeTwneWh0vyTmBJHrFnkicE05SgjdwYklyTiaCPUH5/Ccv5+Tm85We5ALLgk5wPufs5gvaPvjfxOecTH2BE689KZcE/S0En5RjRaKTrAKJ3+sJJHyPCB5PabLxiCAA5ZKWfz0EJdNyiQvs33YgeTtJuqjElEnkobrytB7cfE7tJqgSkZXGmOyK1rlxO28CMAS4zhjziYg8RLlLRcYYIyIV/tUYY57AJhWys7Oj9pflc76tBU5ARfaEU1BUQlGJoWXjJNKaNSI1JRGPJ0zj2cPAGPut+GB+MQcLijiQX8TB/CIOFhTZZc5ru9xuE1hXYLeNof+/FWqU4CElyUvjRC/JSV5SEr00TvLSLDmBts0a0TjJS0qSl5TEBFKSPKQkeklJSnCePc5yL8kJI5kEZb4pVvVts9hnKCmp+Jtmic+Ufnt2vh16PZ6gFof/BCQU5t5El+XXsODUneR1nxTYLsFT+u06KcF+Q0zyeklMkEAS8J/8/Sed+iLwRcH/BcD/xQIwvooSkv2q7SufrCpJYGVO8kGtPI8H29rzn/Rj6P8q2PhLfMYmjGITlDiCW3uGohIf7ZolRyQGN5JCDpBjjPnEef8SNinkiki6MWaXiKQDeyIVwPqdB3nl8xznxO6zz4UlFBT7Lyn4OHbcyT+0DO71CK2b2ATRpql92Nely/zPNUkgxhiOFpawP7+IH44Usv9oEfvzC/nhaBEHjtrnH44WcsB53p9fxP6j9qReXM1ZPSXRS4uURJqnJNA8OZG2zZI5IS2B5imJdnlyIk2TE8p8c/KIlF46kNL35deJELikELxdYF3QJQeP+F8TuOTg39YrggQuO5Tu13+ppl4zF8PWxxn4zVMw9md2XusGTkTwOv/+qpQ4l9kSvB4I460xNRH1pGCM2S0i20WktzFmIzAGeylpPTADuNt5fjVSMXz3/RGeXf6d8+3QE/h2ab9hJtCqiTewLsVZ1yjRv42H5ERnfaJ9JHiE748UsvfwMfIOHWPv4WPsPVxI3qFjbMo9xN7DxygqqeCShUdo3TSpTPJokZLI4YLioBO7kwCOFlFYUnliapzkpWXjJFIbJ5LaOJH01BRS/Sd158TePCUhcJK3yxJolpxIUkJ8jkyOGSJw6k3w4kz46jXoP8ntiFQci3qfAgT6Ff6OzYXfAJdiR0K9AHQGvgUuMMZ8X9V+srOzzYoVKyIcbd0ZYziYX0ze4QLyDpVPHqUJZO/hYxzIL6Jpo4QyJ3j7Osl5nUiLlCRaNk6kZZMke+JvnEijhIb/7bJB85XA/w0DbxJc+WH4SmooVYFY61PAGPMFUFFA1Ux2Wz+JCC0a25P3CW3djkbFJI/XDo195eew8Q3oM87tiFSc0usGSsWKAZPtzXTL/lyzCYCUCiNNCkrFCm+CnRlu5yr4+l23o6lfivLhm6Xwzh0wdwIsvQcKDrodVb3kSp9CuNSXPgWlQlZcCA9nQWoXuOwNt6OJXcWFNnluXWYf2z+xU52KF9r0gryv7LSrI34BQ38GSeG/yas+i7k+BaVUJRKS7InsjV/Bto9sKQxlO+J3ry5NAt9+DEVHALGlyE/+OXQ7DToPg0bNYOfndq6Mt2fDx4/Z/poTZ0JiZMb2NyTaUlAq1hTlw4OZ0K4f/CRiI7NjmzF2Zjx/Etj2ARQcsOvS+kC3U+2jy4iqqx1/9wm8d6fdR/OOcOosyJoe3vkx6qGqWgqaFJSKRf99BN66FS5/25bZbuiMgR+2liaBrcvgSJ5d17KrkwROg64joVn7mu//m/ftXNvbP7GX5kbdDAMviNs52jUpKFXfHDsMDw6ETkPhovluRxNeRfmQtxH2fAV71tvn3HVwyJnoqGl76H6aTQRdT4GWXcJzXGNgy9vw7h9g15fQuiecfgv0mwSe+Bpzo30KStU3jZrC8Kvh3TvtCSx9kNsR1VxJEezb4pz4N5QmgO+/wZbkw96s16a3bQF0Ptm2BlqfEJmb90Sg55lwwhmwYaFtObx0GbS9H0b/L/QepzcNoi0FpWJXwQH4y0DoMcrOEx2rfD7Yv63sN/89X8HezeArstuIx57s2/aFtv3sc1pfaNXdvUs4Ph+sexmW/skmrw6DYfSt0GNMg08O2lJQqj5KbgEnXwHL7oOPHoYTxtgTqduXOg7udDp/P7TTluZthKKjpetTO9sTf6+zSxNA656xN/LH44GBk6Hf+bB6vp1179kfQ+fhNjl0Hel2hK7QloJSsezo9/DMRDscE6BxG+h2SmnHa6vukf9We2SvHf3j7wDet8UuT2lpL2v5T/xt+0FabzsktD4qLoTP/2nvKD+0y/5+R/+uQXb0a0ezUvXd/u1lT8wHd9jlzTuWDs/segqkdqr7sQoOwLf/LT1W7lq7PKmpHQLqP167Ae63WiKhKB9W/AM+fMCOgKrJENh6QpOCUg2JMbazduv7zon7Azi6165r1d0mB/9JrGkIFRgLj8L25aVJYOfndqabhGTodHJpq6RDFngTI/uzxZJjh23LYcvbNkkWHSVws5z/d+K/Wa6e0aSgVEPm89myDoEbvT6EY07dn7S+QS2JEfaST3Eh7FgRVCLiU9sh7EmAjJNKt++YHXv9AG6prKyGJwE6nlj6O8sYWi9+Z5oUlIonJcWw+8uyJSGK8wGx1/z3f1f6rbdDVukJrdMwOxRWVa8o3yYG/+94xyowJeBtZO8t6ebcZ9FxSEy2rjQpKBXPAi2DDyDnM2jdw7k+/iPbclB1V3AQvvvYSRLvw+41dnliE/t7Du6HiYG7qDUpKKVUNB3ZB99+WNqS2LvJLvcm2SqubfuWvWejReeodtrrfQpKKRVNTVpDv4n2AXBwl3Nfxxp7Y993y2HNi6XbJzaBtn3KJoq2/aBpu6jfSKdJQSmlIq15OmROAaaULis44NSAWl96N/imN+HzZ0u3SWkZlCT894L0ieiwWE0KSinlhuQWtlO609Cyyw/n2dFkgbIhG2D1i3DsQOk2zdJh+DXwo+vCHpYmBaWUiiVN0+yj26mly4yx5UWC60s1S4/I4TUpKKVUrBOBFh3to+cZET1UA7xHXSmlVG25lhRExCsin4vIf5z33UTkExHZIiLzRSS+58tTSikXuNlSuAH4Kuj9PcBfjDEnAD8Al7sSlVJKxTFXkoKIZADnAn933gswGnjJ2WQucL4bsSmlVDxzq6XwIPBrwOe8bw3sN8YUO+9zgI4VfVBErhCRFSKyIi8vL/KRKqVUHIl6UhCR8cAeY8zK2nzeGPOEMSbbGJOdlpYW5uiUUiq+uTEkdQQwQUTGAclAc+AhIFVEEpzWQgaww4XYlFIqrkW9pWCMucUYk2GM6QpMBd41xlwMvAdMdjabAbwa7diUUireuVolVURGAbOMMeNFpDvwPNAK+ByYbow5Vs3n84Bva3n4NsDeWn42kmI1Lojd2DSumtG4aqYhxtXFGFPh9fd6XTq7LkRkRWWlY90Uq3FB7MamcdWMxlUz8RaX3tGslFIqQJOCUkqpgHhOCk+4HUAlYjUuiN3YNK6a0bhqJq7iits+BaWUUseL55aCUkqpcjQpKKWUCojLpCAiY0Vko1Om+2a34wEQkU4i8p6IrBeRdSJyg9sxBStf6jwWiEiqiLwkIhtE5CsRGe52TAAicqPzb7hWROaJSLJLcTwtIntEZG3QslYiskRENjvPLWMkrj87/46rReQVEUmNdlyVxRa07iYRMSLSJlbiEpHrnN/bOhG5NxzHirukICJe4DHgHKAfME1E+rkbFQDFwE3GmH7AMOCaGInLr3yp81jwELDYGNMHGEQMxCciHYHrgWxjzADAi71z3w1zgLHllt0MvGOM6Qm847yPtjkcH9cSYIAxJhPYBNwS7aAcczg+NkSkE3AW8F20A3LMoVxcInI6MBEYZIzpD9wXjgPFXVIAhgJbjDHfGGMKsXdRT3Q5Jowxu4wxq5zXh7AnuAorxUZb+VLnsUBEWgCnAk8BGGMKjTH73Y0qIAFIEZEEoDGw040gjDHLgO/LLZ6ILU0PLpWoryguY8xbQVWSl2Prn0VdJb8zgL9gKzu7MjKnkriuAu72V34wxuwJx7HiMSl0BLYHva+0TLdbRKQrMBj4xN1IAsqXOo8F3YA84B/OZa2/i0gTt4MyxuzAfmP7DtgFHDDGvOVuVGW0M8bscl7vBtq5GUwlLgPecDsIPxGZCOwwxnzpdizl9AJOcWasfF9ETgrHTuMxKcQ0EWkK/Bv4hTHmYAzEU6dS5xGUAAwB/mqMGQwcwZ1LIWU41+gnYpNWB6CJiEx3N6qKGTsePabGpIvI/2IvpT7ndiwAItIY+C1wm9uxVCABWytuGPAr4AVnwrI6iceksAPoFPQ+Zsp0i0giNiE8Z4x52e14HP5S59uwl9pGi8iz7oYE2BZejjHG35p6CZsk3HYGsNUYk2eMKQJeBn7kckzBckUkHcB5Dsslh3AQkZnAeOBiEzs3UPXAJvgvnf8DGcAqEWnvalRWDvCysT7FtuTr3Akej0nhM6CniHQTkSRsJ+BrLsfkn5L0KeArY8wDbsfjV0mpc9e/+RpjdgPbRaS3s2gMsN7FkPy+A4aJSGPn33QMMdABHuQ1bGl6iKES9SIyFnuJcoIx5qjb8fgZY9YYY9oaY7o6/wdygCHO35/bFgCnA4hILyCJMFRzjbuk4HRmXQu8if3P+oIxZp27UQH2G/kl2G/iXziPcW4HFeOuA54TkdVAFvBHl+PBabm8BKwC1mD/j7lSJkFE5gEfA71FJEdELgfuBs4Ukc3YVs3dMRLXo0AzYInzt/94tOOqIjbXVRLX00B3Z5jq88CMcLSwtMyFUkqpgLhrKSillKqcJgWllFIBmhSUUkoFaFJQSikVoElBKaVUgCYFpaJIREbFUqVZpcrTpKCUUipAk4JSFRCR6SLyqXMj1d+c+SQOi8hfnNr174hImrNtlogsD5oLoKWz/AQReVtEvhSRVSLSw9l906B5IJ7z16sRkbvFzqexWkTCUgZZqZrSpKBUOSLSF7gQGGGMyQJKgIuBJsAKp3b9+8Bs5yPPAL9x5gJYE7T8OeAxY8wgbP0jf3XSwcAvsPN5dAdGiEhrYBLQ39nPnZH9KZWqmCYFpY43BjgR+ExEvnDed8cWHJvvbPMsMNKZ1yHVGPO+s3wucKqINAM6GmNeATDGFATV9PnUGJNjjPEBXwBdgQNAAfCUiPwPEDP1f1R80aSg1PEEmGuMyXIevY0xt1ewXW1rxBwLel0CJDg1uYZi6yaNBxbXct9K1YkmBaWO9w4wWUTaQmBe4y7Y/y+TnW0uAj40xhwAfhCRU5zllwDvO7Pn5YjI+c4+Gjm1+SvkzKPRwhizCLgRO72oUlGX4HYASsUaY8x6EbkVeEtEPEARcA12Ip+hzro92H4HsCWoH3dO+t8AlzrLLwH+JiJ3OPuYUsVhmwGvikgytqXyyzD/WEqFRKukKhUiETlsjGnqdhxKRZJePlJKKRWgLQWllFIB2lJQSikVoElBKaVUgCYFpZRSAZoUlFJKBWhSUEopFfD/MkO2PeMPmzcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "Epochs=range(1,18)\n",
    "msec={}\n",
    "msec['MSE-COSINE-SIMILARITY']=[ 57.898399,58.339805, 57.279348, 58.127807,58.862261, 58.971174, 59.096178,59.566813,\n",
    "    59.393306, 59.582851, 59.891960, 59.829067,59.680285, 59.614254, 59.790584, 59.844391, 59.562348] \n",
    "msec['Invalid numbers']=[63,127,165,104,82,69,81,60,74,50, 47,50,52, 55,49,46,44]\n",
    "msec=pd.DataFrame.from_dict(msec)\n",
    "import matplotlib.pyplot as plt\n",
    "fig,ax = plt.subplots()\n",
    "\n",
    "for name in msec.columns:\n",
    "    ax.plot(msec[name].values,label=name)\n",
    "\n",
    "ax.set_xlabel(\"epochs\")\n",
    "ax.set_ylabel(\"Performance\")\n",
    "ax.legend(loc='best')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "_cell_guid": "265fd2f2-cd5f-590d-1fa3-cf6eeedf89fe",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "SRKF7S2a-fPe",
    "outputId": "958c839a-fa21-4c0c-e3dc-e36dfde0484d",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "----Prepare evaluation\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 721, 179, 6983, 399, 106, 460, 781, 2291, 3503, 5983, 1528, 5320, 359, 664, 71, 1738, 2802, 1646, 36803]\n",
      "  Input Words: good strong coffee modified cup pack works well keurig brewer suppose less wasteful cannot beat price especially subscribe save\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [0, 179, 137, 71]\n",
      "  Response Words: good coffee at price\n",
      " Ground Truth: good coffee good price\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "\n",
      "...prepare evaluation finished.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#random = np.random.randint(0,3000)\n",
    "#text = news_texts_filtered[random]\n",
    "model_predictions = []\n",
    "ground_truth = []\n",
    "invalid_number_prediction_count = 0\n",
    "checkpoint = logs_path + 'best_so_far_model.ckpt' \n",
    "print(\"\\n----Prepare evaluation\")\n",
    "for ii in range(100):\n",
    "    random = np.random.randint(21700,27125)\n",
    "    text = news_texts_filtered[random]\n",
    "\n",
    "    #random = np.random.randint(3000,len(news_texts_filtered))\n",
    "        \n",
    "    loaded_graph = tf.Graph()\n",
    "    with tf.Session(graph=loaded_graph) as sess:\n",
    "                  loader = tf.train.import_meta_graph(checkpoint + '.meta')\n",
    "                  loader.restore(sess, checkpoint)\n",
    "                  input_data = loaded_graph.get_tensor_by_name('input_data:0')\n",
    "                  logits = loaded_graph.get_tensor_by_name('predictions:0')\n",
    "                  text_length = loaded_graph.get_tensor_by_name('text_len:0')\n",
    "                  summary_length = loaded_graph.get_tensor_by_name('summary_len:0')\n",
    "                  keep_prob = loaded_graph.get_tensor_by_name('dropout_probs:0')\n",
    "                  result_logits = sess.run(logits, {input_data: [text]*batch_size, \n",
    "                  summary_length: [np.random.randint(5,8)], \n",
    "                  text_length: [len(text)]*batch_size,\n",
    "                  keep_prob: 1.0})[0] \n",
    "\n",
    "                  pad = word2int[TOKEN_PAD] \n",
    "                  eos = word2int[TOKEN_EOS] \n",
    "                  Y=''.join(int2word[i] for i in result_logits if i !=pad)\n",
    "                  O=''.join(int2word[i] for i in news_summaries_filtered[random] if i!=pad)\n",
    "                     \n",
    "                  if mean_absolute_error_texts([Y],[O]) > 90:\n",
    "                  \n",
    "                     print('        ')\n",
    "                     print('        ')\n",
    "                     #print('\\nOriginal Text:', input_sentence)\n",
    "\n",
    "                     print('\\nText')\n",
    "                     print('  Word Ids:    {}'.format([i for i in text]))\n",
    "                     print('  Input Words: {}'.format(\" \".join( int2word[i] for i in text if i!=eos)))\n",
    "\n",
    "                     print('\\nSummary')\n",
    "                     print('  Word Ids:       {}'.format([i for i in result_logits if i != pad]))\n",
    "                     print('  Response Words: {}'.format(\" \".join(int2word[i] for i in result_logits if i!=pad )))\n",
    "                     print(' Ground Truth: {}'.format(\" \".join(int2word[i] for i in news_summaries_filtered[random] if i!=pad)))\n",
    "                     print('        ')\n",
    "                     print('        ')\n",
    " \n",
    "model_predictions.append(result_logits)\n",
    "ground_truth.append(news_summaries_filtered[random])         \n",
    "print('\\n...prepare evaluation finished.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "4UN0LwG3r7FP",
    "outputId": "be32f2b2-b9bb-4cde-f7c1-cc6d6261b2ab"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finish \n"
     ]
    }
   ],
   "source": [
    "print(\"Finish \")"
   ]
  }
 ],
 "metadata": {
  "_change_revision": 0,
  "_is_fork": false,
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "02_example_amazon_reviews_seq2seq_colab_with_attetionMay26.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}