Character Level Language Model .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <u>Neural Language Translator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sequence to sequence model for language translation using Deep $LSTM$ network. \n",
    "<br>The following model translates from $English$ to $French$. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Model, load_model\n",
    "from keras.layers import Input, LSTM, Dense\n",
    "from keras.callbacks import TensorBoard, ModelCheckpoint\n",
    "import numpy as np\n",
    "import os.path\n",
    "from lang_trans_utility import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# constants for the model\n",
    "batch_size = 64  # batch size \n",
    "num_epochs = 70  # total epochs to train\n",
    "latent_dim = 256  # no. of activation units or latent dimensionality of encoder\n",
    "num_samples = 10000 # no. of samples to train on\n",
    "\n",
    "# path to look for the data file \n",
    "data_path = r'data/fra.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_texts = []  # for storing the input text data\n",
    "target_texts = []  # for storing the target text data\n",
    "input_chars = set()  # for storing the unique chars in input text data\n",
    "target_chars = set()  # for storing the unique chars in target text data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Variable Initialization\n",
    "encoder_unique_tokens = 0 # unique tokens in encoder input \n",
    "decoder_unique_tokens = 0 # unique tokens in decoder output\n",
    "Tx = 0 # max length of input sequence for encoder\n",
    "Ty = 0 # max length of output sequence for decoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total no. of lines of Original Text data: 149862\n",
      "Number of samples: 10000\n",
      "Number of unique input tokens: 71\n",
      "Number of unique output tokens: 94\n",
      "Max sequence length for inputs: 16\n",
      "Max sequence length for Target outputs: 59\n"
     ]
    }
   ],
   "source": [
    "encoder_unique_tokens, decoder_unique_tokens, input_chars, target_chars, input_texts, target_texts, Tx, Ty = load_dataset(data_path, num_samples )\n",
    "print('Number of samples:', len(input_texts))\n",
    "print('Number of unique input tokens:', encoder_unique_tokens)\n",
    "print('Number of unique output tokens:', decoder_unique_tokens)\n",
    "print('Max sequence length for inputs:', Tx)\n",
    "print('Max sequence length for Target outputs:', Ty)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[' ', '!', '$', '&', \"'\", ',', '-', '.', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'Y', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '’']\n",
      "\n",
      "\n",
      "\n",
      "['\\t', '\\n', ' ', '!', '$', '&', \"'\", '(', ')', ',', '-', '.', '0', '1', '3', '5', '6', '8', '9', ':', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'Y', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '\\xa0', '«', '»', 'À', 'Ç', 'É', 'Ê', 'à', 'â', 'ç', 'è', 'é', 'ê', 'ë', 'î', 'ï', 'ô', 'ù', 'û', 'œ', '\\u2009', '‘', '’', '\\u202f']\n"
     ]
    }
   ],
   "source": [
    "print(input_chars)\n",
    "print('\\n\\n')\n",
    "print(target_chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No. of keys in input data char to idx dict: 71\n",
      "{' ': 0, '!': 1, '$': 2, '&': 3, \"'\": 4, ',': 5, '-': 6, '.': 7, '0': 8, '1': 9, '2': 10, '3': 11, '4': 12, '5': 13, '6': 14, '7': 15, '8': 16, '9': 17, ':': 18, '?': 19, 'A': 20, 'B': 21, 'C': 22, 'D': 23, 'E': 24, 'F': 25, 'G': 26, 'H': 27, 'I': 28, 'J': 29, 'K': 30, 'L': 31, 'M': 32, 'N': 33, 'O': 34, 'P': 35, 'Q': 36, 'R': 37, 'S': 38, 'T': 39, 'U': 40, 'V': 41, 'W': 42, 'Y': 43, 'a': 44, 'b': 45, 'c': 46, 'd': 47, 'e': 48, 'f': 49, 'g': 50, 'h': 51, 'i': 52, 'j': 53, 'k': 54, 'l': 55, 'm': 56, 'n': 57, 'o': 58, 'p': 59, 'q': 60, 'r': 61, 's': 62, 't': 63, 'u': 64, 'v': 65, 'w': 66, 'x': 67, 'y': 68, 'z': 69, '’': 70}\n",
      "\n",
      "\n",
      "\n",
      "No. of keys in target data char to idx dict: 94\n",
      "{'\\t': 0, '\\n': 1, ' ': 2, '!': 3, '$': 4, '&': 5, \"'\": 6, '(': 7, ')': 8, ',': 9, '-': 10, '.': 11, '0': 12, '1': 13, '3': 14, '5': 15, '6': 16, '8': 17, '9': 18, ':': 19, '?': 20, 'A': 21, 'B': 22, 'C': 23, 'D': 24, 'E': 25, 'F': 26, 'G': 27, 'H': 28, 'I': 29, 'J': 30, 'K': 31, 'L': 32, 'M': 33, 'N': 34, 'O': 35, 'P': 36, 'Q': 37, 'R': 38, 'S': 39, 'T': 40, 'U': 41, 'V': 42, 'Y': 43, 'a': 44, 'b': 45, 'c': 46, 'd': 47, 'e': 48, 'f': 49, 'g': 50, 'h': 51, 'i': 52, 'j': 53, 'k': 54, 'l': 55, 'm': 56, 'n': 57, 'o': 58, 'p': 59, 'q': 60, 'r': 61, 's': 62, 't': 63, 'u': 64, 'v': 65, 'w': 66, 'x': 67, 'y': 68, 'z': 69, '\\xa0': 70, '«': 71, '»': 72, 'À': 73, 'Ç': 74, 'É': 75, 'Ê': 76, 'à': 77, 'â': 78, 'ç': 79, 'è': 80, 'é': 81, 'ê': 82, 'ë': 83, 'î': 84, 'ï': 85, 'ô': 86, 'ù': 87, 'û': 88, 'œ': 89, '\\u2009': 90, '‘': 91, '’': 92, '\\u202f': 93}\n"
     ]
    }
   ],
   "source": [
    "input_char_idx, input_idx_char, target_char_idx, target_idx_char = create_mappings(input_chars, target_chars)\n",
    "print('No. of keys in input data char to idx dict: ' + str(len(input_char_idx)))\n",
    "print(input_char_idx)\n",
    "print('\\n\\n')\n",
    "print('No. of keys in target data char to idx dict: ' + str(len(target_char_idx)))\n",
    "print(target_char_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# now we will prepare data for the model\n",
    "# no. of training examples\n",
    "m = len(input_texts)\n",
    "# encoder input data\n",
    "enc_input_data = np.zeros((m, Tx, encoder_unique_tokens), dtype = 'float32')\n",
    "# decoder input data\n",
    "dec_input_data = np.zeros((m, Ty, decoder_unique_tokens), dtype = 'float32')\n",
    "# decoder output target data\n",
    "dec_target_data = np.zeros((m, Ty, decoder_unique_tokens), dtype = 'float32')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create training examples\n",
    "Training examples will be of format (X, Y), where X is input and Y is target output.\n",
    "\n",
    "For X we will take text sentences from **input_texts** and for Y we will take text sentences from **target_texts**.<br>\n",
    "But for machine translation we will be using an Architecture where the output from the encoder network is given to the decoder network and using that it produces the target output in the 1st time step , then that produced output is again fed to the decoder network in the next time step, this continues till we get **'\\n'** or exceed max sequence length. \n",
    "\n",
    "For the decoder network the output in each time step is one time step ahead of the input. The 1st input is **start_char** to the decoder and the output for that time step is the input for the next time step.\n",
    "\n",
    "For input we will be using **One Hot encoding(OHE)** for the encoder network. Similarly for the decoder network we will be using **OHE** for input and output representations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating training examples\n",
    "for i, (input_text, target_text) in enumerate(zip(input_texts, target_texts)):\n",
    "    # for encoder network\n",
    "    # make the indices for the chars hot i.e, 1 in input text\n",
    "    for curr_timestep, char in enumerate(input_text):\n",
    "        enc_input_data[i, curr_timestep, input_char_idx[char]] = 1\n",
    "        \n",
    "    # for decoder network\n",
    "    # make the indices for the chars hot i.e, 1 in target input text\n",
    "    for curr_timestep, char in enumerate(target_text):\n",
    "        dec_input_data[i, curr_timestep, target_char_idx[char]] = 1\n",
    "    if curr_timestep > 0:     \n",
    "        # make the indices for the chars hot i.e, 1 in target text, only this will \n",
    "        # be one time step ahead of decoder input\n",
    "        for curr_timestep, char in enumerate(target_text):\n",
    "            dec_target_data[i, curr_timestep-1, target_char_idx[char]] = 1    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <u>Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since we want the weights to be same for the different timesteps so for achieving that we do global declaration for the various components.\n",
    "\n",
    "Also we return the state information from the encoder network and use that information for the decoder network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for ENCODER network\n",
    "# for taking input for the encoder network\n",
    "encoder_inputs = Input(shape=(None, encoder_unique_tokens))\n",
    "# we will LSTM units \n",
    "encoder_lstm = LSTM(latent_dim, return_state=True)\n",
    "# we will save the activation and cell mem state information of encoder network\n",
    "# No need to save the outputs\n",
    "_, activation, cell_mem = encoder_lstm(encoder_inputs)\n",
    "encoder_states = [activation, cell_mem]\n",
    "\n",
    "# for DECODER network\n",
    "# we will use the encoder state information as initial state for decoder network\n",
    "decoder_inputs = Input(shape = (None, decoder_unique_tokens))\n",
    "# we will save the state info of decoder network and use it \n",
    "# for making predictions later and return the output from decoder network units\n",
    "decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True)\n",
    "# get the LSTM outputs\n",
    "decoder_outputs, _, _ = decoder_lstm(decoder_inputs, initial_state=encoder_states)\n",
    "# pass the LSTM output to a softmax layer\n",
    "decoder_dense = Dense(decoder_unique_tokens, activation = 'softmax')\n",
    "# get the final output from the softmax layer\n",
    "decoder_outputs = decoder_dense(decoder_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the model\n",
    "model = Model( [encoder_inputs, decoder_inputs], decoder_outputs)\n",
    "model.compile(optimizer = 'Adam', loss = 'categorical_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load any previously saved model\n",
    "model_path = r'models/fra_eng_orig_wt.h5'\n",
    "if os.path.exists(model_path):\n",
    "    model.load_weights(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, None, 71)     0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, None, 94)     0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   [(None, 256), (None, 335872      input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                   [(None, None, 256),  359424      input_2[0][0]                    \n",
      "                                                                 lstm_1[0][1]                     \n",
      "                                                                 lstm_1[0][2]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, None, 94)     24158       lstm_2[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 719,454\n",
      "Trainable params: 719,454\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.4300 - val_loss: 0.5451\n",
      "Epoch 2/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.4178 - val_loss: 0.5371\n",
      "Epoch 3/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.4071 - val_loss: 0.5260\n",
      "Epoch 4/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.3966 - val_loss: 0.5212\n",
      "Epoch 5/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.3863 - val_loss: 0.5104\n",
      "Epoch 6/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.3766 - val_loss: 0.5040\n",
      "Epoch 7/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.3684 - val_loss: 0.5021\n",
      "Epoch 8/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.3596 - val_loss: 0.4917\n",
      "Epoch 9/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.3508 - val_loss: 0.4899\n",
      "Epoch 10/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.3421 - val_loss: 0.4831\n",
      "Epoch 11/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.3346 - val_loss: 0.4798\n",
      "Epoch 12/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.3270 - val_loss: 0.4749\n",
      "Epoch 13/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.3199 - val_loss: 0.4732\n",
      "Epoch 14/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.3121 - val_loss: 0.4680\n",
      "Epoch 15/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.3047 - val_loss: 0.4656\n",
      "Epoch 16/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2982 - val_loss: 0.4632\n",
      "Epoch 17/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2912 - val_loss: 0.4616\n",
      "Epoch 18/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2841 - val_loss: 0.4652\n",
      "Epoch 19/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2777 - val_loss: 0.4586\n",
      "Epoch 20/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2708 - val_loss: 0.4567\n",
      "Epoch 21/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2652 - val_loss: 0.4587\n",
      "Epoch 22/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2587 - val_loss: 0.4557\n",
      "Epoch 23/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2530 - val_loss: 0.4557\n",
      "Epoch 24/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2471 - val_loss: 0.4583\n",
      "Epoch 25/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2411 - val_loss: 0.4590\n",
      "Epoch 26/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2361 - val_loss: 0.4599\n",
      "Epoch 27/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2304 - val_loss: 0.4584\n",
      "Epoch 28/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2250 - val_loss: 0.4602\n",
      "Epoch 29/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2201 - val_loss: 0.4641\n",
      "Epoch 30/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2150 - val_loss: 0.4641\n",
      "Epoch 31/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2095 - val_loss: 0.4652\n",
      "Epoch 32/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2052 - val_loss: 0.4722\n",
      "Epoch 33/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.2015 - val_loss: 0.4712\n",
      "Epoch 34/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1955 - val_loss: 0.4753\n",
      "Epoch 35/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1911 - val_loss: 0.4756\n",
      "Epoch 36/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1871 - val_loss: 0.4832\n",
      "Epoch 37/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1832 - val_loss: 0.4822\n",
      "Epoch 38/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1784 - val_loss: 0.4858\n",
      "Epoch 39/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1764 - val_loss: 0.4944\n",
      "Epoch 40/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1722 - val_loss: 0.4943\n",
      "Epoch 41/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1673 - val_loss: 0.4963\n",
      "Epoch 42/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1640 - val_loss: 0.5027\n",
      "Epoch 43/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1600 - val_loss: 0.5051\n",
      "Epoch 44/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1567 - val_loss: 0.5078\n",
      "Epoch 45/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1532 - val_loss: 0.5108\n",
      "Epoch 46/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1493 - val_loss: 0.5137\n",
      "Epoch 47/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1463 - val_loss: 0.5205\n",
      "Epoch 48/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1436 - val_loss: 0.5249\n",
      "Epoch 49/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1405 - val_loss: 0.5301\n",
      "Epoch 50/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1380 - val_loss: 0.5340\n",
      "Epoch 51/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1344 - val_loss: 0.5369\n",
      "Epoch 52/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1320 - val_loss: 0.5384\n",
      "Epoch 53/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1289 - val_loss: 0.5482\n",
      "Epoch 54/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1254 - val_loss: 0.5479\n",
      "Epoch 55/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1231 - val_loss: 0.5513\n",
      "Epoch 56/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1212 - val_loss: 0.5582\n",
      "Epoch 57/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1200 - val_loss: 0.5590\n",
      "Epoch 58/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1171 - val_loss: 0.5619\n",
      "Epoch 59/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1138 - val_loss: 0.5720\n",
      "Epoch 60/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1113 - val_loss: 0.5782\n",
      "Epoch 61/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1099 - val_loss: 0.5863\n",
      "Epoch 62/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1081 - val_loss: 0.5857\n",
      "Epoch 63/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1059 - val_loss: 0.5866\n",
      "Epoch 64/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1037 - val_loss: 0.5892\n",
      "Epoch 65/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1018 - val_loss: 0.6173\n",
      "Epoch 66/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.1039 - val_loss: 0.6019\n",
      "Epoch 67/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.0973 - val_loss: 0.6032\n",
      "Epoch 68/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.0950 - val_loss: 0.6112\n",
      "Epoch 69/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.0931 - val_loss: 0.6144\n",
      "Epoch 70/70\n",
      "8000/8000 [==============================] - 38s 5ms/step - loss: 0.0917 - val_loss: 0.6229\n"
     ]
    }
   ],
   "source": [
    "# start training the model\n",
    "model.fit( [enc_input_data, dec_input_data], dec_target_data,\n",
    "         batch_size = batch_size, epochs = num_epochs, \n",
    "         validation_split = 0.2)\n",
    "# save the model\n",
    "model.save_weights(model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction and Sampling\n",
    "To see how good the model is working we sample outputs from it, so we are going to do that.\n",
    "\n",
    "Our Model can be divided into two parts: Encoder and Decoder. <br>\n",
    "1. We pass the text input through the encoder network to get the states.\n",
    "2. We use the encoder states as initial states for the decoder network.\n",
    "3. We feed start character '\\t' as the input for the first time step to the decoder network and then the predicted output is fed as input to the next time step.\n",
    "4. We do this till we get '\\n' or exceed max char length."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# take the encoder part \n",
    "# this basically takes the encoder part of the training model as \n",
    "# we are saying that the input is encoder_inputs and as output we get encoder_outputs\n",
    "encoder_model = Model( encoder_inputs, encoder_states)\n",
    "\n",
    "# for the decoder part\n",
    "decoder_activation_state_input = Input(shape=(latent_dim,))\n",
    "decoder_mem_state_input = Input(shape=(latent_dim,))\n",
    "decoder_states_inputs = [decoder_activation_state_input, decoder_mem_state_input]\n",
    "# using decoder lstm\n",
    "decoder_outputs, activation_state, mem_state = decoder_lstm(\n",
    "    decoder_inputs, initial_state=decoder_states_inputs)\n",
    "\n",
    "decoder_states = [activation_state, mem_state]\n",
    "# the activations go through the softmax layer\n",
    "decoder_outputs = decoder_dense(decoder_outputs)\n",
    "# model compilation\n",
    "decoder_model = Model(\n",
    "    [decoder_inputs] + decoder_states_inputs,\n",
    "    [decoder_outputs] + decoder_states)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make Translations\n",
    "Now we will make translations for the input sequence. For that we first pass the input sequence through the encoder and then pass its state info to the decoder network and do decoding."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# decode the output from the encoder network\n",
    "def do_translation(input_text):\n",
    "    \n",
    "    # encoder state values\n",
    "    state_vals = encoder_model.predict(input_text)\n",
    "    \n",
    "    # make a target input consisting of start character '\\t'\n",
    "    dec_input_seq = np.zeros((1,1, decoder_unique_tokens), dtype = 'float32')\n",
    "    dec_input_seq[0, 0, target_char_idx['\\t']] = 1\n",
    "    \n",
    "    # now we start the translation process by sampling out the predictions\n",
    "    # each time sampling a single character\n",
    "    translated_text = ''\n",
    "    # decides whether to continue generating samples, becomes false\n",
    "    # on encountering '\\n' or when the the output sequence length exceeds max limit\n",
    "    run_loop = True\n",
    "    \n",
    "    while run_loop:\n",
    "        output_tokens, acti, mem = decoder_model.predict([dec_input_seq] + state_vals)\n",
    "        \n",
    "        # sample a char token\n",
    "        # since we get softmax prob. from the output layer, we pick the \n",
    "        # index with max prob.\n",
    "        sampled_token_idx = np.argmax(output_tokens[0, -1, :])\n",
    "        sampled_token = target_idx_char[sampled_token_idx]\n",
    "        translated_text += sampled_token\n",
    "        \n",
    "        # check for loop condition\n",
    "        if len(translated_text)> Ty or sampled_token == '\\n':\n",
    "            run_loop = False\n",
    "        \n",
    "        # now update the decoder input for the next time step\n",
    "        dec_input_seq = np.zeros((1,1, decoder_unique_tokens), dtype = 'float32')\n",
    "        dec_input_seq[0, 0, sampled_token_idx] = 1\n",
    "        \n",
    "        # update state values\n",
    "        state_vals = [acti, mem]\n",
    "        \n",
    "    return translated_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input sentence: Run!\n",
      "Decoded sentence: Atteque la votte !\n",
      "\n",
      "Input sentence: Wait!\n",
      "Decoded sentence: Attaquez !\n",
      "\n",
      "Input sentence: Grab this.\n",
      "Decoded sentence: Attrape ça !\n",
      "\n",
      "Input sentence: I felt safe.\n",
      "Decoded sentence: Je me suis sentie freille.\n",
      "\n",
      "Input sentence: I went twice.\n",
      "Decoded sentence: Je vous ai eu.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "indices = [2, 11, 500, 2001, 3478]\n",
    "for seq_index in indices:\n",
    "    # Take one sequence (part of the training set)\n",
    "    # for trying out decoding.\n",
    "    input_seq = enc_input_data[seq_index: seq_index + 1]\n",
    "    decoded_sentence = do_translation(input_seq)\n",
    "    print('Input sentence:', input_texts[seq_index])\n",
    "    print('Decoded sentence:', decoded_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# takes user input in english and translates the phrase to french\n",
    "def take_input():\n",
    "    print('Enter in English (max length ' + str(Tx) + ')')\n",
    "    user_input = input()\n",
    "    \n",
    "    # create a One Hot representation\n",
    "    enc_input_data = to_OHE(user_input, Tx, encoder_unique_tokens, input_char_idx)\n",
    "    \n",
    "    decoded_sentence = do_translation(enc_input_data)\n",
    "    print('Input English sentence: ', user_input)\n",
    "    print('Decoded French sentence: ', decoded_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter in English (max length 16)\n",
      "hello\n",
      "Input English sentence:  hello\n",
      "Decoded French sentence:  Tiens ceci !\n",
      "\n"
     ]
    }
   ],
   "source": [
    "take_input()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**For bigger phrases translations will appear way out of line since the training data was very very less.  **"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### <u>Credits:\n",
    "The code is heavly based on \n",
    "https://blog.keras.io/a-ten-minute-introduction-to-sequence-to-sequence-learning-in-keras.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}