Recurrent Neural Networks/LSTM-Character-Modelling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text generation using RNN/LSTM\n",
    "This code implements a Recurrent Neural Network with LSTM/RNN units for training/sampling from character-level language models. In other words, the model takes a text file as input and trains the RNN network that learns to predict the next character in a sequence.  \n",
    "The RNN can then be used to generate text, character by character that will look like the original training data. \n",
    "\n",
    "The code is based on this [blog](http://karpathy.github.io/2015/05/21/rnn-effectiveness/), and it is an step-by-step implimentation of the [character-level implimentation](https://github.com/crazydonkey200/tensorflow-char-rnn)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, import the requiered libraries:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import time\n",
    "import codecs\n",
    "import os\n",
    "import collections\n",
    "from six.moves import cPickle\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data loader\n",
    "The following cell is a class that help to read data from input file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class TextLoader():\n",
    "    def __init__(self, data_dir, batch_size, seq_length, encoding='utf-8'):\n",
    "        self.data_dir = data_dir\n",
    "        self.batch_size = batch_size\n",
    "        self.seq_length = seq_length\n",
    "        self.encoding = encoding\n",
    "\n",
    "        input_file = os.path.join(data_dir, \"input.txt\")\n",
    "        vocab_file = os.path.join(data_dir, \"vocab.pkl\")\n",
    "        tensor_file = os.path.join(data_dir, \"data.npy\")\n",
    "\n",
    "        if not (os.path.exists(vocab_file) and os.path.exists(tensor_file)):\n",
    "            print(\"reading text file\")\n",
    "            self.preprocess(input_file, vocab_file, tensor_file)\n",
    "        else:\n",
    "            print(\"loading preprocessed files\")\n",
    "            self.load_preprocessed(vocab_file, tensor_file)\n",
    "        self.create_batches()\n",
    "        self.reset_batch_pointer()\n",
    "\n",
    "    def preprocess(self, input_file, vocab_file, tensor_file):\n",
    "        with codecs.open(input_file, \"r\", encoding=self.encoding) as f:\n",
    "            data = f.read()\n",
    "        counter = collections.Counter(data)\n",
    "        count_pairs = sorted(counter.items(), key=lambda x: -x[1])\n",
    "        self.chars, _ = zip(*count_pairs)\n",
    "        self.vocab_size = len(self.chars)\n",
    "        self.vocab = dict(zip(self.chars, range(len(self.chars))))\n",
    "        with open(vocab_file, 'wb') as f:\n",
    "            cPickle.dump(self.chars, f)\n",
    "        self.tensor = np.array(list(map(self.vocab.get, data)))\n",
    "        np.save(tensor_file, self.tensor)\n",
    "\n",
    "    def load_preprocessed(self, vocab_file, tensor_file):\n",
    "        with open(vocab_file, 'rb') as f:\n",
    "            self.chars = cPickle.load(f)\n",
    "        self.vocab_size = len(self.chars)\n",
    "        self.vocab = dict(zip(self.chars, range(len(self.chars))))\n",
    "        self.tensor = np.load(tensor_file)\n",
    "        self.num_batches = int(self.tensor.size / (self.batch_size * self.seq_length))\n",
    "\n",
    "    def create_batches(self):\n",
    "        self.num_batches = int(self.tensor.size / (self.batch_size * self.seq_length))\n",
    "\n",
    "        # When the data (tensor) is too small.\n",
    "        if self.num_batches==0:\n",
    "            assert False, \"Not enough data. Make seq_length and batch_size small.\"\n",
    "\n",
    "        self.tensor = self.tensor[:self.num_batches * self.batch_size * self.seq_length]\n",
    "        xdata = self.tensor\n",
    "        ydata = np.copy(self.tensor)\n",
    "        ydata[:-1] = xdata[1:]\n",
    "        ydata[-1] = xdata[0]\n",
    "        self.x_batches = np.split(xdata.reshape(self.batch_size, -1), self.num_batches, 1)\n",
    "        self.y_batches = np.split(ydata.reshape(self.batch_size, -1), self.num_batches, 1)\n",
    "\n",
    "    def next_batch(self):\n",
    "        x, y = self.x_batches[self.pointer], self.y_batches[self.pointer]\n",
    "        self.pointer += 1\n",
    "        return x, y\n",
    "\n",
    "    def reset_batch_pointer(self):\n",
    "        self.pointer = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parameters\n",
    "#### Batch, number_of_batch, batch_size and seq_length\n",
    "\n",
    "Lets assume the input is this sentence: '__here is an example__'. Then:\n",
    "- txt_length = 18  \n",
    "- seq_length = 3  \n",
    "- batch_size = 2  \n",
    "- number_of_batchs = 18 / 3 * 2 = 3\n",
    "- batch = array(['h', 'e', 'r'], ['e', ' ', 'i'])\n",
    "- sample Seq = 'her'  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ok, now, lets look at a real dataset, with real parameters. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seq_length = 50  # RNN sequence length.\n",
    "batch_size = 60  # Minibatch size, i.e. size of data in each epoch.\n",
    "num_epochs = 125\n",
    "learning_rate = 0.002\n",
    "decay_rate = 0.97\n",
    "rnn_size = 128  # Size of RNN hidden state (output dimension).\n",
    "num_layers = 2  # Number of layers in the RNN."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Print part of the input file:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First Citizen:\n",
      "Before we proceed any further, hear me speak.\n",
      "\n",
      "All:\n",
      "Speak, speak.\n",
      "\n",
      "First Citizen:\n",
      "You\n"
     ]
    }
   ],
   "source": [
    "with open('input.txt', 'r') as f:\n",
    "    read_data = f.read()\n",
    "    print(read_data[0:100])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we can read the data at batches using the `TextLoader`. It will convert the characters to numbers, and represent each sequence as a vector in batches:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading preprocessed files\n",
      "vocabulary size: 65\n",
      "Characters: (' ', 'e', 't', 'o', 'a', 'h', 's', 'r', 'n', 'i', '\\n', 'l', 'd', 'u', 'm', 'y', ',', 'w', 'f', 'c', 'g', 'I', 'b', 'p', ':', '.', 'A', 'v', 'k', 'T', \"'\", 'E', 'O', 'N', 'R', 'S', 'L', 'C', ';', 'W', 'U', 'H', 'M', 'B', '?', 'G', '!', 'D', '-', 'F', 'Y', 'P', 'K', 'V', 'j', 'q', 'x', 'z', 'J', 'Q', 'Z', 'X', '3', '&', '$')\n",
      "vocab number of 'F': 49\n",
      "Character sequences (first batch): [[49  9  7 ...  1  4  7]\n",
      " [19  4 14 ... 14  9 20]\n",
      " [ 8 20 10 ...  8 10 18]\n",
      " ...\n",
      " [21  2  0 ...  0 21  0]\n",
      " [ 9  7  7 ...  0  2  3]\n",
      " [ 3  7  0 ...  5  9 23]]\n"
     ]
    }
   ],
   "source": [
    "data_loader = TextLoader('', batch_size, seq_length)\n",
    "vocab_size = data_loader.vocab_size\n",
    "print(\"vocabulary size:\", data_loader.vocab_size)\n",
    "print(\"Characters:\", data_loader.chars)\n",
    "print(\"vocab number of 'F':\", data_loader.vocab['F'])\n",
    "print(\"Character sequences (first batch):\", data_loader.x_batches[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Input and output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[49,  9,  7, ...,  1,  4,  7],\n",
       "       [19,  4, 14, ..., 14,  9, 20],\n",
       "       [ 8, 20, 10, ...,  8, 10, 18],\n",
       "       ...,\n",
       "       [21,  2,  0, ...,  0, 21,  0],\n",
       "       [ 9,  7,  7, ...,  0,  2,  3],\n",
       "       [ 3,  7,  0, ...,  5,  9, 23]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x, y = data_loader.next_batch()\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60, 50)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape  # batch_size = 60, seq_length = 50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here, `y` is the next character for each character in `x`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 9,  7,  6, ...,  4,  7,  0],\n",
       "       [ 4, 14, 22, ...,  9, 20,  5],\n",
       "       [20, 10, 29, ..., 10, 18,  4],\n",
       "       ...,\n",
       "       [ 2,  0,  6, ..., 21,  0,  6],\n",
       "       [ 7,  7,  4, ...,  2,  3,  0],\n",
       "       [ 7,  0, 33, ...,  9, 23,  0]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSTM Architecture\n",
    "Each LSTM cell has 5 parts:\n",
    "1. Input\n",
    "2. prv_state\n",
    "3. prv_output\n",
    "4. new_state\n",
    "5. new_output\n",
    "\n",
    "\n",
    "- Each LSTM cell has an input layer, which size is 128 units in our case. The input vector's dimension also is 128, which is the dimensionality of embedding vector, so called, dimension size of W2V/embedding, for each character/word.\n",
    "- Each LSTM cell has a hidden layer, where there are some hidden units. The argument `n_hidden = 128` of BasicLSTMCell is the number of hidden units of the LSTM (inside A). It keeps the size of the output and state vector. It is also known as, rnn_size, num_units, num_hidden_units, and LSTM size\n",
    "- An LSTM keeps two pieces of information as it propagates through time: \n",
    "    - __hidden state__ vector: Each LSTM cell accepts a vector, called __hidden state__ vector, of size `n_hidden = 128`, and its value is returned to the LSTM cell in the next step. The __hidden state__ vector which is the memory of the LSTM, accumulates using its (forget, input and output) gates through time. \"num_units\" is equivalant to \"size of RNN hidden state\". number of hidden units is the dimensianality of the output (dimesianality of the state) of the LSTM cell.\n",
    "    - __previous time-step output__: For each LSTM cell that we initialize, we need to supply a value (128 in this case) for the hidden dimension, or as some people like to call it, the number of units in the LSTM cell. \n",
    "\n",
    "\n",
    "`num_layers = 2`  \n",
    "- Number of layers in the RNN, is defined by num_layers\n",
    "- An input of `MultiRNNCell` is `cells` which is a list of RNNCells that will be composed in this order."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining stacked RNN Cell"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "`BasicRNNCell` is the most basic RNN cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cell = tf.contrib.rnn.BasicRNNCell(rnn_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# A two layer cell.\n",
    "stacked_cell = tf.contrib.rnn.MultiRNNCell([cell] * num_layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "128"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Hidden state size.\n",
    "stacked_cell.output_size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`state` varibale keeps output and new_state of the LSTM, so it is a tuple of size:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(128, 128)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stacked_cell.state_size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lets define input data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Placeholder:0' shape=(60, 50) dtype=int32>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_data = tf.placeholder(tf.int32, [batch_size, seq_length])\n",
    "input_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And target data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Placeholder_1:0' shape=(60, 50) dtype=int32>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "targets = tf.placeholder(tf.int32, [batch_size, seq_length])\n",
    "targets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The memory state of the network is initialized with a vector of zeros and gets updated after reading each character.\n",
    "\n",
    "`BasicRNNCell.zero_state(batch_size, dtype)` returns zero-filled state tensor(s). In this function, batch_size\n",
    "representing the batch size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "initial_state = stacked_cell.zero_state(batch_size, tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Lets check the value of the input_data again:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[49,  9,  7, ...,  1,  4,  7],\n",
       "       [19,  4, 14, ..., 14,  9, 20],\n",
       "       [ 8, 20, 10, ...,  8, 10, 18],\n",
       "       ...,\n",
       "       [21,  2,  0, ...,  0, 21,  0],\n",
       "       [ 9,  7,  7, ...,  0,  2,  3],\n",
       "       [ 3,  7,  0, ...,  5,  9, 23]], dtype=int32)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "session = tf.Session()\n",
    "feed_dict = {input_data: x, targets: y}\n",
    "session.run(input_data, feed_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Embedding\n",
    "In this section, we build a 128-dim vector for each character. As we have 60 batches, and 50 characters in each sequence, it will generate a [60, 50, 128] matrix."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Notice:__ The function `tf.get_variable()` is used to share a variable and to initialize it in one place. `tf.get_variable()` is used to get or create a variable instead of a direct call to `tf.Variable`. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope('rnnlm', reuse=False):\n",
    "    softmax_w = tf.get_variable(\"softmax_w\", [rnn_size, vocab_size])  # 128x65\n",
    "    softmax_b = tf.get_variable(\"softmax_b\", [vocab_size])  # 1x65\n",
    "        \n",
    "    # Embedding variable is initialized randomly.\n",
    "    embedding = tf.get_variable(\"embedding\", [vocab_size, rnn_size])  # 65x128\n",
    "\n",
    "    # embedding_lookup goes to each row of input_data, and for each character in the row,\n",
    "    # finds the correspond vector in embedding.\n",
    "    # Creates a 60*50*[1*128] matrix.\n",
    "    # So, the first elemnt of em, is a matrix of 50x128, which each row of it is vector representing that character.\n",
    "    em = tf.nn.embedding_lookup(embedding, input_data)  # em is 60x50x[1*128]\n",
    "    # split: Splits a tensor into sub tensors.\n",
    "    # It will split the 60x50x[1x128] matrix into 50 matrix of 60x[1*128]\n",
    "    inputs = tf.split(em, seq_length, 1)\n",
    "    # Convert the list to 50 matrix of [60x128]\n",
    "    inputs = [tf.squeeze(input_, [1]) for input_ in inputs]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lets take a look at the __embedding__, __em__ and __inputs__ variables:\n",
    "\n",
    "Embedding variable is initialized with random values:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.1722724 , -0.08362447, -0.12620343, ..., -0.09066197,\n",
       "        -0.10572398,  0.03630777],\n",
       "       [ 0.0302359 , -0.16806553, -0.10838205, ...,  0.16656597,\n",
       "         0.11187743, -0.04912362],\n",
       "       [ 0.07656111,  0.07549156,  0.06535269, ..., -0.02334714,\n",
       "        -0.03505114, -0.075584  ],\n",
       "       ...,\n",
       "       [-0.15861222, -0.04039523, -0.00890502, ..., -0.1521869 ,\n",
       "        -0.14659537,  0.06569719],\n",
       "       [-0.15520895,  0.11477567, -0.14152746, ...,  0.11538155,\n",
       "         0.05942373, -0.03053328],\n",
       "       [-0.0547492 ,  0.15497552, -0.04752883, ...,  0.07466076,\n",
       "         0.10189132, -0.10777494]], dtype=float32)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "session.run(tf.global_variables_initializer())\n",
    "session.run(embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The first element of em, is a matrix of 50x128, which each row of it is a vector representing that character."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60, 50, 128)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 0.08352269,  0.10425128,  0.13842289, ..., -0.12345897,\n",
       "        -0.06512738, -0.07250892],\n",
       "       [-0.02447218,  0.08035068, -0.04622024, ...,  0.14011173,\n",
       "         0.03743592,  0.11483796],\n",
       "       [ 0.02744865,  0.14238094, -0.10165575, ..., -0.09894219,\n",
       "         0.12508132, -0.14850871],\n",
       "       ...,\n",
       "       [ 0.0302359 , -0.16806553, -0.10838205, ...,  0.16656597,\n",
       "         0.11187743, -0.04912362],\n",
       "       [-0.14158408, -0.08662911,  0.05569063, ...,  0.06909324,\n",
       "        -0.04474066, -0.14188258],\n",
       "       [ 0.02744865,  0.14238094, -0.10165575, ..., -0.09894219,\n",
       "         0.12508132, -0.14850871]], dtype=float32)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "em = tf.nn.embedding_lookup(embedding, input_data)\n",
    "emp = session.run(em, feed_dict={input_data: x})\n",
    "print(emp.shape)\n",
    "emp[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's consider each sequence as a sentence of length 50 characters, then, the first item in __inputs__ is a [60x128] vector which represents the first characters of 60 sentences."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'Squeeze:0' shape=(60, 128) dtype=float32>,\n",
       " <tf.Tensor 'Squeeze_1:0' shape=(60, 128) dtype=float32>,\n",
       " <tf.Tensor 'Squeeze_2:0' shape=(60, 128) dtype=float32>,\n",
       " <tf.Tensor 'Squeeze_3:0' shape=(60, 128) dtype=float32>,\n",
       " <tf.Tensor 'Squeeze_4:0' shape=(60, 128) dtype=float32>]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs = tf.split(em, seq_length, 1)\n",
    "inputs = [tf.squeeze(input_, [1]) for input_ in inputs]\n",
    "inputs[0:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feeding a batch of 50 sequences to a RNN:\n",
    "\n",
    "The feeding process for inputs is as following:\n",
    "\n",
    "- Step 1:  first character of each of the 50 sentences (in a batch) is entered in parallel.  \n",
    "- Step 2:  second character of each of the 50 sentences is input in parallel. \n",
    "- Step n: nth character of each of the 50 sentences is input in parallel.  \n",
    "\n",
    "The parallelism is only for efficiency.  Each character in a batch is handled in parallel,  but the network sees one character of a sequence at a time and does the computations accordingly. All the computations involving the characters of all sequences in a batch at a given time step are done in parallel. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.08352269,  0.10425128,  0.13842289, ..., -0.12345897,\n",
       "        -0.06512738, -0.07250892],\n",
       "       [-0.037874  , -0.07817087,  0.1164002 , ...,  0.07249571,\n",
       "        -0.0046092 ,  0.01694898],\n",
       "       [ 0.03743163, -0.02072446, -0.12352531, ...,  0.11484666,\n",
       "         0.13091497,  0.02999784],\n",
       "       ...,\n",
       "       [ 0.11660947, -0.1103497 ,  0.09357838, ...,  0.0761029 ,\n",
       "        -0.03518528,  0.07144852],\n",
       "       [-0.02447218,  0.08035068, -0.04622024, ...,  0.14011173,\n",
       "         0.03743592,  0.11483796],\n",
       "       [-0.03696737,  0.14969508, -0.12469925, ..., -0.168251  ,\n",
       "         0.14400227, -0.07804833]], dtype=float32)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "session.run(inputs[0], feed_dict={input_data: x})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Feeding the RNN with one batch, we can check the new output and new state of network:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor 'rnnlm_1/rnnlm/multi_rnn_cell/cell_0/cell_0/basic_rnn_cell/Tanh_98:0' shape=(60, 128) dtype=float32>,\n",
       " <tf.Tensor 'rnnlm_1/rnnlm/multi_rnn_cell/cell_0/cell_0/basic_rnn_cell/Tanh_99:0' shape=(60, 128) dtype=float32>)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# outputs is 50x[60*128]\n",
    "outputs, new_state = tf.contrib.legacy_seq2seq.rnn_decoder(inputs, initial_state, stacked_cell, loop_function=None, scope='rnnlm')\n",
    "new_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'rnnlm_1/rnnlm/multi_rnn_cell/cell_0/cell_0/basic_rnn_cell/Tanh_1:0' shape=(60, 128) dtype=float32>,\n",
       " <tf.Tensor 'rnnlm_1/rnnlm/multi_rnn_cell/cell_0/cell_0/basic_rnn_cell/Tanh_3:0' shape=(60, 128) dtype=float32>,\n",
       " <tf.Tensor 'rnnlm_1/rnnlm/multi_rnn_cell/cell_0/cell_0/basic_rnn_cell/Tanh_5:0' shape=(60, 128) dtype=float32>,\n",
       " <tf.Tensor 'rnnlm_1/rnnlm/multi_rnn_cell/cell_0/cell_0/basic_rnn_cell/Tanh_7:0' shape=(60, 128) dtype=float32>,\n",
       " <tf.Tensor 'rnnlm_1/rnnlm/multi_rnn_cell/cell_0/cell_0/basic_rnn_cell/Tanh_9:0' shape=(60, 128) dtype=float32>]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs[0:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's check the output of the network after feeding it with first batch:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.07270692,  0.00914923,  0.07391221, ...,  0.00625553,\n",
       "        -0.10549624, -0.11863969],\n",
       "       [-0.06421256,  0.13520257,  0.01071523, ...,  0.03852994,\n",
       "         0.10306417, -0.03523347],\n",
       "       [-0.13965774,  0.04955263, -0.03708952, ...,  0.00821678,\n",
       "        -0.1010602 , -0.1300817 ],\n",
       "       ...,\n",
       "       [ 0.15044728, -0.03523955, -0.02970305, ..., -0.00524335,\n",
       "        -0.14697067,  0.02718503],\n",
       "       [-0.0813082 , -0.0428643 ,  0.04957348, ..., -0.09574681,\n",
       "        -0.14821191,  0.10273302],\n",
       "       [ 0.05410901,  0.0038401 ,  0.05570202, ..., -0.01997526,\n",
       "         0.0489104 ,  0.1196968 ]], dtype=float32)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_output = outputs[0]\n",
    "session.run(tf.global_variables_initializer())\n",
    "session.run(first_output, feed_dict={input_data: x})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As it was explained, __outputs__ variable is a 50x[60x128] tensor. We need to reshape it back to [60x50x128] to be able to calculate the probablity of the next character using the softmax. The `softmax_w` shape is [rnn_size, vocab_size], which is [128x65] in our case. Threfore, we have a fully connected layer on top of LSTM cells, which help us to decode the next charachter. We can use the `softmax(output * softmax_w + softmax_b)` for this purpose. The shape of the matrices would be:\n",
    "\n",
    "softmax([60x50x128]x[128x65]+[1x65]) = [60x50x65]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can do it step-by-step:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Reshape:0' shape=(3000, 128) dtype=float32>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output = tf.reshape(tf.concat(outputs, 1), [-1, rnn_size])\n",
    "output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'add:0' shape=(3000, 65) dtype=float32>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits = tf.matmul(output, softmax_w) + softmax_b\n",
    "logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Softmax:0' shape=(3000, 65) dtype=float32>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probs = tf.nn.softmax(logits)\n",
    "probs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here is the probablity of the next chracter in all batches:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.01596195, 0.01497225, 0.01251268, ..., 0.0152396 , 0.01854368,\n",
       "        0.01418732],\n",
       "       [0.0152303 , 0.01380554, 0.01368246, ..., 0.01589741, 0.01796742,\n",
       "        0.01024323],\n",
       "       [0.0144489 , 0.01359341, 0.01239462, ..., 0.01483944, 0.01449357,\n",
       "        0.01601436],\n",
       "       ...,\n",
       "       [0.0166013 , 0.01857689, 0.01334734, ..., 0.01840278, 0.01606724,\n",
       "        0.01974845],\n",
       "       [0.01209294, 0.01238278, 0.01320278, ..., 0.01759913, 0.01553547,\n",
       "        0.01237708],\n",
       "       [0.01651255, 0.01159354, 0.01127395, ..., 0.01653617, 0.01862071,\n",
       "        0.0123069 ]], dtype=float32)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "session.run(tf.global_variables_initializer())\n",
    "session.run(probs, feed_dict={input_data: x})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we are in the position to calculate the cost of training with __loss function__, and keep feedng the network to learn it. But, the question is: what the LSTM networks learn?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'rnnlm/softmax_w:0' shape=(128, 65) dtype=float32_ref>,\n",
       " <tf.Variable 'rnnlm/softmax_b:0' shape=(65,) dtype=float32_ref>,\n",
       " <tf.Variable 'rnnlm/embedding:0' shape=(65, 128) dtype=float32_ref>,\n",
       " <tf.Variable 'rnnlm/multi_rnn_cell/cell_0/basic_rnn_cell/kernel:0' shape=(256, 128) dtype=float32_ref>,\n",
       " <tf.Variable 'rnnlm/multi_rnn_cell/cell_0/basic_rnn_cell/bias:0' shape=(128,) dtype=float32_ref>]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grad_clip = 5.\n",
    "tvars = tf.trainable_variables()\n",
    "tvars"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# All together\n",
    "Now, let's put all of parts together in a class, and train the model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class LSTMModel():\n",
    "    def __init__(self, sample=False):\n",
    "        rnn_size = 128  # Size of RNN hidden state vector.\n",
    "        batch_size = 60  # Minibatch size, i.e. size of dataset in each epoch.\n",
    "        seq_length = 50  # RNN sequence length.\n",
    "        num_layers = 2  # Number of layers in the RNN.\n",
    "        vocab_size = 65\n",
    "        grad_clip = 5.\n",
    "        if sample:\n",
    "            print(\">> sample mode:\")\n",
    "            batch_size = 1\n",
    "            seq_length = 1\n",
    "        # The core of the model consists of a LSTM cell that processes one char at a time and\n",
    "        # computes probabilities of the possible continuations of the char. \n",
    "        basic_cell = tf.contrib.rnn.BasicRNNCell(rnn_size)\n",
    "        self.stacked_cell = tf.contrib.rnn.MultiRNNCell([basic_cell] * num_layers)\n",
    "\n",
    "        self.input_data = tf.placeholder(tf.int32, [batch_size, seq_length], name=\"input_data\")\n",
    "        self.targets = tf.placeholder(tf.int32, [batch_size, seq_length], name=\"targets\")\n",
    "        # Initial state of the LSTM memory.\n",
    "        # The memory state of the network is initialized with a vector of zeros and gets updated\n",
    "        # after reading each char. \n",
    "        self.initial_state = stacked_cell.zero_state(batch_size, tf.float32)\n",
    "\n",
    "        with tf.variable_scope('rnnlm_class1'):\n",
    "            softmax_w = tf.get_variable(\"softmax_w\", [rnn_size, vocab_size])  # 128x65\n",
    "            softmax_b = tf.get_variable(\"softmax_b\", [vocab_size])  # 1x65\n",
    "            with tf.device(\"/cpu:0\"):\n",
    "                embedding = tf.get_variable(\"embedding\", [vocab_size, rnn_size])  # 65x128\n",
    "                inputs = tf.split(tf.nn.embedding_lookup(embedding, self.input_data), seq_length, 1)\n",
    "                inputs = [tf.squeeze(input_, [1]) for input_ in inputs]\n",
    "\n",
    "        # The value of state is updated after processing each batch of chars.\n",
    "        outputs, last_state = tf.contrib.legacy_seq2seq.rnn_decoder(inputs, self.initial_state, self.stacked_cell, loop_function=None, scope='rnnlm_class1')\n",
    "        output = tf.reshape(tf.concat(outputs, 1), [-1, rnn_size])\n",
    "        self.logits = tf.matmul(output, softmax_w) + softmax_b\n",
    "        self.probs = tf.nn.softmax(self.logits)\n",
    "        loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example([self.logits],\n",
    "                [tf.reshape(self.targets, [-1])],\n",
    "                [tf.ones([batch_size * seq_length])],\n",
    "                vocab_size)\n",
    "        self.cost = tf.reduce_sum(loss) / batch_size / seq_length\n",
    "        self.final_state = last_state\n",
    "        self.lr = tf.Variable(0.0, trainable=False)\n",
    "        tvars = tf.trainable_variables()\n",
    "        grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, tvars), grad_clip)\n",
    "        optimizer = tf.train.AdamOptimizer(self.lr)\n",
    "        self.train_op = optimizer.apply_gradients(zip(grads, tvars))\n",
    "    \n",
    "    def sample(self, sess, chars, vocab, num=200, prime='The ', sampling_type=1):\n",
    "        state = sess.run(self.stacked_cell.zero_state(1, tf.float32))\n",
    "        for char in prime[:-1]:\n",
    "            x = np.zeros((1, 1))\n",
    "            x[0, 0] = vocab[char]\n",
    "            feed = {self.input_data: x, self.initial_state: state}\n",
    "            [state] = sess.run([self.final_state], feed)\n",
    "\n",
    "        def weighted_pick(weights):\n",
    "            t = np.cumsum(weights)\n",
    "            s = np.sum(weights)\n",
    "            return(int(np.searchsorted(t, np.random.rand(1)*s)))\n",
    "\n",
    "        ret = prime\n",
    "        char = prime[-1]\n",
    "        for n in range(num):\n",
    "            x = np.zeros((1, 1))\n",
    "            x[0, 0] = vocab[char]\n",
    "            feed = {self.input_data: x, self.initial_state: state}\n",
    "            [probs, state] = sess.run([self.probs, self.final_state], feed)\n",
    "            p = probs[0]\n",
    "\n",
    "            if sampling_type == 0:\n",
    "                sample = np.argmax(p)\n",
    "            elif sampling_type == 2:\n",
    "                if char == ' ':\n",
    "                    sample = weighted_pick(p)\n",
    "                else:\n",
    "                    sample = np.argmax(p)\n",
    "            else:\n",
    "                sample = weighted_pick(p)\n",
    "\n",
    "            pred = chars[sample]\n",
    "            ret += pred\n",
    "            char = pred\n",
    "        return ret"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Creating the LSTM object\n",
    "Now we create a LSTM model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope(\"rnn\"):\n",
    "    model = LSTMModel()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train using LSTMModel\n",
    "We can train our model through feeding batches:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "370/46375 (epoch 0), train_loss = 1.912, time/batch = 0.050\n",
      ">> sample mode:\n",
      "The sagrenous a man; aid to Advarge our lanjacnant one\n",
      "----------------------------------\n",
      "741/46375 (epoch 1), train_loss = 1.750, time/batch = 0.060\n",
      ">> sample mode:\n",
      "The within, I was head, offy dring, ane mers thill of \n",
      "----------------------------------\n",
      "1112/46375 (epoch 2), train_loss = 1.677, time/batch = 0.047\n",
      ">> sample mode:\n",
      "The pray of throous oftentiou, you alouth belike the w\n",
      "----------------------------------\n",
      "1483/46375 (epoch 3), train_loss = 1.638, time/batch = 0.062\n",
      ">> sample mode:\n",
      "The her\n",
      "First from a slears;\n",
      "With; theu, late among fa\n",
      "----------------------------------\n",
      "1854/46375 (epoch 4), train_loss = 1.612, time/batch = 0.049\n",
      ">> sample mode:\n",
      "The so not he shamest you send and ever hold you, but \n",
      "----------------------------------\n",
      "2225/46375 (epoch 5), train_loss = 1.596, time/batch = 0.046\n",
      ">> sample mode:\n",
      "The knots you stose beggamen;\n",
      "And thou hand unrear hil\n",
      "----------------------------------\n",
      "2596/46375 (epoch 6), train_loss = 1.585, time/batch = 0.054\n",
      ">> sample mode:\n",
      "The daar.\n",
      "How tho, and been, guarest to his depelfed b\n",
      "----------------------------------\n",
      "2967/46375 (epoch 7), train_loss = 1.577, time/batch = 0.051\n",
      ">> sample mode:\n",
      "The doubts;\n",
      "There beaning, fet both and she and depock\n",
      "----------------------------------\n",
      "3338/46375 (epoch 8), train_loss = 1.569, time/batch = 0.047\n",
      ">> sample mode:\n",
      "The saw'd and Romeature; to condemn thy littly.\n",
      "\n",
      "Secon\n",
      "----------------------------------\n",
      "3709/46375 (epoch 9), train_loss = 1.561, time/batch = 0.085\n",
      ">> sample mode:\n",
      "The year be foried: and, like will, gentle fled the ve\n",
      "----------------------------------\n",
      "4080/46375 (epoch 10), train_loss = 1.553, time/batch = 0.062\n",
      ">> sample mode:\n",
      "The is might,\n",
      "In us burnorder; within apmorghtion piac\n",
      "----------------------------------\n",
      "4451/46375 (epoch 11), train_loss = 1.545, time/batch = 0.047\n",
      ">> sample mode:\n",
      "The chaarefore that too,\n",
      "The may not, I mean my louds\n",
      "\n",
      "----------------------------------\n",
      "4822/46375 (epoch 12), train_loss = 1.539, time/batch = 0.052\n",
      ">> sample mode:\n",
      "The married, many nome?\n",
      "It shall sir, goes.\n",
      "To came yo\n",
      "----------------------------------\n",
      "5193/46375 (epoch 13), train_loss = 1.532, time/batch = 0.050\n",
      ">> sample mode:\n",
      "The day olf too under, good, and ass'd attendable beat\n",
      "----------------------------------\n",
      "5564/46375 (epoch 14), train_loss = 1.527, time/batch = 0.107\n",
      ">> sample mode:\n",
      "The shamst. Ot the fool.\n",
      "\n",
      "CLAULE:\n",
      "Dew thouspy pleace i\n",
      "----------------------------------\n",
      "5935/46375 (epoch 15), train_loss = 1.521, time/batch = 0.056\n",
      ">> sample mode:\n",
      "The inkingmence you speak, you, from from the druek ch\n",
      "----------------------------------\n",
      "6306/46375 (epoch 16), train_loss = 1.517, time/batch = 0.060\n",
      ">> sample mode:\n",
      "The king, be\n",
      "Procotit, let's crossh it, unspery wisk, \n",
      "----------------------------------\n",
      "6677/46375 (epoch 17), train_loss = 1.512, time/batch = 0.060\n",
      ">> sample mode:\n",
      "The is speeds but too sack'd,\n",
      "His botter mean in my bu\n",
      "----------------------------------\n",
      "7048/46375 (epoch 18), train_loss = 1.508, time/batch = 0.048\n",
      ">> sample mode:\n",
      "The vistiage\n",
      "As ever the purset brought upon dischalk,\n",
      "----------------------------------\n",
      "7419/46375 (epoch 19), train_loss = 1.504, time/batch = 0.050\n",
      ">> sample mode:\n",
      "The over sheffilently dank's vork down.\n",
      "\n",
      "Shepherd:\n",
      "My \n",
      "----------------------------------\n",
      "7790/46375 (epoch 20), train_loss = 1.500, time/batch = 0.057\n",
      ">> sample mode:\n",
      "The dowing are crave as not so delice,\n",
      "For thy bounds:\n",
      "----------------------------------\n",
      "8161/46375 (epoch 21), train_loss = 1.497, time/batch = 0.048\n",
      ">> sample mode:\n",
      "The a poor Corsion\n",
      "Tooth excuse\n",
      "To flike thee he spide\n",
      "----------------------------------\n",
      "8532/46375 (epoch 22), train_loss = 1.494, time/batch = 0.057\n",
      ">> sample mode:\n",
      "The uneld beard's head me and his purmicape,\n",
      "And but, \n",
      "----------------------------------\n",
      "8903/46375 (epoch 23), train_loss = 1.491, time/batch = 0.065\n",
      ">> sample mode:\n",
      "The jest; and I was peace, not have lage; go mace laug\n",
      "----------------------------------\n",
      "9274/46375 (epoch 24), train_loss = 1.489, time/batch = 0.046\n",
      ">> sample mode:\n",
      "The poor\n",
      "Of my your embrace is in?\n",
      "\n",
      "KANGOBELO:\n",
      "Find of\n",
      "----------------------------------\n",
      "9645/46375 (epoch 25), train_loss = 1.486, time/batch = 0.052\n",
      ">> sample mode:\n",
      "The accors, fourl,\n",
      "With me him refore her curty to be \n",
      "----------------------------------\n",
      "10016/46375 (epoch 26), train_loss = 1.484, time/batch = 0.064\n",
      ">> sample mode:\n",
      "The reasom, let me a peacter you remain\n",
      "Shall being jo\n",
      "----------------------------------\n",
      "10387/46375 (epoch 27), train_loss = 1.482, time/batch = 0.062\n",
      ">> sample mode:\n",
      "The prison, my necchers, look off buttains;\n",
      "Or we have\n",
      "----------------------------------\n",
      "10758/46375 (epoch 28), train_loss = 1.480, time/batch = 0.052\n",
      ">> sample mode:\n",
      "The husble, I not buy\n",
      "I hope the very them faith, unle\n",
      "----------------------------------\n",
      "11129/46375 (epoch 29), train_loss = 1.478, time/batch = 0.055\n",
      ">> sample mode:\n",
      "The o'er obey. Know him child.\n",
      "\n",
      "MENENIUS:\n",
      "The take not\n",
      "----------------------------------\n",
      "11500/46375 (epoch 30), train_loss = 1.476, time/batch = 0.058\n",
      ">> sample mode:\n",
      "The daughter in Marcius,\n",
      "That impariors not, crack-don\n",
      "----------------------------------\n",
      "11871/46375 (epoch 31), train_loss = 1.474, time/batch = 0.061\n",
      ">> sample mode:\n",
      "The mirich up thy mother wont Langerous!\n",
      "\n",
      "HERMIONE:\n",
      "Mo\n",
      "----------------------------------\n",
      "12242/46375 (epoch 32), train_loss = 1.473, time/batch = 0.047\n",
      ">> sample mode:\n",
      "The Awrated, for her bur'd\n",
      "A motwerry wied of that bee\n",
      "----------------------------------\n",
      "12613/46375 (epoch 33), train_loss = 1.471, time/batch = 0.049\n",
      ">> sample mode:\n",
      "The rich, lords, not of one sweet take they do not, oa\n",
      "----------------------------------\n",
      "12984/46375 (epoch 34), train_loss = 1.469, time/batch = 0.079\n",
      ">> sample mode:\n",
      "The instant olds, busion. Gentleman\n",
      "That you they entr\n",
      "----------------------------------\n",
      "13355/46375 (epoch 35), train_loss = 1.468, time/batch = 0.063\n",
      ">> sample mode:\n",
      "The known nobloods it bobring of your more thus part, \n",
      "----------------------------------\n",
      "13726/46375 (epoch 36), train_loss = 1.466, time/batch = 0.053\n",
      ">> sample mode:\n",
      "The Saint.\n",
      "\n",
      "DUCHESS OF CATETHAMNEDW:\n",
      "No, I canst untie\n",
      "----------------------------------\n",
      "14097/46375 (epoch 37), train_loss = 1.465, time/batch = 0.052\n",
      ">> sample mode:\n",
      "The or;\n",
      "Muster is bound it.\n",
      "\n",
      "SICINIUS:\n",
      "Sir, or godside\n",
      "----------------------------------\n",
      "14468/46375 (epoch 38), train_loss = 1.464, time/batch = 0.057\n",
      ">> sample mode:\n",
      "The roars? who, unto my father not and no gates at mak\n",
      "----------------------------------\n",
      "14839/46375 (epoch 39), train_loss = 1.463, time/batch = 0.057\n",
      ">> sample mode:\n",
      "The degoman?\n",
      "\n",
      "DUKE VINCENTIO:\n",
      "Didst thou wilt his prin\n",
      "----------------------------------\n",
      "15210/46375 (epoch 40), train_loss = 1.461, time/batch = 0.071\n",
      ">> sample mode:\n",
      "The soon of swean to hillous;\n",
      "For high for the clinner\n",
      "----------------------------------\n",
      "15581/46375 (epoch 41), train_loss = 1.460, time/batch = 0.058\n",
      ">> sample mode:\n",
      "The news?\n",
      "\n",
      "QUEEN MARGARET:\n",
      "No; Let me holy and prove w\n",
      "----------------------------------\n",
      "15952/46375 (epoch 42), train_loss = 1.459, time/batch = 0.067\n",
      ">> sample mode:\n",
      "The world? I traim\n",
      "tham myself way as the issue.\n",
      "\n",
      "DUCH\n",
      "----------------------------------\n",
      "16323/46375 (epoch 43), train_loss = 1.458, time/batch = 0.125\n",
      ">> sample mode:\n",
      "The bloodsly.\n",
      "Forget Freigns.\n",
      "Woy least you not; a hou\n",
      "----------------------------------\n",
      "16694/46375 (epoch 44), train_loss = 1.457, time/batch = 0.054\n",
      ">> sample mode:\n",
      "The kindred from blood nobly, good thanks;\n",
      "Why, you ki\n",
      "----------------------------------\n",
      "17065/46375 (epoch 45), train_loss = 1.456, time/batch = 0.055\n",
      ">> sample mode:\n",
      "The sure, to ungentle to give, but, speak too lord? ou\n",
      "----------------------------------\n",
      "17436/46375 (epoch 46), train_loss = 1.455, time/batch = 0.055\n",
      ">> sample mode:\n",
      "The runsting fried?\n",
      "-Lay, to wear,\n",
      "And thus?\n",
      "\n",
      "Good and\n",
      "----------------------------------\n",
      "17807/46375 (epoch 47), train_loss = 1.455, time/batch = 0.050\n",
      ">> sample mode:\n",
      "The time to me?\n",
      "Now, Kath at poor as 'twelt, 't.\n",
      "What,\n",
      "----------------------------------\n",
      "18178/46375 (epoch 48), train_loss = 1.454, time/batch = 0.052\n",
      ">> sample mode:\n",
      "The sworn not: concuse my father, being see\n",
      "Will, with\n",
      "----------------------------------\n",
      "18549/46375 (epoch 49), train_loss = 1.453, time/batch = 0.052\n",
      ">> sample mode:\n",
      "The more at death, well, Rosans a sent many merre.\n",
      "\n",
      "LE\n",
      "----------------------------------\n",
      "18920/46375 (epoch 50), train_loss = 1.452, time/batch = 0.069\n",
      ">> sample mode:\n",
      "The done! happily to thou the\n",
      "eintle house our royard;\n",
      "----------------------------------\n",
      "19291/46375 (epoch 51), train_loss = 1.451, time/batch = 0.053\n",
      ">> sample mode:\n",
      "The senson. Pault our affalled?\n",
      "\n",
      "QUEEN ELIZABETH:\n",
      "Pray\n",
      "----------------------------------\n",
      "19662/46375 (epoch 52), train_loss = 1.450, time/batch = 0.063\n",
      ">> sample mode:\n",
      "The twenty'd conquerent foushies the pity and speed in\n",
      "----------------------------------\n",
      "20033/46375 (epoch 53), train_loss = 1.450, time/batch = 0.058\n",
      ">> sample mode:\n",
      "The execut tears;\n",
      "if myself wratt,\n",
      "That I desperate, I\n",
      "----------------------------------\n",
      "20404/46375 (epoch 54), train_loss = 1.449, time/batch = 0.055\n",
      ">> sample mode:\n",
      "The welcome it be paintly better\n",
      "To citizen to succeas\n",
      "----------------------------------\n",
      "20775/46375 (epoch 55), train_loss = 1.448, time/batch = 0.058\n",
      ">> sample mode:\n",
      "The alled,\n",
      "And fourtstain be little?\n",
      "\n",
      "DUKE VINCENTIO:\n",
      "\n",
      "----------------------------------\n",
      "21146/46375 (epoch 56), train_loss = 1.447, time/batch = 0.089\n",
      ">> sample mode:\n",
      "The words on her:\n",
      "No, I seems, I am you, kings, are ha\n",
      "----------------------------------\n",
      "21517/46375 (epoch 57), train_loss = 1.447, time/batch = 0.068\n",
      ">> sample mode:\n",
      "The visio, myself plead' 'fong'de take away, for of a \n",
      "----------------------------------\n",
      "21888/46375 (epoch 58), train_loss = 1.446, time/batch = 0.069\n",
      ">> sample mode:\n",
      "The wates! Shall she been such most his piece\n",
      "In his w\n",
      "----------------------------------\n",
      "22259/46375 (epoch 59), train_loss = 1.445, time/batch = 0.062\n",
      ">> sample mode:\n",
      "The littly.\n",
      "\n",
      "LEONTES:\n",
      "I citle bosom, are and there?\n",
      "Ta\n",
      "----------------------------------\n",
      "22630/46375 (epoch 60), train_loss = 1.445, time/batch = 0.081\n",
      ">> sample mode:\n",
      "The water was feel her good up,\n",
      "Heave of fool;\n",
      "wour of\n",
      "----------------------------------\n",
      "23001/46375 (epoch 61), train_loss = 1.444, time/batch = 0.072\n",
      ">> sample mode:\n",
      "The hard, I prevent and send my oard.\n",
      "\n",
      "PETRUCHIO:\n",
      "Make\n",
      "----------------------------------\n",
      "23372/46375 (epoch 62), train_loss = 1.443, time/batch = 0.079\n",
      ">> sample mode:\n",
      "The appens,\n",
      "Lords to request; murder them my horse to \n",
      "----------------------------------\n",
      "23743/46375 (epoch 63), train_loss = 1.443, time/batch = 0.073\n",
      ">> sample mode:\n",
      "The dudst a man\n",
      "Undowivar answers on himself and senat\n",
      "----------------------------------\n",
      "24114/46375 (epoch 64), train_loss = 1.442, time/batch = 0.228\n",
      ">> sample mode:\n",
      "The count you?\n",
      "\n",
      "LADY ANNE:\n",
      "Ere, and callius, your grea\n",
      "----------------------------------\n",
      "24485/46375 (epoch 65), train_loss = 1.441, time/batch = 0.082\n",
      ">> sample mode:\n",
      "The rightor, end some down\n",
      "thy bord;\n",
      "Arziry the late, \n",
      "----------------------------------\n",
      "24856/46375 (epoch 66), train_loss = 1.441, time/batch = 0.092\n",
      ">> sample mode:\n",
      "The raise for this tearse-prince taker, let it?\n",
      "\n",
      "LEONT\n",
      "----------------------------------\n",
      "25227/46375 (epoch 67), train_loss = 1.440, time/batch = 0.052\n",
      ">> sample mode:\n",
      "The day, no' his ort\n",
      "Magour,\n",
      "Thou hast title hand\n",
      "Tyr \n",
      "----------------------------------\n",
      "25598/46375 (epoch 68), train_loss = 1.440, time/batch = 0.060\n",
      ">> sample mode:\n",
      "The yet, to for friend,\n",
      "Justing by a good,--\n",
      "I have fi\n",
      "----------------------------------\n",
      "25969/46375 (epoch 69), train_loss = 1.439, time/batch = 0.053\n",
      ">> sample mode:\n",
      "The back of your work,\n",
      "As now on the hollowing sustom,\n",
      "----------------------------------\n",
      "26340/46375 (epoch 70), train_loss = 1.439, time/batch = 0.054\n",
      ">> sample mode:\n",
      "The wails must be torchary tent of beneful, yet not 't\n",
      "----------------------------------\n",
      "26711/46375 (epoch 71), train_loss = 1.438, time/batch = 0.054\n",
      ">> sample mode:\n",
      "The seiseh: not they with good father ventle part of t\n",
      "----------------------------------\n",
      "27082/46375 (epoch 72), train_loss = 1.437, time/batch = 0.051\n",
      ">> sample mode:\n",
      "The procesing prest\n",
      "At his fat you, Camillow's,\n",
      "Or eye\n",
      "----------------------------------\n",
      "27453/46375 (epoch 73), train_loss = 1.437, time/batch = 0.060\n",
      ">> sample mode:\n",
      "The got\n",
      "How third Warwick, so.\n",
      "\n",
      "PETRUCHIO:\n",
      "Daughter th\n",
      "----------------------------------\n",
      "27824/46375 (epoch 74), train_loss = 1.436, time/batch = 0.055\n",
      ">> sample mode:\n",
      "The bleed; of our person, the double draw-to wilt shal\n",
      "----------------------------------\n",
      "28195/46375 (epoch 75), train_loss = 1.436, time/batch = 0.069\n",
      ">> sample mode:\n",
      "The hunder To the sore he\n",
      "is itself is hunts but this \n",
      "----------------------------------\n",
      "28566/46375 (epoch 76), train_loss = 1.436, time/batch = 0.047\n",
      ">> sample mode:\n",
      "The way you thank'd,\n",
      "And thou ho, powerw'd forgets, fr\n",
      "----------------------------------\n",
      "28937/46375 (epoch 77), train_loss = 1.435, time/batch = 0.072\n",
      ">> sample mode:\n",
      "The Paulinopatien times hasse,\n",
      "I'll know, the bid not \n",
      "----------------------------------\n",
      "29308/46375 (epoch 78), train_loss = 1.435, time/batch = 0.059\n",
      ">> sample mode:\n",
      "The king's it. Camillo. From grief,\n",
      "And they shall the\n",
      "----------------------------------\n",
      "29679/46375 (epoch 79), train_loss = 1.434, time/batch = 0.062\n",
      ">> sample mode:\n",
      "The peace me?\n",
      "But we do it enough on; and here,\n",
      "More t\n",
      "----------------------------------\n",
      "30050/46375 (epoch 80), train_loss = 1.434, time/batch = 0.056\n",
      ">> sample mode:\n",
      "The owarful's hopes, that Warwicks,\n",
      "I'll be bear you t\n",
      "----------------------------------\n",
      "30421/46375 (epoch 81), train_loss = 1.433, time/batch = 0.054\n",
      ">> sample mode:\n",
      "The rather prolouculing. Most not, deat is, my lords y\n",
      "----------------------------------\n",
      "30792/46375 (epoch 82), train_loss = 1.433, time/batch = 0.079\n",
      ">> sample mode:\n",
      "The woe, which above this dead; buoth their marriagess\n",
      "----------------------------------\n",
      "31163/46375 (epoch 83), train_loss = 1.433, time/batch = 0.059\n",
      ">> sample mode:\n",
      "The roars and doth seldesd\n",
      "My lord.\n",
      "\n",
      "BRAy:\n",
      "I think, I \n",
      "----------------------------------\n",
      "31534/46375 (epoch 84), train_loss = 1.432, time/batch = 0.055\n",
      ">> sample mode:\n",
      "The more.\n",
      "\n",
      "WERShARDINCH:\n",
      "God find me.\n",
      "\n",
      "CORIOLANUS:\n",
      "Now\n",
      "----------------------------------\n",
      "31905/46375 (epoch 85), train_loss = 1.432, time/batch = 0.060\n",
      ">> sample mode:\n",
      "The proclaim'd to the jadlow'st to Rome and no friends\n",
      "----------------------------------\n",
      "32276/46375 (epoch 86), train_loss = 1.432, time/batch = 0.061\n",
      ">> sample mode:\n",
      "The soud queen for fear solication\n",
      "Do the mark your qu\n",
      "----------------------------------\n",
      "32647/46375 (epoch 87), train_loss = 1.431, time/batch = 0.058\n",
      ">> sample mode:\n",
      "The self.\n",
      "\n",
      "CAPULET:\n",
      "I'll haste;\n",
      "For unpicius in place,\n",
      "----------------------------------\n",
      "33018/46375 (epoch 88), train_loss = 1.431, time/batch = 0.057\n",
      ">> sample mode:\n",
      "The Plantagenets all possward princied after, what wil\n",
      "----------------------------------\n",
      "33389/46375 (epoch 89), train_loss = 1.431, time/batch = 0.052\n",
      ">> sample mode:\n",
      "The Vial;\n",
      "O'er Claudio coat well am flowers?\n",
      "\n",
      "ISABELLA\n",
      "----------------------------------\n",
      "33760/46375 (epoch 90), train_loss = 1.430, time/batch = 0.062\n",
      ">> sample mode:\n",
      "The inlife.\n",
      "\n",
      "DUKE VINCENTIO:\n",
      "Good Jovert! ho's aimisle\n",
      "----------------------------------\n",
      "34131/46375 (epoch 91), train_loss = 1.430, time/batch = 0.082\n",
      ">> sample mode:\n",
      "The teme you we rid?\n",
      "\n",
      "HORTENSIO:\n",
      "But as he shad I live\n",
      "----------------------------------\n",
      "34502/46375 (epoch 92), train_loss = 1.430, time/batch = 0.049\n",
      ">> sample mode:\n",
      "The day, would havour is fled, and mank, make blast ch\n",
      "----------------------------------\n",
      "34873/46375 (epoch 93), train_loss = 1.430, time/batch = 0.065\n",
      ">> sample mode:\n",
      "The time no goldavions by confident,\n",
      "That summial,\n",
      "and\n",
      "----------------------------------\n",
      "35244/46375 (epoch 94), train_loss = 1.429, time/batch = 0.059\n",
      ">> sample mode:\n",
      "The lady;\n",
      "Ged friend, asknelling fiends proved state a\n",
      "----------------------------------\n",
      "35615/46375 (epoch 95), train_loss = 1.429, time/batch = 0.052\n",
      ">> sample mode:\n",
      "The year nothing; Musires thy weary life'ses, tell me,\n",
      "----------------------------------\n",
      "35986/46375 (epoch 96), train_loss = 1.429, time/batch = 0.110\n",
      ">> sample mode:\n",
      "The cap avoid.\n",
      "\n",
      "Second OwI:\n",
      "Your did meet your gentlem\n",
      "----------------------------------\n",
      "36357/46375 (epoch 97), train_loss = 1.429, time/batch = 0.054\n",
      ">> sample mode:\n",
      "The whom his nent.\n",
      "\n",
      "CAPULET:\n",
      "MEd unbarred of them the \n",
      "----------------------------------\n",
      "36728/46375 (epoch 98), train_loss = 1.428, time/batch = 0.072\n",
      ">> sample mode:\n",
      "The hanging him, call on thy father!\n",
      "You now; you have\n",
      "----------------------------------\n",
      "37099/46375 (epoch 99), train_loss = 1.428, time/batch = 0.082\n",
      ">> sample mode:\n",
      "The match;\n",
      "Richarde would I say, desire ushoo, to the \n",
      "----------------------------------\n",
      "37470/46375 (epoch 100), train_loss = 1.428, time/batch = 0.061\n",
      ">> sample mode:\n",
      "The godle?\n",
      "\n",
      "HENRY BOLINGBROKE:\n",
      "Why, be controwing dria\n",
      "----------------------------------\n",
      "37841/46375 (epoch 101), train_loss = 1.428, time/batch = 0.058\n",
      ">> sample mode:\n",
      "The naperit infanting,\n",
      "Whered, let'd not?\n",
      "\n",
      "Conabled in\n",
      "----------------------------------\n",
      "38212/46375 (epoch 102), train_loss = 1.428, time/batch = 0.053\n",
      ">> sample mode:\n",
      "The daughter,\n",
      "So did now, he will along but my father,\n",
      "----------------------------------\n",
      "38583/46375 (epoch 103), train_loss = 1.427, time/batch = 0.060\n",
      ">> sample mode:\n",
      "The roted thy countrys, honour. Neighbland.\n",
      "Let me Luc\n",
      "----------------------------------\n",
      "38954/46375 (epoch 104), train_loss = 1.427, time/batch = 0.056\n",
      ">> sample mode:\n",
      "The house censures his, madam, needs, vain, feed of hi\n",
      "----------------------------------\n",
      "39325/46375 (epoch 105), train_loss = 1.427, time/batch = 0.060\n",
      ">> sample mode:\n",
      "The yours,\n",
      "Unless make, he lie to sanger him.\n",
      "\n",
      "LUCIO:\n",
      "\n",
      "----------------------------------\n",
      "39696/46375 (epoch 106), train_loss = 1.427, time/batch = 0.109\n",
      ">> sample mode:\n",
      "The duke'er subds\n",
      "Are I to that have well, what morest\n",
      "----------------------------------\n",
      "40067/46375 (epoch 107), train_loss = 1.427, time/batch = 0.068\n",
      ">> sample mode:\n",
      "The king's unless house: piece, a do live,\n",
      "That sufped\n",
      "----------------------------------\n",
      "40438/46375 (epoch 108), train_loss = 1.427, time/batch = 0.096\n",
      ">> sample mode:\n",
      "The leisure spoke a cloud's love is dear up the circen\n",
      "----------------------------------\n",
      "40809/46375 (epoch 109), train_loss = 1.426, time/batch = 0.095\n",
      ">> sample mode:\n",
      "The forgets,\n",
      "And sworn all as if my love hithers are d\n",
      "----------------------------------\n",
      "41180/46375 (epoch 110), train_loss = 1.426, time/batch = 0.072\n",
      ">> sample mode:\n",
      "The chambined and graps the door soveread\n",
      "You have tha\n",
      "----------------------------------\n",
      "41551/46375 (epoch 111), train_loss = 1.426, time/batch = 0.055\n",
      ">> sample mode:\n",
      "The things as they deserve, when I will not kind no th\n",
      "----------------------------------\n",
      "41922/46375 (epoch 112), train_loss = 1.426, time/batch = 0.073\n",
      ">> sample mode:\n",
      "The advences.\n",
      "\n",
      "BUCKINGHAM:\n",
      "I servon; I\n",
      "no Land:\n",
      "I was \n",
      "----------------------------------\n",
      "42293/46375 (epoch 113), train_loss = 1.426, time/batch = 0.067\n",
      ">> sample mode:\n",
      "The sacress being lamb, was is the odds\n",
      "I thoughts it \n",
      "----------------------------------\n",
      "42664/46375 (epoch 114), train_loss = 1.426, time/batch = 0.052\n",
      ">> sample mode:\n",
      "The Foic,\n",
      "This sovereign;\n",
      "And intends to did that my c\n",
      "----------------------------------\n",
      "43035/46375 (epoch 115), train_loss = 1.425, time/batch = 0.080\n",
      ">> sample mode:\n",
      "The hand art speak peace:\n",
      "Besides,\n",
      "Can keare is heaven\n",
      "----------------------------------\n",
      "43406/46375 (epoch 116), train_loss = 1.425, time/batch = 0.053\n",
      ">> sample mode:\n",
      "The seightow me at him by one good with hut's son.\n",
      "I w\n",
      "----------------------------------\n",
      "43777/46375 (epoch 117), train_loss = 1.425, time/batch = 0.055\n",
      ">> sample mode:\n",
      "The souts how that he shall, yet what one hence\n",
      "Lord C\n",
      "----------------------------------\n",
      "44148/46375 (epoch 118), train_loss = 1.425, time/batch = 0.059\n",
      ">> sample mode:\n",
      "The many ake withing you, gentlice.\n",
      "This shall stand t\n",
      "----------------------------------\n",
      "44519/46375 (epoch 119), train_loss = 1.425, time/batch = 0.050\n",
      ">> sample mode:\n",
      "The away be cons a fortingans as we were capsing and h\n",
      "----------------------------------\n",
      "44890/46375 (epoch 120), train_loss = 1.425, time/batch = 0.067\n",
      ">> sample mode:\n",
      "The aims\n",
      "Speaks the king's commonded: else thou revone\n",
      "----------------------------------\n",
      "45261/46375 (epoch 121), train_loss = 1.425, time/batch = 0.073\n",
      ">> sample mode:\n",
      "The tun hath points that that all\n",
      "In many on Sice to h\n",
      "----------------------------------\n",
      "45632/46375 (epoch 122), train_loss = 1.425, time/batch = 0.067\n",
      ">> sample mode:\n",
      "The peniteness,\n",
      "And he is the goose to be dirsh of sor\n",
      "----------------------------------\n",
      "46003/46375 (epoch 123), train_loss = 1.424, time/batch = 0.051\n",
      ">> sample mode:\n",
      "The hands.\n",
      "\n",
      "GONZALO:\n",
      "Coting help myselved his parth of\n",
      "----------------------------------\n",
      "46374/46375 (epoch 124), train_loss = 1.424, time/batch = 0.047\n",
      ">> sample mode:\n",
      "The kind-rights borne, I would never led, go fury and \n",
      "----------------------------------\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for e in range(num_epochs):\n",
    "        sess.run(tf.assign(model.lr, learning_rate * (decay_rate ** e)))\n",
    "        data_loader.reset_batch_pointer()\n",
    "        state = sess.run(model.initial_state)  # (2x[60x128])\n",
    "        for b in range(data_loader.num_batches):\n",
    "            start = time.time()\n",
    "            x, y = data_loader.next_batch()\n",
    "            feed = {model.input_data: x, model.targets: y, model.initial_state: state}\n",
    "            train_loss, state, _ = sess.run([model.cost, model.final_state, model.train_op], feed)\n",
    "            end = time.time()\n",
    "        print(\"{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}\" \\\n",
    "                .format(e * data_loader.num_batches + b, num_epochs * data_loader.num_batches, e, train_loss, end - start))\n",
    "        with tf.variable_scope(\"rnn\", reuse=True):\n",
    "            sample_model = LSTMModel(sample=True)\n",
    "            print(sample_model.sample(sess, data_loader.chars, data_loader.vocab, num=50, prime='The ', sampling_type=1))\n",
    "            print('----------------------------------')"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}