honors_assignment_conv_chat_bot.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Approach:\n",
    "    \n",
    "I have used Generative LSTM based language model due to the unavailability of GPU or higher CPU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dependencies\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "import datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_len = 50\n",
    "data = datasets.readCornellData(\"data/cornell\", max_len = max_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('cornell_data_%d'%max_len, 'wb') as file:\n",
    "    pickle.dump(data, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('cornell_data_50', 'rb') as file:\n",
    "    data = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('gosh if only we could find kat a boyfriend', 'let me see what i can do'),\n",
       " ('cesc ma tete this is my head', 'right see youre ready for the quiz'),\n",
       " ('thats because its such a nice one', 'forget french'),\n",
       " ('there', 'where'),\n",
       " ('you have my word as a gentleman', 'youre sweet'),\n",
       " ('hi', 'looks like things worked out tonight huh'),\n",
       " ('you know chastity', 'i believe we share an art instructor'),\n",
       " ('have fun tonight', 'tons'),\n",
       " ('i was', 'you never wanted to go out with me did you'),\n",
       " ('well no', 'then thats all you had to say')]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "101349\n"
     ]
    }
   ],
   "source": [
    "print (len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "not there\n",
      "not there\n"
     ]
    }
   ],
   "source": [
    "word2idx = {'A' : 1,  'B' : 2}\n",
    "\n",
    "word = 'A B C D'\n",
    "\n",
    "for word in word.split(' '):\n",
    "    if word not in word2idx:\n",
    "        print ('not there')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter out sentences above a max threshold for both the question and response.\n",
    "\n",
    "MAX_LEN = 10\n",
    "\n",
    "def remove_pairs_above_max_thresh(pairs, MAX_LEN):\n",
    "    \n",
    "    if ((len(pairs[0].split(' ')) <= MAX_LEN) & \n",
    "        (len(pairs[1].split(' ')) <= MAX_LEN)):\n",
    "        return True\n",
    "    else:\n",
    "        return False    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_data = [p for p in data if remove_pairs_above_max_thresh(p, MAX_LEN)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('gosh if only we could find kat a boyfriend', 'let me see what i can do'),\n",
       " ('cesc ma tete this is my head', 'right see youre ready for the quiz'),\n",
       " ('thats because its such a nice one', 'forget french'),\n",
       " ('there', 'where'),\n",
       " ('you have my word as a gentleman', 'youre sweet'),\n",
       " ('hi', 'looks like things worked out tonight huh'),\n",
       " ('you know chastity', 'i believe we share an art instructor'),\n",
       " ('have fun tonight', 'tons'),\n",
       " ('i was', 'you never wanted to go out with me did you'),\n",
       " ('well no', 'then thats all you had to say')]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_data[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "97104\n"
     ]
    }
   ],
   "source": [
    "print (len(new_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[0][0] +  ' '  + data[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['well', 'no', 'then', 'thats', 'all', 'you', 'had', 'to', 'say']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(data[9][0] +  ' '  + data[9][1]).split(\" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# need to pack pair query, response into one corpus of words\n",
    "from collections import Counter\n",
    "\n",
    "total_vocab = Counter()\n",
    "\n",
    "combined = []\n",
    "for j in range(len(data)):\n",
    "    for word in (data[j][0] + ' ' + data[j][0]).split():\n",
    "        total_vocab[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('hay', 2),\n",
       " ('conti', 2),\n",
       " ('nental', 2),\n",
       " ('mmmmmmmmm', 2),\n",
       " ('fuchsmachen', 2),\n",
       " ('sweetheartis', 2),\n",
       " ('schwanzstucker', 2),\n",
       " ('tenthirty', 2),\n",
       " ('grrrhmmnnnjkjmmmnn', 2),\n",
       " ('noggs', 2)]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_vocab.most_common()[-10:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab size is: 21,800\n"
     ]
    }
   ],
   "source": [
    "# total size of the vocab\n",
    "\n",
    "vocab = set(total_vocab.keys())\n",
    "print (\"Vocab size is: {:,}\".format(len(vocab)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11086\n",
      "10714\n"
     ]
    }
   ],
   "source": [
    "# let's drop any word which has frequency below a min count\n",
    "# in our case, let's use 5 for example\n",
    "\n",
    "keep_words = []\n",
    "drop_words = []\n",
    "min_cnt = 3\n",
    "\n",
    "for k, v in total_vocab.items():\n",
    "    if v < min_cnt:\n",
    "        drop_words.append(k)\n",
    "    elif v >= min_cnt:\n",
    "        keep_words.append(k)\n",
    "        \n",
    "print (len(drop_words))\n",
    "print (len(keep_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['cesc',\n",
       " 'tete',\n",
       " 'unbalanced',\n",
       " '9th',\n",
       " 'perish',\n",
       " 'tutor',\n",
       " 'shrew',\n",
       " 'biancas',\n",
       " 'smokers',\n",
       " 'assail']"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drop_words[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['PAD', 'SOS', 'EOS', 'we', 'could', 'find', 'kat', 'a', 'boyfriend', 'ma']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# adding special words to the keep_words list\n",
    "\n",
    "keep_words[0] = \"PAD\"\n",
    "keep_words[1] = \"SOS\"\n",
    "keep_words[2] = \"EOS\"\n",
    "\n",
    "keep_words[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2idx = {}\n",
    "idx2word = {}\n",
    "\n",
    "for i, word in enumerate(keep_words):\n",
    "    word2idx[word] = i\n",
    "    idx2word[i] = word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'PAD': 0,\n",
       " 'SOS': 1,\n",
       " 'EOS': 2,\n",
       " 'we': 3,\n",
       " 'could': 4,\n",
       " 'find': 5,\n",
       " 'kat': 6,\n",
       " 'a': 7,\n",
       " 'boyfriend': 8,\n",
       " 'ma': 9,\n",
       " 'this': 10,\n",
       " 'is': 11,\n",
       " 'my': 12,\n",
       " 'head': 13,\n",
       " 'thats': 14,\n",
       " 'because': 15}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample = {k: word2idx[k] for k in list(word2idx)[:16]}\n",
    "sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The number of Question Answer pairs in new_data is: 97,104\n",
      "Trummed from 97,104 pairs to 75,352, 0.78 of total\n"
     ]
    }
   ],
   "source": [
    "# let's try to filter out pairs that do not contain the words in keep words\n",
    "\n",
    "print (\"The number of Question Answer pairs in new_data is: {:,}\".format(len(new_data)))\n",
    "new_data2 = []\n",
    "\n",
    "for p in range(len(new_data)):\n",
    "    \n",
    "    input_sen = new_data[p][0]\n",
    "    output_sen = new_data[p][1]\n",
    "    \n",
    "    keep_input = True\n",
    "    keep_output = True\n",
    "    \n",
    "    for word in input_sen.split():\n",
    "        \n",
    "        if word not in keep_words:\n",
    "            keep_input = False\n",
    "            break\n",
    "            \n",
    "    for word in output_sen.split():\n",
    "        \n",
    "        if word not in keep_words:\n",
    "            keep_output = False\n",
    "            break\n",
    "            \n",
    "    if keep_input and keep_output:        \n",
    "        new_data2.append((input_sen, output_sen))\n",
    "        \n",
    "print (\"Trummed from {:,} pairs to {:,}, {:.2f} of total\".format(len(new_data), len(new_data2),\n",
    "                                                                len(new_data2)/len(new_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('thats because its such a nice one', 'forget french'),\n",
       " ('there', 'where'),\n",
       " ('you have my word as a gentleman', 'youre sweet'),\n",
       " ('hi', 'looks like things worked out tonight huh'),\n",
       " ('have fun tonight', 'tons')]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_data2[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3, 5, 10, 0, 0, 0, 0, 0, 0, 0]\n",
      "[4, 9, 1, 3, 10, 0, 0, 0, 0, 0]\n",
      "[3, 3, 4, 5, 1, 6, 2, 1, 7, 10]\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "list1 = [3, 5, 10]\n",
    "list2 = [4, 9, 1, 3, 10]\n",
    "list3 = [3, 3, 4, 5, 1, 6, 2, 1, 7, 10]\n",
    "\n",
    "def test_ex(mylist, ml, val):\n",
    "    if len(mylist) < ml:\n",
    "        return mylist + [val] * (ml-len(mylist))\n",
    "    else:\n",
    "        return mylist\n",
    "    \n",
    "print (test_ex(list1, ml=10, val=0))\n",
    "print (test_ex(list2, ml=10, val=0))\n",
    "print (test_ex(list3, ml=10, val=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[14, 15, 16, 17, 7, 18, 19, 2, 0, 0]\n",
      "[241, 172, 2, 0, 0, 0, 0, 0, 0, 0]\n",
      "[20, 2, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "[61, 2, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "[21, 22, 12, 23, 24, 7, 25, 2, 0, 0]\n",
      "[200, 80, 2, 0, 0, 0, 0, 0, 0, 0]\n",
      "[26, 2, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "[1358, 87, 1183, 2820, 100, 30, 637, 2, 0, 0]\n",
      "[22, 29, 30, 2, 0, 0, 0, 0, 0, 0]\n",
      "[7514, 2, 0, 0, 0, 0, 0, 0, 0, 0]\n"
     ]
    }
   ],
   "source": [
    "# converting the input and output sentences into tensors and train with mini-batches\n",
    "# data processing: since max_length = 10, any sentence shorter than 10 will need to be padded\n",
    "# with zeros after EOS token\n",
    "\n",
    "\n",
    "# padding function\n",
    "\n",
    "def padding_func(mylist, ml, val):\n",
    "    if len(mylist) < ml:\n",
    "        return mylist + [val] * (ml-len(mylist))\n",
    "    else:\n",
    "        return mylist\n",
    "\n",
    "# adding EOS \n",
    "\n",
    "max_len = 10\n",
    "input_index = []\n",
    "output_index = []  \n",
    "\n",
    "pad_token = word2idx['PAD']\n",
    "eos_token = word2idx['EOS']\n",
    "    \n",
    "for p in range(len(new_data2)):\n",
    "    \n",
    "    input_sen = new_data2[p][0]\n",
    "    output_sen = new_data2[p][1]\n",
    "    \n",
    "    # convert both sentences to indexes using the above defined word2idx\n",
    "    \n",
    "    p_input = []\n",
    "    p_output = []\n",
    "    \n",
    "    for word in input_sen.split():\n",
    "        p_input.append(word2idx[word])        \n",
    "        \n",
    "    p_input.append(eos_token)\n",
    "    p_input = padding_func(p_input, ml=max_len, val=pad_token)    \n",
    "    input_index.append(p_input)\n",
    "    \n",
    "    \n",
    "    for word in output_sen.split():\n",
    "            p_output.append(word2idx[word])        \n",
    "\n",
    "    p_output.append(eos_token)\n",
    "    p_output = padding_func(p_output, ml=max_len, val=pad_token)    \n",
    "    output_index.append(p_output)\n",
    "    \n",
    "    \n",
    "for j in range(5):\n",
    "    \n",
    "    print (input_index[j])\n",
    "    print (output_index[j])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "75352\n",
      "75352\n"
     ]
    }
   ],
   "source": [
    "print (len(input_index))\n",
    "print (len(output_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def true_len(mylist):    \n",
    "    return len(list(filter(lambda a: a != 0, mylist)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, [14, 15, 16, 17, 7, 18, 19, 2, 0, 0]) 8\n",
      "(1, [20, 2, 0, 0, 0, 0, 0, 0, 0, 0]) 2\n",
      "(2, [21, 22, 12, 23, 24, 7, 25, 2, 0, 0]) 8\n",
      "(3, [26, 2, 0, 0, 0, 0, 0, 0, 0, 0]) 2\n"
     ]
    }
   ],
   "source": [
    "for i in enumerate(input_index[:4]):\n",
    "    print (i, true_len(i[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "72853\n",
      "72853\n"
     ]
    }
   ],
   "source": [
    "# sorting the pairs by length of the questions\n",
    "\n",
    "sorted_questions = []\n",
    "sorted_answers = []\n",
    "\n",
    "for length in range(1, max_len+1):\n",
    "    \n",
    "    for i in enumerate(input_index):\n",
    "        if true_len(i[1]) == length:\n",
    "            sorted_questions.append(input_index[i[0]])\n",
    "            sorted_answers.append(output_index[i[0]])\n",
    "\n",
    "print(len(sorted_questions))\n",
    "print(len(sorted_answers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[20, 2, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "[61, 2, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "******************************\n",
      "[26, 2, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "[1358, 87, 1183, 2820, 100, 30, 637, 2, 0, 0]\n",
      "******************************\n",
      "[40, 2, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "[21, 258, 86, 10, 8097, 2, 0, 0, 0, 0]\n",
      "******************************\n"
     ]
    }
   ],
   "source": [
    "for i in range(3):\n",
    "    print(sorted_questions[i])\n",
    "    print(sorted_answers[i])\n",
    "    print ('*' * 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_inputs():\n",
    "    \n",
    "    \n",
    "    input_data = tf.placeholder(tf.int32, [None, None], name = 'input')\n",
    "    targets = tf.placeholder(tf.int32, [None, None], name = 'targets')\n",
    "    lr = tf.placeholder(tf.float32, name = 'learning_rate')\n",
    "    keep_prob = tf.placeholder(tf.float32, name  ='keep_prob')\n",
    "\n",
    "    return input_data, targets, lr, keep_prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_encoding_input(target_data, vocab_to_int, batch_size):\n",
    "    '''Remove the last word id from each batch and concat the SOS to the begining of each batch'''\n",
    "    \n",
    "    ending = tf.strided_slice(target_data, [0, 0], [batch_size, -1], [1, 1])\n",
    "    dec_input = tf.concat([tf.fill([batch_size, 1], vocab_to_int['SOS']), ending], 1)\n",
    "\n",
    "    return dec_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encoding_layer(rnn_inputs, rnn_size, num_layers, keep_prob, sequence_length):\n",
    "    '''Create the encoding layer'''\n",
    "    \n",
    "    lstm = tf.contrib.rnn.BasicLSTMCell(rnn_size)\n",
    "    drop = tf.contrib.rnn.DropoutWrapper(lstm, input_keep_prob = keep_prob)\n",
    "    enc_cell = tf.contrib.rnn.MultiRNNCell([drop] * num_layers)\n",
    "    _, enc_state = tf.nn.bidirectional_dynamic_rnn(cell_fw = enc_cell,\n",
    "                                                   cell_bw = enc_cell,\n",
    "                                                   sequence_length = sequence_length,\n",
    "                                                   inputs = rnn_inputs, \n",
    "                                                   dtype=tf.float32)\n",
    "    return enc_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decoding_layer_train(encoder_state, dec_cell, dec_embed_input, sequence_length, decoding_scope,\n",
    "                         output_fn, keep_prob, batch_size):\n",
    "    '''Decode the training data'''\n",
    "    \n",
    "    attention_states = tf.zeros([batch_size, 1, dec_cell.output_size])\n",
    "    \n",
    "    att_keys, att_vals, att_score_fn, att_construct_fn = tf.contrib.seq2seq.prepare_attention(attention_states,\n",
    "                                                 attention_option=\"bahdanau\",\n",
    "                                                 num_units=dec_cell.output_size)\n",
    "    \n",
    "    train_decoder_fn = tf.contrib.seq2seq.attention_decoder_fn_train(encoder_state[0],\n",
    "                                                                     att_keys,\n",
    "                                                                     att_vals,\n",
    "                                                                     att_score_fn,\n",
    "                                                                     att_construct_fn,\n",
    "                                                                     name = \"attn_dec_train\")\n",
    "    train_pred, _, _ = tf.contrib.seq2seq.dynamic_rnn_decoder(dec_cell, \n",
    "                                                              train_decoder_fn, \n",
    "                                                              dec_embed_input, \n",
    "                                                              sequence_length, \n",
    "                                                              scope=decoding_scope)\n",
    "    train_pred_drop = tf.nn.dropout(train_pred, keep_prob)\n",
    "    return output_fn(train_pred_drop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decoding_layer_infer(encoder_state, dec_cell, dec_embeddings, start_of_sequence_id, end_of_sequence_id,\n",
    "                         maximum_length, vocab_size, decoding_scope, output_fn, keep_prob, batch_size):\n",
    "    '''Decode the prediction data'''\n",
    "    \n",
    "    attention_states = tf.zeros([batch_size, 1, dec_cell.output_size])\n",
    "    \n",
    "    att_keys, att_vals, att_score_fn, att_construct_fn =             tf.contrib.seq2seq.prepare_attention(attention_states,\n",
    "                                                 attention_option=\"bahdanau\",\n",
    "                                                 num_units=dec_cell.output_size)\n",
    "    \n",
    "    infer_decoder_fn = tf.contrib.seq2seq.attention_decoder_fn_inference(output_fn, \n",
    "                                                                         encoder_state[0], \n",
    "                                                                         att_keys, \n",
    "                                                                         att_vals, \n",
    "                                                                         att_score_fn, \n",
    "                                                                         att_construct_fn, \n",
    "                                                                         dec_embeddings,\n",
    "                                                                         start_of_sequence_id, \n",
    "                                                                         end_of_sequence_id, \n",
    "                                                                         maximum_length, \n",
    "                                                                         vocab_size, \n",
    "                                                                         name = \"attn_dec_inf\")\n",
    "    infer_logits, _, _ = tf.contrib.seq2seq.dynamic_rnn_decoder(dec_cell, \n",
    "                                                                infer_decoder_fn, \n",
    "                                                                scope=decoding_scope)\n",
    "    \n",
    "    return infer_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decoding_layer(dec_embed_input, dec_embeddings, encoder_state, vocab_size, sequence_length, rnn_size,\n",
    "                   num_layers, vocab_to_int, keep_prob, batch_size):\n",
    "    '''Create the decoding cell and input the parameters for the training and inference decoding layers'''\n",
    "    \n",
    "    with tf.variable_scope(\"decoding\") as decoding_scope:\n",
    "        lstm = tf.contrib.rnn.BasicLSTMCell(rnn_size)\n",
    "        drop = tf.contrib.rnn.DropoutWrapper(lstm, input_keep_prob = keep_prob)\n",
    "        dec_cell = tf.contrib.rnn.MultiRNNCell([drop] * num_layers)\n",
    "        \n",
    "        weights = tf.truncated_normal_initializer(stddev=0.1)\n",
    "        biases = tf.zeros_initializer()\n",
    "        output_fn = lambda x: tf.contrib.layers.fully_connected(x, \n",
    "                                                                vocab_size, \n",
    "                                                                None, \n",
    "                                                                scope=decoding_scope,\n",
    "                                                                weights_initializer = weights,\n",
    "                                                                biases_initializer = biases)\n",
    "\n",
    "        train_logits = decoding_layer_train(encoder_state, \n",
    "                                            dec_cell, \n",
    "                                            dec_embed_input, \n",
    "                                            sequence_length, \n",
    "                                            decoding_scope, \n",
    "                                            output_fn, \n",
    "                                            keep_prob, \n",
    "                                            batch_size)\n",
    "        decoding_scope.reuse_variables()\n",
    "        infer_logits = decoding_layer_infer(encoder_state, \n",
    "                                            dec_cell, \n",
    "                                            dec_embeddings, \n",
    "                                            vocab_to_int['SOS'],\n",
    "                                            vocab_to_int['EOS'], \n",
    "                                            sequence_length - 1, \n",
    "                                            vocab_size,\n",
    "                                            decoding_scope, \n",
    "                                            output_fn, keep_prob, \n",
    "                                            batch_size)\n",
    "\n",
    "    return train_logits, infer_logits\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "def seq2seq_model(input_data, target_data, keep_prob, batch_size, sequence_length, answers_vocab_size, \n",
    "                  questions_vocab_size, enc_embedding_size, dec_embedding_size, rnn_size, num_layers, \n",
    "                  questions_vocab_to_int):\n",
    "    \n",
    "    '''Use the previous functions to create the training and inference logits'''\n",
    "    \n",
    "    enc_embed_input = tf.contrib.layers.embed_sequence(input_data, \n",
    "                                                       answers_vocab_size+1, \n",
    "                                                       enc_embedding_size,\n",
    "                                                       initializer = tf.random_uniform_initializer(0,1))\n",
    "    enc_state = encoding_layer(enc_embed_input, rnn_size, num_layers, keep_prob, sequence_length)\n",
    "\n",
    "    dec_input = process_encoding_input(target_data, questions_vocab_to_int, batch_size)\n",
    "    dec_embeddings = tf.Variable(tf.random_uniform([questions_vocab_size+1, dec_embedding_size], 0, 1))\n",
    "    dec_embed_input = tf.nn.embedding_lookup(dec_embeddings, dec_input)\n",
    "    \n",
    "    train_logits, infer_logits = decoding_layer(dec_embed_input, \n",
    "                                                dec_embeddings, \n",
    "                                                enc_state, \n",
    "                                                questions_vocab_size, \n",
    "                                                sequence_length, \n",
    "                                                rnn_size, \n",
    "                                                num_layers, \n",
    "                                                questions_vocab_to_int, \n",
    "                                                keep_prob, \n",
    "                                                batch_size)\n",
    "    return train_logits, infer_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the Hyperparameters\n",
    "epochs = 100\n",
    "batch_size = 128\n",
    "rnn_size = 512\n",
    "num_layers = 2\n",
    "encoding_embedding_size = 512\n",
    "decoding_embedding_size = 512\n",
    "learning_rate = 0.005\n",
    "learning_rate_decay = 0.9\n",
    "min_learning_rate = 0.0001\n",
    "keep_probability = 0.75"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "module 'tensorflow' has no attribute 'reset_default_graph'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-79-05c772cd4600>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# Reset the graph to ensure that it is ready for training\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset_default_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;31m# Start the session\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0msess\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mInteractiveSession\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: module 'tensorflow' has no attribute 'reset_default_graph'"
     ]
    }
   ],
   "source": [
    "# Reset the graph to ensure that it is ready for training\n",
    "tf.reset_default_graph()\n",
    "# Start the session\n",
    "sess = tf.InteractiveSession()\n",
    "    \n",
    "# Load the model inputs    \n",
    "input_data, targets, lr, keep_prob = model_inputs()\n",
    "\n",
    "# Sequence length will be the max line length for each batch\n",
    "sequence_length = tf.placeholder_with_default(max_line_length, None, name='sequence_length')\n",
    "\n",
    "# Find the shape of the input data for sequence_loss\n",
    "input_shape = tf.shape(input_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}