NLP_Research1_old.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\310267647\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential, Model\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers import Input, Activation, Dense, Permute, Dropout, add, dot, concatenate\n",
    "from keras.layers import LSTM\n",
    "from keras.utils.data_utils import get_file\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import load_model\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn import metrics\n",
    "from functools import reduce\n",
    "import pickle\n",
    "import tarfile\n",
    "import numpy as np\n",
    "import re\n",
    "import os\n",
    "import time\n",
    "\n",
    "\n",
    "def tokenize(sent):\n",
    "    '''Return the tokens of a sentence including punctuation.\n",
    "    >>> tokenize('Bob dropped the apple. Where is the apple?')\n",
    "    ['Bob', 'dropped', 'the', 'apple', '.', 'Where', 'is', 'the', 'apple', '?']\n",
    "    '''\n",
    "    return [x.strip() for x in re.split('(\\W+)', sent) if x.strip()]\n",
    "\n",
    "\n",
    "def parse_stories(lines, only_supporting=False):\n",
    "    '''Parse stories provided in the bAbi tasks format\n",
    "    If only_supporting is true, only the sentences\n",
    "    that support the answer are kept.\n",
    "    '''\n",
    "    data = []\n",
    "    story = []\n",
    "    for line in lines:\n",
    "        line = line.decode('utf-8').strip()\n",
    "        nid, line = line.split(' ', 1)\n",
    "        nid = int(nid)\n",
    "        if nid == 1:\n",
    "            story = []\n",
    "        if '\\t' in line:\n",
    "            q, a, supporting = line.split('\\t')\n",
    "            q = tokenize(q)\n",
    "            substory = None\n",
    "            if only_supporting:\n",
    "                # Only select the related substory\n",
    "                supporting = map(int, supporting.split())\n",
    "                substory = [story[i - 1] for i in supporting]\n",
    "            else:\n",
    "                # Provide all the substories\n",
    "                substory = [x for x in story if x]\n",
    "            data.append((substory, q, a))\n",
    "            story.append('')\n",
    "        else:\n",
    "            sent = tokenize(line)\n",
    "            story.append(sent)\n",
    "    return data\n",
    "\n",
    "\n",
    "def get_stories(f, only_supporting=False, max_length=None):\n",
    "    '''Given a file name, read the file,\n",
    "    retrieve the stories,\n",
    "    and then convert the sentences into a single story.\n",
    "    If max_length is supplied,\n",
    "    any stories longer than max_length tokens will be discarded.\n",
    "    '''\n",
    "    data = parse_stories(f.readlines(), only_supporting=only_supporting)\n",
    "    flatten = lambda data: reduce(lambda x, y: x + y, data)\n",
    "    data = [(flatten(story), q, answer) for story, q, answer in data if not max_length or len(flatten(story)) < max_length]\n",
    "    return data\n",
    "\n",
    "\n",
    "def vectorize_stories(data):\n",
    "    inputs, queries, answers = [], [], []\n",
    "    for story, query, answer in data:\n",
    "        inputs.append([word_idx[w] for w in story])\n",
    "        queries.append([word_idx[w] for w in query])\n",
    "        answers.append(word_idx[answer])\n",
    "    return (pad_sequences(inputs, maxlen=story_maxlen),\n",
    "            pad_sequences(queries, maxlen=query_maxlen),\n",
    "            np.array(answers))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting stories for the challenge: single_supporting_fact_10k\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    path = 'C:/Users/310267647/.keras/datasets/babi-tasks-v1-2.tar.gz'\n",
    "except:\n",
    "    print('Error downloading dataset, please download it manually:\\n'\n",
    "          '$ wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\\n'\n",
    "          '$ mv tasks_1-20_v1-2.tar.gz ~/.keras/datasets/babi-tasks-v1-2.tar.gz')\n",
    "    raise\n",
    "tar = tarfile.open(path)\n",
    "\n",
    "challenges = {\n",
    "    # QA1 with 10,000 samples\n",
    "    'single_supporting_fact_10k': 'tasks_1-20_v1-2/en-10k/qa1_single-supporting-fact_{}.txt',\n",
    "    # QA2 with 10,000 samples\n",
    "    'two_supporting_facts_10k': 'tasks_1-20_v1-2/en-10k/qa2_two-supporting-facts_{}.txt',\n",
    "}\n",
    "challenge_type = 'single_supporting_fact_10k'\n",
    "challenge = challenges[challenge_type]\n",
    "\n",
    "print('Extracting stories for the challenge:', challenge_type)\n",
    "train_stories = get_stories(tar.extractfile(challenge.format('train')))\n",
    "test_stories = get_stories(tar.extractfile(challenge.format('test')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Story: Mary moved to the bathroom . John went to the hallway .\n",
      "Query: Where is Mary ?\n",
      "Answer: bathroom\n",
      "---\n",
      "Story: Mary moved to the bathroom . John went to the hallway . Daniel went back to the hallway . Sandra moved to the garden .\n",
      "Query: Where is Daniel ?\n",
      "Answer: hallway\n",
      "---\n",
      "Story: Mary moved to the bathroom . John went to the hallway . Daniel went back to the hallway . Sandra moved to the garden . John moved to the office . Sandra journeyed to the bathroom .\n",
      "Query: Where is Daniel ?\n",
      "Answer: hallway\n",
      "---\n",
      "Story: Mary moved to the bathroom . John went to the hallway . Daniel went back to the hallway . Sandra moved to the garden . John moved to the office . Sandra journeyed to the bathroom . Mary moved to the hallway . Daniel travelled to the office .\n",
      "Query: Where is Daniel ?\n",
      "Answer: office\n",
      "---\n",
      "Story: Mary moved to the bathroom . John went to the hallway . Daniel went back to the hallway . Sandra moved to the garden . John moved to the office . Sandra journeyed to the bathroom . Mary moved to the hallway . Daniel travelled to the office . John went back to the garden . John moved to the bedroom .\n",
      "Query: Where is Sandra ?\n",
      "Answer: bathroom\n",
      "---\n",
      "Story: Sandra travelled to the office . Sandra went to the bathroom .\n",
      "Query: Where is Sandra ?\n",
      "Answer: bathroom\n",
      "---\n",
      "Story: Sandra travelled to the office . Sandra went to the bathroom . Mary went to the bedroom . Daniel moved to the hallway .\n",
      "Query: Where is Sandra ?\n",
      "Answer: bathroom\n",
      "---\n",
      "Story: Sandra travelled to the office . Sandra went to the bathroom . Mary went to the bedroom . Daniel moved to the hallway . John went to the garden . John travelled to the office .\n",
      "Query: Where is Sandra ?\n",
      "Answer: bathroom\n",
      "---\n",
      "Story: Sandra travelled to the office . Sandra went to the bathroom . Mary went to the bedroom . Daniel moved to the hallway . John went to the garden . John travelled to the office . Daniel journeyed to the bedroom . Daniel travelled to the hallway .\n",
      "Query: Where is John ?\n",
      "Answer: office\n",
      "---\n",
      "Story: Sandra travelled to the office . Sandra went to the bathroom . Mary went to the bedroom . Daniel moved to the hallway . John went to the garden . John travelled to the office . Daniel journeyed to the bedroom . Daniel travelled to the hallway . John went to the bedroom . John travelled to the office .\n",
      "Query: Where is Daniel ?\n",
      "Answer: hallway\n",
      "---\n",
      "Story: Mary went to the bedroom . John journeyed to the bathroom .\n",
      "Query: Where is John ?\n",
      "Answer: bathroom\n",
      "---\n",
      "Story: Mary went to the bedroom . John journeyed to the bathroom . Sandra journeyed to the hallway . John journeyed to the garden .\n",
      "Query: Where is Mary ?\n",
      "Answer: bedroom\n",
      "---\n",
      "Story: Mary went to the bedroom . John journeyed to the bathroom . Sandra journeyed to the hallway . John journeyed to the garden . John journeyed to the bathroom . Sandra journeyed to the garden .\n",
      "Query: Where is John ?\n",
      "Answer: bathroom\n",
      "---\n",
      "Story: Mary went to the bedroom . John journeyed to the bathroom . Sandra journeyed to the hallway . John journeyed to the garden . John journeyed to the bathroom . Sandra journeyed to the garden . Sandra went back to the bedroom . Daniel travelled to the bathroom .\n",
      "Query: Where is John ?\n",
      "Answer: bathroom\n",
      "---\n",
      "Story: Mary went to the bedroom . John journeyed to the bathroom . Sandra journeyed to the hallway . John journeyed to the garden . John journeyed to the bathroom . Sandra journeyed to the garden . Sandra went back to the bedroom . Daniel travelled to the bathroom . John went to the office . Mary moved to the office .\n",
      "Query: Where is Sandra ?\n",
      "Answer: bedroom\n",
      "---\n",
      "Story: Daniel journeyed to the kitchen . Daniel journeyed to the bedroom .\n",
      "Query: Where is Daniel ?\n",
      "Answer: bedroom\n",
      "---\n",
      "Story: Daniel journeyed to the kitchen . Daniel journeyed to the bedroom . Sandra moved to the garden . Sandra went back to the kitchen .\n",
      "Query: Where is Sandra ?\n",
      "Answer: kitchen\n",
      "---\n",
      "Story: Daniel journeyed to the kitchen . Daniel journeyed to the bedroom . Sandra moved to the garden . Sandra went back to the kitchen . Daniel travelled to the office . Sandra went back to the hallway .\n",
      "Query: Where is Sandra ?\n",
      "Answer: hallway\n",
      "---\n",
      "Story: Daniel journeyed to the kitchen . Daniel journeyed to the bedroom . Sandra moved to the garden . Sandra went back to the kitchen . Daniel travelled to the office . Sandra went back to the hallway . Daniel journeyed to the hallway . Daniel went to the bathroom .\n",
      "Query: Where is Sandra ?\n",
      "Answer: hallway\n",
      "---\n",
      "Story: Daniel journeyed to the kitchen . Daniel journeyed to the bedroom . Sandra moved to the garden . Sandra went back to the kitchen . Daniel travelled to the office . Sandra went back to the hallway . Daniel journeyed to the hallway . Daniel went to the bathroom . John travelled to the garden . Daniel moved to the hallway .\n",
      "Query: Where is Daniel ?\n",
      "Answer: hallway\n",
      "---\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# See what the data looks like# See w \n",
    "\n",
    "for i in range(20):\n",
    "    print(\"Story: {}\".format(' '.join(train_stories[i][0])))\n",
    "    print(\"Query: {}\".format(' '.join(train_stories[i][1])))\n",
    "    print(\"Answer: {}\".format(train_stories[i][2]))\n",
    "    print(\"---\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-\n",
      "Vocab size: 22 unique words\n",
      "Story max length: 68 words\n",
      "Query max length: 4 words\n",
      "Number of training stories: 10000\n",
      "Number of test stories: 1000\n",
      "-\n",
      "Here's what a \"story\" tuple looks like (input, query, answer):\n",
      "(['Mary', 'moved', 'to', 'the', 'bathroom', '.', 'John', 'went', 'to', 'the', 'hallway', '.'], ['Where', 'is', 'Mary', '?'], 'bathroom')\n",
      "-\n",
      "(0, '.')\n",
      "(1, '?')\n",
      "(2, 'Daniel')\n",
      "(3, 'John')\n",
      "(4, 'Mary')\n",
      "(5, 'Sandra')\n",
      "(6, 'Where')\n",
      "(7, 'back')\n",
      "(8, 'bathroom')\n",
      "(9, 'bedroom')\n",
      "(10, 'garden')\n",
      "(11, 'hallway')\n",
      "(12, 'is')\n",
      "(13, 'journeyed')\n",
      "(14, 'kitchen')\n",
      "(15, 'moved')\n",
      "(16, 'office')\n",
      "(17, 'the')\n",
      "(18, 'to')\n",
      "(19, 'travelled')\n",
      "(20, 'went')\n"
     ]
    }
   ],
   "source": [
    "vocab = set()\n",
    "for story, q, answer in train_stories + test_stories:\n",
    "    vocab |= set(story + q + [answer])\n",
    "vocab = sorted(vocab)\n",
    "\n",
    "# Reserve 0 for masking via pad_sequences\n",
    "vocab_size = len(vocab) + 1\n",
    "story_maxlen = max(map(len, (x for x, _, _ in train_stories + test_stories)))\n",
    "query_maxlen = max(map(len, (x for _, x, _ in train_stories + test_stories)))\n",
    "\n",
    "print('-')\n",
    "print('Vocab size:', vocab_size, 'unique words')\n",
    "print('Story max length:', story_maxlen, 'words')\n",
    "print('Query max length:', query_maxlen, 'words')\n",
    "print('Number of training stories:', len(train_stories))\n",
    "print('Number of test stories:', len(test_stories))\n",
    "print('-')\n",
    "print('Here\\'s what a \"story\" tuple looks like (input, query, answer):')\n",
    "print(train_stories[0])\n",
    "print('-')\n",
    "\n",
    "\n",
    "for s in list(enumerate(vocab)):\n",
    "    print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vectorizing the word sequences...\n",
      "-\n",
      "inputs: integer tensor of shape (samples, max_length)\n",
      "inputs_train shape: (10000, 68)\n",
      "inputs_test shape: (1000, 68)\n",
      "-\n",
      "queries: integer tensor of shape (samples, max_length)\n",
      "queries_train shape: (10000, 4)\n",
      "queries_test shape: (1000, 4)\n",
      "-\n",
      "answers: binary (1 or 0) tensor of shape (samples, vocab_size)\n",
      "answers_train shape: (10000,)\n",
      "answers_test shape: (1000,)\n",
      "-\n"
     ]
    }
   ],
   "source": [
    "print('Vectorizing the word sequences...')\n",
    "word_idx = dict((c, i + 1) for i, c in enumerate(vocab))\n",
    "inputs_train, queries_train, answers_train = vectorize_stories(train_stories)\n",
    "inputs_test, queries_test, answers_test = vectorize_stories(test_stories)\n",
    "\n",
    "print('-')\n",
    "print('inputs: integer tensor of shape (samples, max_length)')\n",
    "print('inputs_train shape:', inputs_train.shape)\n",
    "print('inputs_test shape:', inputs_test.shape)\n",
    "print('-')\n",
    "print('queries: integer tensor of shape (samples, max_length)')\n",
    "print('queries_train shape:', queries_train.shape)\n",
    "print('queries_test shape:', queries_test.shape)\n",
    "print('-')\n",
    "print('answers: binary (1 or 0) tensor of shape (samples, vocab_size)')\n",
    "print('answers_train shape:', answers_train.shape)\n",
    "print('answers_test shape:', answers_test.shape)\n",
    "print('-')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Story (x): [ 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0  0  0  0  0  0  0  5 16 19 18  9  1  4 21 19 18 12  1]\n",
      "Question (x): [ 7 13  5  2]\n",
      "Answer: 9\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# See individual training element.# See i \n",
    "\n",
    "print(\"Story (x): {}\".format(inputs_train[0]))\n",
    "print(\"Question (x): {}\".format(queries_train[0]))\n",
    "print(\"Answer: {}\".format(answers_train[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Compiling...\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "print('Compiling...')\n",
    "\n",
    "# placeholders\n",
    "input_sequence = Input((story_maxlen,))\n",
    "question = Input((query_maxlen,))\n",
    "\n",
    "# encoders\n",
    "# embed the input sequence into a sequence of vectors\n",
    "input_encoder_m = Sequential()\n",
    "input_encoder_m.add(Embedding(input_dim=vocab_size,\n",
    "                              output_dim=64))\n",
    "input_encoder_m.add(Dropout(0.3))\n",
    "# output: (samples, story_maxlen, embedding_dim)\n",
    "\n",
    "# embed the input into a sequence of vectors of size query_maxlen\n",
    "input_encoder_c = Sequential()\n",
    "input_encoder_c.add(Embedding(input_dim=vocab_size,\n",
    "                              output_dim=query_maxlen))\n",
    "input_encoder_c.add(Dropout(0.3))\n",
    "# output: (samples, story_maxlen, query_maxlen)\n",
    "\n",
    "# embed the question into a sequence of vectors\n",
    "question_encoder = Sequential()\n",
    "question_encoder.add(Embedding(input_dim=vocab_size,\n",
    "                               output_dim=64,\n",
    "                               input_length=query_maxlen))\n",
    "question_encoder.add(Dropout(0.3))\n",
    "# output: (samples, query_maxlen, embedding_dim)\n",
    "\n",
    "# encode input sequence and questions (which are indices)\n",
    "# to sequences of dense vectors\n",
    "input_encoded_m = input_encoder_m(input_sequence)\n",
    "input_encoded_c = input_encoder_c(input_sequence)\n",
    "question_encoded = question_encoder(question)\n",
    "\n",
    "# compute a 'match' between the first input vector sequence\n",
    "# and the question vector sequence\n",
    "# shape: `(samples, story_maxlen, query_maxlen)`\n",
    "match = dot([input_encoded_m, question_encoded], axes=(2, 2))\n",
    "match = Activation('softmax')(match)\n",
    "\n",
    "# add the match matrix with the second input vector sequence\n",
    "response = add([match, input_encoded_c])  # (samples, story_maxlen, query_maxlen)\n",
    "response = Permute((2, 1))(response)  # (samples, query_maxlen, story_maxlen)\n",
    "\n",
    "# concatenate the match matrix with the question vector sequence\n",
    "answer = concatenate([response, question_encoded])\n",
    "\n",
    "# the original paper uses a matrix multiplication for this reduction step.\n",
    "# we choose to use a RNN instead.\n",
    "answer = LSTM(32)(answer)  # (samples, 32)\n",
    "\n",
    "# one regularization layer -- more would probably be needed.\n",
    "answer = Dropout(0.3)(answer)\n",
    "answer = Dense(vocab_size)(answer)  # (samples, vocab_size)\n",
    "# we output a probability distribution over the vocabulary\n",
    "answer = Activation('softmax')(answer)\n",
    "\n",
    "# build the final model\n",
    "model = Model([input_sequence, question], answer)\n",
    "model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "print(\"Done.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 10000 samples, validate on 1000 samples\n",
      "Epoch 1/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0442 - acc: 0.9857 - val_loss: 0.1131 - val_acc: 0.9690\n",
      "Epoch 2/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0444 - acc: 0.9846 - val_loss: 0.1178 - val_acc: 0.9720\n",
      "Epoch 3/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0459 - acc: 0.9861 - val_loss: 0.1079 - val_acc: 0.97200.\n",
      "Epoch 4/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0411 - acc: 0.9878 - val_loss: 0.1494 - val_acc: 0.9590\n",
      "Epoch 5/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0375 - acc: 0.9869 - val_loss: 0.1218 - val_acc: 0.9680\n",
      "Epoch 6/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0411 - acc: 0.9870 - val_loss: 0.1119 - val_acc: 0.9700\n",
      "Epoch 7/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0376 - acc: 0.9893 - val_loss: 0.1221 - val_acc: 0.9680\n",
      "Epoch 8/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0344 - acc: 0.9887 - val_loss: 0.1319 - val_acc: 0.9690\n",
      "Epoch 9/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0338 - acc: 0.9889 - val_loss: 0.1394 - val_acc: 0.9650\n",
      "Epoch 10/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0323 - acc: 0.9903 - val_loss: 0.1065 - val_acc: 0.9710\n",
      "Epoch 11/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0305 - acc: 0.9906 - val_loss: 0.1369 - val_acc: 0.9690\n",
      "Epoch 12/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0341 - acc: 0.9890 - val_loss: 0.1206 - val_acc: 0.9720\n",
      "Epoch 13/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0345 - acc: 0.9894 - val_loss: 0.1191 - val_acc: 0.9710\n",
      "Epoch 14/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0346 - acc: 0.9894 - val_loss: 0.1094 - val_acc: 0.9740\n",
      "Epoch 15/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0291 - acc: 0.9888 - val_loss: 0.1086 - val_acc: 0.9760\n",
      "Epoch 16/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0332 - acc: 0.9889 - val_loss: 0.1117 - val_acc: 0.9730\n",
      "Epoch 17/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0351 - acc: 0.9897 - val_loss: 0.1063 - val_acc: 0.9710\n",
      "Epoch 18/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0290 - acc: 0.9908 - val_loss: 0.1438 - val_acc: 0.9720\n",
      "Epoch 19/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0271 - acc: 0.9912 - val_loss: 0.1362 - val_acc: 0.9740\n",
      "Epoch 20/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0290 - acc: 0.9915 - val_loss: 0.1263 - val_acc: 0.9720\n",
      "Epoch 21/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0289 - acc: 0.9896 - val_loss: 0.1333 - val_acc: 0.9680\n",
      "Epoch 22/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0326 - acc: 0.9896 - val_loss: 0.1071 - val_acc: 0.9750\n",
      "Epoch 23/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0280 - acc: 0.9918 - val_loss: 0.1125 - val_acc: 0.9700\n",
      "Epoch 24/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0275 - acc: 0.9914 - val_loss: 0.1318 - val_acc: 0.9720\n",
      "Epoch 25/120\n",
      "10000/10000 [==============================] - 3s - loss: 0.0300 - acc: 0.9908 - val_loss: 0.1472 - val_acc: 0.9670\n",
      "Epoch 26/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0285 - acc: 0.9927 - val_loss: 0.1280 - val_acc: 0.9730\n",
      "Epoch 27/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0256 - acc: 0.9919 - val_loss: 0.1193 - val_acc: 0.9720\n",
      "Epoch 28/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0300 - acc: 0.9917 - val_loss: 0.1204 - val_acc: 0.9790\n",
      "Epoch 29/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0193 - acc: 0.9924 - val_loss: 0.1120 - val_acc: 0.9760\n",
      "Epoch 30/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0283 - acc: 0.9927 - val_loss: 0.1441 - val_acc: 0.9680\n",
      "Epoch 31/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0274 - acc: 0.9910 - val_loss: 0.1335 - val_acc: 0.9710\n",
      "Epoch 32/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0241 - acc: 0.9932 - val_loss: 0.1235 - val_acc: 0.9710\n",
      "Epoch 33/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0256 - acc: 0.9918 - val_loss: 0.1345 - val_acc: 0.9700\n",
      "Epoch 34/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0306 - acc: 0.9917 - val_loss: 0.1204 - val_acc: 0.9760\n",
      "Epoch 35/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0277 - acc: 0.9914 - val_loss: 0.1325 - val_acc: 0.9690\n",
      "Epoch 36/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0228 - acc: 0.9934 - val_loss: 0.1095 - val_acc: 0.9750\n",
      "Epoch 37/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0294 - acc: 0.9917 - val_loss: 0.1003 - val_acc: 0.9790\n",
      "Epoch 38/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0224 - acc: 0.9926 - val_loss: 0.1096 - val_acc: 0.9770\n",
      "Epoch 39/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0245 - acc: 0.9926 - val_loss: 0.1042 - val_acc: 0.9780\n",
      "Epoch 40/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0206 - acc: 0.9928 - val_loss: 0.1418 - val_acc: 0.9730\n",
      "Epoch 41/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0283 - acc: 0.9921 - val_loss: 0.1058 - val_acc: 0.9750\n",
      "Epoch 42/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0249 - acc: 0.9931 - val_loss: 0.1162 - val_acc: 0.9760\n",
      "Epoch 43/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0240 - acc: 0.9921 - val_loss: 0.1236 - val_acc: 0.9720\n",
      "Epoch 44/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0270 - acc: 0.9923 - val_loss: 0.1223 - val_acc: 0.9770\n",
      "Epoch 45/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0222 - acc: 0.9940 - val_loss: 0.1334 - val_acc: 0.9750 - ETA: 1s - loss: 0.02 - ETA: 0s - loss: 0.0226 - \n",
      "Epoch 46/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0234 - acc: 0.9929 - val_loss: 0.1402 - val_acc: 0.9740\n",
      "Epoch 47/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0305 - acc: 0.9919 - val_loss: 0.1138 - val_acc: 0.9780\n",
      "Epoch 48/120\n",
      "10000/10000 [==============================] - 6s - loss: 0.0240 - acc: 0.9925 - val_loss: 0.1612 - val_acc: 0.97100.0239 - acc:\n",
      "Epoch 49/120\n",
      "10000/10000 [==============================] - 6s - loss: 0.0192 - acc: 0.9931 - val_loss: 0.1394 - val_acc: 0.9750\n",
      "Epoch 50/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0272 - acc: 0.9926 - val_loss: 0.1460 - val_acc: 0.9700\n",
      "Epoch 51/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0234 - acc: 0.9933 - val_loss: 0.1522 - val_acc: 0.9730\n",
      "Epoch 52/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0165 - acc: 0.9955 - val_loss: 0.1381 - val_acc: 0.9730\n",
      "Epoch 53/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0214 - acc: 0.9936 - val_loss: 0.1348 - val_acc: 0.9760\n",
      "Epoch 54/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0218 - acc: 0.9942 - val_loss: 0.1348 - val_acc: 0.9740\n",
      "Epoch 55/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0196 - acc: 0.9947 - val_loss: 0.1456 - val_acc: 0.9680\n",
      "Epoch 56/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0234 - acc: 0.9945 - val_loss: 0.1214 - val_acc: 0.9830\n",
      "Epoch 57/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0187 - acc: 0.9939 - val_loss: 0.1287 - val_acc: 0.9780\n",
      "Epoch 58/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0245 - acc: 0.9932 - val_loss: 0.1133 - val_acc: 0.9790\n",
      "Epoch 59/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0214 - acc: 0.9939 - val_loss: 0.1335 - val_acc: 0.9750\n",
      "Epoch 60/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0216 - acc: 0.9936 - val_loss: 0.1649 - val_acc: 0.9660\n",
      "Epoch 61/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0262 - acc: 0.9932 - val_loss: 0.1284 - val_acc: 0.97500.99\n",
      "Epoch 62/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0193 - acc: 0.9929 - val_loss: 0.1334 - val_acc: 0.9730\n",
      "Epoch 63/120\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 4s - loss: 0.0287 - acc: 0.9932 - val_loss: 0.1320 - val_acc: 0.9740\n",
      "Epoch 64/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0208 - acc: 0.9943 - val_loss: 0.1312 - val_acc: 0.9750\n",
      "Epoch 65/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0208 - acc: 0.9942 - val_loss: 0.1398 - val_acc: 0.9740\n",
      "Epoch 66/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0186 - acc: 0.9957 - val_loss: 0.1328 - val_acc: 0.9760\n",
      "Epoch 67/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0277 - acc: 0.9926 - val_loss: 0.1250 - val_acc: 0.9750\n",
      "Epoch 68/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0197 - acc: 0.9938 - val_loss: 0.1380 - val_acc: 0.9760\n",
      "Epoch 69/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0200 - acc: 0.9938 - val_loss: 0.1380 - val_acc: 0.9700\n",
      "Epoch 70/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0211 - acc: 0.9944 - val_loss: 0.1282 - val_acc: 0.9750\n",
      "Epoch 71/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0217 - acc: 0.9948 - val_loss: 0.1126 - val_acc: 0.9800\n",
      "Epoch 72/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0216 - acc: 0.9938 - val_loss: 0.1364 - val_acc: 0.9730\n",
      "Epoch 73/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0205 - acc: 0.9947 - val_loss: 0.1147 - val_acc: 0.9760\n",
      "Epoch 74/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0173 - acc: 0.9946 - val_loss: 0.1292 - val_acc: 0.9770\n",
      "Epoch 75/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0185 - acc: 0.9943 - val_loss: 0.1544 - val_acc: 0.9720\n",
      "Epoch 76/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0243 - acc: 0.9940 - val_loss: 0.1253 - val_acc: 0.9780\n",
      "Epoch 77/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0205 - acc: 0.9947 - val_loss: 0.1514 - val_acc: 0.9700\n",
      "Epoch 78/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0234 - acc: 0.9942 - val_loss: 0.1081 - val_acc: 0.9760\n",
      "Epoch 79/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0242 - acc: 0.9946 - val_loss: 0.1075 - val_acc: 0.9790\n",
      "Epoch 80/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0261 - acc: 0.9939 - val_loss: 0.1213 - val_acc: 0.9740\n",
      "Epoch 81/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0195 - acc: 0.9956 - val_loss: 0.1280 - val_acc: 0.9760\n",
      "Epoch 82/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0229 - acc: 0.9947 - val_loss: 0.1336 - val_acc: 0.9760\n",
      "Epoch 83/120\n",
      "10000/10000 [==============================] - 4s - loss: 0.0252 - acc: 0.9932 - val_loss: 0.1349 - val_acc: 0.9750\n",
      "Epoch 84/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0156 - acc: 0.9955 - val_loss: 0.1181 - val_acc: 0.9780\n",
      "Epoch 85/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0219 - acc: 0.9942 - val_loss: 0.1272 - val_acc: 0.9790\n",
      "Epoch 86/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0183 - acc: 0.9950 - val_loss: 0.1180 - val_acc: 0.9750\n",
      "Epoch 87/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0231 - acc: 0.9942 - val_loss: 0.1142 - val_acc: 0.9760\n",
      "Epoch 88/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0183 - acc: 0.9958 - val_loss: 0.1295 - val_acc: 0.9790\n",
      "Epoch 89/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0194 - acc: 0.9955 - val_loss: 0.1156 - val_acc: 0.97900.9\n",
      "Epoch 90/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0223 - acc: 0.9947 - val_loss: 0.1410 - val_acc: 0.9750\n",
      "Epoch 91/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0131 - acc: 0.9960 - val_loss: 0.1535 - val_acc: 0.9730\n",
      "Epoch 92/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0138 - acc: 0.9957 - val_loss: 0.1292 - val_acc: 0.9710 0s - loss: 0.0138 - acc: 0.995\n",
      "Epoch 93/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0216 - acc: 0.9947 - val_loss: 0.1230 - val_acc: 0.9750\n",
      "Epoch 94/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0195 - acc: 0.9950 - val_loss: 0.1196 - val_acc: 0.9740\n",
      "Epoch 95/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0200 - acc: 0.9954 - val_loss: 0.1684 - val_acc: 0.9750\n",
      "Epoch 96/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0201 - acc: 0.9941 - val_loss: 0.1329 - val_acc: 0.9720\n",
      "Epoch 97/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0175 - acc: 0.9952 - val_loss: 0.1230 - val_acc: 0.9800\n",
      "Epoch 98/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0195 - acc: 0.9945 - val_loss: 0.1456 - val_acc: 0.9730\n",
      "Epoch 99/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0167 - acc: 0.9961 - val_loss: 0.1373 - val_acc: 0.9760\n",
      "Epoch 100/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0216 - acc: 0.9946 - val_loss: 0.1185 - val_acc: 0.9760\n",
      "Epoch 101/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0228 - acc: 0.9943 - val_loss: 0.1288 - val_acc: 0.9740\n",
      "Epoch 102/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0183 - acc: 0.9951 - val_loss: 0.1415 - val_acc: 0.9720\n",
      "Epoch 103/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0226 - acc: 0.9948 - val_loss: 0.1381 - val_acc: 0.9740\n",
      "Epoch 104/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0187 - acc: 0.9960 - val_loss: 0.1134 - val_acc: 0.9750\n",
      "Epoch 105/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0155 - acc: 0.9962 - val_loss: 0.1218 - val_acc: 0.9750\n",
      "Epoch 106/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0169 - acc: 0.9960 - val_loss: 0.1301 - val_acc: 0.9770\n",
      "Epoch 107/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0216 - acc: 0.9946 - val_loss: 0.1163 - val_acc: 0.9780\n",
      "Epoch 108/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0229 - acc: 0.9948 - val_loss: 0.1106 - val_acc: 0.9810\n",
      "Epoch 109/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0157 - acc: 0.9958 - val_loss: 0.1273 - val_acc: 0.9790\n",
      "Epoch 110/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0205 - acc: 0.9949 - val_loss: 0.1189 - val_acc: 0.9800\n",
      "Epoch 111/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0220 - acc: 0.9946 - val_loss: 0.1720 - val_acc: 0.9660\n",
      "Epoch 112/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0244 - acc: 0.9947 - val_loss: 0.1160 - val_acc: 0.9780\n",
      "Epoch 113/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0172 - acc: 0.9951 - val_loss: 0.1327 - val_acc: 0.9760\n",
      "Epoch 114/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0240 - acc: 0.9951 - val_loss: 0.1747 - val_acc: 0.9740\n",
      "Epoch 115/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0176 - acc: 0.9957 - val_loss: 0.1348 - val_acc: 0.9770\n",
      "Epoch 116/120\n",
      "10000/10000 [==============================] - 6s - loss: 0.0144 - acc: 0.9953 - val_loss: 0.1699 - val_acc: 0.9710\n",
      "Epoch 117/120\n",
      "10000/10000 [==============================] - 6s - loss: 0.0303 - acc: 0.9943 - val_loss: 0.1279 - val_acc: 0.9800\n",
      "Epoch 118/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0173 - acc: 0.9958 - val_loss: 0.1230 - val_acc: 0.9810\n",
      "Epoch 119/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0240 - acc: 0.9948 - val_loss: 0.1536 - val_acc: 0.9760\n",
      "Epoch 120/120\n",
      "10000/10000 [==============================] - 5s - loss: 0.0133 - acc: 0.9955 - val_loss: 0.1472 - val_acc: 0.9760\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "# train\n",
    "model.fit([inputs_train, queries_train], answers_train,\n",
    "          batch_size=32,\n",
    "          epochs=120,\n",
    "          validation_data=([inputs_test, queries_test], answers_test))\n",
    "\n",
    "# save\n",
    "save_path = \"C:/Users/310267647/.keras/datasets/\"\n",
    "# save entire network to HDF5 (save everything, suggested)\n",
    "model.save(os.path.join(save_path,\"chatbot.h5\"))\n",
    "# save the vocab too, indexes must be the same\n",
    "pickle.dump( vocab, open( os.path.join(save_path,\"vocab.pkl\"), \"wb\" ) )\n",
    "\n",
    "elapsed_time = time.time() - start_time\n",
    "#print(\"Elapsed time: {}\".format(hms_string(elapsed_time)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Load the model, if it exists, load vocab too\n",
    "save_path = \"C:/Users/310267647/.keras/datasets/\"\n",
    "model = load_model(os.path.join(save_path,\"chatbot.h5\"))\n",
    "vocab = pickle.load( open( os.path.join(save_path,\"vocab.pkl\"), \"rb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2.84454407e-22 2.88222764e-22 2.75384173e-22 ... 2.73462180e-22\n",
      "  2.66911504e-22 2.76148951e-22]\n",
      " [8.63731656e-18 6.34245243e-18 6.72251628e-18 ... 9.12310319e-18\n",
      "  8.20846312e-18 8.69752538e-18]\n",
      " [1.68333274e-21 1.34126125e-21 1.42787408e-21 ... 1.05572793e-21\n",
      "  1.75524353e-21 1.72172737e-21]\n",
      " ...\n",
      " [1.56481343e-20 2.03151843e-20 1.71734004e-20 ... 1.43914515e-20\n",
      "  1.53216080e-20 1.98714746e-20]\n",
      " [1.17170505e-20 7.35907533e-21 7.11325971e-21 ... 7.91727280e-21\n",
      "  6.58933893e-21 6.75051718e-21]\n",
      " [5.90645762e-16 5.53329990e-16 5.77884205e-16 ... 5.10483199e-16\n",
      "  6.29598372e-16 6.90903758e-16]]\n"
     ]
    }
   ],
   "source": [
    "pred = model.predict([inputs_test, queries_test])\n",
    "# See what the predictions look like, they are just probabilities of each class.\n",
    "print(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[12  9 15 12 15 12 11 12 17 17 15 11 17 10 10 15 11 15 17 12 12 17 17 12\n",
      " 10 10 10 15 15 15 12 17 15 15  9 17  9 11 15 10  9 11 12 12 11 15 12  9\n",
      " 17 10 10 17 11 11 15 10 15 12 11 12 12 17  9 11 11 15 15  9 17 17 11 12\n",
      " 11 10  9 15 15 15 12 17 11  9 10 11 17 11 11 12 15 15  9  9 12  9  9 15\n",
      " 15 15 10 10 10  9 17 11 11 12 10  9 17 10  9  9 15 12 15 11 17 15  9 17\n",
      " 10  9 15 12 10 17 11 11 11  9 11 12  9 11 10  9 17 15  9  9 17 12 12 17\n",
      " 10 15 15 15 15 15 11 11  9 17 10  9 11 10 17 12 10 10 10 11  9 11 11 11\n",
      " 15 10 17 17 11  9  9 17 15 10 17 17 12 12 12 12 10  9 17 11 15 17 12 15\n",
      " 11 12 12 11 17 15  9 10 12 17 17 10 12  9 15 12  9 12  9 10 12 10 12 10\n",
      "  9 12 12  9  9 12 15 12 10 17 12 11  9  9 17 11 12 17 12 10 10 15 10 10\n",
      " 12 15 11 11 15 15 11 11  9 11 10 17  9  9 10 10 17 17 15 11 11 17 12 17\n",
      " 17  9 11 10  9 10 15 17 17 15  9 17 15 12 17  9  9 11 17 17 11 17  9 17\n",
      " 11  9 15 15 10 15 15 11 11 12 12  9  9 12 15 17 12 11  9 17 17 15 11 11\n",
      " 17 10 11 17 10 17 10 11 11 15 15 11 11  9 10 11 11 10 15 17 15 10 17 17\n",
      " 12 17 12 12 12 11 17 15 11 17 17 12 10 15 10 11 11 10 10  9 17 12 11 12\n",
      " 11  9 11 11 11 11 11 12 12 12 12 12 11 11 17 10 15  9  9 10  9 10 10 17\n",
      " 17 15 17 17 17 15 15 12  9 10 17 17 10 17  9 11 10 11 10 15 17 15 12 10\n",
      " 11 15 10 11  9 12 17 10 10 15 12  9 12 15  9 12 10 15 15 17 10 10 17 15\n",
      " 15 10 17 11 15  9 10 11 10 12  9 11 10 11 10 12 11 15 15 15 15 11  9 15\n",
      " 10 12 11 12  9 11 10 17 11 11 10 10 10 17 12 17 17  9 15 11  9 15 11  9\n",
      " 10 10 12 12 11 10  9 12 10  9 15 10 10 11 11 15 15 10 12 12 15 11  9 12\n",
      "  9 15 10 17 17 10 10 17 10 10 17  9  9 10 12 11 17 17 12 12 11 12 17 11\n",
      " 11 12 11 11 11 17  9 11 10 12 12 10 11 17 12 12 15 11 15 15 12 10 10  9\n",
      " 17 10 15 12 15 10 10 17 11 10 11 15 10  9  9 15 10 12 10 10 15 15 17  9\n",
      " 10 12 10 10 12 12 11 15 10 10 11 12 10 10 17 12 17 11  9  9 17 17 11 10\n",
      " 10 10 15 15 17 11 15 12 11 10 15 17  9 11  9 17 15 12 15 17 11 17 17 15\n",
      "  9 10  9  9 10 10  9 10 10 15 11 11 11 15 12  9 11 11 10 17 11 11 12 17\n",
      " 15 17  9 11 15 15 11  9 15 15 11 15 15 15 11 10 10  9 17 17 17 17 10 10\n",
      " 17  9 15  9 17 12 12 12 15 15 15 17  9 11  9 17  9 12 11  9  9 17 11 11\n",
      " 11 10 15 10 12 12  9  9 17 12 12  9 17 12 11 11 15 15 12 11 12 17 12 17\n",
      " 11 12 11 17 17 11 17  9 11  9 10 10 12 17 10 12  9  9  9 15 11 11 15 17\n",
      "  9 10  9 12 17 15 17 17 12 10 10 11 11 17 12  9 17 10  9 11 12 15  9 15\n",
      " 12 10  9 10 17 17 15 15  9  9  9 17 10 17 17  9 10 17 17 12 12 11 11 11\n",
      " 11 17 11 17 17 17 17 17 12  9 17 10 17 11 10 10 12 17 11 11 17 12 12 17\n",
      " 17 15 11 17 11 11  9 12  9 17  9 10  9 15 11  9 11 15 10  9  9  9 17 12\n",
      " 11 11 17 15 15 10 10 11 11 17 10 12  9 10 10 10 10 10 15 17  9 11 17 15\n",
      " 12 11 10 10 15 12 10 11 12 12 11 11 11 15 12 17 10 12 17 17 12 10 10 12\n",
      " 17 10 11 17 11 12 17 17 11 12 11 17  9  9 11 17 11 11 15 15 17 17 10 10\n",
      " 17 12 11 15 12 11 11 10  9 12 12 15 10  9 17 15 17  9 17 17 10 17 15 12\n",
      " 12 11 10 10 10 17 17 15 12 17  9  9 15 10 12 15 15 15 17 17 15 11 15 10\n",
      " 12  9  9 10 11 11  9  9 12 12  9 15  9 12 12 10 17 17 10 12 15  9 10 17\n",
      " 11 11 11 17  9  9 11 15 11 11 11 15 12 17  9 10]\n"
     ]
    }
   ],
   "source": [
    "# Use argmax to turn those into actual predictions.  The class (word) with the highest\n",
    "# probability is the answer.\n",
    "\n",
    "pred = np.argmax(pred,axis=1)\n",
    "print(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final accuracy: 0.976\n"
     ]
    }
   ],
   "source": [
    "score = metrics.accuracy_score(answers_test, pred)\n",
    "print(\"Final accuracy: {}\".format(score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remember, I only know these words: ['.', '?', 'Daniel', 'John', 'Mary', 'Sandra', 'Where', 'back', 'bathroom', 'bedroom', 'garden', 'hallway', 'is', 'journeyed', 'kitchen', 'moved', 'office', 'the', 'to', 'travelled', 'went']\n",
      "\n",
      "[2.0395073e-21 2.6753854e-21 2.3785207e-21 2.7368878e-21 2.1030766e-21\n",
      " 2.4609603e-21 1.9202861e-21 2.1721708e-21 2.4134715e-21 9.6075252e-11\n",
      " 5.0006704e-14 4.1893827e-13 1.4033311e-10 2.6519946e-21 2.3997192e-21\n",
      " 5.1556041e-13 2.2921702e-21 1.0000000e+00 2.4891525e-21 1.9497972e-21\n",
      " 2.0082861e-21 2.7105386e-21]\n",
      "Answer: office([17])\n"
     ]
    }
   ],
   "source": [
    "print(\"Remember, I only know these words: {}\".format(vocab))\n",
    "print()\n",
    "story = \"Daniel went to the bathroom. Daniel to journeyed to the office.  Mary went to the kitchen.\"\n",
    "query = \"Where is Daniel?\"\n",
    "\n",
    "adhoc_stories = (tokenize(story), tokenize(query), '?')\n",
    "\n",
    "adhoc_train, adhoc_query, adhoc_answer = vectorize_stories([adhoc_stories])\n",
    "\n",
    "pred = model.predict([adhoc_train, adhoc_query])\n",
    "print(pred[0])\n",
    "pred = np.argmax(pred,axis=1)\n",
    "print(\"Answer: {}({})\".format(vocab[pred[0]-1],pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}