03B_NLP_tensorflow_IPLE_Complete.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TensorFlow for Natural Language Processing with Neural Networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Python 3, Tensorflow 1.12"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Word Embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Word embeddings, or word vectors, provide a way of mapping words from a vocabulary into a low-dimensional space, where words with similar meanings are grouped together. Here, we will look at the implementation of the two main methods to train word embeddings in an unsupervised manner: skip-gram and CBOW models. \n",
    "\n",
    "#### Preparing the text data\n",
    "#1. Download the training corpus (e.g., Wikipedia dump, GoogleNews, Bookcorpus);\n",
    "#2. Build a vocabulary by choosing those top frequent words (can be as many as a billion);\n",
    "#3. Build the two dictionaries: ixtoword, wordtoix;\n",
    "#4. Split the corpus into short text sequences (e.g. five consecutive words)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#4. get the input and output lists for training\n",
    "from random import randint\n",
    "\n",
    "sequences = [['i', 'love', 'coding', 'in', 'tensorflow'], ['the', 'dog', 'barked', 'at', 'him']]\n",
    "# Skip-gram model\n",
    "skip_input = []\n",
    "skip_output = []\n",
    "for s in sequences:\n",
    "    skip_input.append(s[len(s)//2])\n",
    "    temp = s[:len(s)//2] + s[len(s)//2+1:]\n",
    "    skip_output.append(temp[randint(0, len(temp)-1)])\n",
    "print('skip-gram training input: ', skip_input)\n",
    "print('skip-gram training output: ', skip_output)\n",
    "\n",
    "# CBOW model\n",
    "cbow_input = []\n",
    "cbow_output = []\n",
    "for s in sequences: \n",
    "    cbow_input.append(s[:len(s)//2] + s[len(s)//2+1:])\n",
    "    cbow_output.append(s[len(s)//2])\n",
    "print('CBOW training input: ', cbow_input)\n",
    "print('CBOW training output: ', cbow_output)\n",
    "\n",
    "# Transfer the input and output lists to indexes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write the Skip-gram model\n",
    "import tensorflow as tf\n",
    "import math\n",
    "\n",
    "vocabulary_size = 10000\n",
    "embedding_size = 300\n",
    "batch_size = 128\n",
    "\n",
    "# setup TensorFlow placeholders\n",
    "train_inputs = tf.placeholder(tf.int32, shape=[batch_size])\n",
    "train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])\n",
    "\n",
    "# Look up embeddings for inputs\n",
    "embeddings = tf.Variable(tf.random_uniform(\n",
    "    [vocabulary_size, embedding_size], -1.0, 1.0))\n",
    "embed = tf.nn.embedding_lookup(embeddings, train_inputs)\n",
    "\n",
    "# Construct the variables for the softmax\n",
    "weights = tf.Variable(tf.truncated_normal([vocabulary_size, embedding_size],\n",
    "                          stddev=1.0 / math.sqrt(embedding_size)))\n",
    "biases = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "hidden_out = tf.matmul(embed, tf.transpose(weights)) + biases\n",
    "\n",
    "# convert train_context to a one-hot format\n",
    "train_one_hot = tf.one_hot(train_labels, vocabulary_size)\n",
    "print('train_one_hot shape: %s'%(train_one_hot.get_shape().as_list()))\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=hidden_out, \n",
    "    labels=train_one_hot))\n",
    "\n",
    "# Construct the SGD optimizer using a learning rate of 1.0.\n",
    "optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write the CBOW model\n",
    "vocabulary_size = 10000\n",
    "embedding_size = 300\n",
    "batch_size = 128\n",
    "skip_window = 2 # how many words to consider left and right.\n",
    "\n",
    "# setup TensorFlow placeholders\n",
    "train_inputs = tf.placeholder(tf.int32, shape=[batch_size, 2*skip_window])\n",
    "train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])\n",
    "\n",
    "# Look up embeddings for inputs\n",
    "embeddings = tf.Variable(tf.random_uniform(\n",
    "    [vocabulary_size, embedding_size], -1.0, 1.0))\n",
    "embeds = None\n",
    "for i in range(2*skip_window):\n",
    "    embedding_i = tf.nn.embedding_lookup(embeddings, train_inputs[:,i])\n",
    "    #print('embedding %d shape: %s'%(i,embedding_i.get_shape().as_list()))\n",
    "    emb_x, emb_y = embedding_i.get_shape().as_list()\n",
    "    if embeds is None:\n",
    "        embeds = tf.reshape(embedding_i,[emb_x,emb_y,1])\n",
    "        print('embedding shape: %s'%(embeds.get_shape().as_list()))\n",
    "    else:\n",
    "        embeds = tf.concat([embeds,tf.reshape(embedding_i,[emb_x,emb_y,1])], 2)\n",
    "        print('embedding shape: %s'%(embeds.get_shape().as_list()))\n",
    "\n",
    "avg_embed =  tf.reduce_mean(embeds,2,keep_dims=False)\n",
    "print('avg_embed shape: %s'%(avg_embed.get_shape().as_list()))\n",
    "\n",
    "# Construct the variables for the softmax\n",
    "weights = tf.Variable(tf.truncated_normal([vocabulary_size, embedding_size],\n",
    "                         stddev=1.0 / math.sqrt(embedding_size)))\n",
    "biases = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "hidden_out = tf.matmul(embed, tf.transpose(weights)) + biases\n",
    "\n",
    "# convert train_context to a one-hot format\n",
    "train_one_hot = tf.one_hot(train_labels, vocabulary_size)\n",
    "print('train_one_hot shape: %s'%(train_one_hot.get_shape().as_list()))\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=hidden_out, \n",
    "    labels=train_one_hot))\n",
    "\n",
    "# Construct the SGD optimizer using a learning rate of 1.0.\n",
    "optimizer = tf.train.AdagradOptimizer(1.0).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### A toy example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "# choose which GPU to use\n",
    "import os\n",
    "\n",
    "GPUID = 1\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = str(GPUID)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus_raw = 'He is the king . The king is royal . She is the royal  queen '\n",
    "# corpus_raw = 'He is the king . The king is royal . She is the queen. The queen is royal '\n",
    "# convert to lower case\n",
    "corpus_raw = corpus_raw.lower()\n",
    "\n",
    "words = []\n",
    "for word in corpus_raw.split():\n",
    "    if word != '.': # because we don't want to treat . as a word\n",
    "        words.append(word)\n",
    "\n",
    "words = set(words) # so that all duplicate words are removed\n",
    "word2int = {}\n",
    "int2word = {}\n",
    "vocab_size = len(words) # gives the total number of unique words\n",
    "\n",
    "for i,word in enumerate(words):\n",
    "    word2int[word] = i\n",
    "    int2word[i] = word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# raw sentences is a list of sentences.\n",
    "raw_sentences = corpus_raw.split('.')\n",
    "sentences = []\n",
    "for sentence in raw_sentences:\n",
    "    sentences.append(sentence.split())\n",
    "\n",
    "WINDOW_SIZE = 2\n",
    "\n",
    "data = []\n",
    "for sentence in sentences:\n",
    "    for word_index, word in enumerate(sentence):\n",
    "        for nb_word in sentence[max(word_index - WINDOW_SIZE, 0) : min(word_index + WINDOW_SIZE, len(sentence)) + 1] : \n",
    "            if nb_word != word:\n",
    "                data.append([word, nb_word])\n",
    "\n",
    "print(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to convert numbers to one hot vectors\n",
    "def to_one_hot(data_point_index, vocab_size):\n",
    "    temp = np.zeros(vocab_size)\n",
    "    temp[data_point_index] = 1\n",
    "    return temp\n",
    "\n",
    "x_train = [] # input word\n",
    "y_train = [] # output word\n",
    "\n",
    "for data_word in data:\n",
    "    x_train.append(to_one_hot(word2int[ data_word[0] ], vocab_size))\n",
    "    y_train.append(to_one_hot(word2int[ data_word[1] ], vocab_size))\n",
    "\n",
    "# convert them to numpy arrays\n",
    "x_train = np.asarray(x_train)\n",
    "y_train = np.asarray(y_train)\n",
    "print('x:', x_train[:4])\n",
    "print('y:', y_train[:4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# making placeholders for x_train and y_train\n",
    "x = tf.placeholder(tf.float32, shape=(None, vocab_size))\n",
    "y_label = tf.placeholder(tf.float32, shape=(None, vocab_size))\n",
    "\n",
    "EMBEDDING_DIM = 5 # you can choose your own number\n",
    "W1 = tf.Variable(tf.random_normal([vocab_size, EMBEDDING_DIM]))\n",
    "b1 = tf.Variable(tf.random_normal([EMBEDDING_DIM])) #bias\n",
    "hidden_representation = tf.add(tf.matmul(x,W1), b1)\n",
    "# hidden_representation = tf.matmul(x,W1)\n",
    "\n",
    "W2 = tf.Variable(tf.random_normal([EMBEDDING_DIM, vocab_size]))\n",
    "b2 = tf.Variable(tf.random_normal([vocab_size]))\n",
    "prediction = tf.nn.softmax(tf.add( tf.matmul(hidden_representation, W2), b2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Set up the configuration for using the utility\n",
    "config = tf.ConfigProto(\n",
    "    log_device_placement = False, \n",
    "    allow_soft_placement = True, \n",
    "    graph_options=tf.GraphOptions(build_cost_model=1))\n",
    "# config.gpu_options.allow_growth = True\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.1\n",
    "\n",
    "sess = tf.Session()\n",
    "init = tf.global_variables_initializer()\n",
    "sess.run(init) #make sure you do this!\n",
    "\n",
    "# define the loss function:\n",
    "cross_entropy_loss = tf.reduce_mean(-tf.reduce_sum(y_label * tf.log(prediction), reduction_indices=[1]))\n",
    "\n",
    "# define the training step:\n",
    "train_step = tf.train.GradientDescentOptimizer(0.1).minimize(cross_entropy_loss)\n",
    "\n",
    "n_iters = 10000\n",
    "# train for n_iter iterations\n",
    "\n",
    "for i in range(n_iters):\n",
    "    _, _loss = sess.run([train_step,cross_entropy_loss], feed_dict={x: x_train, y_label: y_train})\n",
    "    if i % 500 == 0:\n",
    "        print('loss is : ', _loss)\n",
    "\n",
    "vectors = sess.run(W1 + b1)\n",
    "# vectors = sess.run(W1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "from sklearn import preprocessing\n",
    "import matplotlib.pyplot as plt\n",
    "def euclidean_dist(vec1, vec2):\n",
    "    return np.sqrt(np.sum((vec1-vec2)**2))\n",
    "\n",
    "def find_closest(word_index, vectors):\n",
    "    min_dist = 10000 # to act like positive infinity\n",
    "    min_index = -1\n",
    "    query_vector = vectors[word_index]\n",
    "    for index, vector in enumerate(vectors):\n",
    "        if euclidean_dist(vector, query_vector) < min_dist and not np.array_equal(vector, query_vector):\n",
    "            min_dist = euclidean_dist(vector, query_vector)\n",
    "            min_index = index\n",
    "    return min_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = TSNE(n_components=2, random_state=0)\n",
    "np.set_printoptions(suppress=True)\n",
    "tsne_vectors = model.fit_transform(vectors) \n",
    "\n",
    "normalizer = preprocessing.Normalizer()\n",
    "tsne_vectors =  normalizer.fit_transform(tsne_vectors, 'l2')\n",
    "\n",
    "print(tsne_vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots()\n",
    "print(words)\n",
    "for word in words:\n",
    "    print(word, tsne_vectors[word2int[word]][0], tsne_vectors[word2int[word]][1])\n",
    "    ax.annotate(word, (tsne_vectors[word2int[word]][0],tsne_vectors[word2int[word]][1] ))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(int2word[find_closest(word2int['king'], vectors)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using exsiting word2vec word embeddings\n",
    "For the following part, we will look at how we can load pretrained word2vec word embeddings. Some interesting properties of these word vectors will also be exhibited.\n",
    "\n",
    "First step: download the embedding file from https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM/edit, and then unzip the .gz file. Put the file in the same folder as this jupyter file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "\n",
    "# Load Google's pre-trained Word2Vec model\n",
    "# model = gensim.models.Word2Vec.load_word2vec_format('./GoogleNews-vectors-negative300.bin', binary=True)\n",
    "model = gensim.models.KeyedVectors.load_word2vec_format('./GoogleNews-vectors-negative300.bin', binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# getting word vectors of a word\n",
    "vector = model['computer']\n",
    "print(vector.shape)\n",
    "print(vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# performing king queen magic\n",
    "print(model.most_similar(positive=['woman', 'king'], negative=['man'], topn=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# picking odd one out\n",
    "print(model.doesnt_match(\"breakfast cereal dinner lunch\".split()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# printing similarity index\n",
    "print(model.similarity('woman', 'man'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find the most similar words\n",
    "w = 'usa'\n",
    "model.most_similar(positive=w, topn=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find the most similar words\n",
    "w = 'dirty'\n",
    "model.most_similar(positive=w, topn=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find the most similar words\n",
    "w = 'obama'\n",
    "model.most_similar(positive=w, topn=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Language Modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will show how to train a recurrent neural network on a challenging task of language modeling. The goal of the problem is to fit a probabilistic model which assigns probabilities to each word within a sentence. \n",
    "\n",
    "The cornerstone for this NLP problem is the Recurrent Neual Networks (RNNs). To improve RNNs' ability to capture long-term dependency, LSTMs are typically employed."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Recurrent Neural Networks (RNNs)\n",
    "\n",
    "In the context of deep learning, natural language is commonly modeled with Recurrent Neural Networks (RNNs).\n",
    "RNNs pass the output of a neuron back to the input of the next time step of the same neuron.\n",
    "These directed cycles in the RNN architecture gives them the ability to model temporal dynamics, making them particularly suited for modeling sequences (e.g. text).\n",
    "We can visualize an RNN layer as follows:\n",
    "\n",
    "<img src=\"Figures/basic_RNN.PNG\" alt=\"basic_RNN\" style=\"width: 80px;\"/>\n",
    "<center>Figure from *Understanding LSTMs*. https://colah.github.io/posts/2015-08-Understanding-LSTMs/</center>\n",
    "\n",
    "We can unroll an RNN through time, making the sequence aspect of them more obvious:\n",
    "\n",
    "<img src=\"Figures/unrolled_RNN.PNG\" alt=\"basic_RNN\" style=\"width: 400px;\"/>\n",
    "<center>Figure from *Understanding LSTMs*. https://colah.github.io/posts/2015-08-Understanding-LSTMs/</center>\n",
    "\n",
    "#### RNNs in TensorFlow\n",
    "How would we implement an RNN in TensorFlow? Given the different forms of RNNs, there are quite a few ways, but we'll stick to a simple one. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preparing the text data\n",
    "#1. Download the training corpus (e.g., Yelp reviews, news articles);\n",
    "#2. Truncate each sentence to a maximum length (typically 20 or 25 words);\n",
    "#3. Build a vocabulary by choosing those top frequent words (typically within the range of 10k ~ 100k);\n",
    "#4. Build the two dictionaries: ixtoword, wordtoix;\n",
    "#5. Convert each sentence to a list of indexes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "batch_size = 64   # number of samples within a batch\n",
    "num_steps = 25  # the maximum length of sentences\n",
    "vocab_size = 10000   # number of words in the vocabulary\n",
    "embedding_size = 300   # word embedding dimension\n",
    "hidden_size = 512  # the number of hidden states for LSTM\n",
    "keep_prob = 0.8   # the percentage of words left after the dropout layer\n",
    "num_layers = 3 # number of LSTM layers\n",
    "\n",
    "# placeholders for data\n",
    "train_inputs = tf.placeholder(tf.int32, shape=[batch_size, num_steps])\n",
    "train_targets = tf.placeholder(tf.int32, shape=[batch_size, num_steps])\n",
    "    \n",
    "# create the word embeddings\n",
    "embedding = tf.Variable(tf.random_uniform([vocab_size, embedding_size], -1.0, 1.0))\n",
    "inputs = tf.nn.embedding_lookup(embedding, train_inputs)\n",
    "inputs = tf.nn.dropout(inputs, keep_prob)\n",
    "print('inputs shape: %s'%(inputs.get_shape().as_list()))  # batch_size * num_steps * embedding_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RNN cell\n",
    "rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)\n",
    "initial_state = rnn_cell.zero_state(batch_size, dtype=tf.float32)\n",
    "outputs, state = tf.nn.dynamic_rnn(rnn_cell, inputs,\n",
    "                                   initial_state=initial_state,\n",
    "                                   dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Or multilayer LSTM\n",
    "rnn_layers = [tf.nn.rnn_cell.LSTMCell(size) for size in [hidden_size, hidden_size, hidden_size]]\n",
    "\n",
    "# create a RNN cell composed sequentially of a number of RNNCells\n",
    "multi_rnn_cell = tf.nn.rnn_cell.MultiRNNCell(rnn_layers)\n",
    "\n",
    "# 'outputs' is a tensor of shape [batch_size, max_time, 256]\n",
    "# 'state' is a N-tuple where N is the number of LSTMCells containing a\n",
    "# tf.contrib.rnn.LSTMStateTuple for each cell\n",
    "outputs, state = tf.nn.dynamic_rnn(cell=multi_rnn_cell,\n",
    "                                   inputs=inputs,\n",
    "                                   dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs, state = tf.nn.dynamic_rnn(cell=multi_rnn_cell,\n",
    "                                   inputs=inputs,\n",
    "                                   dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# claculate the output probabilities\n",
    "# a linear transformation for the LSTM output\n",
    "output = tf.reshape(outputs, [-1, hidden_size])\n",
    "softmax_w = tf.Variable(tf.random_uniform([hidden_size, vocab_size]))\n",
    "softmax_b = tf.Variable(tf.random_uniform([vocab_size]))\n",
    "logits = tf.nn.xw_plus_b(output, softmax_w, softmax_b)\n",
    "\n",
    "# Reshape logits to be a 3-D tensor for sequence loss\n",
    "logits = tf.reshape(logits, [batch_size, num_steps, vocab_size])\n",
    "print('logits shape: %s'%(logits.get_shape().as_list()))\n",
    "# output shape: batch_size * sentence_length * vocub_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use the contrib sequence loss and average over the batches;\n",
    "# constitutes the weighting of each prediction in the sequence; \n",
    "# When using weights as masking, set all valid timesteps to 1 and all padded timesteps to 0.\n",
    "loss = tf.contrib.seq2seq.sequence_loss(\n",
    "            logits,\n",
    "            train_targets,\n",
    "            tf.ones([batch_size, num_steps], dtype=tf.float32),\n",
    "            average_across_timesteps=False,\n",
    "            average_across_batch=True)\n",
    "\n",
    "# Update the cost\n",
    "cost = tf.reduce_sum(loss)\n",
    "\n",
    "# Construct the SGD optimizer using a learning rate of 1.0. \n",
    "optimizer = tf.train.AdagradOptimizer(1.0).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Some samples generated from a trained LSTM language model (I was training the LSTM model on EMNLP news dataset\n",
    "# for about 10 hours):\n",
    "print(\"Generated Text: We got to a bus station in the evening , but our connection didn ' t leave until the following morning .\")\n",
    "print(\"\\n\")\n",
    "print(\"Generated Text: An estimated 80 million people across 20 states are facing a second day of being trapped inside due to heavy snow and dangerous conditions , which are expected to last until Sunday .\")\n",
    "print(\"\\n\")\n",
    "print(\"Generated Text: The security guard claimed he suffered back pain and shock on his way home later that day and was taken to hospital .   \")\n",
    "print(\"\\n\")\n",
    "print(\"Generated Text: But these are all things that save me time and that I ' m happy to share with other iPhone users .  \")\n",
    "print(\"\\n\")\n",
    "print(\"Generated Text: We just have to keep putting our hands up , both of us , and put in a good performance .\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some samples generated from a trained LSTM language model (I was training the LSTM model on Yelp Reviews dataset\n",
    "# for about 17 hours):\n",
    "print(\"Generated Text: ate dinner here last night, the portions were small, i had the chicken and waffles, but i was not impressed .\")\n",
    "print(\"\\n\")\n",
    "print(\"Generated Text: this is my favorite restaurant in the valley, with a great view of the strict, my parents and i have been going to this place for years .\")\n",
    "print(\"\\n\")\n",
    "print(\"Generated Text: we go there for eakfast, i ve been here 3 times and it s always good, the hot dogs are delicious, and the hot dogs are delicious . \")\n",
    "print(\"\\n\")\n",
    "print(\"Generated Text: i was in vegas and was told to see a show in vegas, i was told it would be a great show, but i was told to see the show, i was told it would be the best show in vegas .\")\n",
    "print(\"\\n\")\n",
    "print(\"Generated Text: this place needs to be shut down, one of the worst experiences i have ever had, the manager was very rude .\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}