highway_dc_network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The dataset used in this project is MNIST dataset, you can download it by using built-in tensorflow functions.\n",
    "\n",
    "\n",
    "For more information about Highway Networks read:\n",
    "\n",
    "[Highway Networks](https://arxiv.org/pdf/1505.00387.pdf)\n",
    "\n",
    "[This blog post](https://medium.com/jim-fleming/highway-networks-with-tensorflow-1e6dfa667daa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 1. Load the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data\\train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist_data = input_data.read_data_sets(\"MNIST_data\", one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 2. Define helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weights_init(shape):\n",
    "    '''\n",
    "    Weights initialization helper function.\n",
    "    \n",
    "    Input(s): shape - Type: int list, Example: [5, 5, 32, 32], This parameter is used to define dimensions of weights tensor\n",
    "    \n",
    "    Output: tensor of weights in shape defined with the input to this function\n",
    "    '''\n",
    "    return tf.Variable(tf.truncated_normal(shape, stddev=0.05))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bias_init(shape, bias_value=0.05):\n",
    "    '''\n",
    "    Bias initialization helper function.\n",
    "    \n",
    "    Input(s): shape - Type: int list, Example: [32], This parameter is used to define dimensions of bias tensor.\n",
    "              bias_value - Type: float number, Example: 0.01, This parameter is set to be value of bias tensor.\n",
    "    \n",
    "    Output: tensor of biases in shape defined with the input to this function\n",
    "    '''\n",
    "    return tf.Variable(tf.constant(bias_value, shape=shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def convd2_custom(input, filter_size, number_of_channels, number_of_filters, max_pool=False, padding='SAME', \n",
    "                activation=tf.nn.relu):\n",
    "    '''\n",
    "    This function is used to define a convolutional layer for a network,\n",
    "    \n",
    "    Input(s): input - this is input into convolutional layer (Previous layer or an image)\n",
    "              filter_size - also called kernel size, kernel is moved (convolved) across an image. Example: 3\n",
    "              number_of_channels - how many channels the input tensor has\n",
    "              number_of_filters - this is hyperparameter, and this will set one of dimensions of the output tensor from \n",
    "                                  this layer. Note: this number will be number_of_channels for the layer after this one\n",
    "              max_pool - if this is True, output tensor will be 2x smaller in size. Max pool is there to decrease spartial \n",
    "                        dimensions of our output tensor, so computation is less expensive.\n",
    "              padding - the way that we pad input tensor with zeros (\"SAME\" or \"VALID\")\n",
    "              activation - the non-linear function used at this layer.\n",
    "              \n",
    "              \n",
    "    Output: Convolutional layer with input parameters.\n",
    "    '''\n",
    "    weights = weights_init([filter_size, filter_size, number_of_channels, number_of_filters])\n",
    "    bias = bias_init([number_of_filters])\n",
    "    \n",
    "    layer = tf.nn.conv2d(input, filter=weights, strides=[1, 1, 1, 1], padding=padding) + bias\n",
    "    \n",
    "    layer = activation(layer)\n",
    "    \n",
    "    if max_pool:\n",
    "        layer = tf.nn.max_pool(layer, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "        \n",
    "    return layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def dense_custom(input, input_shape, output_shape, activation=tf.nn.relu, dropout=None):\n",
    "    '''\n",
    "    This function is used to define a fully connected layer for a network,\n",
    "    \n",
    "    Input(s): input - this is input into fully connected (Dense) layer (Previous layer or an image)\n",
    "              input_size - how many neurons/features the input tensor has. Example: input.shape[1]\n",
    "              output_shape - how many neurons this layer will have\n",
    "              activation - the non-linear function used at this layer.    \n",
    "              dropout - the regularization method used to prevent overfitting. The way it works, we randomly turn off\n",
    "                        some neurons in this layer\n",
    "              \n",
    "    Output: fully connected layer with input parameters.\n",
    "    '''\n",
    "    weights = weights_init([input_shape, output_shape])\n",
    "    bias = bias_init([output_shape])\n",
    "    \n",
    "    layer = tf.matmul(input, weights) + bias\n",
    "    \n",
    "    if activation != None:\n",
    "        layer = activation(layer)\n",
    "    \n",
    "    if dropout != None:\n",
    "        layer = tf.nn.dropout(layer, dropout)\n",
    "        \n",
    "    return layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def flatten(layer):\n",
    "    '''\n",
    "    This method is used to convert convolutional output (4 dimensional tensor) into 2 dimensional tensor.\n",
    "    \n",
    "    Input(s): layer - the output from last conv layer in your network (4d tensor)\n",
    "    \n",
    "    Output(s): reshaped - reshaped layer, 2 dimensional matrix\n",
    "               elements_num - number of features for this layer\n",
    "    '''\n",
    "    shape = layer.get_shape()\n",
    "    \n",
    "    elements_num = shape[1:4].num_elements()\n",
    "    \n",
    "    reshaped = tf.reshape(layer, [-1, elements_num])\n",
    "    return reshaped, elements_num"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The highway layer what we want are two “gates” that control the flow of information. The “transform” gate controls how much of the activation we pass through and the “carry” gate controls how much of the unmodified input we pass through.\n",
    "\n",
    "The formula for the highway layer:\n",
    "![](formula_highway_layer.png?raw=true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def highway_conv2d(input, filter_size, number_of_channels, number_of_filters, max_pool=False, padding='SAME', \n",
    "                activation=tf.nn.relu, transf_bias_value=-1.0):\n",
    "    \n",
    "    '''\n",
    "    This function is used to define a highway convolutional layer for a network,\n",
    "    \n",
    "    Input(s): input - this is input into highway convolutional layer (Previous layer or an image)\n",
    "              filter_size - also called kernel size, kernel is moved (convolved) across an image. Example: 3\n",
    "              number_of_channels - how many channels the input tensor has\n",
    "              number_of_filters - this is hyperparameter, and this will set one of dimensions of the output tensor from \n",
    "                                  this layer. Note: this number will be number_of_channels for the layer after this one\n",
    "              max_pool - if this is True, output tensor will be 2x smaller in size. Max pool is there to decrease spartial \n",
    "                        dimensions of our output tensor, so computation is less expensive.\n",
    "              padding - the way that we pad input tensor with zeros (\"SAME\" or \"VALID\")\n",
    "              activation - the non-linear function used at this layer.\n",
    "              transf_bias_value - Float number -  Bias value in Transform gate\n",
    "              \n",
    "    Output: Highway Convolutional layer with input parameters.\n",
    "    '''\n",
    "    \n",
    "    weights = weights_init([filter_size, filter_size, number_of_channels, number_of_filters])\n",
    "    bias = bias_init([number_of_filters])\n",
    "    \n",
    "    layer_H = tf.nn.conv2d(input, filter=weights, strides=[1, 1, 1, 1], padding=padding) + bias\n",
    "    layer_H = activation(layer_H)\n",
    "    \n",
    "    weights_transform = weights_init([filter_size, filter_size, number_of_channels, number_of_filters])\n",
    "    bias_transform = bias_init([number_of_filters], bias_value=transf_bias_value)\n",
    "    \n",
    "    gate_T = tf.nn.conv2d(input, filter=weights_transform, strides=[1, 1, 1, 1], padding=padding) + bias_transform\n",
    "    gate_T = tf.nn.sigmoid(gate_T)\n",
    "    \n",
    "    gate_C = tf.subtract(1.0, gate_T)\n",
    "    \n",
    "    y = tf.add(tf.multiply(layer_H, gate_T), tf.multiply(input, gate_C), name=\"conv2d_highway\")\n",
    "    return y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 3. Highway Deep Conv network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Inputs to the Highway DC network\n",
    "inputs = tf.placeholder(tf.float32, [None, 28, 28, 1], name='inputs')\n",
    "targets = tf.placeholder(tf.float32, [None, 10], name='inputs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Defining the network\n",
    "layer_1 = convd2_custom(inputs, 5, 1, 32)\n",
    "layer_2 = highway_conv2d(layer_1, 3, 32, 32)\n",
    "layer_3 = highway_conv2d(layer_2, 3, 32, 32, max_pool=True)\n",
    "drop = tf.nn.dropout(layer_3, 0.8)\n",
    "\n",
    "layer_4 = highway_conv2d(drop, 3, 32, 32)\n",
    "layer_5 = highway_conv2d(layer_4, 3, 32, 32)\n",
    "layer_6 = highway_conv2d(layer_5, 3, 32, 32, max_pool=True)\n",
    "drop = tf.nn.dropout(layer_6, 0.6)\n",
    "\n",
    "layer_7 = highway_conv2d(drop, 3, 32, 32)\n",
    "layer_8 = highway_conv2d(layer_7, 3, 32, 32)\n",
    "layer_9 = highway_conv2d(layer_8, 3, 32, 32, max_pool=True)\n",
    "drop = tf.nn.dropout(layer_9, 0.4)\n",
    "\n",
    "layer_10 = highway_conv2d(drop, 3, 32, 32)\n",
    "layer_11 = highway_conv2d(layer_10, 3, 32, 32)\n",
    "layer_12 = highway_conv2d(layer_11, 3, 32, 32, max_pool=True)\n",
    "drop = tf.nn.dropout(layer_12, 0.2)\n",
    "\n",
    "flat, num_elements = flatten(drop)\n",
    "fc_1 = dense_custom(flat, num_elements, 512, activation=tf.nn.relu)\n",
    "output = dense_custom(fc_1, 512, 10, activation=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Loss function and optimizer for the network\n",
    "cost = tf.reduce_sum(tf.nn.softmax_cross_entropy_with_logits(logits=output, labels=targets))\n",
    "optimizer = tf.train.AdamOptimizer(0.01).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Testing accuracy of the network\n",
    "correct_prediction = tf.equal(tf.argmax(tf.nn.softmax(output), 1), tf.argmax(targets, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 4. Train, test functions for the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "epochs = 20\n",
    "batch_size = 64\n",
    "def optimize():\n",
    "    \n",
    "    for i in range(epochs):\n",
    "        epoch_cost = []\n",
    "        epoch_time = time.time()\n",
    "        for ii in range(mnist_data.train.num_examples//batch_size):\n",
    "            batch = mnist_data.train.next_batch(batch_size)\n",
    "            imgs = batch[0].reshape((-1, 28, 28, 1))\n",
    "            labs = batch[1]\n",
    "            \n",
    "            c, _ = session.run([cost, optimizer], feed_dict={inputs:imgs, targets:labs})\n",
    "\n",
    "            epoch_cost.append(c)\n",
    "        print(\"Epoch: {}/{}\".format(i+1, epochs), \" | Current loss: {}\".format(np.mean(epoch_cost)),\n",
    "             \"  |  Epoch time: {:.2f}s\".format(time.time() - epoch_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size_validate = 1000\n",
    "def validate_model():\n",
    "    accuracy_per_batch = []\n",
    "    for ii in range(mnist_data.validation.num_examples//batch_size_validate):\n",
    "        batch = mnist_data.validation.next_batch(batch_size_validate)\n",
    "        imgs = batch[0].reshape((-1, 28, 28, 1))\n",
    "        labs = batch[1]\n",
    "\n",
    "        accuracy_per_batch.append(session.run(accuracy, feed_dict={inputs:imgs, targets:labs}))\n",
    "\n",
    "    print(\"Validation accuracy {}\".format(accuracy_per_batch))\n",
    "    print(\"Validation accuracy average: {}\".format(np.mean(accuracy_per_batch)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "batch_size_test = 1000\n",
    "def test_model():\n",
    "    accuracy_per_batch = []\n",
    "    for ii in range(mnist_data.test.num_examples//batch_size_test):\n",
    "        batch = mnist_data.test.next_batch(batch_size_test)\n",
    "        imgs = batch[0].reshape((-1, 28, 28, 1))\n",
    "        labs = batch[1]\n",
    "\n",
    "        accuracy_per_batch.append(session.run(accuracy, feed_dict={inputs:imgs, targets:labs}))\n",
    "\n",
    "    print(\"Test accuracy {}\".format(accuracy_per_batch))\n",
    "    print(\"Test accuracy average: {}\".format(np.mean(accuracy_per_batch)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 5. Training and testint the Highway DC network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "session = tf.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "session.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1/20  | Current loss: 9.693116188049316   |  Epoch time: 59.19s\n",
      "Epoch: 2/20  | Current loss: 8.4194917678833   |  Epoch time: 59.06s\n",
      "Epoch: 3/20  | Current loss: 7.4638142585754395   |  Epoch time: 59.09s\n",
      "Epoch: 4/20  | Current loss: 6.606746673583984   |  Epoch time: 59.44s\n",
      "Epoch: 5/20  | Current loss: 6.440258979797363   |  Epoch time: 58.99s\n",
      "Epoch: 6/20  | Current loss: 6.112809658050537   |  Epoch time: 59.10s\n",
      "Epoch: 7/20  | Current loss: 5.248502254486084   |  Epoch time: 59.00s\n",
      "Epoch: 8/20  | Current loss: 9.59684944152832   |  Epoch time: 59.01s\n",
      "Epoch: 9/20  | Current loss: 5.251861095428467   |  Epoch time: 59.17s\n",
      "Epoch: 10/20  | Current loss: 9.391960144042969   |  Epoch time: 59.26s\n",
      "Epoch: 11/20  | Current loss: 12.334983825683594   |  Epoch time: 59.29s\n",
      "Epoch: 12/20  | Current loss: 24.280094146728516   |  Epoch time: 59.21s\n",
      "Epoch: 13/20  | Current loss: 12.361469268798828   |  Epoch time: 59.22s\n",
      "Epoch: 14/20  | Current loss: 8.058774948120117   |  Epoch time: 59.14s\n",
      "Epoch: 15/20  | Current loss: 6.652774810791016   |  Epoch time: 59.35s\n",
      "Epoch: 16/20  | Current loss: 167.5113067626953   |  Epoch time: 58.93s\n",
      "Epoch: 17/20  | Current loss: 40.09580612182617   |  Epoch time: 58.58s\n",
      "Epoch: 18/20  | Current loss: 30.640457153320312   |  Epoch time: 58.87s\n",
      "Epoch: 19/20  | Current loss: 23.376251220703125   |  Epoch time: 58.67s\n",
      "Epoch: 20/20  | Current loss: 16.44678497314453   |  Epoch time: 59.47s\n"
     ]
    }
   ],
   "source": [
    "optimize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy [0.93900001, 0.93000007, 0.93300003, 0.93599999, 0.93199998, 0.92500001, 0.926, 0.94300008, 0.93400002, 0.94300002]\n",
      "Test accuracy average: 0.9340999722480774\n"
     ]
    }
   ],
   "source": [
    "test_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy [0.92700011, 0.93400002, 0.92400002, 0.93000007, 0.93000007]\n",
      "Validation accuracy average: 0.9290000796318054\n"
     ]
    }
   ],
   "source": [
    "validate_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# session.close()\n",
    "#Close the session after testing the network"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}