4a_CNN_MNIST_160712.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\* *[Notice] I wrote thie code while following the examples in [Choi's Tesorflow-101 tutorial](https://github.com/sjchoi86/Tensorflow-101). And,  as I know, most of Choi's examples originally come from [Aymeric Damien's](https://github.com/aymericdamien/TensorFlow-Examples/) and  [Nathan Lintz's ](https://github.com/nlintz/TensorFlow-Tutorials) tutorials.*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Convolutional Neural Network with MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "# %matplotlib inline  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/train-images-idx3-ubyte.gz\n",
      "Extracting data/train-labels-idx1-ubyte.gz\n",
      "Extracting data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist      = input_data.read_data_sets('data', one_hot=True)\n",
    "X_train   = mnist.train.images\n",
    "Y_train = mnist.train.labels\n",
    "X_test    = mnist.test.images\n",
    "Y_test  = mnist.test.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of (X_train, X_test, Y_train, Y_test)\n",
      "((55000, 784), (10000, 784), (55000, 10), (10000, 10))\n"
     ]
    }
   ],
   "source": [
    "dimX = X_train.shape[1]\n",
    "dimY = Y_train.shape[1]\n",
    "nTrain = X_train.shape[0]\n",
    "nTest = X_test.shape[0]\n",
    "print (\"Shape of (X_train, X_test, Y_train, Y_test)\")\n",
    "print (X_train.shape, X_test.shape, Y_train.shape, Y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set parameters for my CNN structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pp = {\n",
    "    'nLayerIn': dimX,\n",
    "    'nLayerOut':  dimY,\n",
    "    'sigma_init': 0.1, \n",
    "    'myDropProb': 0.7, \n",
    "\n",
    "    'nWin_conv1': 3,\n",
    "    'nStr_conv1': 1,\n",
    "    'nPad_conv1': 'SAME', # or 'VALID'\n",
    "    'nWin_pool1': 2,\n",
    "    'nStr_pool1': 2,\n",
    "    'nPad_pool1': 'SAME',\n",
    "    'nFeat1': 64,\n",
    "\n",
    "    'nWin_conv2': 3,\n",
    "    'nStr_conv2': 1,\n",
    "    'nPad_conv2': 'SAME',\n",
    "    'nWin_pool2': 2,\n",
    "    'nStr_pool2': 2,\n",
    "    'nPad_pool2': 'SAME',\n",
    "    'nFeat2': 128,\n",
    "\n",
    "    'dimX_mat': 28,   # 28*28 = 784\n",
    "    'nDimReduce': 7,  # dimX_mat/nWin_pool1/nWin_pool2\n",
    "    'nFull': 1024\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build my CNN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "W = {\n",
    "    'W_conv1': tf.Variable(tf.truncated_normal([pp['nWin_conv1'], pp['nWin_conv1'], 1, pp['nFeat1']], stddev=pp['sigma_init'])),\n",
    "    'W_conv2': tf.Variable(tf.truncated_normal([pp['nWin_conv2'], pp['nWin_conv2'], pp['nFeat1'], pp['nFeat2']], stddev=pp['sigma_init'])),\n",
    "    'W_full': tf.Variable(tf.truncated_normal([pp['nDimReduce']*pp['nDimReduce']*pp['nFeat2'], pp['nFull']], stddev=pp['sigma_init'])),\n",
    "    'W_out': tf.Variable(tf.truncated_normal([pp['nFull'], pp['nLayerOut']], stddev=pp['sigma_init']))\n",
    "    \n",
    "}\n",
    "b = {\n",
    "    'b_conv1': tf.Variable(tf.truncated_normal([pp['nFeat1']], stddev=pp['sigma_init'])),\n",
    "    'b_conv2': tf.Variable(tf.truncated_normal([pp['nFeat2']], stddev=pp['sigma_init'])),\n",
    "    'b_full': tf.Variable(tf.truncated_normal([pp['nFull']], stddev=pp['sigma_init'])),\n",
    "    'b_out': tf.Variable(tf.truncated_normal([pp['nLayerOut']], stddev=pp['sigma_init']))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def model_myCNN(_X, _W, _B, _dropout_prob, _pp):\n",
    "       \n",
    "    _X_mat = tf.reshape(_X, shape=[-1, _pp['dimX_mat'], _pp['dimX_mat'], 1])\n",
    "\n",
    "    # L1: Convolution\n",
    "    _L1_conv = tf.nn.relu(tf.nn.bias_add(\n",
    "            tf.nn.conv2d(_X_mat, _W['W_conv1'], strides=[1, _pp['nStr_conv1'], _pp['nStr_conv1'], 1], padding=_pp['nPad_conv1'])\n",
    "            , _B['b_conv1']))\n",
    "    _L1_pool = tf.nn.max_pool(_L1_conv, ksize=[1, _pp['nWin_pool1'], _pp['nWin_pool1'], 1], strides=[1, _pp['nStr_pool1'], _pp['nStr_pool1'], 1], padding=_pp['nPad_pool1'])\n",
    "    _L1_pool2 = tf.nn.dropout(_L1_pool, _dropout_prob)\n",
    "    \n",
    "    # L2: Convolution\n",
    "    _L2_conv = tf.nn.relu(tf.nn.bias_add(\n",
    "            tf.nn.conv2d(_L1_pool2, _W['W_conv2'], strides=[1, _pp['nStr_conv2'], _pp['nStr_conv2'], 1], padding=_pp['nPad_conv1'])\n",
    "            , _B['b_conv2']))\n",
    "    _L2_pool = tf.nn.max_pool(_L2_conv, ksize=[1, _pp['nWin_pool2'], _pp['nWin_pool2'], 1], strides=[1, _pp['nStr_pool2'], _pp['nStr_pool2'], 1], padding=_pp['nPad_pool1'])\n",
    "    _L2_pool2 = tf.nn.dropout(_L2_pool, _dropout_prob)\n",
    "    \n",
    "\n",
    "    # L_full: Fully-connected\n",
    "    _L2_pool2_vec = tf.reshape(_L2_pool2, [-1, _W['W_full'].get_shape().as_list()[0]])\n",
    "    _L_full = tf.nn.relu(tf.add(tf.matmul(_L2_pool2_vec, _W['W_full']), _B['b_full']))\n",
    "    _L_full2 = tf.nn.dropout(_L_full, _dropout_prob)\n",
    "    \n",
    "    # L_full: Output\n",
    "    _L_out = tf.add(tf.matmul(_L_full2, _W['W_out']), _B['b_out'])\n",
    "    \n",
    "    # Return \n",
    "    out = {\n",
    "        'X_mat': _X_mat,\n",
    "        'L1_conv': _L1_conv, 'L1_pool': _L1_pool, 'L1_pool2': _L1_pool2, # After dropout\n",
    "        'L2_conv': _L2_conv, 'L2_pool': _L2_pool, 'L2_pool2': _L2_pool2,\n",
    "        'L_full': _L_full, 'L_full2': _L_full2, 'L_out': _L_out\n",
    "    }\n",
    "    return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define variables and optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float32, [None, dimX], name=\"input\")\n",
    "Y= tf.placeholder(tf.float32, [None, dimY], name=\"output\")\n",
    "dropout_prob = tf.placeholder(tf.float32, name=\"dropout\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "Y_pred_all = model_myCNN(X, W, b, dropout_prob, pp)\n",
    "Y_pred = Y_pred_all['L_out']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(Y_pred, Y))\n",
    "learning_rate = 0.001\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)\n",
    "training_epochs = 5\n",
    "display_epoch = 1\n",
    "batch_size = 100   # For each time, we will use 100 samples to update ppeters "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(Y_pred, 1), tf.argmax(Y, 1))    \n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Because of the memory allocation problem in evaluation\n",
    "divide_train = 50;\n",
    "divide_test = 10;\n",
    "nTrainSub = (int)(nTrain/divide_train);\n",
    "nTestSub = (int)(nTest/divide_test);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(epoch 1)\n",
      "[Loss / Tranining Accuracy / Test Accuracy] 0.0559 / 0.9830 / 0.9838\n",
      " \n",
      "(epoch 2)\n",
      "[Loss / Tranining Accuracy / Test Accuracy] 0.0339 / 0.9892 / 0.9884\n",
      " \n",
      "(epoch 3)\n",
      "[Loss / Tranining Accuracy / Test Accuracy] 0.0226 / 0.9932 / 0.9920\n",
      " \n",
      "(epoch 4)\n",
      "[Loss / Tranining Accuracy / Test Accuracy] 0.0181 / 0.9945 / 0.9918\n",
      " \n",
      "(epoch 5)\n",
      "[Loss / Tranining Accuracy / Test Accuracy] 0.0124 / 0.9962 / 0.9926\n",
      " \n",
      "[Test Accuracy] 0.9926\n"
     ]
    }
   ],
   "source": [
    "#with tf.Session() as sess:   \n",
    "sess = tf.InteractiveSession()\n",
    "sess.run(tf.initialize_all_variables())\n",
    "\n",
    "for epoch in range(training_epochs):\n",
    "    nBatch  = int(nTrain/batch_size)\n",
    "    #myIdx =  np.random.permutation(nTrain)\n",
    "    for ii in range(nBatch):\n",
    "        X_Batch, Y_Batch = mnist.train.next_batch(batch_size)\n",
    "        #X_Batch = X_train[myIdx[ii*batch_size:(ii+1)*batch_size],:]\n",
    "        #Y_Batch = Y_train[myIdx[ii*batch_size:(ii+1)*batch_size],:]\n",
    "        sess.run(optimizer, feed_dict={X:X_Batch, Y:Y_Batch, dropout_prob:pp['myDropProb']})\n",
    "\n",
    "    if (epoch+1) % display_epoch == 0:\n",
    "        # Because of the memory allocation problem in evaluation\n",
    "        loss_temp = accuracy_train_temp = accuracy_test_temp = 0\n",
    "        for jj in range(divide_train):\n",
    "            myIdx1 = jj*nTrainSub\n",
    "            myIdx2 = (jj+1)*nTrainSub\n",
    "            loss_temp += sess.run(loss, feed_dict={X: X_train[myIdx1:myIdx2,:], Y:Y_train[myIdx1:myIdx2,:], dropout_prob:1.})\n",
    "            accuracy_train_temp += accuracy.eval({X: X_train[myIdx1:myIdx2,:], Y:Y_train[myIdx1:myIdx2,:], dropout_prob:1.})\n",
    "        for kk in range(divide_test):\n",
    "            myIdx1 = kk*nTestSub\n",
    "            myIdx2 = (kk+1)*nTestSub\n",
    "            accuracy_test_temp += accuracy.eval({X: X_test[myIdx1:myIdx2,:], Y: Y_test[myIdx1:myIdx2,:], dropout_prob:1.}) \n",
    "\n",
    "        print \"(epoch {})\".format(epoch+1) \n",
    "        print \"[Loss / Tranining Accuracy / Test Accuracy] {:05.4f} / {:05.4f} / {:05.4f}\".format(loss_temp/divide_train, accuracy_train_temp/divide_train, accuracy_test_temp/divide_test)\n",
    "        print \" \"\n",
    "\n",
    "print \"[Test Accuracy] {:05.4f}\".format(accuracy_test_temp/divide_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Let's see the learned features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nExample = 223\n",
    "\n",
    "Y_pred_all = model_myCNN(X, W, b, dropout_prob, pp)\n",
    "X_mat = sess.run(Y_pred_all['X_mat'], feed_dict={X: X_train[nExample-1:nExample, :]})\n",
    "L1_conv   = sess.run(Y_pred_all['L1_conv'], feed_dict={X: X_train[nExample-1:nExample, :]})\n",
    "\n",
    "#L1_pool   = sess.run(Y_pred_all['L1_pool'], feed_dict={X: X_train[nExample-1:nExample, :]})\n",
    "#L1_pool2   = sess.run(Y_pred_all['L1_pool2'], feed_dict={X: X_train[nExample-1:nExample, :]})\n",
    "#L2_conv   = sess.run(Y_pred_all['L2_conv'], feed_dict={X: X_train[nExample-1:nExample, :]})\n",
    "#L2_pool    = sess.run(Y_pred_all['L2_pool'], feed_dict={X: X_train[nExample-1:nExample, :]})\n",
    "#L2_pool2    = sess.run(Y_pred_all['L2_pool2'], feed_dict={X: X_train[nExample-1:nExample, :]})\n",
    "#L_full   = sess.run(Y_pred_all['L_full'], feed_dict={X: X_train[nExample-1:nExample, :]})\n",
    "#L_full2     = sess.run(Y_pred_all['L_full2'], feed_dict={X: X_train[nExample-1:nExample, :]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# (nExample)th Input\n",
    "plt.matshow(X_mat[0, :, :, 0], cmap=plt.get_cmap('gray'))\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Features\n",
    "nFeature = 40\n",
    "plt.matshow(L1_conv[0, :, :, nFeature], cmap=plt.get_cmap('gray'))\n",
    "plt.colorbar()\n",
    "plt.show() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}