GradProject_NB4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> DS200A Computer Vision Assignment</h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>  Part Four: Extension Activities</h2>\t\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TensorFlow (Optional)- Now, try using TensorFlow to categorize your images. The accuracy should be significantly higher due to the usage of nueral nets\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dataset processing\n",
    "import os, glob\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import cv2\n",
    "import time\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "#Dataset processing\n",
    "def load_data(train_path, img_w, img_h):\n",
    "    print('Reading training images')\n",
    "    img_list, label_list = [], []\n",
    "    classes = os.listdir(train_path)\n",
    "    for index, fld in enumerate(classes):\n",
    "        path = os.path.join(train_path, fld, '*g')\n",
    "        files = glob.glob(path)\n",
    "        for fl in files:\n",
    "            image = cv2.imread(fl)\n",
    "            #resize each image to the standard img_w * img_h \n",
    "            image = cv2.resize(image, (img_w, img_h), cv2.INTER_LINEAR)\n",
    "            img_list.append(image), label_list.append(index)\n",
    "    img_list, label_list = np.array(img_list, dtype=np.uint8), np.array( label_list, dtype=np.int32 )\n",
    "    img_list = img_list.astype('float32')\n",
    "    #scale down the value of pixels to [0, 1] for numeric stability\n",
    "    img_list = img_list / 255\n",
    "    return img_list, label_list\n",
    "\n",
    "#define the directory, pre-processed image size\n",
    "img_w, img_h = 224, 224\n",
    "train_dir, test_dir = \"train/\", \"test/\"\n",
    "\n",
    "data, label = load_data(train_dir, img_w, img_h)\n",
    "print(data.shape, label.shape)\n",
    "idx = np.arange( data.shape[0] )\n",
    "np.random.shuffle(idx)\n",
    "data, label = data[idx], label[idx]\n",
    "\n",
    "#randomly split the training and validation set, we use the corresponding validation set to tune the parameters\n",
    "split_ratio = 0.9\n",
    "split = np.int(data.shape[0]* split_ratio)\n",
    "x_train, y_train = data[:split], label[:split]\n",
    "x_val, y_val =data[split:], label[split:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tensorflow model\n",
    "def inference(input_tensor, batch_size, n_classes, train):\n",
    "    # the first convolution layer with relu as activation function \n",
    "    with tf.variable_scope(\"layer1-conv1\") as scope:\n",
    "        weights = tf.get_variable(\"weights\", shape=[3, 3, 3, 16], dtype=tf.float32, initializer=tf.truncated_normal_initializer(stddev=0.1, dtype=tf.float32))\n",
    "        biases = tf.get_variable(\"biases\", shape=[16], dtype=tf.float32, initializer=tf.constant_initializer(0.1))\n",
    "        conv = tf.nn.conv2d(input_tensor, weights, strides=[1, 1, 1, 1], padding=\"SAME\")\n",
    "        activation = tf.nn.bias_add(conv, biases)\n",
    "        conv1 = tf.nn.relu(activation, name=\"conv1\")\n",
    "\n",
    "    # the second max pooling and normalization layer to aggregate the conv features\n",
    "    with tf.variable_scope(\"layer2-pooling_lrn_1\") as scope:\n",
    "        pool1 = tf.nn.max_pool(conv1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding=\"SAME\", name=\"pooling1\")\n",
    "        norm1 = tf.nn.lrn(pool1, depth_radius=4, bias=1.0, alpha=0.001/9.0, beta=0.75, name='layer_norm1')\n",
    "\n",
    "    # the third convolution layer with relu as activation function \n",
    "    with tf.variable_scope(\"layer3-conv2\") as scope:\n",
    "        weights = tf.get_variable(\"weights\", shape=[3, 3, 16, 16], dtype=tf.float32, initializer=tf.truncated_normal_initializer(stddev=0.1, dtype=tf.float32))\n",
    "        biases = tf.get_variable(\"biases\", shape=[16], dtype=tf.float32, initializer=tf.constant_initializer(0.1))\n",
    "        conv = tf.nn.conv2d(norm1, weights, strides=[1, 1, 1, 1], padding=\"SAME\")\n",
    "        activation = tf.nn.bias_add(conv, biases)\n",
    "        conv2 = tf.nn.relu(activation, name=\"conv2\")\n",
    "\n",
    "    # the fourth max pooling and normalization layer to aggregate the conv features\n",
    "    with tf.variable_scope(\"layer4-pooling2_lrn\") as scope:\n",
    "        pool2 = tf.nn.avg_pool(conv2, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding=\"SAME\", name=\"pooling2\")\n",
    "        norm2 = tf.nn.lrn(pool2, depth_radius=4, bias=1.0, alpha=0.001/9.0, beta=0.75, name='layer_norm2')\n",
    "        print(norm2.shape)\n",
    "        \n",
    "    # the fifth full connected layer to extract the feature from the previous layers\n",
    "    # add dropout layer (towards intra-ensmebling of model itself) for training, and regularizer for scaling down the\n",
    "    # value of parameter to make it more generalizable\n",
    "    with tf.variable_scope(\"layer5-fc1\") as scope:\n",
    "        reshape = tf.reshape(norm2, shape=[batch_size, -1])\n",
    "        dim = reshape.get_shape()[1].value\n",
    "        weights = tf.get_variable(\"weights\", shape=[50176, 128], dtype=tf.float32, initializer=tf.truncated_normal_initializer(stddev=0.005, dtype=tf.float32))\n",
    "        biases = tf.get_variable(\"biases\", shape=[128], dtype=tf.float32, initializer=tf.constant_initializer(0.1))\n",
    "        fc1 = tf.nn.relu(tf.matmul(reshape, weights) + biases, name=\"fc1\")\n",
    "        if regularizer != None: tf.add_to_collection('losses', regularizer(biases))\n",
    "        if train: fc1 = tf.nn.dropout(fc1, 0.5)\n",
    "    \n",
    "    # the sixth full connected layer to extract the feature from the full-connected layer\n",
    "    with tf.variable_scope(\"layer6-fc2\") as scope:\n",
    "        weights = tf.get_variable(\"weights\", shape=[128, 128], dtype=tf.float32, initializer=tf.truncated_normal_initializer(stddev=0.005, dtype=tf.float32))\n",
    "        biases = tf.get_variable(\"biases\", shape=[128], dtype=tf.float32, initializer=tf.constant_initializer(0.1))\n",
    "        fc2 = tf.nn.relu(tf.matmul(fc1, weights) + biases, name=\"fc2\")\n",
    "        if train: fc2 = tf.nn.dropout(fc2, 0.5)\n",
    "        if regularizer != None: tf.add_to_collection('losses', regularizer(biases))\n",
    "\n",
    "    # the final prediction layer w/o dropout to predict the classes of image batches\n",
    "    with tf.variable_scope(\"layer7-fc3\") as scope:\n",
    "        weights = tf.get_variable(\"weights\", shape=[128, n_classes], dtype=tf.float32, initializer=tf.truncated_normal_initializer(stddev=0.005, dtype=tf.float32))\n",
    "        biases = tf.get_variable(\"biases\", shape=[n_classes], dtype=tf.float32, initializer=tf.constant_initializer(0.1))\n",
    "        logit = tf.add(tf.matmul(fc2, weights), biases, name=\"fc3\")\n",
    "        if regularizer != None: tf.add_to_collection('losses', regularizer(biases))\n",
    "    return logit\n",
    "\n",
    "#basic config setting about the #chaneel, #classes, batch_size and learning rates\n",
    "channel, num_classes = 3, 20\n",
    "model_path = \"model/model.ckpt\"\n",
    "batch_size = 64\n",
    "lr = 0.001\n",
    "\n",
    "#the input and output tensor placeholder for image batches\n",
    "x = tf.placeholder(tf.float32, shape=[None, img_w, img_h, channel], name='x')\n",
    "labels = tf.placeholder(tf.int32, shape=[None,], name='label')\n",
    "\n",
    "#the regularizer with scale as 0.0001\n",
    "regularizer = tf.contrib.layers.l2_regularizer(0.0001)\n",
    "\n",
    "#the final outputs and inference logits\n",
    "logits = inference(x, batch_size=batch_size, n_classes=num_classes, train=True)\n",
    "b = tf.constant(value=1,dtype=tf.float32)\n",
    "logits_eval = tf.multiply(logits,b,name='logits_eval') \n",
    "\n",
    "#define the sparse_softmax_cross_entropy loss and optimazation method as adam\n",
    "loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels)\n",
    "train_op = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss)\n",
    "gold = tf.equal(tf.cast(tf.argmax(logits, 1), tf.int32), labels)    \n",
    "acc = tf.reduce_mean(tf.cast(gold, tf.float32))\n",
    "\n",
    "#tidy the training data into batches, (shuffle the whole data when training)\n",
    "def minibatches(inputs=None, targets=None, batch_size=None, shuffle=False):\n",
    "    assert len(inputs) == len(targets)\n",
    "    if shuffle:\n",
    "        indices = np.arange(len(inputs))\n",
    "        np.random.shuffle(indices)\n",
    "    for start_idx in range(0, len(inputs) - batch_size + 1, batch_size):\n",
    "        if shuffle:\n",
    "            excerpt = indices[start_idx:start_idx + batch_size]\n",
    "        else:\n",
    "            excerpt = slice(start_idx, start_idx + batch_size)\n",
    "        yield inputs[excerpt], targets[excerpt]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#training process\n",
    "n_epoch = 30       \n",
    "best_val_acc = 0.0\n",
    "saver = tf.train.Saver()\n",
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "for epoch in range(n_epoch):\n",
    "    start_time = time.time()\n",
    "    #training\n",
    "    train_loss, train_acc, n_batch = 0, 0, 0\n",
    "    for x_train_a, y_train_a in minibatches(x_train, y_train, batch_size, shuffle=True):\n",
    "        _,err,ac=sess.run([train_op,loss,acc], feed_dict={x: x_train_a, labels: y_train_a})\n",
    "        train_loss += err; train_acc += ac; n_batch += 1\n",
    "    print(\"------------Epoch: %d------------\" % epoch)\n",
    "    print(\"   train loss: %f\" % (np.sum(train_loss)/ n_batch))\n",
    "    print(\"   train acc: %f\" % ( np.sum(train_acc)/ n_batch))\n",
    "\n",
    "    #validation\n",
    "    val_loss, val_acc, n_batch = 0, 0, 0\n",
    "    for x_val_a, y_val_a in minibatches(x_val, y_val, batch_size, shuffle=False):\n",
    "        err, ac = sess.run([loss,acc], feed_dict={x: x_val_a, labels: y_val_a})\n",
    "        val_loss += err; val_acc += ac; n_batch += 1\n",
    "    print(\"   validation loss: %f\" % (np.sum(val_loss)/ n_batch))\n",
    "    print(\"   validation acc: %f\" % (np.sum(val_acc)/ n_batch))\n",
    "    #save the model w/ the best performance on validation set, which is 39.06% in terms of accuracy\n",
    "    if np.sum(val_acc)/n_batch > best_val_acc:\n",
    "        best_val_acc = np.sum(val_acc)/n_batch\n",
    "        saver.save(sess, model_path)\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#loading the test data and following the same data preprocessing schema\n",
    "def load_test(train_path, test_path, img_w, img_h):\n",
    "    print('Reading testing images')\n",
    "    img_list, label_list = [], []\n",
    "    classes = os.listdir(train_path)\n",
    "    files = glob.glob(test_path + '*g')\n",
    "    for fl in files:\n",
    "        image = cv2.imread(fl)\n",
    "        image = cv2.resize(image, (img_w, img_h), cv2.INTER_LINEAR)\n",
    "        img_list.append( image )\n",
    "    img_list = np.array(img_list, dtype=np.uint8)\n",
    "    img_list = img_list.astype('float32')\n",
    "    img_list = img_list / 255\n",
    "    return img_list, classes, files\n",
    "\n",
    "test_data, _, test_files = load_test(train_dir, test_dir, img_w, img_h)\n",
    "print(test_data.shape)\n",
    "total_outputs = []\n",
    "with tf.Session() as sess:\n",
    "    #load the saved model\n",
    "    saver = tf.train.import_meta_graph('model/model.ckpt.meta')\n",
    "    saver.restore(sess,tf.train.latest_checkpoint('model/'))\n",
    "    graph = tf.get_default_graph()\n",
    "    x = graph.get_tensor_by_name(\"x:0\")\n",
    "    #pack the test data into batches and tidy the outputs accordingly\n",
    "    for idx in range(0, test_data.shape[0], batch_size):\n",
    "        if idx + batch_size > test_data.shape[0]:\n",
    "            cur_pack = test_data[-batch_size:]\n",
    "            cur_left = test_data.shape[0] % batch_size\n",
    "        else:\n",
    "            cur_pack = test_data[idx:idx+batch_size]\n",
    "            cur_left = batch_size\n",
    "        feed_dict = {x:cur_pack}\n",
    "        logits = graph.get_tensor_by_name(\"logits_eval:0\")\n",
    "        classification_result = sess.run(logits,feed_dict)\n",
    "        output = tf.argmax(classification_result, 1).eval()\n",
    "        total_outputs.extend( output.tolist()[-cur_left:] )    \n",
    "print(len(total_outputs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "#save the prediction outputs for the test set\n",
    "res = {x:y for x, y in zip(test_files, total_outputs)}\n",
    "open(\"test.json\", \"w\").write( json.dumps( res ) )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}