Multi-Task Learning Tensorflow Part 1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multi Task Learning in Tensorflow Part 1\n",
    "## https://jg8610.github.io/Multi-Task/\n",
    "\n",
    "### The following code is just an introduction to a very simple computation in Tensorflow, designed to clarify the concept of a graph. Please see the blog for a more thorough explanation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.4753139 ]\n",
      " [ 0.29372749]\n",
      " [ 0.23547001]\n",
      " [ 0.12686814]\n",
      " [ 0.48484427]\n",
      " [ 0.16205794]\n",
      " [ 0.20848498]\n",
      " [ 0.66798812]\n",
      " [ 0.01020519]\n",
      " [ 0.11265998]]\n"
     ]
    }
   ],
   "source": [
    "# Import Tensorflow and Numpy\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "# ======================\n",
    "# Define the Graph\n",
    "# ======================\n",
    "\n",
    "# Create Placeholders For X And Y (for feeding in data)\n",
    "X = tf.placeholder(\"float\",[10, 10],name=\"X\") # Our input is 10x10\n",
    "Y = tf.placeholder(\"float\", [10, 1],name=\"Y\") # Our output is 10x1\n",
    "\n",
    "# Create a Trainable Variable, \"W\", our weights for the linear transformation\n",
    "initial_W = np.zeros((10,1))\n",
    "W = tf.Variable(initial_W, name=\"W\", dtype=\"float32\")\n",
    "\n",
    "# Define Your Loss Function\n",
    "Loss = tf.pow(tf.add(Y,-tf.matmul(X,W)),2,name=\"Loss\")\n",
    "\n",
    "with tf.Session() as sess: # set up the session\n",
    "    sess.run(tf.initialize_all_variables())\n",
    "    Model_Loss = sess.run(\n",
    "                Loss, # the first argument is the name of the tensorflow variabl you want to return\n",
    "                { # the second argument is the data for the placeholders\n",
    "                  X: np.random.rand(10,10),\n",
    "                  Y: np.random.rand(10).reshape(-1,1)\n",
    "                })\n",
    "    print(Model_Loss)\n",
    "\n",
    "    sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi-Task Example\n",
    "### This is an example of a graph that has three ways of training - just task 1, just task 2 or both tasks jointly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#  GRAPH CODE\n",
    "# ============\n",
    "\n",
    "# Import Tensorflow and Numpy\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "# ======================\n",
    "# Define the Graph\n",
    "# ======================\n",
    "\n",
    "# Define the Placeholders\n",
    "X = tf.placeholder(\"float\", [10, 10], name=\"X\")\n",
    "Y1 = tf.placeholder(\"float\", [10, 20], name=\"Y1\")\n",
    "Y2 = tf.placeholder(\"float\", [10, 20], name=\"Y2\")\n",
    "\n",
    "# Define the weights for the layers\n",
    "\n",
    "initial_shared_layer_weights = np.random.rand(10,20)\n",
    "initial_Y1_layer_weights = np.random.rand(20,20)\n",
    "initial_Y2_layer_weights = np.random.rand(20,20)\n",
    "\n",
    "shared_layer_weights = tf.Variable(initial_shared_layer_weights, name=\"share_W\", dtype=\"float32\")\n",
    "Y1_layer_weights = tf.Variable(initial_Y1_layer_weights, name=\"share_Y1\", dtype=\"float32\")\n",
    "Y2_layer_weights = tf.Variable(initial_Y2_layer_weights, name=\"share_Y2\", dtype=\"float32\")\n",
    "\n",
    "# Construct the Layers with RELU Activations\n",
    "shared_layer = tf.nn.relu(tf.matmul(X,shared_layer_weights))\n",
    "Y1_layer = tf.nn.relu(tf.matmul(shared_layer,Y1_layer_weights))\n",
    "Y2_layer = tf.nn.relu(tf.matmul(shared_layer,Y2_layer_weights))\n",
    "\n",
    "# Calculate Loss\n",
    "Y1_Loss = tf.nn.l2_loss(Y1-Y1_layer)\n",
    "Y2_Loss = tf.nn.l2_loss(Y2-Y2_layer)\n",
    "Joint_Loss = Y1_Loss + Y2_Loss\n",
    "\n",
    "# optimisers\n",
    "Optimiser = tf.train.AdamOptimizer().minimize(Joint_Loss)\n",
    "Y1_op = tf.train.AdamOptimizer().minimize(Y1_Loss)\n",
    "Y2_op = tf.train.AdamOptimizer().minimize(Y2_Loss)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This is a demonstration of how you would train both tasks alternately, sharing a representation on the bottom layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.49855e+06\n",
      "4.88283e+06\n",
      "4.73779e+06\n",
      "4.78109e+06\n",
      "5.83479e+06\n",
      "5.43606e+06\n",
      "6.98611e+06\n",
      "5.45781e+06\n",
      "5.56779e+06\n",
      "5.84073e+06\n"
     ]
    }
   ],
   "source": [
    "# Alternate Training\n",
    "# Calculation (Session) Code\n",
    "# ==========================\n",
    "\n",
    "# open the session\n",
    "\n",
    "with tf.Session() as session:\n",
    "    session.run(tf.initialize_all_variables())\n",
    "    for iters in range(10):\n",
    "        if np.random.rand() < 0.5:\n",
    "            _, Y1_loss = session.run([Y1_op, Y1_Loss],\n",
    "                            {\n",
    "                              X: np.random.rand(10,10)*10,\n",
    "                              Y1: np.random.rand(10,20)*10,\n",
    "                              Y2: np.random.rand(10,20)*10\n",
    "                              })\n",
    "            print(Y1_loss)\n",
    "        else:\n",
    "            _, Y2_loss = session.run([Y2_op, Y2_Loss],\n",
    "                            {\n",
    "                              X: np.random.rand(10,10)*10,\n",
    "                              Y1: np.random.rand(10,20)*10,\n",
    "                              Y2: np.random.rand(10,20)*10\n",
    "                              })\n",
    "            print(Y2_loss)\n",
    "    session.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This is a demonstration of how you would train both tasks jointly, sharing a representation on the bottom layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.15856e+07\n"
     ]
    }
   ],
   "source": [
    "# Joint Training\n",
    "# Calculation (Session) Code\n",
    "# ==========================\n",
    "\n",
    "# open the session\n",
    "\n",
    "with tf.Session() as session:\n",
    "    session.run(tf.initialize_all_variables())\n",
    "    _, Joint_loss = session.run([Optimiser, Joint_Loss],\n",
    "                    {\n",
    "                      X: np.random.rand(10,10)*10,\n",
    "                      Y1: np.random.rand(10,20)*10,\n",
    "                      Y2: np.random.rand(10,20)*10\n",
    "                      })\n",
    "    print(Joint_loss)\n",
    "    session.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}