NN with Shared Variables.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#http://cs231n.stanford.edu/slides/winter1516_lecture12.pdf\n",
    "\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "import numpy as np\n",
    "\n",
    "num_data = 64\n",
    "num_dim = 1000\n",
    "num_hid1 = 100\n",
    "num_output = 10\n",
    "\n",
    "x = T.matrix('x')\n",
    "y = T.vector('y', dtype = 'int64')\n",
    "w1 = theano.shared(1e-3 * np.random.randn(num_dim, num_hid1).astype('float32'), name='w1')\n",
    "w2 = theano.shared(1e-3 * np.random.randn(num_hid1, num_output).astype('float32'), name='w2')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hid1 = x.dot(w1)\n",
    "hid1_relu = T.nnet.relu(hid1)\n",
    "output = hid1_relu.dot(w2)\n",
    "\n",
    "output_prob = T.nnet.softmax(output)\n",
    "loss = T.nnet.categorical_crossentropy(output_prob, y).mean()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dw1, dw2 = T.grad(loss, [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 1e-1\n",
    "num_epoch = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train = theano.function(\n",
    "    inputs = [x, y],\n",
    "    outputs = loss,\n",
    "    updates=(\n",
    "        (w1, w1 - learning_rate * dw1),\n",
    "        (w2, w2 - learning_rate * dw2)\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xx = np.random.randn(num_data, num_dim).astype('float32')\n",
    "yy = np.random.randint(num_output, size=num_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.30256009102\n",
      "2.30241537094\n",
      "2.302267313\n",
      "2.30210924149\n",
      "2.3019349575\n",
      "2.30173707008\n",
      "2.30150723457\n",
      "2.30123519897\n",
      "2.30090904236\n",
      "2.30051469803\n",
      "2.30003404617\n",
      "2.29944396019\n",
      "2.2987165451\n",
      "2.2978143692\n",
      "2.29668974876\n",
      "2.2952837944\n",
      "2.29351425171\n",
      "2.29129290581\n",
      "2.28848266602\n",
      "2.28491687775\n"
     ]
    }
   ],
   "source": [
    "for i in xrange(num_epoch):\n",
    "    loss_val = train(xx, yy)\n",
    "    print loss_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}