08-Neural Networks Training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.cross_validation import train_test_split, KFold, cross_val_score\n",
    "import sklearn.metrics as sk\n",
    "\n",
    "import pandas as pd\n",
    "from collections import Counter\n",
    "import numpy as np\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({u'Black': 1576,\n",
       "         u'Blue': 1573,\n",
       "         u'Green': 1566,\n",
       "         u'Red': 1575,\n",
       "         u'White': 1584})"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "modern = pd.read_pickle('data/5color_modern_no_name_hardmode.pkl')\n",
    "Counter(modern.colors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After all the data munging the classes are still amazingly balanced.\n",
    "\n",
    "## Lets single out blue and red for a binary classification\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>colors</th>\n",
       "      <th>cmc</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1709</th>\n",
       "      <td>Renegade Doppelganger</td>\n",
       "      <td>Blue</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Whenever another creature enters the battlefie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2782</th>\n",
       "      <td>Mercurial Pretender</td>\n",
       "      <td>Blue</td>\n",
       "      <td>5.0</td>\n",
       "      <td>You may have This enter the battlefield as a c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2907</th>\n",
       "      <td>Bloodfire Enforcers</td>\n",
       "      <td>Red</td>\n",
       "      <td>4.0</td>\n",
       "      <td>This has first strike and trample as long as a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3106</th>\n",
       "      <td>Coastal Discovery</td>\n",
       "      <td>Blue</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Draw two cards. Awaken 4—{5}{1}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2635</th>\n",
       "      <td>Forgestoker Dragon</td>\n",
       "      <td>Red</td>\n",
       "      <td>6.0</td>\n",
       "      <td>Flying {1}{1}: This deals 1 damage to target c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279</th>\n",
       "      <td>Hearth Kami</td>\n",
       "      <td>Red</td>\n",
       "      <td>2.0</td>\n",
       "      <td>{1}, Sacrifice This: Destroy target artifact w...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       name colors  cmc  \\\n",
       "1709  Renegade Doppelganger   Blue  2.0   \n",
       "2782    Mercurial Pretender   Blue  5.0   \n",
       "2907    Bloodfire Enforcers    Red  4.0   \n",
       "3106      Coastal Discovery   Blue  4.0   \n",
       "2635     Forgestoker Dragon    Red  6.0   \n",
       "279             Hearth Kami    Red  2.0   \n",
       "\n",
       "                                                   text  \n",
       "1709  Whenever another creature enters the battlefie...  \n",
       "2782  You may have This enter the battlefield as a c...  \n",
       "2907  This has first strike and trample as long as a...  \n",
       "3106                   Draw two cards. Awaken 4—{5}{1}   \n",
       "2635  Flying {1}{1}: This deals 1 damage to target c...  \n",
       "279   {1}, Sacrifice This: Destroy target artifact w...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "UG = modern.loc[modern['colors'].isin(['Blue', 'Red'])]\n",
    "\n",
    "UG.reset_index(inplace=True)\n",
    "UG.pop('index')\n",
    "\n",
    "UG[['name', 'colors', 'cmc', 'text']].sample(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Blue</th>\n",
       "      <th>Red</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Blue  Red\n",
       "0   1.0  0.0\n",
       "1   0.0  1.0\n",
       "2   1.0  0.0\n",
       "3   1.0  0.0\n",
       "4   1.0  0.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummies = pd.get_dummies(UG.colors)\n",
    "dummies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2361L, 815L)\n",
      "(2361L, 2L)\n",
      "<type 'numpy.ndarray'>\n",
      "<type 'numpy.ndarray'>\n",
      "(2361L, 815L)\n",
      "(2361L, 2L)\n",
      "<type 'numpy.ndarray'>\n",
      "<type 'numpy.ndarray'>\n",
      "There are 815 words in the vocabulary.\n"
     ]
    }
   ],
   "source": [
    "vectorizer = CountVectorizer()\n",
    "\n",
    "vec_X = vectorizer.fit_transform(UG['text'])\n",
    "\n",
    "xTrain, xTest, yTrain, yTest = train_test_split(vec_X, dummies,\n",
    "                                             random_state=42)\n",
    "\n",
    "xTrain = np.asarray(xTrain.todense())\n",
    "xTest  = np.asarray(xTest.todense())\n",
    "yTrain = np.asarray(yTrain)\n",
    "yTest  = np.asarray(yTest)\n",
    "\n",
    "print xTrain.shape\n",
    "print yTrain.shape\n",
    "print type(xTrain)\n",
    "print type(yTrain)\n",
    "\n",
    "# xTrain = xTrain.reshape(-1, 1, 1, 815)\n",
    "# xTest = xTest.reshape(-1, 1, 1, 815)\n",
    "\n",
    "print xTrain.shape\n",
    "print yTrain.shape\n",
    "print type(xTrain)\n",
    "print type(yTrain)\n",
    "print\n",
    "print \"There are {:,} words in the vocabulary.\".format(len(vectorizer.vocabulary_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0  Test: 0.651842439644  Train: 0.629394324439\n",
      "Round: 30  Test: 0.810673443456  Train: 0.831427361288\n",
      "Round: 60  Test: 0.833545108005  Train: 0.849216433715\n",
      "Round: 90  Test: 0.853875476493  Train: 0.861075815332\n",
      "Round: 120  Test: 0.852604828463  Train: 0.866581956798\n",
      "Round: 150  Test: 0.855146124524  Train: 0.873358746294\n",
      "Round: 180  Test: 0.861499364676  Train: 0.879288437103\n",
      "Round: 210  Test: 0.864040660737  Train: 0.885218127912\n",
      "Round: 240  Test: 0.870393900889  Train: 0.889030072003\n",
      "Round: 270  Test: 0.866581956798  Train: 0.892842016095\n",
      "Round: 300  Test: 0.866581956798  Train: 0.894536213469\n",
      "Round: 330  Test: 0.869123252859  Train: 0.894112664125\n",
      "Round: 360  Test: 0.87166454892  Train: 0.895383312156\n",
      "Round: 390  Test: 0.87166454892  Train: 0.895806861499\n"
     ]
    }
   ],
   "source": [
    "import theano\n",
    "from theano import tensor as T\n",
    "import numpy as np\n",
    "from math import sqrt\n",
    "\n",
    "\n",
    "def floatX(X):\n",
    "    return np.asarray(X, dtype=theano.config.floatX)\n",
    "\n",
    "def init_weights(shape):\n",
    "    (h, w) = shape\n",
    "    normalizer = 2.0 * sqrt(6) / sqrt(h + w) * .2  #factors: correct for uni[0,1], glo, glo, softmax deriv\n",
    "    return theano.shared(floatX((np.random.random_sample(shape) - 0.5) \\\n",
    "                                * normalizer))  #code for using Glorot init\n",
    "    \n",
    "def model(X, w):\n",
    "    return T.nnet.softmax(T.dot(X, w))\n",
    "\n",
    "def adaDelta(cost, params, eta=0.2, rho=0.9, epsilon=1e-6):\n",
    "    grads = T.grad(cost=cost, wrt=params)\n",
    "    updates = []\n",
    "    for p, g in zip(params, grads):\n",
    "        #calc g-squared\n",
    "        gSq = theano.shared(p.get_value() * 0.)\n",
    "        dwSq = theano.shared(p.get_value() * 0.)\n",
    "\n",
    "        #exp smoothed g squared\n",
    "        gSqNew = rho * gSq + (1 - rho) * g * g\n",
    "\n",
    "        #calc dx-squared\n",
    "        dw = eta * T.sqrt(dwSq + epsilon) * g / T.sqrt(gSq + epsilon)\n",
    "        dwSqNew = rho * dwSq + (1 - rho) * dw * dw\n",
    "\n",
    "        updates.append((dwSq, dwSqNew))\n",
    "        updates.append((gSq, gSqNew))\n",
    "        updates.append((p, p - dw))\n",
    "    return updates\n",
    "\n",
    "X = T.fmatrix()\n",
    "Y = T.fmatrix()\n",
    "# grad_list = theano.shared(np.array([0,0]), name='grad_list')\n",
    "\n",
    "w = init_weights((815 , 2))\n",
    "\n",
    "py_x = model(X, w)\n",
    "y_pred = T.argmax(py_x, axis=1)\n",
    "\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(py_x, Y))\n",
    "gradient = T.grad(cost=cost, wrt=w)\n",
    "update = [[w, w - gradient * 0.1]]\n",
    "\n",
    "train = theano.function(inputs=[X, Y], \n",
    "                        outputs=[cost, gradient], \n",
    "                        updates=update, \n",
    "                        allow_input_downcast=True)\n",
    "\n",
    "predict = theano.function(inputs=[X], \n",
    "                          outputs=y_pred, \n",
    "                          allow_input_downcast=True)\n",
    "\n",
    "\n",
    "for i in range(401):\n",
    "# #     for start, end in zip(range(0, xTrain.shape[0], 128), \n",
    "# #                           range(128, xTrain.shape[0], 128)):\n",
    "# #         cost, gradient = train(xTrain[start:end], yTrain[start:end])\n",
    "    cost, gradient = train(xTrain, yTrain)\n",
    "    if i % 30 == 0: \n",
    "        tr = np.mean(np.argmax(yTest, axis=1) == predict(xTest))\n",
    "        trr =  np.mean(np.argmax(yTrain, axis=1) == predict(xTrain))\n",
    "        print 'Round:', i,\" Test:\", tr, ' Train:', trr\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### All Five Vs All Five\n",
    "\n",
    "And now the main event - simply comparing two colors was too easy. Five way classification of all the colors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5905L, 1161L)\n",
      "(5905L, 5L)\n",
      "<type 'numpy.ndarray'>\n",
      "<type 'numpy.ndarray'>\n",
      "(5905L, 1161L)\n",
      "(5905L, 5L)\n",
      "<type 'numpy.ndarray'>\n",
      "<type 'numpy.ndarray'>\n",
      "There are 1,161 words in the vocabulary.\n"
     ]
    }
   ],
   "source": [
    "vectorizer = CountVectorizer()\n",
    "\n",
    "y = pd.get_dummies(modern.colors)\n",
    "\n",
    "X = vectorizer.fit_transform(modern.text)\n",
    "\n",
    "xTrain, xTest, yTrain, yTest = train_test_split(X, y, random_state=42)\n",
    "\n",
    "xTrain = np.asarray(xTrain.todense())\n",
    "xTest  = np.asarray(xTest.todense())\n",
    "yTrain = np.asarray(yTrain)\n",
    "yTest  = np.asarray(yTest)\n",
    "\n",
    "print xTrain.shape\n",
    "print yTrain.shape\n",
    "print type(xTrain)\n",
    "print type(yTrain)\n",
    "\n",
    "# xTrain = xTrain.reshape(-1, 1, 1, 815)\n",
    "# xTest = xTest.reshape(-1, 1, 1, 815)\n",
    "\n",
    "print xTrain.shape\n",
    "print yTrain.shape\n",
    "print type(xTrain)\n",
    "print type(yTrain)\n",
    "\n",
    "print \"There are {:,} words in the vocabulary.\".format(len(vectorizer.vocabulary_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0  Test: 0.228542407313  Train: 0.230482641829\n",
      "Round: 50  Test: 0.475368207212  Train: 0.509398814564\n",
      "Round: 100  Test: 0.501777552057  Train: 0.544623200677\n",
      "Round: 150  Test: 0.523108176739  Train: 0.564267569856\n",
      "Round: 200  Test: 0.540883697308  Train: 0.574767146486\n",
      "Round: 250  Test: 0.557643473845  Train: 0.58899237934\n",
      "Round: 300  Test: 0.57592686643  Train: 0.598306519898\n",
      "Round: 350  Test: 0.585576434738  Train: 0.609652836579\n",
      "Round: 400  Test: 0.590655154901  Train: 0.617781541067\n",
      "Round: 450  Test: 0.594210259015  Train: 0.626248941575\n",
      "Round: 500  Test: 0.599288979177  Train: 0.633700254022\n",
      "Round: 550  Test: 0.605891315389  Train: 0.638441998307\n",
      "Round: 600  Test: 0.608938547486  Train: 0.644199830652\n",
      "Round: 650  Test: 0.611477907567  Train: 0.649280270957\n",
      "Round: 700  Test: 0.613001523616  Train: 0.649788314987\n",
      "Round: 750  Test: 0.614525139665  Train: 0.653175275191\n",
      "Round: 800  Test: 0.618080243779  Train: 0.655884843353\n",
      "Round: 850  Test: 0.621635347892  Train: 0.659102455546\n",
      "Round: 900  Test: 0.625698324022  Train: 0.66299745978\n",
      "Round: 950  Test: 0.625190452006  Train: 0.665029635902\n",
      "Round: 1000  Test: 0.627729812087  Train: 0.669263336156\n",
      "Round: 1050  Test: 0.630777044185  Train: 0.670956816257\n",
      "Round: 1100  Test: 0.632300660234  Train: 0.672819644369\n",
      "Round: 1150  Test: 0.634332148299  Train: 0.676545300593\n",
      "Round: 1200  Test: 0.633824276282  Train: 0.679085520745\n",
      "Round: 1250  Test: 0.634332148299  Train: 0.681287044877\n",
      "Round: 1300  Test: 0.63687150838  Train: 0.682641828959\n",
      "Round: 1350  Test: 0.638395124429  Train: 0.68382726503\n",
      "Round: 1400  Test: 0.64093448451  Train: 0.68416596105\n",
      "Round: 1450  Test: 0.64093448451  Train: 0.685520745131\n",
      "Round: 1500  Test: 0.64093448451  Train: 0.688399661304\n",
      "Round: 1550  Test: 0.641442356526  Train: 0.688569009314\n",
      "Round: 1600  Test: 0.643473844591  Train: 0.689754445385\n",
      "Round: 1650  Test: 0.64499746064  Train: 0.691447925487\n",
      "Round: 1700  Test: 0.645505332656  Train: 0.692633361558\n",
      "Round: 1750  Test: 0.646013204672  Train: 0.693988145639\n",
      "Round: 1800  Test: 0.648044692737  Train: 0.695850973751\n",
      "Round: 1850  Test: 0.650076180802  Train: 0.696867061812\n",
      "Round: 1900  Test: 0.653631284916  Train: 0.699915325995\n",
      "Round: 1950  Test: 0.654647028949  Train: 0.699915325995\n",
      "Round: 2000  Test: 0.654647028949  Train: 0.701100762066\n",
      "Round: 2050  Test: 0.65718638903  Train: 0.702455546147\n",
      "Round: 2100  Test: 0.657694261046  Train: 0.702624894157\n",
      "Round: 2150  Test: 0.657694261046  Train: 0.704487722269\n",
      "Round: 2200  Test: 0.658710005079  Train: 0.70550381033\n",
      "Round: 2250  Test: 0.657694261046  Train: 0.70567315834\n",
      "Round: 2300  Test: 0.657694261046  Train: 0.706858594412\n",
      "Round: 2350  Test: 0.65718638903  Train: 0.707535986452\n",
      "Round: 2400  Test: 0.657694261046  Train: 0.708044030483\n",
      "Round: 2450  Test: 0.659217877095  Train: 0.707874682472\n",
      "Round: 2500  Test: 0.659217877095  Train: 0.708382726503\n",
      "Round: 2550  Test: 0.658710005079  Train: 0.709229466554\n",
      "Round: 2600  Test: 0.660741493144  Train: 0.710245554615\n",
      "Round: 2650  Test: 0.661757237176  Train: 0.711430990686\n",
      "Round: 2700  Test: 0.661757237176  Train: 0.712616426757\n",
      "Round: 2750  Test: 0.662772981209  Train: 0.713124470787\n",
      "Round: 2800  Test: 0.663788725241  Train: 0.713801862828\n",
      "Round: 2850  Test: 0.663788725241  Train: 0.714140558848\n",
      "Round: 2900  Test: 0.664296597257  Train: 0.71532599492\n",
      "Round: 2950  Test: 0.663788725241  Train: 0.716511430991\n",
      "Round: 3000  Test: 0.664804469274  Train: 0.717527519052\n",
      "Round: 3050  Test: 0.66531234129  Train: 0.717866215072\n",
      "Round: 3100  Test: 0.66531234129  Train: 0.718374259102\n",
      "Round: 3150  Test: 0.665820213306  Train: 0.718882303133\n",
      "Round: 3200  Test: 0.667343829355  Train: 0.719559695174\n",
      "Round: 3250  Test: 0.667851701371  Train: 0.720067739204\n",
      "Round: 3300  Test: 0.66937531742  Train: 0.721083827265\n",
      "Round: 3350  Test: 0.671914677501  Train: 0.722099915326\n",
      "Round: 3400  Test: 0.67343829355  Train: 0.723793395428\n",
      "Round: 3450  Test: 0.673946165566  Train: 0.723962743438\n",
      "Round: 3500  Test: 0.675977653631  Train: 0.724132091448\n",
      "Round: 3550  Test: 0.676993397664  Train: 0.724470787468\n",
      "Round: 3600  Test: 0.67750126968  Train: 0.725486875529\n",
      "Round: 3650  Test: 0.678517013713  Train: 0.72599491956\n",
      "Round: 3700  Test: 0.678517013713  Train: 0.72616426757\n",
      "Round: 3750  Test: 0.679532757745  Train: 0.72650296359\n",
      "Round: 3800  Test: 0.680040629761  Train: 0.727011007621\n",
      "Round: 3850  Test: 0.680040629761  Train: 0.727349703641\n",
      "Round: 3900  Test: 0.679532757745  Train: 0.727857747671\n",
      "Round: 3950  Test: 0.679532757745  Train: 0.728535139712\n",
      "Round: 4000  Test: 0.681056373794  Train: 0.729043183743\n",
      "Round: 4050  Test: 0.683087861859  Train: 0.728704487722\n",
      "Round: 4100  Test: 0.683595733875  Train: 0.728873835732\n",
      "Round: 4150  Test: 0.683595733875  Train: 0.729212531753\n",
      "Round: 4200  Test: 0.683595733875  Train: 0.729551227773\n",
      "Round: 4250  Test: 0.684103605891  Train: 0.730228619814\n",
      "Round: 4300  Test: 0.684611477908  Train: 0.730906011854\n",
      "Round: 4350  Test: 0.685119349924  Train: 0.731583403895\n",
      "Round: 4400  Test: 0.68562722194  Train: 0.731752751905\n",
      "Round: 4450  Test: 0.68562722194  Train: 0.732260795936\n",
      "Round: 4500  Test: 0.686135093956  Train: 0.732768839966\n",
      "Round: 4550  Test: 0.687150837989  Train: 0.732768839966\n",
      "Round: 4600  Test: 0.687150837989  Train: 0.733107535986\n",
      "Round: 4650  Test: 0.688166582021  Train: 0.733615580017\n",
      "Round: 4700  Test: 0.688674454038  Train: 0.734123624047\n",
      "Round: 4750  Test: 0.688166582021  Train: 0.734970364098\n",
      "Round: 4800  Test: 0.688166582021  Train: 0.735478408129\n",
      "Round: 4850  Test: 0.687658710005  Train: 0.735817104149\n",
      "Round: 4900  Test: 0.687658710005  Train: 0.73632514818\n",
      "Round: 4950  Test: 0.688166582021  Train: 0.73700254022\n",
      "Round: 5000  Test: 0.688166582021  Train: 0.737679932261\n",
      "Round: 5050  Test: 0.687658710005  Train: 0.738018628281\n",
      "Round: 5100  Test: 0.687658710005  Train: 0.738526672312\n",
      "Round: 5150  Test: 0.687658710005  Train: 0.738526672312\n",
      "Round: 5200  Test: 0.688166582021  Train: 0.738696020322\n",
      "Round: 5250  Test: 0.688166582021  Train: 0.738865368332\n",
      "Round: 5300  Test: 0.688674454038  Train: 0.738865368332\n",
      "Round: 5350  Test: 0.688674454038  Train: 0.739034716342\n",
      "Round: 5400  Test: 0.688674454038  Train: 0.739204064352\n",
      "Round: 5450  Test: 0.688674454038  Train: 0.738696020322\n",
      "Round: 5500  Test: 0.688674454038  Train: 0.738865368332\n",
      "Round: 5550  Test: 0.688674454038  Train: 0.739034716342\n",
      "Round: 5600  Test: 0.688674454038  Train: 0.739373412362\n",
      "Round: 5650  Test: 0.689182326054  Train: 0.739542760373\n",
      "Round: 5700  Test: 0.689182326054  Train: 0.739542760373\n",
      "Round: 5750  Test: 0.688674454038  Train: 0.739712108383\n",
      "Round: 5800  Test: 0.688674454038  Train: 0.740050804403\n",
      "Round: 5850  Test: 0.687658710005  Train: 0.740728196444\n",
      "Round: 5900  Test: 0.687658710005  Train: 0.740897544454\n",
      "Round: 5950  Test: 0.687658710005  Train: 0.741405588484\n",
      "Round: 6000  Test: 0.687658710005  Train: 0.741744284505\n",
      "Round: 6050  Test: 0.688674454038  Train: 0.742252328535\n",
      "Round: 6100  Test: 0.689182326054  Train: 0.742591024555\n",
      "Round: 6150  Test: 0.689182326054  Train: 0.742421676545\n",
      "Round: 6200  Test: 0.68969019807  Train: 0.742760372566\n",
      "Round: 6250  Test: 0.68969019807  Train: 0.742591024555\n",
      "Round: 6300  Test: 0.690198070086  Train: 0.742591024555\n",
      "Round: 6350  Test: 0.690198070086  Train: 0.742591024555\n",
      "Round: 6400  Test: 0.690198070086  Train: 0.742760372566\n",
      "Round: 6450  Test: 0.690198070086  Train: 0.743099068586\n",
      "Round: 6500  Test: 0.691213814119  Train: 0.743437764606\n",
      "Round: 6550  Test: 0.692229558151  Train: 0.743776460627\n",
      "Round: 6600  Test: 0.693245302184  Train: 0.743945808637\n",
      "Round: 6650  Test: 0.693245302184  Train: 0.744284504657\n",
      "Round: 6700  Test: 0.693245302184  Train: 0.744623200677\n",
      "Round: 6750  Test: 0.6937531742  Train: 0.744961896698\n",
      "Round: 6800  Test: 0.6937531742  Train: 0.745300592718\n",
      "Round: 6850  Test: 0.694261046216  Train: 0.745300592718\n",
      "Round: 6900  Test: 0.694261046216  Train: 0.745977984759\n",
      "Round: 6950  Test: 0.694261046216  Train: 0.746147332769\n",
      "Wall time: 4min 37s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "import numpy as np\n",
    "from math import sqrt\n",
    "\n",
    "def floatX(X):\n",
    "    return np.asarray(X, dtype=theano.config.floatX)\n",
    "\n",
    "def init_weights(shape):\n",
    "    (h, w) = shape\n",
    "    normalizer = 2.0 * sqrt(6) / sqrt(h + w) * .2  #factors: correct for uni[0,1], glo, glo, softmax deriv\n",
    "    return theano.shared(floatX((np.random.random_sample(shape) - 0.5) \\\n",
    "                                * normalizer))  #code for using Glorot init\n",
    "    \n",
    "def model(X, w):\n",
    "    return T.nnet.softmax(T.dot(X, w))\n",
    "\n",
    "def adaDelta(cost, params, eta=0.2, rho=0.9, epsilon=1e-6):\n",
    "    grads = T.grad(cost=cost, wrt=params)\n",
    "    updates = []\n",
    "    for p, g in zip(params, grads):\n",
    "        #calc g-squared\n",
    "        gSq = theano.shared(p.get_value() * 0.)\n",
    "        dwSq = theano.shared(p.get_value() * 0.)\n",
    "\n",
    "        #exp smoothed g squared\n",
    "        gSqNew = rho * gSq + (1 - rho) * g * g\n",
    "\n",
    "        #calc dx-squared\n",
    "        dw = eta * T.sqrt(dwSq + epsilon) * g / T.sqrt(gSq + epsilon)\n",
    "        dwSqNew = rho * dwSq + (1 - rho) * dw * dw\n",
    "\n",
    "        updates.append((dwSq, dwSqNew))\n",
    "        updates.append((gSq, gSqNew))\n",
    "        updates.append((p, p - dw))\n",
    "    return updates\n",
    "\n",
    "X = T.fmatrix()\n",
    "Y = T.fmatrix()\n",
    "\n",
    "w = init_weights((len(vectorizer.vocabulary_) , yTest.shape[1]))\n",
    "\n",
    "py_x = model(X, w)\n",
    "y_pred = T.argmax(py_x, axis=1)\n",
    "\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(py_x, Y))\n",
    "gradient = T.grad(cost=cost, wrt=w)\n",
    "update = [[w, w - gradient * 0.1]]\n",
    "\n",
    "train = theano.function(inputs=[X, Y], \n",
    "                        outputs=[cost, gradient], \n",
    "                        updates=update, \n",
    "                        allow_input_downcast=True)\n",
    "\n",
    "predict = theano.function(inputs=[X], \n",
    "                          outputs=y_pred, \n",
    "                          allow_input_downcast=True)\n",
    "\n",
    "p_v1, p_t1, i1 = [], [], []\n",
    "for i in range(7000):\n",
    "    cost, gradient = train(xTrain, yTrain)\n",
    "    if i % 50 == 0: \n",
    "        tr = np.mean(np.argmax(yTest, axis=1) == predict(xTest))\n",
    "        trr =  np.mean(np.argmax(yTrain, axis=1) == predict(xTrain))\n",
    "        p_v1 += [tr]\n",
    "        p_t1 += [trr]\n",
    "        i1 += [i]\n",
    "        print 'Round:', i,\" Test:\", tr, ' Train:', trr\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Amazing results, almost equal to logistic regression at 5min. \n",
    "\n",
    "### 4-Layer Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0  Test: 0.263585576435  Train: 0.28382726503\n",
      "Round: 10  Test: 0.303707465719  Train: 0.310584250635\n",
      "Round: 20  Test: 0.490096495683  Train: 0.498899237934\n",
      "Round: 30  Test: 0.601320467242  Train: 0.63116003387\n",
      "Round: 40  Test: 0.652615540884  Train: 0.693480101609\n",
      "Round: 50  Test: 0.675977653631  Train: 0.732768839966\n",
      "Round: 60  Test: 0.68969019807  Train: 0.756308213378\n",
      "Round: 70  Test: 0.696292534281  Train: 0.775444538527\n",
      "Round: 80  Test: 0.701371254444  Train: 0.78933107536\n",
      "Round: 90  Test: 0.710512950736  Train: 0.806096528366\n",
      "Round: 100  Test: 0.716607414931  Train: 0.817104149026\n",
      "Round: 110  Test: 0.717623158964  Train: 0.830990685859\n",
      "Round: 120  Test: 0.728796343321  Train: 0.844877222693\n",
      "Round: 130  Test: 0.724225495175  Train: 0.854868755292\n",
      "Round: 140  Test: 0.726764855256  Train: 0.86316680779\n",
      "Round: 150  Test: 0.728796343321  Train: 0.871634208298\n",
      "Round: 160  Test: 0.725749111224  Train: 0.878577476715\n",
      "Round: 170  Test: 0.731335703403  Train: 0.886367485182\n",
      "Round: 180  Test: 0.733367191468  Train: 0.891447925487\n",
      "Round: 190  Test: 0.728288471305  Train: 0.897544453853\n",
      "Wall time: 1min 43s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
    "import numpy as np\n",
    "\n",
    "srng = RandomStreams()\n",
    "\n",
    "def floatX(X):\n",
    "    return np.asarray(X, dtype=theano.config.floatX)\n",
    "\n",
    "def init_weights(shape):\n",
    "    return theano.shared(floatX(np.random.randn(*shape) * 0.01))\n",
    "\n",
    "def rectify(X, alpha=1.0):\n",
    "#     return T.maximum(X, 0.)\n",
    "    return T.maximum(X, 0.05*X)  #leaky rectifier\n",
    "#     return T.switch(X > 0, X, alpha * (T.exp(X) - 1)) # ELU\n",
    "\n",
    "def softmax(X):\n",
    "    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n",
    "    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n",
    "\n",
    "def RMSprop(cost, params, lr=0.001, rho=0.9, epsilon=1e-6):\n",
    "    grads = T.grad(cost=cost, wrt=params)\n",
    "    updates = []\n",
    "    for p, g in zip(params, grads):\n",
    "        acc = theano.shared(p.get_value() * 0.)\n",
    "        acc_new = rho * acc + (1 - rho) * g ** 2\n",
    "        gradient_scaling = T.sqrt(acc_new + epsilon)\n",
    "        g = g / gradient_scaling\n",
    "        updates.append((acc, acc_new))\n",
    "        updates.append((p, p - lr * g))\n",
    "    return updates\n",
    "\n",
    "def dropout(X, p=0.):\n",
    "    if p > 0:\n",
    "        retain_prob = 1 - p\n",
    "        X *= srng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX)\n",
    "        X /= retain_prob\n",
    "    return X\n",
    "\n",
    "def model(X, w_h, w_h2, w_o, p_drop_input, p_drop_hidden):\n",
    "    X = dropout(X, p_drop_input)\n",
    "    h = rectify(T.dot(X, w_h))\n",
    "\n",
    "    h = dropout(h, p_drop_hidden)\n",
    "    h2 = rectify(T.dot(h, w_h2))\n",
    "\n",
    "    h2 = dropout(h2, p_drop_hidden)\n",
    "    py_x = softmax(T.dot(h2, w_o))\n",
    "    return h, h2, py_x\n",
    "\n",
    "X = T.fmatrix()\n",
    "Y = T.fmatrix()\n",
    "\n",
    "# w = init_weights((len(vectorizer.vocabulary_) , yTest.shape[1])) # old \n",
    "w_h = init_weights((len(vectorizer.vocabulary_), 600))\n",
    "w_h2 = init_weights((600, 600))\n",
    "w_o = init_weights((600, yTest.shape[1]))\n",
    "\n",
    "noise_h, noise_h2, noise_py_x = model(X, w_h, w_h2, w_o, 0.2, 0.5)\n",
    "h, h2, py_x = model(X, w_h, w_h2, w_o, 0., 0.)\n",
    "y_x = T.argmax(py_x, axis=1)\n",
    "\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n",
    "params = [w_h, w_h2, w_o]\n",
    "updates = RMSprop(cost, params, lr=0.0001)\n",
    "\n",
    "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
    "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)\n",
    "\n",
    "p_v2, p_t2, i2 = [], [], []\n",
    "for i in range(200):\n",
    "    for start, end in zip(range(0, len(xTrain), 128), range(128, len(xTrain), 128)):\n",
    "        cost = train(xTrain[start:end], yTrain[start:end])\n",
    "    if i%10 == 0: \n",
    "        tr = np.mean(np.argmax(yTest, axis=1) == predict(xTest))\n",
    "        trr =  np.mean(np.argmax(yTrain, axis=1) == predict(xTrain))\n",
    "        p_v2 += [tr]\n",
    "        p_t2 += [trr]\n",
    "        i2  += [i]\n",
    "        print 'Round:', i,\" Test:\", tr, ' Train:', trr\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Strong signs of overfitting\n",
    "\n",
    "Next steps:  \n",
    "\n",
    "Leaky RELU swapped out for ELU, alpha = 1.0   \n",
    "Train accuracy dropped by 14% (good)  \n",
    "Test accuracy dropped by 3% (bad)   \n",
    "\n",
    "Leaky RELU swapped out for ELU, alpha = .5  \n",
    "Train accuracy dropped by 7% (good)  \n",
    "Test accuracy dropped by 1% (bad)   \n",
    "\n",
    "Leaky RELU  alpha = .1\n",
    "RHO = .99\n",
    "Train accuracy dropped by 7% (good)  \n",
    "Test accuracy up by 1% (good)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0  Test: 0.4281361097  Train: 0.435055038103\n",
      "Round: 10  Test: 0.594718131031  Train: 0.644199830652\n",
      "Round: 20  Test: 0.65718638903  Train: 0.71566469094\n",
      "Round: 30  Test: 0.680548501778  Train: 0.755122777307\n",
      "Round: 40  Test: 0.693245302184  Train: 0.781202370872\n",
      "Round: 50  Test: 0.703402742509  Train: 0.798814563929\n",
      "Round: 60  Test: 0.715083798883  Train: 0.815410668925\n",
      "Round: 70  Test: 0.720162519045  Train: 0.834546994073\n",
      "Round: 80  Test: 0.722701879126  Train: 0.846909398815\n",
      "Round: 90  Test: 0.72625698324  Train: 0.859610499577\n",
      "Round: 100  Test: 0.735906551549  Train: 0.867231160034\n",
      "Round: 110  Test: 0.728288471305  Train: 0.875867908552\n",
      "Round: 120  Test: 0.726764855256  Train: 0.88433530906\n",
      "Round: 130  Test: 0.731335703403  Train: 0.890431837426\n",
      "Round: 140  Test: 0.732859319451  Train: 0.898391193903\n",
      "Round: 150  Test: 0.733875063484  Train: 0.904487722269\n",
      "Round: 160  Test: 0.732859319451  Train: 0.908721422523\n",
      "Round: 170  Test: 0.730827831386  Train: 0.914140558848\n",
      "Round: 180  Test: 0.732859319451  Train: 0.91600338696\n",
      "Round: 190  Test: 0.732351447435  Train: 0.920575783235\n",
      "Round: 200  Test: 0.729812087354  Train: 0.922607959356\n",
      "Round: 210  Test: 0.726764855256  Train: 0.9266723116\n",
      "Round: 220  Test: 0.732351447435  Train: 0.929043183743\n",
      "Round: 230  Test: 0.725241239208  Train: 0.931075359865\n",
      "Round: 240  Test: 0.722701879126  Train: 0.934292972058\n",
      "Round: 250  Test: 0.725749111224  Train: 0.934801016088\n",
      "Round: 260  Test: 0.723209751143  Train: 0.937510584251\n",
      "Round: 270  Test: 0.723209751143  Train: 0.940389500423\n",
      "Round: 280  Test: 0.721686135094  Train: 0.941574936494\n",
      "Round: 290  Test: 0.719146775013  Train: 0.942082980525\n",
      "Round: 300  Test: 0.723209751143  Train: 0.944453852667\n",
      "Wall time: 2min 35s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "\"\"\" glorot 4-layer: dropout, dropout, dropout \"\"\"\n",
    "\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
    "import numpy as np\n",
    "\n",
    "srng = RandomStreams()\n",
    "\n",
    "def floatX(X):\n",
    "    return np.asarray(X, dtype=theano.config.floatX)\n",
    "\n",
    "def init_weights(shape):\n",
    "    (h, w) = shape\n",
    "    # Glorot normalization - last factor depends on non-linearity\n",
    "    # 0.25 for sigmoid and 0.1 for softmax, 1.0 for tanh or Relu\n",
    "    normalizer = 2.0 * np.sqrt(6) / np.sqrt(h + w) * 1.0\n",
    "    return theano.shared(floatX((np.random.random_sample(shape) - 0.5) * normalizer))\n",
    "\n",
    "\n",
    "def rectify(X, alpha=1.0):\n",
    "#     return T.maximum(X, 0.)\n",
    "    return T.maximum(X, 0.1*X)  #leaky rectifier\n",
    "#     return T.switch(X > 0, X, alpha * (T.exp(X) - 1)) # ELU\n",
    "\n",
    "def softmax(X):\n",
    "    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n",
    "    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n",
    "\n",
    "def RMSprop(cost, params, lr=0.001, rho=0.99, epsilon=1e-6):\n",
    "    grads = T.grad(cost=cost, wrt=params)\n",
    "    updates = []\n",
    "    for p, g in zip(params, grads):\n",
    "        acc = theano.shared(p.get_value() * 0.)\n",
    "        acc_new = rho * acc + (1 - rho) * g ** 2\n",
    "        gradient_scaling = T.sqrt(acc_new + epsilon)\n",
    "        g = g / gradient_scaling\n",
    "        updates.append((acc, acc_new))\n",
    "        updates.append((p, p - lr * g))\n",
    "    return updates\n",
    "\n",
    "def dropout(X, p=0.):\n",
    "    if p > 0:\n",
    "        retain_prob = 1 - p\n",
    "        X *= srng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX)\n",
    "        X /= retain_prob\n",
    "    return X\n",
    "\n",
    "def model(X, w_h, b_h, w_h2, b_h2, w_o, p_drop_input, p_drop_hidden):\n",
    "    X = dropout(X, p_drop_input)\n",
    "    h = rectify(T.dot(X, w_h) + b_h)\n",
    "\n",
    "    h = dropout(h, p_drop_hidden)\n",
    "    h2 = rectify(T.dot(h, w_h2) + b_h2)\n",
    "\n",
    "    h2 = dropout(h2, p_drop_hidden)\n",
    "    py_x = softmax(T.dot(h2, w_o))\n",
    "    return h, h2, py_x\n",
    "\n",
    "\n",
    "\n",
    "X = T.fmatrix()\n",
    "Y = T.fmatrix()\n",
    "\n",
    "w_h = init_weights((len(vectorizer.vocabulary_), 600))\n",
    "b_h = theano.shared(floatX(np.zeros(600,)))\n",
    "w_h2 = init_weights((600, 600))\n",
    "b_h2 = theano.shared(floatX(np.zeros(600,)))\n",
    "w_o = init_weights((600, yTest.shape[1]))\n",
    "# b_values = numpy.zeros((600,), dtype=theano.config.floatX)\n",
    "\n",
    "noise_h, noise_h2, noise_py_x = model(X, w_h, b_h, w_h2, b_h2, w_o, 0.2, 0.5)\n",
    "h, h2, py_x = model(X, w_h, b_h, w_h2, b_h2, w_o, 0., 0.)\n",
    "y_x = T.argmax(py_x, axis=1)\n",
    "\n",
    "\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n",
    "params = [w_h, b_h, w_h2, b_h2, w_o]\n",
    "updates = RMSprop(cost, params, lr=0.0001)\n",
    "\n",
    "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
    "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)\n",
    "\n",
    "for i in range(301):\n",
    "    for start, end in zip(range(0, len(xTrain), 128), range(128, len(xTrain), 128)):\n",
    "        cost = train(xTrain[start:end], yTrain[start:end])\n",
    "\n",
    "    if i%10 == 0: \n",
    "        tr = np.mean(np.argmax(yTest, axis=1) == predict(xTest))\n",
    "        trr =  np.mean(np.argmax(yTrain, axis=1) == predict(xTrain))\n",
    "        print 'Round:', i,\" Test:\", tr, ' Train:', trr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Next, batch normalization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG: nvcc STDOUT mod.cu\r\n",
      "   Creating library C:/Users/hollis_win/AppData/Local/Theano/compiledir_Windows-10-10.0.10586-Intel64_Family_6_Model_58_Stepping_9_GenuineIntel-2.7.11-64/tmpjufiuq/449ea1846932454968e0663d59c262c0.lib and object C:/Users/hollis_win/AppData/Local/Theano/compiledir_Windows-10-10.0.10586-Intel64_Family_6_Model_58_Stepping_9_GenuineIntel-2.7.11-64/tmpjufiuq/449ea1846932454968e0663d59c262c0.exp\r\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0  Test: 0.44032503809  Train: 0.458933107536\n",
      "Round: 5  Test: 0.561198577958  Train: 0.591871295512\n",
      "Round: 10  Test: 0.6124936516  Train: 0.661303979678\n",
      "Round: 15  Test: 0.647536820721  Train: 0.709229466554\n",
      "Round: 20  Test: 0.660741493144  Train: 0.739712108383\n",
      "Round: 25  Test: 0.681056373794  Train: 0.762912785775\n",
      "Round: 30  Test: 0.692737430168  Train: 0.780694326842\n",
      "Round: 35  Test: 0.698831894363  Train: 0.795596951736\n",
      "Round: 40  Test: 0.704926358558  Train: 0.807451312447\n",
      "Round: 45  Test: 0.711528694769  Train: 0.82133784928\n",
      "Round: 50  Test: 0.716607414931  Train: 0.833022861981\n",
      "Round: 55  Test: 0.726764855256  Train: 0.844030482642\n",
      "Round: 60  Test: 0.723209751143  Train: 0.855546147333\n",
      "Round: 65  Test: 0.725749111224  Train: 0.867061812024\n",
      "Round: 70  Test: 0.727780599289  Train: 0.874513124471\n",
      "Round: 75  Test: 0.730827831386  Train: 0.881456392887\n",
      "Round: 80  Test: 0.73031995937  Train: 0.887552921253\n",
      "Round: 85  Test: 0.730827831386  Train: 0.892972057578\n",
      "Round: 90  Test: 0.73031995937  Train: 0.896697713802\n",
      "Round: 95  Test: 0.730827831386  Train: 0.902286198137\n",
      "Round: 100  Test: 0.731335703403  Train: 0.907366638442\n",
      "Round: 105  Test: 0.7343829355  Train: 0.912447078747\n",
      "Round: 110  Test: 0.735906551549  Train: 0.91566469094\n",
      "Round: 115  Test: 0.732859319451  Train: 0.917358171041\n",
      "Round: 120  Test: 0.737938039614  Train: 0.919220999153\n"
     ]
    }
   ],
   "source": [
    "\"\"\" glorot 4-layer: batch, dropout, dropout \"\"\"\n",
    "\n",
    "# batch normalization code adapted from \n",
    "# https://groups.google.com/forum/#!topic/theano-users/dMV6aabL1Ds \n",
    "\n",
    "\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
    "from theano.tensor.nnet.bn import batch_normalization\n",
    "import numpy as np\n",
    "\n",
    "srng = RandomStreams()\n",
    "\n",
    "def floatX(X):\n",
    "    return np.asarray(X, dtype=theano.config.floatX)\n",
    "\n",
    "def init_weights(shape):\n",
    "    (h, w) = shape\n",
    "    # Glorot normalization - last factor depends on non-linearity\n",
    "    # 0.25 for sigmoid and 0.1 for softmax, 1.0 for tanh or Relu\n",
    "    normalizer = 2.0 * np.sqrt(6) / np.sqrt(h + w) * 1.0\n",
    "    return theano.shared(floatX((np.random.random_sample(shape) - 0.5) * normalizer))\n",
    "\n",
    "def rectify(X, alpha=1.0):\n",
    "#     return T.maximum(X, 0.)\n",
    "    return T.maximum(X, 0.1*X)  #leaky rectifier\n",
    "#     return T.switch(X > 0, X, alpha * (T.exp(X) - 1)) # ELU\n",
    "\n",
    "def softmax(X):\n",
    "    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n",
    "    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n",
    "\n",
    "def RMSprop(cost, params, lr=0.001, rho=0.99, epsilon=1e-6):\n",
    "    grads = T.grad(cost=cost, wrt=params)\n",
    "    updates = []\n",
    "    for p, g in zip(params, grads):\n",
    "        acc = theano.shared(p.get_value() * 0.)\n",
    "        acc_new = rho * acc + (1 - rho) * g ** 2\n",
    "        gradient_scaling = T.sqrt(acc_new + epsilon)\n",
    "        g = g / gradient_scaling\n",
    "        updates.append((acc, acc_new))\n",
    "        updates.append((p, p - lr * g))\n",
    "    return updates\n",
    "\n",
    "def dropout(X, p=0.):\n",
    "    if p > 0:\n",
    "        retain_prob = 1 - p\n",
    "        X *= srng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX)\n",
    "        X /= retain_prob\n",
    "    return X\n",
    "\n",
    "def model(X, w_h, b_h, gamma, beta, w_h2, b_h2, w_o, p_drop_input, p_drop_hidden):\n",
    "    X = T.dot(X, w_h) + b_h\n",
    "    mean = X.mean((1,), keepdims=True)\n",
    "    std = T.ones_like(X.var((0,), keepdims = True))\n",
    "    X = batch_normalization(X, gamma= gamma, beta= beta, \n",
    "                            mean= mean, \n",
    "                            std= std, mode='high_mem')    \n",
    "    h = rectify(X)\n",
    "\n",
    "    h = dropout(h, p_drop_hidden)\n",
    "    h2 = rectify(T.dot(h, w_h2) + b_h2)\n",
    "\n",
    "    h2 = dropout(h2, p_drop_hidden)\n",
    "    py_x = softmax(T.dot(h2, w_o))\n",
    "    return h, h2, py_x\n",
    "\n",
    "\n",
    "\n",
    "X = T.fmatrix()\n",
    "Y = T.fmatrix()\n",
    "\n",
    "w_h = init_weights((len(vectorizer.vocabulary_), 600))\n",
    "b_h = theano.shared(floatX(np.zeros(600)))\n",
    "gamma = theano.shared(floatX(np.ones(600)))\n",
    "beta = theano.shared(floatX(np.zeros(600)))\n",
    "\n",
    "w_h2 = init_weights((600, 600))\n",
    "b_h2 = theano.shared(floatX(np.zeros(600,)))\n",
    "w_o = init_weights((600, yTest.shape[1]))\n",
    "\n",
    "noise_h, noise_h2, noise_py_x = model(X, w_h, b_h, gamma, beta, w_h2, b_h2, w_o, 0.2, 0.5)\n",
    "h, h2, py_x = model(X, w_h, b_h, gamma, beta, w_h2, b_h2, w_o, 0., 0.)\n",
    "y_x = T.argmax(py_x, axis=1)\n",
    "\n",
    "\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n",
    "params = [w_h, b_h, gamma, beta, w_h2, b_h2, w_o]\n",
    "updates = RMSprop(cost, params, lr=0.0001)\n",
    "\n",
    "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
    "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)\n",
    "\n",
    "p_v3, p_t3, i3 = [], [], []\n",
    "for i in range(121):\n",
    "    for start, end in zip(range(0, len(xTrain), 128), range(128, len(xTrain), 128)):\n",
    "        cost = train(xTrain[start:end], yTrain[start:end])\n",
    "\n",
    "    if i%5 == 0: \n",
    "        tr = np.mean(np.argmax(yTest, axis=1) == predict(xTest))\n",
    "        trr =  np.mean(np.argmax(yTrain, axis=1) == predict(xTrain))\n",
    "        p_v3 += [tr]\n",
    "        p_t3 += [trr]\n",
    "        i3  += [i]        \n",
    "        print 'Round:', i,\" Test:\", tr, ' Train:', trr\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 74% !!! \n",
    "\n",
    "Lets see if thats repeatable.... "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0  Test: 0.419502285424  Train: 0.459102455546\n",
      "Round: 5  Test: 0.539867953276  Train: 0.585266723116\n",
      "Round: 10  Test: 0.598781107161  Train: 0.661473327688\n",
      "Round: 15  Test: 0.647536820721  Train: 0.706689246401\n",
      "Round: 20  Test: 0.668359573388  Train: 0.743099068586\n",
      "Round: 25  Test: 0.687658710005  Train: 0.766299745978\n",
      "Round: 30  Test: 0.695784662265  Train: 0.788484335309\n",
      "Round: 35  Test: 0.704418486541  Train: 0.80033869602\n",
      "Round: 40  Test: 0.715083798883  Train: 0.812870448772\n",
      "Round: 45  Test: 0.717623158964  Train: 0.828281117697\n",
      "Round: 50  Test: 0.723717623159  Train: 0.838441998307\n",
      "Round: 55  Test: 0.725241239208  Train: 0.848602878916\n",
      "Round: 60  Test: 0.730827831386  Train: 0.856392887384\n",
      "Round: 65  Test: 0.734890807517  Train: 0.865876375953\n",
      "Round: 70  Test: 0.737430167598  Train: 0.876883996613\n",
      "Round: 75  Test: 0.737430167598  Train: 0.881964436918\n",
      "Round: 80  Test: 0.73844591163  Train: 0.890093141406\n",
      "Round: 85  Test: 0.740985271712  Train: 0.89466553768\n",
      "Round: 90  Test: 0.735398679533  Train: 0.900423370025\n",
      "Round: 95  Test: 0.734890807517  Train: 0.90482641829\n",
      "Round: 100  Test: 0.733367191468  Train: 0.909060118544\n"
     ]
    }
   ],
   "source": [
    "\"\"\" glorot 4-layer: batch, dropout, dropout \"\"\"\n",
    "\n",
    "# batch normalization code adapted from \n",
    "# https://groups.google.com/forum/#!topic/theano-users/dMV6aabL1Ds \n",
    "\n",
    "\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
    "from theano.tensor.nnet.bn import batch_normalization\n",
    "import numpy as np\n",
    "\n",
    "srng = RandomStreams()\n",
    "\n",
    "def floatX(X):\n",
    "    return np.asarray(X, dtype=theano.config.floatX)\n",
    "\n",
    "def init_weights(shape):\n",
    "    (h, w) = shape\n",
    "    # Glorot normalization - last factor depends on non-linearity\n",
    "    # 0.25 for sigmoid and 0.1 for softmax, 1.0 for tanh or Relu\n",
    "    normalizer = 2.0 * np.sqrt(6) / np.sqrt(h + w) * 1.0\n",
    "    return theano.shared(floatX((np.random.random_sample(shape) - 0.5) * normalizer))\n",
    "\n",
    "def rectify(X, alpha=1.0):\n",
    "#     return T.maximum(X, 0.)\n",
    "    return T.maximum(X, 0.1*X)  #leaky rectifier\n",
    "#     return T.switch(X > 0, X, alpha * (T.exp(X) - 1)) # ELU\n",
    "\n",
    "def softmax(X):\n",
    "    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n",
    "    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n",
    "\n",
    "def RMSprop(cost, params, lr=0.001, rho=0.99, epsilon=1e-6):\n",
    "    grads = T.grad(cost=cost, wrt=params)\n",
    "    updates = []\n",
    "    for p, g in zip(params, grads):\n",
    "        acc = theano.shared(p.get_value() * 0.)\n",
    "        acc_new = rho * acc + (1 - rho) * g ** 2\n",
    "        gradient_scaling = T.sqrt(acc_new + epsilon)\n",
    "        g = g / gradient_scaling\n",
    "        updates.append((acc, acc_new))\n",
    "        updates.append((p, p - lr * g))\n",
    "    return updates\n",
    "\n",
    "def dropout(X, p=0.):\n",
    "    if p > 0:\n",
    "        retain_prob = 1 - p\n",
    "        X *= srng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX)\n",
    "        X /= retain_prob\n",
    "    return X\n",
    "\n",
    "def model(X, w_h, b_h, gamma, beta, w_h2, b_h2, w_o, p_drop_input, p_drop_hidden):\n",
    "    X = T.dot(X, w_h) + b_h\n",
    "    mean = X.mean((0,), keepdims=True)\n",
    "    std = T.ones_like(X.var((0,), keepdims = True))\n",
    "    X = batch_normalization(X, gamma= gamma, beta= beta, \n",
    "                            mean= mean, #X.mean((0,), keepdims=True), \n",
    "                            std= std, mode='high_mem')    \n",
    "    h = rectify(X)\n",
    "\n",
    "    h = dropout(h, p_drop_hidden)\n",
    "    h2 = rectify(T.dot(h, w_h2) + b_h2)\n",
    "\n",
    "    h2 = dropout(h2, p_drop_hidden)\n",
    "    py_x = softmax(T.dot(h2, w_o))\n",
    "    return h, h2, py_x\n",
    "\n",
    "\n",
    "\n",
    "X = T.fmatrix()\n",
    "Y = T.fmatrix()\n",
    "\n",
    "w_h = init_weights((len(vectorizer.vocabulary_), 600))\n",
    "b_h = theano.shared(floatX(np.zeros(600)))\n",
    "gamma = theano.shared(floatX(np.ones(600)))\n",
    "beta = theano.shared(floatX(np.zeros(600)))\n",
    "\n",
    "w_h2 = init_weights((600, 600))\n",
    "b_h2 = theano.shared(floatX(np.zeros(600,)))\n",
    "w_o = init_weights((600, yTest.shape[1]))\n",
    "\n",
    "noise_h, noise_h2, noise_py_x = model(X, w_h, b_h, gamma, beta, w_h2, b_h2, w_o, 0.2, 0.5)\n",
    "h, h2, py_x = model(X, w_h, b_h, gamma, beta, w_h2, b_h2, w_o, 0., 0.)\n",
    "y_x = T.argmax(py_x, axis=1)\n",
    "\n",
    "\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n",
    "params = [w_h, b_h, gamma, beta, w_h2, b_h2, w_o]\n",
    "updates = RMSprop(cost, params, lr=0.0001)\n",
    "\n",
    "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
    "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)\n",
    "\n",
    "for i in range(101):\n",
    "    for start, end in zip(range(0, len(xTrain), 128), range(128, len(xTrain), 128)):\n",
    "        cost = train(xTrain[start:end], yTrain[start:end])\n",
    "\n",
    "    if i%5 == 0: \n",
    "        tr = np.mean(np.argmax(yTest, axis=1) == predict(xTest))\n",
    "        trr =  np.mean(np.argmax(yTrain, axis=1) == predict(xTrain))\n",
    "        print 'Round:', i,\" Test:\", tr, ' Train:', trr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 74% again!\n",
    "\n",
    "Next, 3-layer batch normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 0: GeForce GTX 970 (CNMeM is disabled, CuDNN not available)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0  Test: 0.483494159472  Train: 0.528196443692\n",
      "Round: 1  Test: 0.527171152869  Train: 0.573581710415\n",
      "Round: 2  Test: 0.566277298121  Train: 0.613547840813\n",
      "Round: 3  Test: 0.596749619096  Train: 0.650973751058\n",
      "Round: 4  Test: 0.617572371762  Train: 0.681625740898\n",
      "Round: 5  Test: 0.641442356526  Train: 0.710584250635\n",
      "Round: 6  Test: 0.656678517014  Train: 0.731414055885\n",
      "Round: 7  Test: 0.668359573388  Train: 0.750211685013\n",
      "Round: 8  Test: 0.675977653631  Train: 0.764944961897\n",
      "Round: 9  Test: 0.688674454038  Train: 0.776121930567\n",
      "Round: 10  Test: 0.694768918233  Train: 0.791363251482\n",
      "Round: 11  Test: 0.70187912646  Train: 0.802201524132\n",
      "Round: 12  Test: 0.713052310818  Train: 0.813378492803\n",
      "Round: 13  Test: 0.717115286948  Train: 0.819983065199\n",
      "Round: 14  Test: 0.716099542915  Train: 0.828281117697\n",
      "Round: 15  Test: 0.72219400711  Train: 0.836409822185\n",
      "Round: 16  Test: 0.724225495175  Train: 0.845893310754\n",
      "Round: 17  Test: 0.723209751143  Train: 0.85249788315\n",
      "Round: 18  Test: 0.726764855256  Train: 0.858594411516\n",
      "Round: 19  Test: 0.729812087354  Train: 0.86316680779\n",
      "Round: 20  Test: 0.731335703403  Train: 0.868755292125\n",
      "Round: 21  Test: 0.731335703403  Train: 0.8733276884\n",
      "Round: 22  Test: 0.73031995937  Train: 0.878238780694\n",
      "Round: 23  Test: 0.730827831386  Train: 0.883488569009\n",
      "Round: 24  Test: 0.730827831386  Train: 0.889246401355\n",
      "Round: 25  Test: 0.73031995937  Train: 0.89449618967\n",
      "Round: 26  Test: 0.729304215338  Train: 0.896528365792\n",
      "Round: 27  Test: 0.728796343321  Train: 0.900254022015\n",
      "Round: 28  Test: 0.728288471305  Train: 0.903302286198\n",
      "Round: 29  Test: 0.728288471305  Train: 0.906181202371\n",
      "Round: 30  Test: 0.727780599289  Train: 0.909398814564\n",
      "Round: 31  Test: 0.728288471305  Train: 0.912447078747\n",
      "Round: 32  Test: 0.728796343321  Train: 0.91583403895\n",
      "Round: 33  Test: 0.727780599289  Train: 0.919220999153\n",
      "Round: 34  Test: 0.727780599289  Train: 0.921930567316\n",
      "Round: 35  Test: 0.727780599289  Train: 0.924132091448\n",
      "Round: 36  Test: 0.72625698324  Train: 0.92684165961\n",
      "Round: 37  Test: 0.72625698324  Train: 0.928196443692\n",
      "Round: 38  Test: 0.725749111224  Train: 0.929720575783\n",
      "Round: 39  Test: 0.726764855256  Train: 0.931075359865\n",
      "Round: 40  Test: 0.72625698324  Train: 0.932599491956\n",
      "Round: 41  Test: 0.724225495175  Train: 0.934123624047\n",
      "Round: 42  Test: 0.723717623159  Train: 0.935986452159\n",
      "Round: 43  Test: 0.723717623159  Train: 0.937679932261\n",
      "Round: 44  Test: 0.723209751143  Train: 0.938187976291\n",
      "Round: 45  Test: 0.721178263078  Train: 0.938865368332\n",
      "Round: 46  Test: 0.722701879126  Train: 0.940050804403\n",
      "Round: 47  Test: 0.721686135094  Train: 0.941066892464\n",
      "Round: 48  Test: 0.721686135094  Train: 0.941744284505\n",
      "Round: 49  Test: 0.72219400711  Train: 0.941913632515\n"
     ]
    }
   ],
   "source": [
    "\"\"\" glorot 4-layer: batch, batch, batch \"\"\"\n",
    "\n",
    "# batch normalization code adapted from \n",
    "# https://groups.google.com/forum/#!topic/theano-users/dMV6aabL1Ds \n",
    "\n",
    "\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
    "from theano.tensor.nnet.bn import batch_normalization\n",
    "import numpy as np\n",
    "\n",
    "srng = RandomStreams()\n",
    "\n",
    "def floatX(X):\n",
    "    return np.asarray(X, dtype=theano.config.floatX)\n",
    "\n",
    "def init_weights(shape):\n",
    "    (h, w) = shape\n",
    "    # Glorot normalization - last factor depends on non-linearity\n",
    "    # 0.25 for sigmoid and 0.1 for softmax, 1.0 for tanh or Relu\n",
    "    normalizer = 2.0 * np.sqrt(6) / np.sqrt(h + w) * 1.0\n",
    "    return theano.shared(floatX((np.random.random_sample(shape) - 0.5) * normalizer))\n",
    "\n",
    "def rectify(X, alpha=1.0):\n",
    "#     return T.maximum(X, 0.)\n",
    "    return T.maximum(X, 0.1*X)  #leaky rectifier\n",
    "#     return T.switch(X > 0, X, alpha * (T.exp(X) - 1)) # ELU\n",
    "\n",
    "def softmax(X):\n",
    "    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n",
    "    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n",
    "\n",
    "def RMSprop(cost, params, lr=0.001, rho=0.99, epsilon=1e-6):\n",
    "    grads = T.grad(cost=cost, wrt=params)\n",
    "    updates = []\n",
    "    for p, g in zip(params, grads):\n",
    "        acc = theano.shared(p.get_value() * 0.)\n",
    "        acc_new = rho * acc + (1 - rho) * g ** 2\n",
    "        gradient_scaling = T.sqrt(acc_new + epsilon)\n",
    "        g = g / gradient_scaling\n",
    "        updates.append((acc, acc_new))\n",
    "        updates.append((p, p - lr * g))\n",
    "    return updates\n",
    "\n",
    "\n",
    "def model(X, w_h, b_h, g_h, bb_h, w_h2, b_h2, g_h2, bb_h2, w_o, b_ho, g_ho, bb_ho):\n",
    "    X = T.dot(X, w_h) + b_h\n",
    "    X = batch_normalization(X, gamma= g_h, beta= bb_h, \n",
    "                            mean= X.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(X.var((0,), keepdims = True)), \n",
    "                            mode='high_mem')    \n",
    "    h = rectify(X)\n",
    "\n",
    "    h  = T.dot(h, w_h2) + b_h2\n",
    "    h = batch_normalization(h, gamma= g_h2, beta= bb_h2, \n",
    "                            mean= h.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h.var((0,), keepdims = True)), \n",
    "                            mode='high_mem')       \n",
    "    h2 = rectify(h)\n",
    "\n",
    "    h2 = T.dot(h2, w_o) + b_ho\n",
    "    h2 = batch_normalization(h2, gamma= g_ho, beta= bb_ho, \n",
    "                            mean= h2.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h2.var((0,), keepdims = True)), \n",
    "                            mode='high_mem')   \n",
    "    py_x = softmax(h2)\n",
    "    return h, h2, py_x\n",
    "\n",
    "\n",
    "X = T.fmatrix()\n",
    "Y = T.fmatrix()\n",
    "\n",
    "h1_size = 600\n",
    "h2_size = 550\n",
    "\n",
    "w_h = init_weights((len(vectorizer.vocabulary_), h1_size))\n",
    "b_h = theano.shared(floatX(np.zeros(h1_size)))\n",
    "g_h = theano.shared(floatX(np.ones(h1_size)))\n",
    "bb_h = theano.shared(floatX(np.zeros(h1_size)))\n",
    "\n",
    "w_h2 = init_weights((h1_size, h2_size))\n",
    "b_h2 = theano.shared(floatX(np.zeros(h2_size)))\n",
    "g_h2 = theano.shared(floatX(np.ones(h2_size)))\n",
    "bb_h2 = theano.shared(floatX(np.zeros(h2_size)))\n",
    "\n",
    "w_o = init_weights((h2_size, yTest.shape[1]))\n",
    "b_ho = theano.shared(floatX(np.zeros(yTest.shape[1])))\n",
    "g_ho = theano.shared(floatX(np.ones(yTest.shape[1])))\n",
    "bb_ho = theano.shared(floatX(np.zeros(yTest.shape[1])))\n",
    "\n",
    "noise_h, noise_h2, noise_py_x = model(X, w_h, b_h, g_h, bb_h, \n",
    "                                      w_h2, b_h2, g_h2, bb_h2, \n",
    "                                      w_o, b_ho, g_ho, bb_ho)\n",
    "\n",
    "h, h2, py_x = model(X, w_h, b_h, g_h, bb_h, \n",
    "                    w_h2, b_h2, g_h2, bb_h2, \n",
    "                    w_o, b_ho, g_ho, bb_ho)\n",
    "\n",
    "y_x = T.argmax(py_x, axis=1)\n",
    "\n",
    "\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n",
    "params = [w_h, b_h, g_h, bb_h, w_h2, b_h2, g_h2, bb_h2, w_o, b_ho, g_ho, bb_ho]\n",
    "updates = RMSprop(cost, params, lr=0.0001)\n",
    "\n",
    "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
    "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)\n",
    "\n",
    "batch_size = 60\n",
    "\n",
    "for i in range(50):\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):\n",
    "        cost = train(xTrain[start:end], yTrain[start:end])\n",
    "\n",
    "    tr = np.mean(np.argmax(yTest, axis=1) == predict(xTest))\n",
    "    trr =  np.mean(np.argmax(yTrain, axis=1) == predict(xTrain))\n",
    "    print 'Round:', i,\" Test:\", tr, ' Train:', trr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3558 5373 1030]\n",
      "[[0 0 0 ..., 0 0 0]\n",
      " [0 0 0 ..., 0 0 0]\n",
      " [0 0 0 ..., 0 0 0]]\n",
      "[[ 0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.]\n",
      " [ 0.  0.  0.  1.  0.]]\n"
     ]
    }
   ],
   "source": [
    "# random indexing for batches \n",
    "\n",
    "index = np.random.choice(xTrain.shape[0], 3)\n",
    "print index\n",
    "print xTrain[index]\n",
    "print yTrain[index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0  Test: 0.479939055358  Train: 0.51532599492\n",
      "Round: 1  Test: 0.515490096496  Train: 0.5583403895\n",
      "Round: 2  Test: 0.558151345861  Train: 0.613547840813\n",
      "Round: 3  Test: 0.583544946673  Train: 0.638272650296\n",
      "Round: 4  Test: 0.605383443372  Train: 0.667908552075\n",
      "Round: 5  Test: 0.633316404266  Train: 0.698221845893\n",
      "Round: 6  Test: 0.660741493144  Train: 0.725486875529\n",
      "Round: 7  Test: 0.671914677501  Train: 0.744453852667\n",
      "Round: 8  Test: 0.682579989843  Train: 0.762574089754\n",
      "Round: 9  Test: 0.684103605891  Train: 0.769348010161\n",
      "Round: 10  Test: 0.692229558151  Train: 0.785944115157\n",
      "Round: 11  Test: 0.701371254444  Train: 0.79966130398\n",
      "Round: 12  Test: 0.70594210259  Train: 0.806943268417\n",
      "Round: 13  Test: 0.71406805485  Train: 0.818120237087\n",
      "Round: 14  Test: 0.719146775013  Train: 0.824386113463\n",
      "Round: 15  Test: 0.717115286948  Train: 0.834716342083\n",
      "Round: 16  Test: 0.724733367191  Train: 0.844538526672\n",
      "Round: 17  Test: 0.724225495175  Train: 0.853852667231\n",
      "Round: 18  Test: 0.721178263078  Train: 0.85300592718\n",
      "Round: 19  Test: 0.726764855256  Train: 0.860287891617\n",
      "Round: 20  Test: 0.731843575419  Train: 0.864182895851\n",
      "Round: 21  Test: 0.724225495175  Train: 0.872311600339\n",
      "Round: 22  Test: 0.724225495175  Train: 0.878069432684\n",
      "Round: 23  Test: 0.729812087354  Train: 0.879762912786\n",
      "Round: 24  Test: 0.733875063484  Train: 0.887214225233\n"
     ]
    }
   ],
   "source": [
    "\"\"\" glorot 4-layer: batch, batch, batch \n",
    "    random indexing                       \"\"\"\n",
    "\n",
    "# batch normalization code adapted from \n",
    "# https://groups.google.com/forum/#!topic/theano-users/dMV6aabL1Ds \n",
    "\n",
    "\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
    "from theano.tensor.nnet.bn import batch_normalization\n",
    "import numpy as np\n",
    "\n",
    "srng = RandomStreams()\n",
    "\n",
    "def floatX(X):\n",
    "    return np.asarray(X, dtype=theano.config.floatX)\n",
    "\n",
    "def init_weights(shape):\n",
    "    (h, w) = shape\n",
    "    # Glorot normalization - last factor depends on non-linearity\n",
    "    # 0.25 for sigmoid and 0.1 for softmax, 1.0 for tanh or Relu\n",
    "    normalizer = 2.0 * np.sqrt(6) / np.sqrt(h + w) * 1.0\n",
    "    return theano.shared(floatX((np.random.random_sample(shape) - 0.5) * normalizer))\n",
    "\n",
    "def rectify(X, alpha=1.0):\n",
    "#     return T.maximum(X, 0.)\n",
    "    return T.maximum(X, 0.1*X)  #leaky rectifier\n",
    "#     return T.switch(X > 0, X, alpha * (T.exp(X) - 1)) # ELU\n",
    "\n",
    "def softmax(X):\n",
    "    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n",
    "    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n",
    "\n",
    "def RMSprop(cost, params, lr=0.001, rho=0.99, epsilon=1e-6):\n",
    "    grads = T.grad(cost=cost, wrt=params)\n",
    "    updates = []\n",
    "    for p, g in zip(params, grads):\n",
    "        acc = theano.shared(p.get_value() * 0.)\n",
    "        acc_new = rho * acc + (1 - rho) * g ** 2\n",
    "        gradient_scaling = T.sqrt(acc_new + epsilon)\n",
    "        g = g / gradient_scaling\n",
    "        updates.append((acc, acc_new))\n",
    "        updates.append((p, p - lr * g))\n",
    "    return updates\n",
    "\n",
    "\n",
    "def model(X, w_h, b_h, g_h, bb_h, w_h2, b_h2, g_h2, bb_h2, w_o, b_ho, g_ho, bb_ho):\n",
    "    X = T.dot(X, w_h) + b_h\n",
    "    X = batch_normalization(X, gamma= g_h, beta= bb_h, \n",
    "                            mean= X.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(X.var((0,), keepdims = True)), \n",
    "                            mode='high_mem')    \n",
    "    h = rectify(X)\n",
    "\n",
    "    h  = T.dot(h, w_h2) + b_h2\n",
    "    h = batch_normalization(h, gamma= g_h2, beta= bb_h2, \n",
    "                            mean= h.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h.var((0,), keepdims = True)), \n",
    "                            mode='high_mem')       \n",
    "    h2 = rectify(h)\n",
    "\n",
    "    h2 = T.dot(h2, w_o) + b_ho\n",
    "    h2 = batch_normalization(h2, gamma= g_ho, beta= bb_ho, \n",
    "                            mean= h2.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h2.var((0,), keepdims = True)), \n",
    "                            mode='high_mem')   \n",
    "    py_x = softmax(h2)\n",
    "    return h, h2, py_x\n",
    "\n",
    "\n",
    "X = T.fmatrix()\n",
    "Y = T.fmatrix()\n",
    "\n",
    "h1_size = 600\n",
    "h2_size = 550\n",
    "\n",
    "w_h = init_weights((len(vectorizer.vocabulary_), h1_size))\n",
    "b_h = theano.shared(floatX(np.zeros(h1_size)))\n",
    "g_h = theano.shared(floatX(np.ones(h1_size)))\n",
    "bb_h = theano.shared(floatX(np.zeros(h1_size)))\n",
    "\n",
    "w_h2 = init_weights((h1_size, h2_size))\n",
    "b_h2 = theano.shared(floatX(np.zeros(h2_size)))\n",
    "g_h2 = theano.shared(floatX(np.ones(h2_size)))\n",
    "bb_h2 = theano.shared(floatX(np.zeros(h2_size)))\n",
    "\n",
    "w_o = init_weights((h2_size, yTest.shape[1]))\n",
    "b_ho = theano.shared(floatX(np.zeros(yTest.shape[1])))\n",
    "g_ho = theano.shared(floatX(np.ones(yTest.shape[1])))\n",
    "bb_ho = theano.shared(floatX(np.zeros(yTest.shape[1])))\n",
    "\n",
    "noise_h, noise_h2, noise_py_x = model(X, w_h, b_h, g_h, bb_h, \n",
    "                                      w_h2, b_h2, g_h2, bb_h2, \n",
    "                                      w_o, b_ho, g_ho, bb_ho)\n",
    "\n",
    "h, h2, py_x = model(X, w_h, b_h, g_h, bb_h, \n",
    "                    w_h2, b_h2, g_h2, bb_h2, \n",
    "                    w_o, b_ho, g_ho, bb_ho)\n",
    "\n",
    "y_x = T.argmax(py_x, axis=1)\n",
    "\n",
    "\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n",
    "params = [w_h, b_h, g_h, bb_h, w_h2, b_h2, g_h2, bb_h2, w_o, b_ho, g_ho, bb_ho]\n",
    "updates = RMSprop(cost, params, lr=0.0001)\n",
    "\n",
    "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
    "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)\n",
    "\n",
    "batch_size = 60\n",
    "\n",
    "for i in range(26):\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):\n",
    "        index = np.random.choice(xTrain.shape[0], batch_size, replace=False)\n",
    "        cost = train(xTrain[index], yTrain[index])\n",
    "\n",
    "    tr = np.mean(np.argmax(yTest, axis=1) == predict(xTest))\n",
    "    trr =  np.mean(np.argmax(yTrain, axis=1) == predict(xTrain))\n",
    "    print 'Round:', i,\" Test:\", tr, ' Train:', trr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random idexing (with replacement) no effect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0  Test: 0.470797359066  Train: 0.517696867062\n",
      "Round: 1  Test: 0.511427120366  Train: 0.55766299746\n",
      "Round: 2  Test: 0.544438801422  Train: 0.598306519898\n",
      "Round: 3  Test: 0.585068562722  Train: 0.644877222693\n",
      "Round: 4  Test: 0.61655662773  Train: 0.677561388654\n",
      "Round: 5  Test: 0.626714068055  Train: 0.696697713802\n",
      "Round: 6  Test: 0.64499746064  Train: 0.725148179509\n",
      "Round: 7  Test: 0.672422549518  Train: 0.749872988992\n",
      "Round: 8  Test: 0.670391061453  Train: 0.764436917866\n",
      "Round: 9  Test: 0.687658710005  Train: 0.77866215072\n",
      "Round: 10  Test: 0.691721686135  Train: 0.795258255715\n",
      "Round: 11  Test: 0.69781615033  Train: 0.807620660457\n",
      "Round: 12  Test: 0.703910614525  Train: 0.816934801016\n",
      "Round: 13  Test: 0.709497206704  Train: 0.827265029636\n",
      "Round: 14  Test: 0.712036566785  Train: 0.836240474174\n",
      "Round: 15  Test: 0.716099542915  Train: 0.841320914479\n",
      "Round: 16  Test: 0.715591670899  Train: 0.851989839119\n",
      "Round: 17  Test: 0.71406805485  Train: 0.859779847587\n",
      "Round: 18  Test: 0.721686135094  Train: 0.868077900085\n",
      "Round: 19  Test: 0.72219400711  Train: 0.876545300593\n",
      "Round: 20  Test: 0.726764855256  Train: 0.883319220999\n",
      "Round: 21  Test: 0.720670391061  Train: 0.886028789162\n",
      "Round: 22  Test: 0.727780599289  Train: 0.889246401355\n",
      "Round: 23  Test: 0.725241239208  Train: 0.892633361558\n",
      "Round: 24  Test: 0.73031995937  Train: 0.900254022015\n",
      "Round: 25  Test: 0.731335703403  Train: 0.901608806097\n",
      "Round: 26  Test: 0.733367191468  Train: 0.904657070279\n",
      "Round: 27  Test: 0.73031995937  Train: 0.907197290432\n",
      "Round: 28  Test: 0.730827831386  Train: 0.913971210838\n",
      "Round: 29  Test: 0.728796343321  Train: 0.916680779001\n",
      "Round: 30  Test: 0.73031995937  Train: 0.918712955123\n",
      "Round: 31  Test: 0.731843575419  Train: 0.922607959356\n",
      "Round: 32  Test: 0.729812087354  Train: 0.925148179509\n",
      "Round: 33  Test: 0.730827831386  Train: 0.927519051651\n"
     ]
    }
   ],
   "source": [
    "\"\"\" glorot 4-layer: batch, batch, batch \n",
    "    random indexing, ELU                \"\"\"\n",
    "\n",
    "# batch normalization code adapted from \n",
    "# https://groups.google.com/forum/#!topic/theano-users/dMV6aabL1Ds \n",
    "\n",
    "\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
    "from theano.tensor.nnet.bn import batch_normalization\n",
    "import numpy as np\n",
    "\n",
    "srng = RandomStreams()\n",
    "\n",
    "def floatX(X):\n",
    "    return np.asarray(X, dtype=theano.config.floatX)\n",
    "\n",
    "def init_weights(shape):\n",
    "    (h, w) = shape\n",
    "    # Glorot normalization - last factor depends on non-linearity\n",
    "    # 0.25 for sigmoid and 0.1 for softmax, 1.0 for tanh or Relu\n",
    "    normalizer = 2.0 * np.sqrt(6) / np.sqrt(h + w) * 1.0\n",
    "    return theano.shared(floatX((np.random.random_sample(shape) - 0.5) * normalizer))\n",
    "\n",
    "def rectify(X, alpha=0.01):\n",
    "#     return T.maximum(X, 0.)\n",
    "#    return T.maximum(X, 0.1*X)  #leaky rectifier\n",
    "     return T.switch(X > 0, X, alpha * (T.exp(X) - 1)) # ELU\n",
    "\n",
    "def softmax(X):\n",
    "    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n",
    "    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n",
    "\n",
    "def RMSprop(cost, params, lr=0.001, rho=0.99, epsilon=1e-6):\n",
    "    grads = T.grad(cost=cost, wrt=params)\n",
    "    updates = []\n",
    "    for p, g in zip(params, grads):\n",
    "        acc = theano.shared(p.get_value() * 0.)\n",
    "        acc_new = rho * acc + (1 - rho) * g ** 2\n",
    "        gradient_scaling = T.sqrt(acc_new + epsilon)\n",
    "        g = g / gradient_scaling\n",
    "        updates.append((acc, acc_new))\n",
    "        updates.append((p, p - lr * g))\n",
    "    return updates\n",
    "\n",
    "\n",
    "def model(X, w_h, b_h, g_h, bb_h, w_h2, b_h2, g_h2, bb_h2, w_o, b_ho, g_ho, bb_ho):\n",
    "    X = T.dot(X, w_h) + b_h\n",
    "    X = batch_normalization(X, gamma= g_h, beta= bb_h, \n",
    "                            mean= X.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(X.var((0,), keepdims = True)), \n",
    "                            mode='high_mem')    \n",
    "    h = rectify(X)\n",
    "\n",
    "    h  = T.dot(h, w_h2) + b_h2\n",
    "    h = batch_normalization(h, gamma= g_h2, beta= bb_h2, \n",
    "                            mean= h.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h.var((0,), keepdims = True)), \n",
    "                            mode='high_mem')       \n",
    "    h2 = rectify(h)\n",
    "\n",
    "    h2 = T.dot(h2, w_o) + b_ho\n",
    "    h2 = batch_normalization(h2, gamma= g_ho, beta= bb_ho, \n",
    "                            mean= h2.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h2.var((0,), keepdims = True)), \n",
    "                            mode='high_mem')   \n",
    "    py_x = softmax(h2)\n",
    "    return h, h2, py_x\n",
    "\n",
    "\n",
    "X = T.fmatrix()\n",
    "Y = T.fmatrix()\n",
    "\n",
    "h1_size = 600\n",
    "h2_size = 550\n",
    "\n",
    "w_h = init_weights((len(vectorizer.vocabulary_), h1_size))\n",
    "b_h = theano.shared(floatX(np.zeros(h1_size)))\n",
    "g_h = theano.shared(floatX(np.ones(h1_size)))\n",
    "bb_h = theano.shared(floatX(np.zeros(h1_size)))\n",
    "\n",
    "w_h2 = init_weights((h1_size, h2_size))\n",
    "b_h2 = theano.shared(floatX(np.zeros(h2_size)))\n",
    "g_h2 = theano.shared(floatX(np.ones(h2_size)))\n",
    "bb_h2 = theano.shared(floatX(np.zeros(h2_size)))\n",
    "\n",
    "w_o = init_weights((h2_size, yTest.shape[1]))\n",
    "b_ho = theano.shared(floatX(np.zeros(yTest.shape[1])))\n",
    "g_ho = theano.shared(floatX(np.ones(yTest.shape[1])))\n",
    "bb_ho = theano.shared(floatX(np.zeros(yTest.shape[1])))\n",
    "\n",
    "noise_h, noise_h2, noise_py_x = model(X, w_h, b_h, g_h, bb_h, \n",
    "                                      w_h2, b_h2, g_h2, bb_h2, \n",
    "                                      w_o, b_ho, g_ho, bb_ho)\n",
    "\n",
    "h, h2, py_x = model(X, w_h, b_h, g_h, bb_h, \n",
    "                    w_h2, b_h2, g_h2, bb_h2, \n",
    "                    w_o, b_ho, g_ho, bb_ho)\n",
    "\n",
    "y_x = T.argmax(py_x, axis=1)\n",
    "\n",
    "\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n",
    "params = [w_h, b_h, g_h, bb_h, w_h2, b_h2, g_h2, bb_h2, w_o, b_ho, g_ho, bb_ho]\n",
    "updates = RMSprop(cost, params, lr=0.0001)\n",
    "\n",
    "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
    "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)\n",
    "\n",
    "batch_size = 60\n",
    "\n",
    "for i in range(34):\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):\n",
    "        index = np.random.choice(xTrain.shape[0], batch_size, replace=False)\n",
    "        cost = train(xTrain[index], yTrain[index])\n",
    "\n",
    "    tr = np.mean(np.argmax(yTest, axis=1) == predict(xTest))\n",
    "    trr =  np.mean(np.argmax(yTrain, axis=1) == predict(xTrain))\n",
    "    print 'Round:', i,\" Test:\", tr, ' Train:', trr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ELU, no effect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0     Test: 0.558333333333 Train: 0.597619047619\n",
      "Round: 1     Test: 0.599479166667 Train: 0.666836734694\n",
      "Round: 2     Test: 0.640625       Train: 0.711734693878\n",
      "Round: 3     Test: 0.6734375      Train: 0.740646258503\n",
      "Round: 4     Test: 0.688020833333 Train: 0.76343537415\n",
      "Round: 5     Test: 0.693229166667 Train: 0.786394557823\n",
      "Round: 6     Test: 0.704166666667 Train: 0.801700680272\n",
      "Round: 7     Test: 0.710416666667 Train: 0.820408163265\n",
      "Round: 8     Test: 0.717708333333 Train: 0.825680272109\n",
      "Round: 9     Test: 0.722916666667 Train: 0.837244897959\n",
      "Round: 10    Test: 0.711458333333 Train: 0.851530612245\n",
      "Round: 11    Test: 0.711979166667 Train: 0.866326530612\n",
      "Round: 12    Test: 0.722916666667 Train: 0.868367346939\n",
      "Round: 13    Test: 0.7265625      Train: 0.876870748299\n",
      "Round: 14    Test: 0.725          Train: 0.879591836735\n",
      "Round: 15    Test: 0.719791666667 Train: 0.892176870748\n",
      "Round: 16    Test: 0.727604166667 Train: 0.897789115646\n",
      "Round: 17    Test: 0.728125       Train: 0.899489795918\n",
      "Round: 18    Test: 0.73125        Train: 0.903401360544\n",
      "Round: 19    Test: 0.730729166667 Train: 0.910714285714\n",
      "Round: 20    Test: 0.725          Train: 0.909013605442\n",
      "Round: 21    Test: 0.721875       Train: 0.915476190476\n",
      "Round: 22    Test: 0.733854166667 Train: 0.914115646259\n",
      "Round: 23    Test: 0.7234375      Train: 0.919727891156\n",
      "Round: 24    Test: 0.71875        Train: 0.92619047619\n",
      "Round: 25    Test: 0.723958333333 Train: 0.924659863946\n",
      "Round: 26    Test: 0.721875       Train: 0.922619047619\n",
      "Round: 27    Test: 0.7296875      Train: 0.926530612245\n",
      "Round: 28    Test: 0.725          Train: 0.930612244898\n",
      "Round: 29    Test: 0.723958333333 Train: 0.930952380952\n",
      "Round: 30    Test: 0.719270833333 Train: 0.931802721088\n",
      "Round: 31    Test: 0.71875        Train: 0.933503401361\n",
      "Round: 32    Test: 0.7171875      Train: 0.937414965986\n",
      "Round: 33    Test: 0.711979166667 Train: 0.93843537415\n",
      "Round: 34    Test: 0.726041666667 Train: 0.938945578231\n",
      "Round: 35    Test: 0.719270833333 Train: 0.938265306122\n",
      "Round: 36    Test: 0.719791666667 Train: 0.939965986395\n",
      "Round: 37    Test: 0.720833333333 Train: 0.942857142857\n",
      "Round: 38    Test: 0.717708333333 Train: 0.939285714286\n",
      "Round: 39    Test: 0.7234375      Train: 0.940306122449\n",
      "Round: 40    Test: 0.70625        Train: 0.944557823129\n",
      "\n",
      "Wall time: 1min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "\"\"\" glorot 4-layer: batch, batch, batch \n",
    "    random indexing without replacement, ELU                \n",
    "    epsilon=1e-9  \"\"\"\n",
    "\n",
    "# batch normalization code adapted from \n",
    "# https://groups.google.com/forum/#!topic/theano-users/dMV6aabL1Ds \n",
    "\n",
    "\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
    "from theano.tensor.nnet.bn import batch_normalization\n",
    "import numpy as np\n",
    "\n",
    "srng = RandomStreams()\n",
    "\n",
    "def shuffle(x, y):\n",
    "    # helper function to shuffle indicies each loop \n",
    "    index = np.random.choice(len(x), len(x), replace=False)\n",
    "    return x[index], y[index]\n",
    "\n",
    "def floatX(X):\n",
    "    return np.asarray(X, dtype=theano.config.floatX)\n",
    "\n",
    "def init_weights(shape):\n",
    "    (h, w) = shape\n",
    "    # Glorot normalization - last factor depends on non-linearity\n",
    "    # 0.25 for sigmoid and 0.1 for softmax, 1.0 for tanh or Relu\n",
    "    normalizer = 2.0 * np.sqrt(6) / np.sqrt(h + w) * 1.0\n",
    "    return theano.shared(floatX((np.random.random_sample(shape) - 0.5) * normalizer))\n",
    "\n",
    "def rectify(X, alpha=0.01):\n",
    "#     return T.maximum(X, 0.)\n",
    "#    return T.maximum(X, 0.1*X)  #leaky rectifier\n",
    "     return T.switch(X > 0, X, alpha * (T.exp(X) - 1)) # ELU\n",
    "\n",
    "def softmax(X):\n",
    "    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n",
    "    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n",
    "\n",
    "def RMSprop(cost, params, lr=0.001, rho=0.99, epsilon=1e-9):\n",
    "    grads = T.grad(cost=cost, wrt=params)\n",
    "    updates = []\n",
    "    for p, g in zip(params, grads):\n",
    "        acc = theano.shared(p.get_value() * 0.)\n",
    "        acc_new = rho * acc + (1 - rho) * g ** 2\n",
    "        gradient_scaling = T.sqrt(acc_new + epsilon)\n",
    "        g = g / gradient_scaling\n",
    "        updates.append((acc, acc_new))\n",
    "        updates.append((p, p - lr * g))\n",
    "    return updates\n",
    "\n",
    "\n",
    "def model(X, w_h, b_h, g_h, bb_h, w_h2, b_h2, g_h2, bb_h2, w_o, b_ho, g_ho, bb_ho):\n",
    "    X = T.dot(X, w_h) + b_h\n",
    "    X = batch_normalization(X, gamma= g_h, beta= bb_h, \n",
    "                            mean= X.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(X.var((0,), keepdims = True)), \n",
    "                            mode='high_mem')    \n",
    "    h = rectify(X)\n",
    "\n",
    "    h  = T.dot(h, w_h2) + b_h2\n",
    "    h = batch_normalization(h, gamma= g_h2, beta= bb_h2, \n",
    "                            mean= h.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h.var((0,), keepdims = True)), \n",
    "                            mode='high_mem')       \n",
    "    h2 = rectify(h)\n",
    "\n",
    "    h2 = T.dot(h2, w_o) + b_ho\n",
    "    h2 = batch_normalization(h2, gamma= g_ho, beta= bb_ho, \n",
    "                            mean= h2.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h2.var((0,), keepdims = True)), \n",
    "                            mode='high_mem')   \n",
    "    py_x = softmax(h2)\n",
    "    return h, h2, py_x\n",
    "\n",
    "\n",
    "X = T.fmatrix()\n",
    "Y = T.fmatrix()\n",
    "\n",
    "batch_size = 60\n",
    "\n",
    "h1_size = 600\n",
    "h2_size = 550\n",
    "\n",
    "w_h = init_weights((len(vectorizer.vocabulary_), h1_size))\n",
    "b_h = theano.shared(floatX(np.zeros(h1_size)))\n",
    "g_h = theano.shared(floatX(np.ones((h1_size))))\n",
    "bb_h = theano.shared(floatX(np.zeros((h1_size))))\n",
    "\n",
    "w_h2 = init_weights((h1_size, h2_size))\n",
    "b_h2 = theano.shared(floatX(np.zeros(h2_size)))\n",
    "g_h2 = theano.shared(floatX(np.ones((h2_size))))\n",
    "bb_h2 = theano.shared(floatX(np.zeros((h2_size))))\n",
    "\n",
    "w_o = init_weights((h2_size, yTest.shape[1]))\n",
    "b_ho = theano.shared(floatX(np.zeros(yTest.shape[1])))\n",
    "g_ho = theano.shared(floatX(np.ones((yTest.shape[1]))))\n",
    "bb_ho = theano.shared(floatX(np.zeros((yTest.shape[1]))))\n",
    "\n",
    "noise_h, noise_h2, noise_py_x = model(X, w_h, b_h, g_h, bb_h, \n",
    "                                      w_h2, b_h2, g_h2, bb_h2, \n",
    "                                      w_o, b_ho, g_ho, bb_ho)\n",
    "\n",
    "h, h2, py_x = model(X, w_h, b_h, g_h, bb_h, \n",
    "                    w_h2, b_h2, g_h2, bb_h2, \n",
    "                    w_o, b_ho, g_ho, bb_ho)\n",
    "\n",
    "y_x = T.argmax(py_x, axis=1)\n",
    "\n",
    "\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n",
    "params = [w_h, b_h, g_h, bb_h, w_h2, b_h2, g_h2, bb_h2, w_o, b_ho, g_ho, bb_ho]\n",
    "updates = RMSprop(cost, params, lr=0.0001)\n",
    "\n",
    "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
    "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)\n",
    "\n",
    "\n",
    "for i in range(41):\n",
    "\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):\n",
    "        cost = train(xTrain[start:end], yTrain[start:end])\n",
    "        \n",
    "    xTrain, yTrain = shuffle(xTrain, yTrain)\n",
    "    xTest, yTest   = shuffle(xTest, yTest)\n",
    "\n",
    "    trr, tr = [], []\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):        \n",
    "        trr += [np.argmax(yTrain[start:end], axis=1) == predict(xTrain[start:end])]\n",
    "\n",
    "    for start, end in zip(range(0, len(xTest), batch_size), range(batch_size, len(xTest), batch_size)):\n",
    "        tr += [np.argmax(yTest[start:end], axis=1) == predict(xTest[start:end])]\n",
    "\n",
    "    print \"Round: %-5s Test: %-14s Train: %-8s\" % (i, np.mean(tr), np.mean(trr))\n",
    "    \n",
    "print"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "70% at round 6!\n",
    "\n",
    "\n",
    "73% at round 18. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0     Test: 0.540625       Train: 0.60119047619\n",
      "Round: 1     Test: 0.601041666667 Train: 0.66768707483\n",
      "Round: 2     Test: 0.647916666667 Train: 0.712414965986\n",
      "Round: 3     Test: 0.665104166667 Train: 0.743027210884\n",
      "Round: 4     Test: 0.680208333333 Train: 0.770918367347\n",
      "Round: 5     Test: 0.696875       Train: 0.787585034014\n",
      "Round: 6     Test: 0.701041666667 Train: 0.803401360544\n",
      "Round: 7     Test: 0.706770833333 Train: 0.821768707483\n",
      "Round: 8     Test: 0.707291666667 Train: 0.833163265306\n",
      "Round: 9     Test: 0.7109375      Train: 0.840646258503\n",
      "Round: 10    Test: 0.718229166667 Train: 0.853231292517\n",
      "Round: 11    Test: 0.719791666667 Train: 0.859013605442\n",
      "Round: 12    Test: 0.727604166667 Train: 0.87380952381\n",
      "Round: 13    Test: 0.719270833333 Train: 0.875170068027\n",
      "Round: 14    Test: 0.7265625      Train: 0.883163265306\n",
      "Round: 15    Test: 0.721875       Train: 0.887755102041\n",
      "Round: 16    Test: 0.7078125      Train: 0.894727891156\n",
      "Round: 17    Test: 0.7234375      Train: 0.898469387755\n",
      "Round: 18    Test: 0.721354166667 Train: 0.901870748299\n",
      "Round: 19    Test: 0.725          Train: 0.911054421769\n",
      "Round: 20    Test: 0.720833333333 Train: 0.910714285714\n",
      "Round: 21    Test: 0.719791666667 Train: 0.916326530612\n",
      "Round: 22    Test: 0.7046875      Train: 0.912585034014\n",
      "Round: 23    Test: 0.716666666667 Train: 0.919047619048\n",
      "Round: 24    Test: 0.708854166667 Train: 0.926020408163\n",
      "Round: 25    Test: 0.730729166667 Train: 0.926360544218\n",
      "Round: 26    Test: 0.723958333333 Train: 0.928401360544\n",
      "Round: 27    Test: 0.714583333333 Train: 0.927891156463\n",
      "Round: 28    Test: 0.715104166667 Train: 0.933333333333\n",
      "Round: 29    Test: 0.725520833333 Train: 0.933843537415\n",
      "Round: 30    Test: 0.722916666667 Train: 0.935544217687\n",
      "Round: 31    Test: 0.706770833333 Train: 0.936054421769\n",
      "Round: 32    Test: 0.7171875      Train: 0.938605442177\n",
      "Round: 33    Test: 0.722395833333 Train: 0.938265306122\n",
      "Round: 34    Test: 0.7125         Train: 0.939455782313\n",
      "Round: 35    Test: 0.7171875      Train: 0.938775510204\n",
      "Round: 36    Test: 0.713541666667 Train: 0.940816326531\n",
      "Round: 37    Test: 0.713541666667 Train: 0.939965986395\n",
      "Round: 38    Test: 0.70625        Train: 0.940646258503\n",
      "Round: 39    Test: 0.708854166667 Train: 0.941836734694\n",
      "Round: 40    Test: 0.713541666667 Train: 0.941496598639\n",
      "\n",
      "Wall time: 1min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "\"\"\" glorot 4-layer: batch, batch, batch \n",
    "    random indexing without replacement, RELU                \n",
    "    epsilon=1e-9  \"\"\"\n",
    "\n",
    "# batch normalization code adapted from \n",
    "# https://groups.google.com/forum/#!topic/theano-users/dMV6aabL1Ds \n",
    "\n",
    "\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
    "from theano.tensor.nnet.bn import batch_normalization\n",
    "import numpy as np\n",
    "\n",
    "srng = RandomStreams()\n",
    "\n",
    "def shuffle(x, y):\n",
    "    # helper function to shuffle indicies each loop \n",
    "    index = np.random.choice(len(x), len(x), replace=False)\n",
    "    return x[index], y[index]\n",
    "\n",
    "def floatX(X):\n",
    "    return np.asarray(X, dtype=theano.config.floatX)\n",
    "\n",
    "def init_weights(shape):\n",
    "    (h, w) = shape\n",
    "    # Glorot normalization - last factor depends on non-linearity\n",
    "    # 0.25 for sigmoid and 0.1 for softmax, 1.0 for tanh or Relu\n",
    "    normalizer = 2.0 * np.sqrt(6) / np.sqrt(h + w) * 1.0\n",
    "    return theano.shared(floatX((np.random.random_sample(shape) - 0.5) * normalizer))\n",
    "\n",
    "def rectify(X, alpha=0.01):\n",
    "#     return T.maximum(X, 0.)\n",
    "#    return T.maximum(X, 0.1*X)  #leaky rectifier\n",
    "     return T.switch(X > 0, X, alpha * (T.exp(X) - 1)) # ELU\n",
    "\n",
    "def softmax(X):\n",
    "    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n",
    "    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n",
    "\n",
    "def RMSprop(cost, params, lr=0.001, rho=0.99, epsilon=1e-9):\n",
    "    grads = T.grad(cost=cost, wrt=params)\n",
    "    updates = []\n",
    "    for p, g in zip(params, grads):\n",
    "        acc = theano.shared(p.get_value() * 0.)\n",
    "        acc_new = rho * acc + (1 - rho) * g ** 2\n",
    "        gradient_scaling = T.sqrt(acc_new + epsilon)\n",
    "        g = g / gradient_scaling\n",
    "        updates.append((acc, acc_new))\n",
    "        updates.append((p, p - lr * g))\n",
    "    return updates\n",
    "\n",
    "\n",
    "def model(X, w_h, b_h, g_h, bb_h, w_h2, b_h2, g_h2, bb_h2, w_o, b_ho, g_ho, bb_ho):\n",
    "    X = T.dot(X, w_h) + b_h\n",
    "    X = batch_normalization(X, gamma= g_h, beta= bb_h, \n",
    "                            mean= X.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(X.var((0,), keepdims = True)), \n",
    "                            mode='high_mem')    \n",
    "    h = rectify(X)\n",
    "\n",
    "    h  = T.dot(h, w_h2) + b_h2\n",
    "    h = batch_normalization(h, gamma= g_h2, beta= bb_h2, \n",
    "                            mean= h.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h.var((0,), keepdims = True)), \n",
    "                            mode='high_mem')       \n",
    "    h2 = rectify(h)\n",
    "\n",
    "    h2 = T.dot(h2, w_o) + b_ho\n",
    "    h2 = batch_normalization(h2, gamma= g_ho, beta= bb_ho, \n",
    "                            mean= h2.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h2.var((0,), keepdims = True)), \n",
    "                            mode='high_mem')   \n",
    "    py_x = softmax(h2)\n",
    "    return h, h2, py_x\n",
    "\n",
    "\n",
    "X = T.fmatrix()\n",
    "Y = T.fmatrix()\n",
    "\n",
    "batch_size = 60\n",
    "\n",
    "h1_size = 600\n",
    "h2_size = 550\n",
    "\n",
    "w_h = init_weights((len(vectorizer.vocabulary_), h1_size))\n",
    "b_h = theano.shared(floatX(np.zeros(h1_size)))\n",
    "g_h = theano.shared(floatX(np.ones((h1_size))))\n",
    "bb_h = theano.shared(floatX(np.zeros((h1_size))))\n",
    "\n",
    "w_h2 = init_weights((h1_size, h2_size))\n",
    "b_h2 = theano.shared(floatX(np.zeros(h2_size)))\n",
    "g_h2 = theano.shared(floatX(np.ones((h2_size))))\n",
    "bb_h2 = theano.shared(floatX(np.zeros((h2_size))))\n",
    "\n",
    "w_o = init_weights((h2_size, yTest.shape[1]))\n",
    "b_ho = theano.shared(floatX(np.zeros(yTest.shape[1])))\n",
    "g_ho = theano.shared(floatX(np.ones((yTest.shape[1]))))\n",
    "bb_ho = theano.shared(floatX(np.zeros((yTest.shape[1]))))\n",
    "\n",
    "noise_h, noise_h2, noise_py_x = model(X, w_h, b_h, g_h, bb_h, \n",
    "                                      w_h2, b_h2, g_h2, bb_h2, \n",
    "                                      w_o, b_ho, g_ho, bb_ho)\n",
    "\n",
    "h, h2, py_x = model(X, w_h, b_h, g_h, bb_h, \n",
    "                    w_h2, b_h2, g_h2, bb_h2, \n",
    "                    w_o, b_ho, g_ho, bb_ho)\n",
    "\n",
    "y_x = T.argmax(py_x, axis=1)\n",
    "\n",
    "\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n",
    "params = [w_h, b_h, g_h, bb_h, w_h2, b_h2, g_h2, bb_h2, w_o, b_ho, g_ho, bb_ho]\n",
    "updates = RMSprop(cost, params, lr=0.0001)\n",
    "\n",
    "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
    "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)\n",
    "\n",
    "\n",
    "for i in range(41):\n",
    "\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):\n",
    "        cost = train(xTrain[start:end], yTrain[start:end])\n",
    "        \n",
    "    xTrain, yTrain = shuffle(xTrain, yTrain)\n",
    "    xTest, yTest   = shuffle(xTest, yTest)\n",
    "\n",
    "    trr, tr = [], []\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):        \n",
    "        trr += [np.argmax(yTrain[start:end], axis=1) == predict(xTrain[start:end])]\n",
    "\n",
    "    for start, end in zip(range(0, len(xTest), batch_size), range(batch_size, len(xTest), batch_size)):\n",
    "        tr += [np.argmax(yTest[start:end], axis=1) == predict(xTest[start:end])]\n",
    "\n",
    "    print \"Round: %-5s Test: %-14s Train: %-8s\" % (i, np.mean(tr), np.mean(trr))\n",
    "    \n",
    "print"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Will adding back dropout get us back to 74%?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0     Test: 0.515625       Train: 0.56462585034\n",
      "Round: 1     Test: 0.569791666667 Train: 0.614965986395\n",
      "Round: 2     Test: 0.609375       Train: 0.666156462585\n",
      "Round: 3     Test: 0.628125       Train: 0.702380952381\n",
      "Round: 4     Test: 0.646875       Train: 0.725170068027\n",
      "Round: 5     Test: 0.669270833333 Train: 0.74574829932\n",
      "Round: 6     Test: 0.678645833333 Train: 0.764115646259\n",
      "Round: 7     Test: 0.686458333333 Train: 0.775680272109\n",
      "Round: 8     Test: 0.693229166667 Train: 0.791836734694\n",
      "Round: 9     Test: 0.7015625      Train: 0.803401360544\n",
      "Round: 10    Test: 0.714583333333 Train: 0.809013605442\n",
      "Round: 11    Test: 0.7125         Train: 0.822619047619\n",
      "Round: 12    Test: 0.715104166667 Train: 0.829421768707\n",
      "Round: 13    Test: 0.716145833333 Train: 0.83537414966\n",
      "Round: 14    Test: 0.721354166667 Train: 0.838775510204\n",
      "Round: 15    Test: 0.713020833333 Train: 0.852551020408\n",
      "Round: 16    Test: 0.71875        Train: 0.853231292517\n",
      "Round: 17    Test: 0.721354166667 Train: 0.860204081633\n",
      "Round: 18    Test: 0.721354166667 Train: 0.865986394558\n",
      "Round: 19    Test: 0.725520833333 Train: 0.867176870748\n",
      "Round: 20    Test: 0.736979166667 Train: 0.873299319728\n",
      "Round: 21    Test: 0.734375       Train: 0.875170068027\n",
      "Round: 22    Test: 0.725520833333 Train: 0.880272108844\n",
      "Round: 23    Test: 0.721875       Train: 0.883163265306\n",
      "Round: 24    Test: 0.717708333333 Train: 0.886224489796\n",
      "Round: 25    Test: 0.733333333333 Train: 0.893027210884\n",
      "Round: 26    Test: 0.73125        Train: 0.89693877551\n",
      "Round: 27    Test: 0.727604166667 Train: 0.894727891156\n",
      "Round: 28    Test: 0.738020833333 Train: 0.901020408163\n",
      "Round: 29    Test: 0.730208333333 Train: 0.90425170068\n",
      "Round: 30    Test: 0.738541666667 Train: 0.901530612245\n",
      "Round: 31    Test: 0.7328125      Train: 0.90612244898\n",
      "Round: 32    Test: 0.729166666667 Train: 0.906972789116\n",
      "Round: 33    Test: 0.730208333333 Train: 0.908503401361\n",
      "Round: 34    Test: 0.725520833333 Train: 0.917346938776\n",
      "Round: 35    Test: 0.7359375      Train: 0.916496598639\n",
      "Round: 36    Test: 0.736979166667 Train: 0.915136054422\n",
      "Round: 37    Test: 0.7265625      Train: 0.917857142857\n",
      "Round: 38    Test: 0.7328125      Train: 0.919727891156\n",
      "Round: 39    Test: 0.727604166667 Train: 0.919047619048\n",
      "Round: 40    Test: 0.727604166667 Train: 0.921768707483\n",
      "Round: 41    Test: 0.729166666667 Train: 0.92074829932\n",
      "Round: 42    Test: 0.732291666667 Train: 0.921428571429\n",
      "Round: 43    Test: 0.732291666667 Train: 0.924319727891\n",
      "Round: 44    Test: 0.7375         Train: 0.92619047619\n",
      "Round: 45    Test: 0.739583333333 Train: 0.926020408163\n",
      "Round: 46    Test: 0.734895833333 Train: 0.926700680272\n",
      "Round: 47    Test: 0.731770833333 Train: 0.925680272109\n",
      "Round: 48    Test: 0.7296875      Train: 0.93112244898\n",
      "Round: 49    Test: 0.727604166667 Train: 0.931972789116\n",
      "Round: 50    Test: 0.7328125      Train: 0.932142857143\n",
      "\n",
      "Wall time: 1min 25s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "\"\"\" glorot 4-layer: batch, drop, batch, drop, batch drop \n",
    "    random indexing without replacement, ELU, epsilon=1e-9  \"\"\"\n",
    "\n",
    "# batch normalization code adapted from \n",
    "# https://groups.google.com/forum/#!topic/theano-users/dMV6aabL1Ds \n",
    "\n",
    "\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
    "from theano.tensor.nnet.bn import batch_normalization\n",
    "import numpy as np\n",
    "\n",
    "srng = RandomStreams()\n",
    "\n",
    "def shuffle(x, y):\n",
    "    # helper function to shuffle indicies each loop \n",
    "    index = np.random.choice(len(x), len(x), replace=False)\n",
    "    return x[index], y[index]\n",
    "\n",
    "def floatX(X):\n",
    "    return np.asarray(X, dtype=theano.config.floatX)\n",
    "\n",
    "def init_weights(shape):\n",
    "    (h, w) = shape\n",
    "    # Glorot normalization - last factor depends on non-linearity\n",
    "    # 0.25 for sigmoid and 0.1 for softmax, 1.0 for tanh or Relu\n",
    "    normalizer = 2.0 * np.sqrt(6) / np.sqrt(h + w) * 1.0\n",
    "    return theano.shared(floatX((np.random.random_sample(shape) - 0.5) * normalizer))\n",
    "\n",
    "def rectify(X, alpha=0.01):\n",
    "#     return T.maximum(X, 0.)\n",
    "#    return T.maximum(X, 0.1*X)  #leaky rectifier\n",
    "     return T.switch(X > 0, X, alpha * (T.exp(X) - 1)) # ELU\n",
    "\n",
    "def softmax(X):\n",
    "    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n",
    "    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n",
    "\n",
    "def RMSprop(cost, params, lr=0.001, rho=0.99, epsilon=1e-9):\n",
    "    grads = T.grad(cost=cost, wrt=params)\n",
    "    updates = []\n",
    "    for p, g in zip(params, grads):\n",
    "        acc = theano.shared(p.get_value() * 0.)\n",
    "        acc_new = rho * acc + (1 - rho) * g ** 2\n",
    "        gradient_scaling = T.sqrt(acc_new + epsilon)\n",
    "        g = g / gradient_scaling\n",
    "        updates.append((acc, acc_new))\n",
    "        updates.append((p, p - lr * g))\n",
    "    return updates\n",
    "\n",
    "def dropout(X, p=0.):\n",
    "    if p > 0:\n",
    "        retain_prob = 1 - p\n",
    "        X *= srng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX)\n",
    "        X /= retain_prob\n",
    "    return X\n",
    "\n",
    "\n",
    "def model(X, w_h, b_h, g_h, bb_h, w_h2, b_h2, g_h2, bb_h2, \n",
    "          w_o, b_ho, g_ho, bb_ho, p_drop_input, p_drop_hidden):\n",
    "    X = T.dot(X, w_h) + b_h\n",
    "    X = batch_normalization(X, gamma= g_h, beta= bb_h, \n",
    "                            mean= X.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(X.var((0,), keepdims = True)), \n",
    "                            mode='high_mem') \n",
    "    X = dropout(X, p_drop_hidden)\n",
    "    h = rectify(X)\n",
    "\n",
    "    h  = T.dot(h, w_h2) + b_h2\n",
    "    h = batch_normalization(h, gamma= g_h2, beta= bb_h2, \n",
    "                            mean= h.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h.var((0,), keepdims = True)), \n",
    "                            mode='high_mem') \n",
    "    h = dropout(h, p_drop_hidden)\n",
    "    h2 = rectify(h)\n",
    "\n",
    "    h2 = T.dot(h2, w_o) + b_ho\n",
    "    h2 = batch_normalization(h2, gamma= g_ho, beta= bb_ho, \n",
    "                            mean= h2.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h2.var((0,), keepdims = True)), \n",
    "                            mode='high_mem') \n",
    "    h2 = dropout(h2, p_drop_hidden)\n",
    "    py_x = softmax(h2)\n",
    "    return h, h2, py_x\n",
    "\n",
    "\n",
    "X = T.fmatrix()\n",
    "Y = T.fmatrix()\n",
    "\n",
    "batch_size = 60\n",
    "\n",
    "h1_size = 600\n",
    "h2_size = 550\n",
    "\n",
    "w_h = init_weights((len(vectorizer.vocabulary_), h1_size))\n",
    "b_h = theano.shared(floatX(np.zeros(h1_size)))\n",
    "g_h = theano.shared(floatX(np.ones((h1_size))))\n",
    "bb_h = theano.shared(floatX(np.zeros((h1_size))))\n",
    "\n",
    "w_h2 = init_weights((h1_size, h2_size))\n",
    "b_h2 = theano.shared(floatX(np.zeros(h2_size)))\n",
    "g_h2 = theano.shared(floatX(np.ones((h2_size))))\n",
    "bb_h2 = theano.shared(floatX(np.zeros((h2_size))))\n",
    "\n",
    "w_o = init_weights((h2_size, yTest.shape[1]))\n",
    "b_ho = theano.shared(floatX(np.zeros(yTest.shape[1])))\n",
    "g_ho = theano.shared(floatX(np.ones((yTest.shape[1]))))\n",
    "bb_ho = theano.shared(floatX(np.zeros((yTest.shape[1]))))\n",
    "\n",
    "noise_h, noise_h2, noise_py_x = model(X, w_h, b_h, g_h, bb_h, \n",
    "                                      w_h2, b_h2, g_h2, bb_h2, \n",
    "                                      w_o, b_ho, g_ho, bb_ho, .0, .2)\n",
    "\n",
    "h, h2, py_x = model(X, w_h, b_h, g_h, bb_h, \n",
    "                    w_h2, b_h2, g_h2, bb_h2, \n",
    "                    w_o, b_ho, g_ho, bb_ho, .0, .0)\n",
    "\n",
    "y_x = T.argmax(py_x, axis=1)\n",
    "\n",
    "\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n",
    "params = [w_h, b_h, g_h, bb_h, w_h2, b_h2, g_h2, bb_h2, \n",
    "          w_o, b_ho, g_ho, bb_ho]\n",
    "updates = RMSprop(cost, params, lr=0.0001)\n",
    "\n",
    "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
    "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)\n",
    "\n",
    "\n",
    "for i in range(51):\n",
    "\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):\n",
    "        cost = train(xTrain[start:end], yTrain[start:end])\n",
    "        \n",
    "    xTrain, yTrain = shuffle(xTrain, yTrain)\n",
    "    xTest, yTest   = shuffle(xTest, yTest)\n",
    "\n",
    "    trr, tr = [], []\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):        \n",
    "        trr += [np.argmax(yTrain[start:end], axis=1) == predict(xTrain[start:end])]\n",
    "\n",
    "    for start, end in zip(range(0, len(xTest), batch_size), range(batch_size, len(xTest), batch_size)):\n",
    "        tr += [np.argmax(yTest[start:end], axis=1) == predict(xTest[start:end])]\n",
    "\n",
    "    print \"Round: %-5s Test: %-14s Train: %-8s\" % (i, np.mean(tr), np.mean(trr))\n",
    "    \n",
    "print\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### High of 73.9% with just a little dropout (0.0, 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0     Test: 0.533333333333 Train: 0.571088435374\n",
      "Round: 1     Test: 0.576041666667 Train: 0.620238095238\n",
      "Round: 2     Test: 0.610416666667 Train: 0.666496598639\n",
      "Round: 3     Test: 0.631770833333 Train: 0.695578231293\n",
      "Round: 4     Test: 0.663020833333 Train: 0.724319727891\n",
      "Round: 5     Test: 0.670833333333 Train: 0.741836734694\n",
      "Round: 6     Test: 0.684895833333 Train: 0.76156462585\n",
      "Round: 7     Test: 0.685416666667 Train: 0.779081632653\n",
      "Round: 8     Test: 0.6859375      Train: 0.791326530612\n",
      "Round: 9     Test: 0.697395833333 Train: 0.799659863946\n",
      "Round: 10    Test: 0.703125       Train: 0.809183673469\n",
      "Round: 11    Test: 0.7125         Train: 0.819387755102\n",
      "Round: 12    Test: 0.70625        Train: 0.82925170068\n",
      "Round: 13    Test: 0.7078125      Train: 0.834353741497\n",
      "Round: 14    Test: 0.721875       Train: 0.842857142857\n",
      "Round: 15    Test: 0.719791666667 Train: 0.852380952381\n",
      "Round: 16    Test: 0.7234375      Train: 0.85425170068\n",
      "Round: 17    Test: 0.7171875      Train: 0.857142857143\n",
      "Round: 18    Test: 0.720833333333 Train: 0.863605442177\n",
      "Round: 19    Test: 0.721875       Train: 0.873469387755\n",
      "Round: 20    Test: 0.727604166667 Train: 0.868027210884\n",
      "Round: 21    Test: 0.728645833333 Train: 0.879081632653\n",
      "Round: 22    Test: 0.723958333333 Train: 0.881632653061\n",
      "Round: 23    Test: 0.726041666667 Train: 0.886054421769\n",
      "Round: 24    Test: 0.7265625      Train: 0.888775510204\n",
      "Round: 25    Test: 0.730208333333 Train: 0.889965986395\n",
      "Round: 26    Test: 0.721354166667 Train: 0.895408163265\n",
      "Round: 27    Test: 0.729166666667 Train: 0.896088435374\n",
      "Round: 28    Test: 0.729166666667 Train: 0.899489795918\n",
      "Round: 29    Test: 0.720833333333 Train: 0.903401360544\n",
      "Round: 30    Test: 0.7234375      Train: 0.905102040816\n",
      "Round: 31    Test: 0.727604166667 Train: 0.90612244898\n",
      "Round: 32    Test: 0.736458333333 Train: 0.911734693878\n",
      "Round: 33    Test: 0.725          Train: 0.906972789116\n",
      "Round: 34    Test: 0.731770833333 Train: 0.912925170068\n",
      "Round: 35    Test: 0.715625       Train: 0.91462585034\n",
      "Round: 36    Test: 0.727604166667 Train: 0.914285714286\n",
      "Round: 37    Test: 0.729166666667 Train: 0.922278911565\n",
      "Round: 38    Test: 0.7234375      Train: 0.919387755102\n",
      "Round: 39    Test: 0.732291666667 Train: 0.923469387755\n",
      "Round: 40    Test: 0.732291666667 Train: 0.924489795918\n",
      "Round: 41    Test: 0.727083333333 Train: 0.920918367347\n",
      "Round: 42    Test: 0.722395833333 Train: 0.925850340136\n",
      "Round: 43    Test: 0.725          Train: 0.926870748299\n",
      "Round: 44    Test: 0.722916666667 Train: 0.928231292517\n",
      "Round: 45    Test: 0.720833333333 Train: 0.928911564626\n",
      "Round: 46    Test: 0.7234375      Train: 0.928571428571\n",
      "Round: 47    Test: 0.729166666667 Train: 0.930442176871\n",
      "Round: 48    Test: 0.727604166667 Train: 0.927891156463\n",
      "Round: 49    Test: 0.727083333333 Train: 0.929931972789\n",
      "Round: 50    Test: 0.727083333333 Train: 0.927380952381\n",
      "\n",
      "Wall time: 1min 28s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "\"\"\" glorot 4-layer: batch, drop, batch, drop, batch drop \n",
    "    random indexing without replacement, ELU, epsilon=1e-9  \"\"\"\n",
    "\n",
    "# batch normalization code adapted from \n",
    "# https://groups.google.com/forum/#!topic/theano-users/dMV6aabL1Ds \n",
    "\n",
    "\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
    "from theano.tensor.nnet.bn import batch_normalization\n",
    "import numpy as np\n",
    "\n",
    "srng = RandomStreams()\n",
    "\n",
    "def shuffle(x, y):\n",
    "    # helper function to shuffle indicies each loop \n",
    "    index = np.random.choice(len(x), len(x), replace=False)\n",
    "    return x[index], y[index]\n",
    "\n",
    "def floatX(X):\n",
    "    return np.asarray(X, dtype=theano.config.floatX)\n",
    "\n",
    "def init_weights(shape):\n",
    "    (h, w) = shape\n",
    "    # Glorot normalization - last factor depends on non-linearity\n",
    "    # 0.25 for sigmoid and 0.1 for softmax, 1.0 for tanh or Relu\n",
    "    normalizer = 2.0 * np.sqrt(6) / np.sqrt(h + w) * 1.0\n",
    "    return theano.shared(floatX((np.random.random_sample(shape) - 0.5) * normalizer))\n",
    "\n",
    "def rectify(X, alpha=0.01):\n",
    "#     return T.maximum(X, 0.)\n",
    "#    return T.maximum(X, 0.1*X)  #leaky rectifier\n",
    "     return T.switch(X > 0, X, alpha * (T.exp(X) - 1)) # ELU\n",
    "\n",
    "def softmax(X):\n",
    "    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n",
    "    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n",
    "\n",
    "def RMSprop(cost, params, lr=0.001, rho=0.99, epsilon=1e-9):\n",
    "    grads = T.grad(cost=cost, wrt=params)\n",
    "    updates = []\n",
    "    for p, g in zip(params, grads):\n",
    "        acc = theano.shared(p.get_value() * 0.)\n",
    "        acc_new = rho * acc + (1 - rho) * g ** 2\n",
    "        gradient_scaling = T.sqrt(acc_new + epsilon)\n",
    "        g = g / gradient_scaling\n",
    "        updates.append((acc, acc_new))\n",
    "        updates.append((p, p - lr * g))\n",
    "    return updates\n",
    "\n",
    "def dropout(X, p=0.):\n",
    "    if p > 0:\n",
    "        retain_prob = 1 - p\n",
    "        X *= srng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX)\n",
    "        X /= retain_prob\n",
    "    return X\n",
    "\n",
    "\n",
    "def model(X, w_h, b_h, g_h, bb_h, w_h2, b_h2, g_h2, bb_h2, \n",
    "          w_o, b_ho, g_ho, bb_ho, p_drop_input, p_drop_hidden):\n",
    "    X = T.dot(X, w_h) + b_h\n",
    "    X = batch_normalization(X, gamma= g_h, beta= bb_h, \n",
    "                            mean= X.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(X.var((0,), keepdims = True)), \n",
    "                            mode='high_mem') \n",
    "    X = dropout(X, p_drop_hidden)\n",
    "    h = rectify(X)\n",
    "\n",
    "    h  = T.dot(h, w_h2) + b_h2\n",
    "    h = batch_normalization(h, gamma= g_h2, beta= bb_h2, \n",
    "                            mean= h.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h.var((0,), keepdims = True)), \n",
    "                            mode='high_mem') \n",
    "    h = dropout(h, p_drop_hidden)\n",
    "    h2 = rectify(h)\n",
    "\n",
    "    h2 = T.dot(h2, w_o) + b_ho\n",
    "    h2 = batch_normalization(h2, gamma= g_ho, beta= bb_ho, \n",
    "                            mean= h2.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h2.var((0,), keepdims = True)), \n",
    "                            mode='high_mem') \n",
    "    h2 = dropout(h2, p_drop_hidden)\n",
    "    py_x = softmax(h2)\n",
    "    return h, h2, py_x\n",
    "\n",
    "\n",
    "X = T.fmatrix()\n",
    "Y = T.fmatrix()\n",
    "\n",
    "batch_size = 60\n",
    "\n",
    "h1_size = 600\n",
    "h2_size = 550\n",
    "\n",
    "w_h = init_weights((len(vectorizer.vocabulary_), h1_size))\n",
    "b_h = theano.shared(floatX(np.zeros(h1_size)))\n",
    "g_h = theano.shared(floatX(np.ones((h1_size))))\n",
    "bb_h = theano.shared(floatX(np.zeros((h1_size))))\n",
    "\n",
    "w_h2 = init_weights((h1_size, h2_size))\n",
    "b_h2 = theano.shared(floatX(np.zeros(h2_size)))\n",
    "g_h2 = theano.shared(floatX(np.ones((h2_size))))\n",
    "bb_h2 = theano.shared(floatX(np.zeros((h2_size))))\n",
    "\n",
    "w_o = init_weights((h2_size, yTest.shape[1]))\n",
    "b_ho = theano.shared(floatX(np.zeros(yTest.shape[1])))\n",
    "g_ho = theano.shared(floatX(np.ones((yTest.shape[1]))))\n",
    "bb_ho = theano.shared(floatX(np.zeros((yTest.shape[1]))))\n",
    "\n",
    "noise_h, noise_h2, noise_py_x = model(X, w_h, b_h, g_h, bb_h, \n",
    "                                      w_h2, b_h2, g_h2, bb_h2, \n",
    "                                      w_o, b_ho, g_ho, bb_ho, .0, .2)\n",
    "\n",
    "h, h2, py_x = model(X, w_h, b_h, g_h, bb_h, \n",
    "                    w_h2, b_h2, g_h2, bb_h2, \n",
    "                    w_o, b_ho, g_ho, bb_ho, .0, .0)\n",
    "\n",
    "y_x = T.argmax(py_x, axis=1)\n",
    "\n",
    "\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n",
    "params = [w_h, b_h, g_h, bb_h, w_h2, b_h2, g_h2, bb_h2, \n",
    "          w_o, b_ho, g_ho, bb_ho]\n",
    "updates = RMSprop(cost, params, lr=0.0001)\n",
    "\n",
    "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
    "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)\n",
    "\n",
    "\n",
    "for i in range(51):\n",
    "\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):\n",
    "        cost = train(xTrain[start:end], yTrain[start:end])\n",
    "        \n",
    "    xTrain, yTrain = shuffle(xTrain, yTrain)\n",
    "    xTest, yTest   = shuffle(xTest, yTest)\n",
    "\n",
    "    trr, tr = [], []\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):        \n",
    "        trr += [np.argmax(yTrain[start:end], axis=1) == predict(xTrain[start:end])]\n",
    "\n",
    "    for start, end in zip(range(0, len(xTest), batch_size), range(batch_size, len(xTest), batch_size)):\n",
    "        tr += [np.argmax(yTest[start:end], axis=1) == predict(xTest[start:end])]\n",
    "\n",
    "    print \"Round: %-5s Test: %-14s Train: %-8s\" % (i, np.mean(tr), np.mean(trr))\n",
    "    \n",
    "print\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0     Test: 0.5296875      Train: 0.56768707483\n",
      "Round: 1     Test: 0.5734375      Train: 0.618537414966\n",
      "Round: 2     Test: 0.6078125      Train: 0.659183673469\n",
      "Round: 3     Test: 0.628125       Train: 0.693367346939\n",
      "Round: 4     Test: 0.652604166667 Train: 0.726360544218\n",
      "Round: 5     Test: 0.665625       Train: 0.744727891156\n",
      "Round: 6     Test: 0.675520833333 Train: 0.763605442177\n",
      "Round: 7     Test: 0.677604166667 Train: 0.773639455782\n",
      "Round: 8     Test: 0.6890625      Train: 0.78843537415\n",
      "Round: 9     Test: 0.693229166667 Train: 0.794557823129\n",
      "Round: 10    Test: 0.715104166667 Train: 0.808333333333\n",
      "Round: 11    Test: 0.7109375      Train: 0.818367346939\n",
      "Round: 12    Test: 0.711458333333 Train: 0.827551020408\n",
      "Round: 13    Test: 0.713020833333 Train: 0.837414965986\n",
      "Round: 14    Test: 0.716145833333 Train: 0.839285714286\n",
      "Round: 15    Test: 0.7203125      Train: 0.848639455782\n",
      "Round: 16    Test: 0.720833333333 Train: 0.855272108844\n",
      "Round: 17    Test: 0.736979166667 Train: 0.855782312925\n",
      "Round: 18    Test: 0.728125       Train: 0.864795918367\n",
      "Round: 19    Test: 0.730208333333 Train: 0.871428571429\n",
      "Round: 20    Test: 0.7296875      Train: 0.874829931973\n",
      "Round: 21    Test: 0.740625       Train: 0.879421768707\n",
      "Round: 22    Test: 0.726041666667 Train: 0.877721088435\n",
      "Round: 23    Test: 0.7296875      Train: 0.882482993197\n",
      "Round: 24    Test: 0.719270833333 Train: 0.886734693878\n",
      "Round: 25    Test: 0.73125        Train: 0.889795918367\n",
      "Round: 26    Test: 0.721875       Train: 0.896088435374\n",
      "Round: 27    Test: 0.715104166667 Train: 0.897448979592\n",
      "Round: 28    Test: 0.7265625      Train: 0.903571428571\n",
      "Round: 29    Test: 0.719791666667 Train: 0.904761904762\n",
      "Round: 30    Test: 0.73125        Train: 0.901020408163\n",
      "Round: 31    Test: 0.727604166667 Train: 0.903911564626\n",
      "Round: 32    Test: 0.727604166667 Train: 0.909353741497\n",
      "Round: 33    Test: 0.723958333333 Train: 0.909353741497\n",
      "Round: 34    Test: 0.724479166667 Train: 0.911394557823\n",
      "Round: 35    Test: 0.719270833333 Train: 0.914795918367\n",
      "Round: 36    Test: 0.7296875      Train: 0.917006802721\n",
      "Round: 37    Test: 0.725          Train: 0.92074829932\n",
      "Round: 38    Test: 0.727083333333 Train: 0.92380952381\n",
      "Round: 39    Test: 0.719270833333 Train: 0.923129251701\n",
      "Round: 40    Test: 0.721875       Train: 0.924489795918\n",
      "Round: 41    Test: 0.73125        Train: 0.922448979592\n",
      "Round: 42    Test: 0.736979166667 Train: 0.921428571429\n",
      "Round: 43    Test: 0.727604166667 Train: 0.927891156463\n",
      "Round: 44    Test: 0.73125        Train: 0.925510204082\n",
      "Round: 45    Test: 0.733333333333 Train: 0.931632653061\n",
      "Round: 46    Test: 0.725520833333 Train: 0.929931972789\n",
      "Round: 47    Test: 0.727083333333 Train: 0.933503401361\n",
      "Round: 48    Test: 0.727604166667 Train: 0.930272108844\n",
      "Round: 49    Test: 0.721354166667 Train: 0.932823129252\n",
      "Round: 50    Test: 0.733333333333 Train: 0.93231292517\n",
      "\n",
      "CPU times: user 4min 49s, sys: 15.3 s, total: 5min 4s\n",
      "Wall time: 3min 23s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "\"\"\" glorot 4-layer: batch, drop, batch, drop, batch drop \n",
    "    random indexing without replacement, ELU, epsilon=1e-9  \"\"\"\n",
    "\n",
    "# batch normalization code adapted from \n",
    "# https://groups.google.com/forum/#!topic/theano-users/dMV6aabL1Ds \n",
    "\n",
    "\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
    "from theano.tensor.nnet.bn import batch_normalization\n",
    "import numpy as np\n",
    "\n",
    "srng = RandomStreams()\n",
    "\n",
    "def shuffle(x, y):\n",
    "    # helper function to shuffle indicies each loop \n",
    "    index = np.random.choice(len(x), len(x), replace=False)\n",
    "    return x[index], y[index]\n",
    "\n",
    "def floatX(X):\n",
    "    return np.asarray(X, dtype=theano.config.floatX)\n",
    "\n",
    "def init_weights(shape):\n",
    "    (h, w) = shape\n",
    "    # Glorot normalization - last factor depends on non-linearity\n",
    "    # 0.25 for sigmoid and 0.1 for softmax, 1.0 for tanh or Relu\n",
    "    normalizer = 2.0 * np.sqrt(6) / np.sqrt(h + w) * 1.0\n",
    "    return theano.shared(floatX((np.random.random_sample(shape) - 0.5) * normalizer))\n",
    "\n",
    "def rectify(X, alpha=0.01):\n",
    "#     return T.maximum(X, 0.)\n",
    "#    return T.maximum(X, 0.1*X)  #leaky rectifier\n",
    "     return T.switch(X > 0, X, alpha * (T.exp(X) - 1)) # ELU\n",
    "\n",
    "def softmax(X):\n",
    "    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n",
    "    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n",
    "\n",
    "def RMSprop(cost, params, lr=0.001, rho=0.99, epsilon=1e-9):\n",
    "    grads = T.grad(cost=cost, wrt=params)\n",
    "    updates = []\n",
    "    for p, g in zip(params, grads):\n",
    "        acc = theano.shared(p.get_value() * 0.)\n",
    "        acc_new = rho * acc + (1 - rho) * g ** 2\n",
    "        gradient_scaling = T.sqrt(acc_new + epsilon)\n",
    "        g = g / gradient_scaling\n",
    "        updates.append((acc, acc_new))\n",
    "        updates.append((p, p - lr * g))\n",
    "    return updates\n",
    "\n",
    "def dropout(X, p=0.):\n",
    "    if p > 0:\n",
    "        retain_prob = 1 - p\n",
    "        X *= srng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX)\n",
    "        X /= retain_prob\n",
    "    return X\n",
    "\n",
    "\n",
    "def model(X, w_h, b_h, g_h, bb_h, w_h2, b_h2, g_h2, bb_h2, \n",
    "          w_o, b_ho, g_ho, bb_ho, p_drop_input, p_drop_hidden):\n",
    "    X = T.dot(X, w_h) + b_h\n",
    "    X = batch_normalization(X, gamma= g_h, beta= bb_h, \n",
    "                            mean= X.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(X.var((0,), keepdims = True)), \n",
    "                            mode='high_mem') \n",
    "    X = dropout(X, p_drop_hidden)\n",
    "    h = rectify(X)\n",
    "\n",
    "    h  = T.dot(h, w_h2) + b_h2\n",
    "    h = batch_normalization(h, gamma= g_h2, beta= bb_h2, \n",
    "                            mean= h.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h.var((0,), keepdims = True)), \n",
    "                            mode='high_mem') \n",
    "    h = dropout(h, p_drop_hidden)\n",
    "    h2 = rectify(h)\n",
    "\n",
    "    h2 = T.dot(h2, w_o) + b_ho\n",
    "    h2 = batch_normalization(h2, gamma= g_ho, beta= bb_ho, \n",
    "                            mean= h2.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h2.var((0,), keepdims = True)), \n",
    "                            mode='high_mem') \n",
    "    h2 = dropout(h2, p_drop_hidden)\n",
    "    py_x = softmax(h2)\n",
    "    return h, h2, py_x\n",
    "\n",
    "\n",
    "X = T.fmatrix()\n",
    "Y = T.fmatrix()\n",
    "\n",
    "batch_size = 60\n",
    "\n",
    "h1_size = 600\n",
    "h2_size = 550\n",
    "\n",
    "w_h = init_weights((len(vectorizer.vocabulary_), h1_size))\n",
    "b_h = theano.shared(floatX(np.zeros(h1_size)))\n",
    "g_h = theano.shared(floatX(np.ones((h1_size))))\n",
    "bb_h = theano.shared(floatX(np.zeros((h1_size))))\n",
    "\n",
    "w_h2 = init_weights((h1_size, h2_size))\n",
    "b_h2 = theano.shared(floatX(np.zeros(h2_size)))\n",
    "g_h2 = theano.shared(floatX(np.ones((h2_size))))\n",
    "bb_h2 = theano.shared(floatX(np.zeros((h2_size))))\n",
    "\n",
    "w_o = init_weights((h2_size, yTest.shape[1]))\n",
    "b_ho = theano.shared(floatX(np.zeros(yTest.shape[1])))\n",
    "g_ho = theano.shared(floatX(np.ones((yTest.shape[1]))))\n",
    "bb_ho = theano.shared(floatX(np.zeros((yTest.shape[1]))))\n",
    "\n",
    "noise_h, noise_h2, noise_py_x = model(X, w_h, b_h, g_h, bb_h, \n",
    "                                      w_h2, b_h2, g_h2, bb_h2, \n",
    "                                      w_o, b_ho, g_ho, bb_ho, .0, .2)\n",
    "\n",
    "h, h2, py_x = model(X, w_h, b_h, g_h, bb_h, \n",
    "                    w_h2, b_h2, g_h2, bb_h2, \n",
    "                    w_o, b_ho, g_ho, bb_ho, .0, .0)\n",
    "\n",
    "y_x = T.argmax(py_x, axis=1)\n",
    "\n",
    "\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n",
    "params = [w_h, b_h, g_h, bb_h, w_h2, b_h2, g_h2, bb_h2, \n",
    "          w_o, b_ho, g_ho, bb_ho]\n",
    "updates = RMSprop(cost, params, lr=0.0001)\n",
    "\n",
    "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
    "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)\n",
    "\n",
    "\n",
    "for i in range(51):\n",
    "\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):\n",
    "        cost = train(xTrain[start:end], yTrain[start:end])\n",
    "        \n",
    "    xTrain, yTrain = shuffle(xTrain, yTrain)\n",
    "    xTest, yTest   = shuffle(xTest, yTest)\n",
    "\n",
    "    trr, tr = [], []\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):        \n",
    "        trr += [np.argmax(yTrain[start:end], axis=1) == predict(xTrain[start:end])]\n",
    "\n",
    "    for start, end in zip(range(0, len(xTest), batch_size), range(batch_size, len(xTest), batch_size)):\n",
    "        tr += [np.argmax(yTest[start:end], axis=1) == predict(xTest[start:end])]\n",
    "\n",
    "    print \"Round: %-5s Test: %-14s Train: %-8s\" % (i, np.mean(tr), np.mean(trr))\n",
    "    \n",
    "print\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0     Test: 0.544791666667 Train: 0.605782312925\n",
      "Round: 1     Test: 0.6046875      Train: 0.66768707483\n",
      "Round: 2     Test: 0.635416666667 Train: 0.71037414966\n",
      "Round: 3     Test: 0.665625       Train: 0.741666666667\n",
      "Round: 4     Test: 0.6734375      Train: 0.767517006803\n",
      "Round: 5     Test: 0.690104166667 Train: 0.790476190476\n",
      "Round: 6     Test: 0.698958333333 Train: 0.804081632653\n",
      "Round: 7     Test: 0.706770833333 Train: 0.822108843537\n",
      "Round: 8     Test: 0.697395833333 Train: 0.828571428571\n",
      "Round: 9     Test: 0.715104166667 Train: 0.844387755102\n",
      "Round: 10    Test: 0.716666666667 Train: 0.848979591837\n",
      "Round: 11    Test: 0.715625       Train: 0.857993197279\n",
      "Round: 12    Test: 0.723958333333 Train: 0.862755102041\n",
      "Round: 13    Test: 0.725520833333 Train: 0.875170068027\n",
      "Round: 14    Test: 0.725          Train: 0.881972789116\n",
      "Round: 15    Test: 0.721875       Train: 0.889965986395\n",
      "Round: 16    Test: 0.729166666667 Train: 0.895238095238\n",
      "Round: 17    Test: 0.717708333333 Train: 0.898299319728\n",
      "Round: 18    Test: 0.730729166667 Train: 0.905442176871\n",
      "Round: 19    Test: 0.73125        Train: 0.908673469388\n",
      "Round: 20    Test: 0.7171875      Train: 0.911734693878\n",
      "Round: 21    Test: 0.722916666667 Train: 0.913095238095\n",
      "Round: 22    Test: 0.728125       Train: 0.921768707483\n",
      "Round: 23    Test: 0.727083333333 Train: 0.921428571429\n",
      "Round: 24    Test: 0.7328125      Train: 0.920918367347\n",
      "Round: 25    Test: 0.728645833333 Train: 0.919557823129\n",
      "Round: 26    Test: 0.730208333333 Train: 0.925340136054\n",
      "Round: 27    Test: 0.717708333333 Train: 0.931802721088\n",
      "Round: 28    Test: 0.71875        Train: 0.925340136054\n",
      "Round: 29    Test: 0.726041666667 Train: 0.933163265306\n",
      "\n",
      "CPU times: user 2min 44s, sys: 6.94 s, total: 2min 51s\n",
      "Wall time: 1min 45s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "\"\"\" glorot 4-layer: batch, drop, batch, drop, batch drop \n",
    "    random indexing without replacement, ELU, epsilon=1e-9  \"\"\"\n",
    "\n",
    "# batch normalization code adapted from \n",
    "# https://groups.google.com/forum/#!topic/theano-users/dMV6aabL1Ds \n",
    "\n",
    "\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
    "from theano.tensor.nnet.bn import batch_normalization\n",
    "import numpy as np\n",
    "\n",
    "srng = RandomStreams()\n",
    "\n",
    "def floatX(X):\n",
    "    return np.asarray(X, dtype=theano.config.floatX)\n",
    "\n",
    "def init_weights(shape):\n",
    "    (h, w) = shape\n",
    "    # Glorot normalization - last factor depends on non-linearity\n",
    "    # 0.25 for sigmoid and 0.1 for softmax, 1.0 for tanh or Relu\n",
    "    normalizer = 2.0 * np.sqrt(6) / np.sqrt(h + w) * 1.0\n",
    "    return theano.shared(floatX((np.random.random_sample(shape) - 0.5) * normalizer))\n",
    "\n",
    "def rectify(X, alpha=0.01):\n",
    "#     return T.maximum(X, 0.)\n",
    "#    return T.maximum(X, 0.1*X)  #leaky rectifier\n",
    "     return T.switch(X > 0, X, alpha * (T.exp(X) - 1)) # ELU\n",
    "\n",
    "def softmax(X):\n",
    "    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n",
    "    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n",
    "\n",
    "def RMSprop(cost, params, lr=0.001, rho=0.99, epsilon=1e-9):\n",
    "    grads = T.grad(cost=cost, wrt=params)\n",
    "    updates = []\n",
    "    for p, g in zip(params, grads):\n",
    "        acc = theano.shared(p.get_value() * 0.)\n",
    "        acc_new = rho * acc + (1 - rho) * g ** 2\n",
    "        gradient_scaling = T.sqrt(acc_new + epsilon)\n",
    "        g = g / gradient_scaling\n",
    "        updates.append((acc, acc_new))\n",
    "        updates.append((p, p - lr * g))\n",
    "    return updates\n",
    "\n",
    "def model(X, w_h, g_h, bb_h, w_h2, g_h2, bb_h2, w_o, g_ho, bb_ho):\n",
    "    \n",
    "    X = T.dot(X, w_h) \n",
    "    X = batch_normalization(X, gamma= g_h, beta= bb_h, \n",
    "                            mean= X.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(X.var((0,), keepdims = True)), \n",
    "                            mode='high_mem') \n",
    "    h = rectify(X)\n",
    "\n",
    "    h  = T.dot(h, w_h2)\n",
    "    h = batch_normalization(h, gamma= g_h2, beta= bb_h2, \n",
    "                            mean= h.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h.var((0,), keepdims = True)), \n",
    "                            mode='high_mem') \n",
    "    h2 = rectify(h)\n",
    "\n",
    "    h2 = T.dot(h2, w_o)\n",
    "    h2 = batch_normalization(h2, gamma= g_ho, beta= bb_ho, \n",
    "                            mean= h2.mean((0,), keepdims=True),\n",
    "                            std= T.ones_like(h2.var((0,), keepdims = True)), \n",
    "                            mode='high_mem') \n",
    "    py_x = softmax(h2)\n",
    "    return h, h2, py_x\n",
    "\n",
    "\n",
    "X = T.fmatrix()\n",
    "Y = T.fmatrix()\n",
    "\n",
    "batch_size = 60\n",
    "\n",
    "h1_size = 600\n",
    "h2_size = 550\n",
    "h3_size = 550\n",
    "\n",
    "w_h = init_weights((len(vectorizer.vocabulary_), h1_size))\n",
    "g_h = theano.shared(floatX(np.ones((h1_size))))\n",
    "bb_h = theano.shared(floatX(np.zeros((h1_size))))\n",
    "\n",
    "w_h2 = init_weights((h1_size, h2_size))\n",
    "g_h2 = theano.shared(floatX(np.ones((h2_size))))\n",
    "bb_h2 = theano.shared(floatX(np.zeros((h2_size))))\n",
    "\n",
    "w_h3 = init_weights((h2_size, h3_size))\n",
    "g_h3 = theano.shared(floatX(np.ones((h3_size))))\n",
    "bb_h3 = theano.shared(floatX(np.zeros((h3_size))))\n",
    "\n",
    "w_o = init_weights((h3_size, yTest.shape[1]))\n",
    "g_ho = theano.shared(floatX(np.ones((yTest.shape[1]))))\n",
    "bb_ho = theano.shared(floatX(np.zeros((yTest.shape[1]))))\n",
    "\n",
    "noise_h, noise_h2, noise_py_x = model(X, w_h, g_h, bb_h, \n",
    "                                      w_h2, g_h2, bb_h2, \n",
    "                                      w_o, g_ho, bb_ho)\n",
    "\n",
    "h, h2, py_x = model(X, w_h, g_h, bb_h, \n",
    "                    w_h2, g_h2, bb_h2, \n",
    "                    w_o, g_ho, bb_ho)\n",
    "\n",
    "y_x = T.argmax(py_x, axis=1)\n",
    "\n",
    "\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n",
    "params = [w_h, g_h, bb_h, w_h2, g_h2, bb_h2, \n",
    "          w_o, g_ho, bb_ho]\n",
    "updates = RMSprop(cost, params, lr=0.0001)\n",
    "\n",
    "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
    "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)\n",
    "\n",
    "\n",
    "for i in range(30):\n",
    "\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):\n",
    "        cost = train(xTrain[start:end], yTrain[start:end])\n",
    "        \n",
    "    xTrain, yTrain = shuffle(xTrain, yTrain)\n",
    "    xTest, yTest   = shuffle(xTest, yTest)\n",
    "\n",
    "    trr, tr = [], []\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):        \n",
    "        trr += [np.argmax(yTrain[start:end], axis=1) == predict(xTrain[start:end])]\n",
    "\n",
    "    for start, end in zip(range(0, len(xTest), batch_size), range(batch_size, len(xTest), batch_size)):\n",
    "        tr += [np.argmax(yTest[start:end], axis=1) == predict(xTest[start:end])]\n",
    "\n",
    "    print \"Round: %-5s Test: %-14s Train: %-8s\" % (i, np.mean(tr), np.mean(trr))\n",
    "    \n",
    "print\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0     Test: 0.622395833333 Train: 0.688605442177\n",
      "Round: 1     Test: 0.6703125      Train: 0.751870748299\n",
      "Round: 2     Test: 0.684895833333 Train: 0.788945578231\n",
      "Round: 3     Test: 0.699479166667 Train: 0.826530612245\n",
      "Round: 4     Test: 0.723958333333 Train: 0.855102040816\n",
      "Round: 5     Test: 0.704166666667 Train: 0.86768707483\n",
      "Round: 6     Test: 0.706770833333 Train: 0.869727891156\n",
      "Round: 7     Test: 0.719791666667 Train: 0.906462585034\n",
      "Round: 8     Test: 0.730729166667 Train: 0.918197278912\n",
      "Round: 9     Test: 0.730729166667 Train: 0.921258503401\n",
      "Round: 10    Test: 0.722916666667 Train: 0.928231292517\n",
      "Round: 11    Test: 0.732291666667 Train: 0.937414965986\n",
      "Round: 12    Test: 0.722916666667 Train: 0.93537414966\n",
      "Round: 13    Test: 0.732291666667 Train: 0.943367346939\n",
      "Round: 14    Test: 0.730208333333 Train: 0.946768707483\n",
      "Round: 15    Test: 0.728125       Train: 0.94880952381\n",
      "Round: 16    Test: 0.733333333333 Train: 0.949489795918\n",
      "Round: 17    Test: 0.721875       Train: 0.94880952381\n",
      "Round: 18    Test: 0.728645833333 Train: 0.950340136054\n",
      "Round: 19    Test: 0.7359375      Train: 0.951360544218\n",
      "Round: 20    Test: 0.726041666667 Train: 0.947789115646\n",
      "Round: 21    Test: 0.718229166667 Train: 0.951530612245\n",
      "Round: 22    Test: 0.726041666667 Train: 0.952721088435\n",
      "Round: 23    Test: 0.719791666667 Train: 0.952721088435\n",
      "Round: 24    Test: 0.713541666667 Train: 0.950850340136\n",
      "Round: 25    Test: 0.723958333333 Train: 0.951360544218\n",
      "Round: 26    Test: 0.721354166667 Train: 0.950510204082\n",
      "Round: 27    Test: 0.71875        Train: 0.950340136054\n",
      "Round: 28    Test: 0.727083333333 Train: 0.952551020408\n",
      "Round: 29    Test: 0.7203125      Train: 0.954421768707\n",
      "Round: 30    Test: 0.715625       Train: 0.952551020408\n",
      "Round: 31    Test: 0.719791666667 Train: 0.952551020408\n",
      "Round: 32    Test: 0.724479166667 Train: 0.951530612245\n",
      "Round: 33    Test: 0.727083333333 Train: 0.954081632653\n",
      "Round: 34    Test: 0.717708333333 Train: 0.95425170068\n",
      "Round: 35    Test: 0.726041666667 Train: 0.95306122449\n",
      "Round: 36    Test: 0.732291666667 Train: 0.954591836735\n",
      "Round: 37    Test: 0.728125       Train: 0.952721088435\n",
      "Round: 38    Test: 0.708854166667 Train: 0.952721088435\n",
      "Round: 39    Test: 0.715625       Train: 0.95306122449\n",
      "Round: 40    Test: 0.716145833333 Train: 0.954931972789\n",
      "\n",
      "Wall time: 1min 57s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "\"\"\" 5-layer \"\"\"\n",
    "\n",
    "# batch normalization code adapted from \n",
    "# https://groups.google.com/forum/#!topic/theano-users/dMV6aabL1Ds \n",
    "\n",
    "\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
    "from theano.tensor.nnet.bn import batch_normalization\n",
    "import numpy as np\n",
    "\n",
    "srng = RandomStreams()\n",
    "\n",
    "def shuffle(x, y):\n",
    "    # helper function to shuffle indicies each loop \n",
    "    index = np.random.choice(len(x), len(x), replace=False)\n",
    "    return x[index], y[index]\n",
    "\n",
    "def floatX(X):\n",
    "    return np.asarray(X, dtype=theano.config.floatX)\n",
    "\n",
    "def init_weights(shape):\n",
    "    (h, w) = shape\n",
    "    # Glorot normalization - last factor depends on non-linearity\n",
    "    # 0.25 for sigmoid and 0.1 for softmax, 1.0 for tanh or Relu\n",
    "    normalizer = 2.0 * np.sqrt(6) / np.sqrt(h + w) * 1.0\n",
    "    return theano.shared(floatX((np.random.random_sample(shape) - 0.5) * normalizer))\n",
    "\n",
    "def rectify(X, alpha=0.01):\n",
    "#     return T.maximum(X, 0.)\n",
    "#    return T.maximum(X, 0.1*X)  #leaky rectifier\n",
    "     return T.switch(X > 0, X, alpha * (T.exp(X) - 1)) # ELU\n",
    "\n",
    "def softmax(X):\n",
    "    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n",
    "    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n",
    "\n",
    "def RMSprop(cost, params, lr=0.001, rho=0.99, epsilon=1e-9):\n",
    "    grads = T.grad(cost=cost, wrt=params)\n",
    "    updates = []\n",
    "    for p, g in zip(params, grads):\n",
    "        acc = theano.shared(p.get_value() * 0.)\n",
    "        acc_new = rho * acc + (1 - rho) * g ** 2\n",
    "        gradient_scaling = T.sqrt(acc_new + epsilon)\n",
    "        g = g / gradient_scaling\n",
    "        updates.append((acc, acc_new))\n",
    "        updates.append((p, p - lr * g))\n",
    "    return updates\n",
    "\n",
    "def model(X, w_h, g_h, bb_h, w_h2, g_h2, bb_h2,\n",
    "          w_h3, g_h3, bb_h3, w_o, g_ho, bb_ho):\n",
    "    \n",
    "    X = T.dot(X, w_h) \n",
    "    X = batch_normalization(X, gamma= g_h, beta= bb_h, \n",
    "                            mean= X.mean((1,), keepdims=True),\n",
    "                            std= T.ones_like(X.std((1,), keepdims = True)), \n",
    "                            mode='high_mem') \n",
    "    h = rectify(X)\n",
    "\n",
    "    h  = T.dot(h, w_h2)\n",
    "    h = batch_normalization(h, gamma= g_h2, beta= bb_h2, \n",
    "                            mean= h.mean((1,), keepdims=True),\n",
    "                            std= T.ones_like(h.std((1,), keepdims = True)), \n",
    "                            mode='high_mem') \n",
    "    h2 = rectify(h)\n",
    "\n",
    "    h2 = T.dot(h2, w_h3)\n",
    "    h2 = batch_normalization(h2, gamma= g_h3, beta= bb_h3, \n",
    "                            mean= h2.mean((1,), keepdims=True),\n",
    "                            std= T.ones_like(h2.std((1,), keepdims = True)), \n",
    "                            mode='high_mem') \n",
    "    h3 = rectify(h2)\n",
    "    \n",
    "    h3 = T.dot(h3, w_o)\n",
    "    h3 = batch_normalization(h3, gamma= g_ho, beta= bb_ho, \n",
    "                            mean= h3.mean((1,), keepdims=True),\n",
    "                            std= T.ones_like(h3.std((1,), keepdims = True)), \n",
    "                            mode='high_mem') \n",
    "    py_x = softmax(h3)\n",
    "    return h, h2, h3, py_x\n",
    "\n",
    "\n",
    "X = T.fmatrix()\n",
    "Y = T.fmatrix()\n",
    "\n",
    "batch_size = 60\n",
    "\n",
    "h1_size = 1000\n",
    "h2_size = 1000\n",
    "h3_size = 1000\n",
    "\n",
    "w_h = init_weights((len(vectorizer.vocabulary_), h1_size))\n",
    "g_h = theano.shared(floatX(np.ones((h1_size))))\n",
    "bb_h = theano.shared(floatX(np.zeros((h1_size))))\n",
    "\n",
    "w_h2 = init_weights((h1_size, h2_size))\n",
    "g_h2 = theano.shared(floatX(np.ones((h2_size))))\n",
    "bb_h2 = theano.shared(floatX(np.zeros((h2_size))))\n",
    "\n",
    "w_h3 = init_weights((h2_size, h3_size))\n",
    "g_h3 = theano.shared(floatX(np.ones((h3_size))))\n",
    "bb_h3 = theano.shared(floatX(np.zeros((h3_size))))\n",
    "\n",
    "w_o = init_weights((h3_size, yTest.shape[1]))\n",
    "g_ho = theano.shared(floatX(np.ones((yTest.shape[1]))))\n",
    "bb_ho = theano.shared(floatX(np.zeros((yTest.shape[1]))))\n",
    "\n",
    "noise_h, noise_h2, noise_h3, noise_py_x = model(X, w_h, g_h, bb_h, \n",
    "                                      w_h2, g_h2, bb_h2, \n",
    "                                       w_h3, g_h3, bb_h3, \n",
    "                                      w_o, g_ho, bb_ho)\n",
    "\n",
    "h, h2, h3, py_x = model(X, w_h, g_h, bb_h, \n",
    "                    w_h2, g_h2, bb_h2, \n",
    "                     w_h3, g_h3, bb_h3, \n",
    "                    w_o, g_ho, bb_ho)\n",
    "\n",
    "y_x = T.argmax(py_x, axis=1)\n",
    "\n",
    "\n",
    "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n",
    "params = [w_h, g_h, bb_h, w_h2, g_h2, bb_h2, \n",
    "           w_h3, g_h3, bb_h3, w_o, g_ho, bb_ho]\n",
    "updates = RMSprop(cost, params, lr=0.0001)\n",
    "\n",
    "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
    "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)\n",
    "\n",
    "p_v4, p_t4, i4 = [], [], []\n",
    "for i in range(41):\n",
    "\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):\n",
    "        cost = train(xTrain[start:end], yTrain[start:end])\n",
    "        \n",
    "    xTrain, yTrain = shuffle(xTrain, yTrain)\n",
    "    xTest, yTest   = shuffle(xTest, yTest)\n",
    "\n",
    "    trr, tr = [], []\n",
    "    for start, end in zip(range(0, len(xTrain), batch_size), range(batch_size, len(xTrain), batch_size)):        \n",
    "        trr += [np.argmax(yTrain[start:end], axis=1) == predict(xTrain[start:end])]\n",
    "\n",
    "    for start, end in zip(range(0, len(xTest), batch_size), range(batch_size, len(xTest), batch_size)):\n",
    "        tr += [np.argmax(yTest[start:end], axis=1) == predict(xTest[start:end])]\n",
    "    \n",
    "    p_v4 += [np.mean(tr)]\n",
    "    p_t4 += [np.mean(trr)]\n",
    "    i4   += [i]  \n",
    "    print \"Round: %-5s Test: %-14s Train: %-8s\" % (i, np.mean(tr), np.mean(trr))\n",
    "    \n",
    "print"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Results from removing T.ones_like \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "Round: 0     Test: 0.203125       Train: 0.198469387755\n",
    "Round: 1     Test: 0.2046875      Train: 0.198979591837\n",
    "Round: 2     Test: 0.205208333333 Train: 0.198979591837\n",
    "Round: 3     Test: 0.203125       Train: 0.198299319728\n",
    "Round: 4     Test: 0.203125       Train: 0.198979591837\n",
    "Round: 5     Test: 0.2046875      Train: 0.198979591837\n",
    "Round: 6     Test: 0.2046875      Train: 0.197959183673\n",
    "Round: 7     Test: 0.203645833333 Train: 0.19880952381\n",
    "Round: 8     Test: 0.205208333333 Train: 0.198979591837\n",
    "Round: 9     Test: 0.203125       Train: 0.198639455782\n",
    "Round: 10    Test: 0.205208333333 Train: 0.198979591837\n",
    "\n",
    "CPU times: user 3min 8s, sys: 6.83 s, total: 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div id=\"4b0d91d3-57a7-4060-983c-032ab3b692eb\" style=\"height: 525; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";Plotly.newPlot(\"4b0d91d3-57a7-4060-983c-032ab3b692eb\", [{\"y\": [0.22854240731335704, 0.47536820721178263, 0.5017775520568817, 0.5231081767394616, 0.5408836973082783, 0.5576434738445911, 0.5759268664296597, 0.5855764347384459, 0.590655154900965, 0.5942102590147282, 0.5992889791772473, 0.6058913153885221, 0.6089385474860335, 0.611477907567293, 0.6130015236160488, 0.6145251396648045, 0.6180802437785679, 0.6216353478923311, 0.6256983240223464, 0.6251904520060945, 0.627729812087354, 0.6307770441848655, 0.6323006602336211, 0.6343321482986287, 0.6338242762823768, 0.6343321482986287, 0.6368715083798883, 0.638395124428644, 0.6409344845099035, 0.6409344845099035, 0.6409344845099035, 0.6414423565261554, 0.6434738445911631, 0.6449974606399187, 0.6455053326561706, 0.6460132046724225, 0.6480446927374302, 0.6500761808024378, 0.6536312849162011, 0.654647028948705, 0.654647028948705, 0.6571863890299644, 0.6576942610462163, 0.6576942610462163, 0.6587100050787201, 0.6576942610462163, 0.6576942610462163, 0.6571863890299644, 0.6576942610462163, 0.659217877094972, 0.659217877094972, 0.6587100050787201, 0.6607414931437278, 0.6617572371762316, 0.6617572371762316, 0.6627729812087354, 0.6637887252412392, 0.6637887252412392, 0.6642965972574911, 0.6637887252412392, 0.664804469273743, 0.665312341289995, 0.665312341289995, 0.6658202133062469, 0.6673438293550026, 0.6678517013712545, 0.6693753174200101, 0.6719146775012697, 0.6734382935500254, 0.6739461655662773, 0.6759776536312849, 0.6769933976637887, 0.6775012696800407, 0.6785170137125445, 0.6785170137125445, 0.6795327577450483, 0.6800406297613002, 0.6800406297613002, 0.6795327577450483, 0.6795327577450483, 0.681056373793804, 0.6830878618588115, 0.6835957338750634, 0.6835957338750634, 0.6835957338750634, 0.6841036058913154, 0.6846114779075673, 0.6851193499238192, 0.6856272219400711, 0.6856272219400711, 0.686135093956323, 0.6871508379888268, 0.6871508379888268, 0.6881665820213306, 0.6886744540375825, 0.6881665820213306, 0.6881665820213306, 0.6876587100050787, 0.6876587100050787, 0.6881665820213306, 0.6881665820213306, 0.6876587100050787, 0.6876587100050787, 0.6876587100050787, 0.6881665820213306, 0.6881665820213306, 0.6886744540375825, 0.6886744540375825, 0.6886744540375825, 0.6886744540375825, 0.6886744540375825, 0.6886744540375825, 0.6886744540375825, 0.6891823260538344, 0.6891823260538344, 0.6886744540375825, 0.6886744540375825, 0.6876587100050787, 0.6876587100050787, 0.6876587100050787, 0.6876587100050787, 0.6886744540375825, 0.6891823260538344, 0.6891823260538344, 0.6896901980700864, 0.6896901980700864, 0.6901980700863383, 0.6901980700863383, 0.6901980700863383, 0.6901980700863383, 0.6912138141188421, 0.6922295581513459, 0.6932453021838497, 0.6932453021838497, 0.6932453021838497, 0.6937531742001015, 0.6937531742001015, 0.6942610462163534, 0.6942610462163534, 0.6942610462163534], \"x\": [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500, 550, 600, 650, 700, 750, 800, 850, 900, 950, 1000, 1050, 1100, 1150, 1200, 1250, 1300, 1350, 1400, 1450, 1500, 1550, 1600, 1650, 1700, 1750, 1800, 1850, 1900, 1950, 2000, 2050, 2100, 2150, 2200, 2250, 2300, 2350, 2400, 2450, 2500, 2550, 2600, 2650, 2700, 2750, 2800, 2850, 2900, 2950, 3000, 3050, 3100, 3150, 3200, 3250, 3300, 3350, 3400, 3450, 3500, 3550, 3600, 3650, 3700, 3750, 3800, 3850, 3900, 3950, 4000, 4050, 4100, 4150, 4200, 4250, 4300, 4350, 4400, 4450, 4500, 4550, 4600, 4650, 4700, 4750, 4800, 4850, 4900, 4950, 5000, 5050, 5100, 5150, 5200, 5250, 5300, 5350, 5400, 5450, 5500, 5550, 5600, 5650, 5700, 5750, 5800, 5850, 5900, 5950, 6000, 6050, 6100, 6150, 6200, 6250, 6300, 6350, 6400, 6450, 6500, 6550, 6600, 6650, 6700, 6750, 6800, 6850, 6900, 6950], \"type\": \"scatter\", \"name\": \"2-Layer\", \"mode\": \"lines\"}, {\"y\": [0.26358557643473846, 0.3037074657186389, 0.4900964956830879, 0.601320467242255, 0.6526155408836973, 0.6759776536312849, 0.6896901980700864, 0.6962925342813611, 0.7013712544438802, 0.7105129507364144, 0.7166074149314373, 0.7176231589639411, 0.728796343321483, 0.7242254951752158, 0.7267648552564754, 0.728796343321483, 0.7257491112239716, 0.7313357034027425, 0.7333671914677501, 0.7282884713052311], \"x\": [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160, 170, 180, 190], \"type\": \"scatter\", \"name\": \"4-Layer Dropout\", \"mode\": \"lines\"}, {\"y\": [0.44032503809040124, 0.5611985779583545, 0.6124936515997969, 0.6475368207211782, 0.6607414931437278, 0.681056373793804, 0.6927374301675978, 0.6988318943626206, 0.7049263585576435, 0.7115286947689182, 0.7166074149314373, 0.7267648552564754, 0.723209751142712, 0.7257491112239716, 0.7277805992889792, 0.7308278313864907, 0.7303199593702387, 0.7308278313864907, 0.7303199593702387, 0.7308278313864907, 0.7313357034027425, 0.7343829355002539, 0.7359065515490096, 0.7328593194514982, 0.7379380396140173], \"x\": [0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100, 105, 110, 115, 120], \"type\": \"scatter\", \"name\": \"4-Layer BN\", \"mode\": \"lines\"}, {\"y\": [0.6223958333333334, 0.6703125, 0.6848958333333334, 0.6994791666666667, 0.7239583333333334, 0.7041666666666667, 0.7067708333333333, 0.7197916666666667, 0.7307291666666667, 0.7307291666666667, 0.7229166666666667, 0.7322916666666667, 0.7229166666666667, 0.7322916666666667, 0.7302083333333333, 0.728125, 0.7333333333333333, 0.721875, 0.7286458333333333, 0.7359375, 0.7260416666666667, 0.7182291666666667, 0.7260416666666667, 0.7197916666666667, 0.7135416666666666, 0.7239583333333334, 0.7213541666666666, 0.71875, 0.7270833333333333, 0.7203125, 0.715625, 0.7197916666666667, 0.7244791666666667, 0.7270833333333333, 0.7177083333333333, 0.7260416666666667, 0.7322916666666667, 0.728125, 0.7088541666666667, 0.715625, 0.7161458333333334], \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40], \"type\": \"scatter\", \"name\": \"5-Layer BN\", \"mode\": \"lines\"}], {\"xaxis\": {\"range\": [0, 400], \"type\": \"linear\"}, \"yaxis\": {\"type\": \"linear\", \"autorange\": true}}, {\"linkText\": \"Export to plot.ly\", \"showLink\": true})});</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import plotly\n",
    "import plotly.graph_objs as go\n",
    "plotly.offline.init_notebook_mode() # run at the start of every notebook\n",
    "\n",
    "trace0 = go.Scatter(\n",
    "    x = i1[:1000],\n",
    "    y = p_v1[:1000],\n",
    "    mode = 'lines',\n",
    "    name = '2-Layer')\n",
    "\n",
    "trace2 = go.Scatter(\n",
    "    x = i2,\n",
    "    y = p_v2,\n",
    "    mode = 'lines',\n",
    "    name = '4-Layer Dropout')\n",
    "\n",
    "trace4 = go.Scatter(\n",
    "    x = i3,\n",
    "    y = p_v3,\n",
    "    mode = 'lines',\n",
    "    name = '4-Layer BN')\n",
    "\n",
    "trace6 = go.Scatter(\n",
    "    x = i4,\n",
    "    y = p_v4,\n",
    "    mode = 'lines',\n",
    "    name = '5-Layer BN')\n",
    "\n",
    "data = [trace0, trace2, trace4, trace6 ]\n",
    "\n",
    "layout = go.Layout(\n",
    "    xaxis=dict(\n",
    "        type='linear',\n",
    "        range=[0,400]\n",
    "    ),\n",
    "    yaxis=dict(\n",
    "        type='linear',\n",
    "        autorange=True\n",
    "    ))\n",
    "fig = go.Figure(data=data, layout=layout)\n",
    "plotly.offline.iplot(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}