SELU_SSN_based_CNN_AlphaDropout.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (60000, 28, 28, 1)\n",
      "60000 train samples\n",
      "10000 test samples\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "60000/60000 [==============================] - 6s - loss: 0.5556 - acc: 0.8263 - val_loss: 0.3215 - val_acc: 0.9443\n",
      "Epoch 2/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.2557 - acc: 0.9242 - val_loss: 0.1914 - val_acc: 0.9682\n",
      "Epoch 3/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.1734 - acc: 0.9487 - val_loss: 0.1399 - val_acc: 0.9771\n",
      "Epoch 4/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.1277 - acc: 0.9618 - val_loss: 0.1158 - val_acc: 0.9807\n",
      "Epoch 5/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.1057 - acc: 0.9683 - val_loss: 0.1014 - val_acc: 0.9852\n",
      "Epoch 6/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0928 - acc: 0.9725 - val_loss: 0.0873 - val_acc: 0.9857\n",
      "Epoch 7/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0825 - acc: 0.9753 - val_loss: 0.0810 - val_acc: 0.9863\n",
      "Epoch 8/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0754 - acc: 0.9771 - val_loss: 0.0908 - val_acc: 0.9870\n",
      "Epoch 9/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0718 - acc: 0.9784 - val_loss: 0.0915 - val_acc: 0.9867\n",
      "Epoch 10/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0666 - acc: 0.9804 - val_loss: 0.0858 - val_acc: 0.9863\n",
      "Epoch 11/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0603 - acc: 0.9820 - val_loss: 0.0872 - val_acc: 0.9874\n",
      "Epoch 12/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0585 - acc: 0.9826 - val_loss: 0.0828 - val_acc: 0.9871\n",
      "Epoch 13/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0561 - acc: 0.9830 - val_loss: 0.0788 - val_acc: 0.9888\n",
      "Epoch 14/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0517 - acc: 0.9838 - val_loss: 0.0767 - val_acc: 0.9893\n",
      "Epoch 15/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0499 - acc: 0.9851 - val_loss: 0.0817 - val_acc: 0.9891\n",
      "Epoch 16/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0489 - acc: 0.9841 - val_loss: 0.0839 - val_acc: 0.9899\n",
      "Epoch 17/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0473 - acc: 0.9854 - val_loss: 0.0764 - val_acc: 0.9901\n",
      "Epoch 18/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0465 - acc: 0.9856 - val_loss: 0.0758 - val_acc: 0.9904\n",
      "Epoch 19/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0450 - acc: 0.9859 - val_loss: 0.0771 - val_acc: 0.9911\n",
      "Epoch 20/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0425 - acc: 0.9868 - val_loss: 0.0745 - val_acc: 0.9900\n",
      "Epoch 21/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0398 - acc: 0.9871 - val_loss: 0.0719 - val_acc: 0.9905\n",
      "Epoch 22/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0399 - acc: 0.9876 - val_loss: 0.0724 - val_acc: 0.9901\n",
      "Epoch 23/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0406 - acc: 0.9871 - val_loss: 0.0837 - val_acc: 0.9887\n",
      "Epoch 24/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0381 - acc: 0.9882 - val_loss: 0.0746 - val_acc: 0.9906\n",
      "Epoch 25/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0385 - acc: 0.9879 - val_loss: 0.0735 - val_acc: 0.9897\n",
      "Epoch 26/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0363 - acc: 0.9885 - val_loss: 0.0763 - val_acc: 0.9899\n",
      "Epoch 27/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0371 - acc: 0.9883 - val_loss: 0.0751 - val_acc: 0.9912\n",
      "Epoch 28/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0365 - acc: 0.9884 - val_loss: 0.0856 - val_acc: 0.9897\n",
      "Epoch 29/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0347 - acc: 0.9896 - val_loss: 0.0801 - val_acc: 0.9903\n",
      "Epoch 30/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0345 - acc: 0.9891 - val_loss: 0.0792 - val_acc: 0.9905\n",
      "Epoch 31/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0332 - acc: 0.9897 - val_loss: 0.0775 - val_acc: 0.9907\n",
      "Epoch 32/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0324 - acc: 0.9902 - val_loss: 0.0889 - val_acc: 0.9898\n",
      "Epoch 33/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0313 - acc: 0.9900 - val_loss: 0.0832 - val_acc: 0.9900\n",
      "Epoch 34/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0308 - acc: 0.9904 - val_loss: 0.0724 - val_acc: 0.9908\n",
      "Epoch 35/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0312 - acc: 0.9899 - val_loss: 0.0772 - val_acc: 0.9909\n",
      "Epoch 36/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0303 - acc: 0.9909 - val_loss: 0.0823 - val_acc: 0.9905\n",
      "Epoch 37/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0301 - acc: 0.9904 - val_loss: 0.0722 - val_acc: 0.9906\n",
      "Epoch 38/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0306 - acc: 0.9905 - val_loss: 0.0797 - val_acc: 0.9909\n",
      "Epoch 39/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0288 - acc: 0.9912 - val_loss: 0.0769 - val_acc: 0.9897\n",
      "Epoch 40/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0289 - acc: 0.9912 - val_loss: 0.0855 - val_acc: 0.9900\n",
      "Epoch 41/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0297 - acc: 0.9905 - val_loss: 0.0843 - val_acc: 0.9891\n",
      "Epoch 42/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0284 - acc: 0.9910 - val_loss: 0.0797 - val_acc: 0.9907\n",
      "Epoch 43/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0265 - acc: 0.9919 - val_loss: 0.0736 - val_acc: 0.9905\n",
      "Epoch 44/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0287 - acc: 0.9912 - val_loss: 0.0795 - val_acc: 0.9904\n",
      "Epoch 45/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0263 - acc: 0.9917 - val_loss: 0.0912 - val_acc: 0.9897\n",
      "Epoch 46/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0277 - acc: 0.9910 - val_loss: 0.0797 - val_acc: 0.9913\n",
      "Epoch 47/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0254 - acc: 0.9924 - val_loss: 0.0784 - val_acc: 0.9905\n",
      "Epoch 48/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0263 - acc: 0.9919 - val_loss: 0.0788 - val_acc: 0.9911\n",
      "Epoch 49/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0270 - acc: 0.9919 - val_loss: 0.0945 - val_acc: 0.9899\n",
      "Epoch 50/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0267 - acc: 0.9916 - val_loss: 0.0781 - val_acc: 0.9917\n",
      "Epoch 51/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0244 - acc: 0.9920 - val_loss: 0.0754 - val_acc: 0.9908\n",
      "Epoch 52/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0252 - acc: 0.9914 - val_loss: 0.0781 - val_acc: 0.9915\n",
      "Epoch 53/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0247 - acc: 0.9920 - val_loss: 0.0826 - val_acc: 0.9919\n",
      "Epoch 54/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0257 - acc: 0.9922 - val_loss: 0.0789 - val_acc: 0.9910\n",
      "Epoch 55/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0234 - acc: 0.9924 - val_loss: 0.0842 - val_acc: 0.9910\n",
      "Epoch 56/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0234 - acc: 0.9926 - val_loss: 0.0861 - val_acc: 0.9902\n",
      "Epoch 57/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0233 - acc: 0.9929 - val_loss: 0.0809 - val_acc: 0.9902\n",
      "Epoch 58/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0254 - acc: 0.9921 - val_loss: 0.0847 - val_acc: 0.9915\n",
      "Epoch 59/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0236 - acc: 0.9928 - val_loss: 0.0719 - val_acc: 0.9906\n",
      "Epoch 60/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0237 - acc: 0.9926 - val_loss: 0.0756 - val_acc: 0.9901\n",
      "Epoch 61/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0224 - acc: 0.9931 - val_loss: 0.0883 - val_acc: 0.9904\n",
      "Epoch 62/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0230 - acc: 0.9929 - val_loss: 0.0804 - val_acc: 0.9900\n",
      "Epoch 63/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000/60000 [==============================] - 5s - loss: 0.0217 - acc: 0.9933 - val_loss: 0.0914 - val_acc: 0.9908\n",
      "Epoch 64/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0240 - acc: 0.9923 - val_loss: 0.0907 - val_acc: 0.9912\n",
      "Epoch 65/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0199 - acc: 0.9934 - val_loss: 0.0790 - val_acc: 0.9918\n",
      "Epoch 66/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0222 - acc: 0.9931 - val_loss: 0.0843 - val_acc: 0.9918\n",
      "Epoch 67/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0235 - acc: 0.9927 - val_loss: 0.0841 - val_acc: 0.9916\n",
      "Epoch 68/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0219 - acc: 0.9928 - val_loss: 0.0808 - val_acc: 0.9915\n",
      "Epoch 69/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0230 - acc: 0.9928 - val_loss: 0.0818 - val_acc: 0.9911\n",
      "Epoch 70/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0217 - acc: 0.9933 - val_loss: 0.0776 - val_acc: 0.9922\n",
      "Epoch 71/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0217 - acc: 0.9931 - val_loss: 0.0892 - val_acc: 0.9904\n",
      "Epoch 72/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0209 - acc: 0.9934 - val_loss: 0.0882 - val_acc: 0.9898\n",
      "Epoch 73/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0209 - acc: 0.9935 - val_loss: 0.0878 - val_acc: 0.9906\n",
      "Epoch 74/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0203 - acc: 0.9937 - val_loss: 0.0806 - val_acc: 0.9911\n",
      "Epoch 75/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0206 - acc: 0.9934 - val_loss: 0.0860 - val_acc: 0.9911\n",
      "Epoch 76/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0207 - acc: 0.9933 - val_loss: 0.0854 - val_acc: 0.9913\n",
      "Epoch 77/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0194 - acc: 0.9941 - val_loss: 0.0841 - val_acc: 0.9915\n",
      "Epoch 78/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0191 - acc: 0.9941 - val_loss: 0.0781 - val_acc: 0.9914\n",
      "Epoch 79/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0176 - acc: 0.9944 - val_loss: 0.0820 - val_acc: 0.9909\n",
      "Epoch 80/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0194 - acc: 0.9942 - val_loss: 0.0832 - val_acc: 0.9913\n",
      "Epoch 81/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0191 - acc: 0.9937 - val_loss: 0.0862 - val_acc: 0.9923\n",
      "Epoch 82/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0195 - acc: 0.9939 - val_loss: 0.0880 - val_acc: 0.9912\n",
      "Epoch 83/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0185 - acc: 0.9940 - val_loss: 0.0917 - val_acc: 0.9913\n",
      "Epoch 84/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0172 - acc: 0.9943 - val_loss: 0.0836 - val_acc: 0.9915\n",
      "Epoch 85/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0172 - acc: 0.9948 - val_loss: 0.0884 - val_acc: 0.9912\n",
      "Epoch 86/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0198 - acc: 0.9938 - val_loss: 0.0885 - val_acc: 0.9909\n",
      "Epoch 87/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0184 - acc: 0.9941 - val_loss: 0.0867 - val_acc: 0.9918\n",
      "Epoch 88/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0186 - acc: 0.9940 - val_loss: 0.0865 - val_acc: 0.9913\n",
      "Epoch 89/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0181 - acc: 0.9944 - val_loss: 0.0832 - val_acc: 0.9913\n",
      "Epoch 90/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0165 - acc: 0.9943 - val_loss: 0.0885 - val_acc: 0.9917\n",
      "Epoch 91/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0183 - acc: 0.9937 - val_loss: 0.0807 - val_acc: 0.9916\n",
      "Epoch 92/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0173 - acc: 0.9944 - val_loss: 0.0879 - val_acc: 0.9917\n",
      "Epoch 93/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0165 - acc: 0.9948 - val_loss: 0.0913 - val_acc: 0.9913\n",
      "Epoch 94/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0159 - acc: 0.9948 - val_loss: 0.0942 - val_acc: 0.9914\n",
      "Epoch 95/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0174 - acc: 0.9947 - val_loss: 0.0900 - val_acc: 0.9915\n",
      "Epoch 96/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0167 - acc: 0.9946 - val_loss: 0.0864 - val_acc: 0.9916\n",
      "Epoch 97/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0176 - acc: 0.9944 - val_loss: 0.0856 - val_acc: 0.9914\n",
      "Epoch 98/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0166 - acc: 0.9949 - val_loss: 0.0874 - val_acc: 0.9913\n",
      "Epoch 99/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0159 - acc: 0.9950 - val_loss: 0.0911 - val_acc: 0.9918\n",
      "Epoch 100/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0163 - acc: 0.9951 - val_loss: 0.0839 - val_acc: 0.9908\n",
      "Epoch 101/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0162 - acc: 0.9948 - val_loss: 0.0894 - val_acc: 0.9924\n",
      "Epoch 102/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0164 - acc: 0.9947 - val_loss: 0.0825 - val_acc: 0.9912\n",
      "Epoch 103/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0158 - acc: 0.9949 - val_loss: 0.0867 - val_acc: 0.9915\n",
      "Epoch 104/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0160 - acc: 0.9949 - val_loss: 0.0893 - val_acc: 0.9916\n",
      "Epoch 105/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0152 - acc: 0.9951 - val_loss: 0.0908 - val_acc: 0.9915\n",
      "Epoch 106/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0167 - acc: 0.9947 - val_loss: 0.0857 - val_acc: 0.9913\n",
      "Epoch 107/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0161 - acc: 0.9950 - val_loss: 0.0854 - val_acc: 0.9916\n",
      "Epoch 108/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0172 - acc: 0.9945 - val_loss: 0.0894 - val_acc: 0.9907\n",
      "Epoch 109/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0153 - acc: 0.9947 - val_loss: 0.0887 - val_acc: 0.9914\n",
      "Epoch 110/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0153 - acc: 0.9952 - val_loss: 0.0854 - val_acc: 0.9916\n",
      "Epoch 111/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0156 - acc: 0.9950 - val_loss: 0.0934 - val_acc: 0.9916\n",
      "Epoch 112/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0151 - acc: 0.9950 - val_loss: 0.0936 - val_acc: 0.9911\n",
      "Epoch 113/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0157 - acc: 0.9950 - val_loss: 0.0829 - val_acc: 0.9919\n",
      "Epoch 114/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0151 - acc: 0.9954 - val_loss: 0.0936 - val_acc: 0.9920\n",
      "Epoch 115/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0151 - acc: 0.9952 - val_loss: 0.0865 - val_acc: 0.9919\n",
      "Epoch 116/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0153 - acc: 0.9950 - val_loss: 0.0850 - val_acc: 0.9920\n",
      "Epoch 117/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0154 - acc: 0.9948 - val_loss: 0.0929 - val_acc: 0.9910\n",
      "Epoch 118/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0155 - acc: 0.9950 - val_loss: 0.0892 - val_acc: 0.9917\n",
      "Epoch 119/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0161 - acc: 0.9953 - val_loss: 0.0894 - val_acc: 0.9916\n",
      "Epoch 120/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0166 - acc: 0.9948 - val_loss: 0.0856 - val_acc: 0.9919\n",
      "Epoch 121/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0160 - acc: 0.9952 - val_loss: 0.0861 - val_acc: 0.9920\n",
      "Epoch 122/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0147 - acc: 0.9952 - val_loss: 0.0879 - val_acc: 0.9916\n",
      "Epoch 123/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0155 - acc: 0.9950 - val_loss: 0.0839 - val_acc: 0.9916\n",
      "Epoch 124/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0160 - acc: 0.9947 - val_loss: 0.0908 - val_acc: 0.9910\n",
      "Epoch 125/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0162 - acc: 0.9948 - val_loss: 0.0799 - val_acc: 0.9917\n",
      "Epoch 126/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000/60000 [==============================] - 5s - loss: 0.0153 - acc: 0.9953 - val_loss: 0.0921 - val_acc: 0.9919\n",
      "Epoch 127/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0152 - acc: 0.9952 - val_loss: 0.0839 - val_acc: 0.9922\n",
      "Epoch 128/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0151 - acc: 0.9951 - val_loss: 0.0894 - val_acc: 0.9917\n",
      "Epoch 129/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0151 - acc: 0.9953 - val_loss: 0.0898 - val_acc: 0.9914\n",
      "Epoch 130/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0158 - acc: 0.9950 - val_loss: 0.0896 - val_acc: 0.9919\n",
      "Epoch 131/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0143 - acc: 0.9956 - val_loss: 0.0955 - val_acc: 0.9911\n",
      "Epoch 132/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0150 - acc: 0.9951 - val_loss: 0.0965 - val_acc: 0.9917\n",
      "Epoch 133/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0149 - acc: 0.9954 - val_loss: 0.0960 - val_acc: 0.9911\n",
      "Epoch 134/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0150 - acc: 0.9953 - val_loss: 0.0935 - val_acc: 0.9919\n",
      "Epoch 135/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0138 - acc: 0.9959 - val_loss: 0.0927 - val_acc: 0.9915\n",
      "Epoch 136/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0150 - acc: 0.9952 - val_loss: 0.0990 - val_acc: 0.9909\n",
      "Epoch 137/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0139 - acc: 0.9956 - val_loss: 0.0889 - val_acc: 0.9921\n",
      "Epoch 138/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0138 - acc: 0.9960 - val_loss: 0.0932 - val_acc: 0.9915\n",
      "Epoch 139/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0139 - acc: 0.9960 - val_loss: 0.0902 - val_acc: 0.9912\n",
      "Epoch 140/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0146 - acc: 0.9954 - val_loss: 0.0961 - val_acc: 0.9914\n",
      "Epoch 141/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0147 - acc: 0.9956 - val_loss: 0.0879 - val_acc: 0.9911\n",
      "Epoch 142/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0138 - acc: 0.9956 - val_loss: 0.0884 - val_acc: 0.9916\n",
      "Epoch 143/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0128 - acc: 0.9960 - val_loss: 0.0911 - val_acc: 0.9914\n",
      "Epoch 144/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0132 - acc: 0.9960 - val_loss: 0.0959 - val_acc: 0.9912\n",
      "Epoch 145/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0132 - acc: 0.9957 - val_loss: 0.0887 - val_acc: 0.9912\n",
      "Epoch 146/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0137 - acc: 0.9953 - val_loss: 0.0899 - val_acc: 0.9915\n",
      "Epoch 147/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0147 - acc: 0.9954 - val_loss: 0.0956 - val_acc: 0.9916\n",
      "Epoch 148/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0136 - acc: 0.9958 - val_loss: 0.0966 - val_acc: 0.9912\n",
      "Epoch 149/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0137 - acc: 0.9959 - val_loss: 0.0946 - val_acc: 0.9915\n",
      "Epoch 150/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0140 - acc: 0.9955 - val_loss: 0.0926 - val_acc: 0.9912\n",
      "Epoch 151/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0135 - acc: 0.9957 - val_loss: 0.0955 - val_acc: 0.9918\n",
      "Epoch 152/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0131 - acc: 0.9959 - val_loss: 0.0897 - val_acc: 0.9905\n",
      "Epoch 153/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0128 - acc: 0.9959 - val_loss: 0.0887 - val_acc: 0.9916\n",
      "Epoch 154/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0142 - acc: 0.9957 - val_loss: 0.0895 - val_acc: 0.9919\n",
      "Epoch 155/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0130 - acc: 0.9960 - val_loss: 0.0925 - val_acc: 0.9920\n",
      "Epoch 156/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0137 - acc: 0.9957 - val_loss: 0.0911 - val_acc: 0.9920\n",
      "Epoch 157/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0138 - acc: 0.9958 - val_loss: 0.0898 - val_acc: 0.9911\n",
      "Epoch 158/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0128 - acc: 0.9957 - val_loss: 0.1008 - val_acc: 0.9909\n",
      "Epoch 159/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0132 - acc: 0.9959 - val_loss: 0.0950 - val_acc: 0.9915\n",
      "Epoch 160/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0124 - acc: 0.9961 - val_loss: 0.0946 - val_acc: 0.9909\n",
      "Epoch 161/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0139 - acc: 0.9958 - val_loss: 0.0993 - val_acc: 0.9906\n",
      "Epoch 162/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0134 - acc: 0.9958 - val_loss: 0.0986 - val_acc: 0.9906\n",
      "Epoch 163/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0132 - acc: 0.9957 - val_loss: 0.0930 - val_acc: 0.9913\n",
      "Epoch 164/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0126 - acc: 0.9963 - val_loss: 0.0862 - val_acc: 0.9918\n",
      "Epoch 165/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0130 - acc: 0.9961 - val_loss: 0.0935 - val_acc: 0.9914\n",
      "Epoch 166/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0116 - acc: 0.9967 - val_loss: 0.0922 - val_acc: 0.9916\n",
      "Epoch 167/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0136 - acc: 0.9957 - val_loss: 0.0994 - val_acc: 0.9911\n",
      "Epoch 168/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0130 - acc: 0.9958 - val_loss: 0.0943 - val_acc: 0.9920\n",
      "Epoch 169/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0122 - acc: 0.9960 - val_loss: 0.0904 - val_acc: 0.9913\n",
      "Epoch 170/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0133 - acc: 0.9956 - val_loss: 0.0924 - val_acc: 0.9913\n",
      "Epoch 171/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0125 - acc: 0.9960 - val_loss: 0.0936 - val_acc: 0.9909\n",
      "Epoch 172/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0128 - acc: 0.9960 - val_loss: 0.0917 - val_acc: 0.9914\n",
      "Epoch 173/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0137 - acc: 0.9959 - val_loss: 0.0882 - val_acc: 0.9918\n",
      "Epoch 174/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0116 - acc: 0.9963 - val_loss: 0.0933 - val_acc: 0.9914\n",
      "Epoch 175/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0131 - acc: 0.9959 - val_loss: 0.0938 - val_acc: 0.9914\n",
      "Epoch 176/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0128 - acc: 0.9960 - val_loss: 0.0950 - val_acc: 0.9912\n",
      "Epoch 177/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0128 - acc: 0.9960 - val_loss: 0.1033 - val_acc: 0.9911\n",
      "Epoch 178/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0129 - acc: 0.9961 - val_loss: 0.0884 - val_acc: 0.9920\n",
      "Epoch 179/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0121 - acc: 0.9960 - val_loss: 0.0927 - val_acc: 0.9922\n",
      "Epoch 180/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0129 - acc: 0.9960 - val_loss: 0.0980 - val_acc: 0.9911\n",
      "Epoch 181/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0130 - acc: 0.9959 - val_loss: 0.0892 - val_acc: 0.9917\n",
      "Epoch 182/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0119 - acc: 0.9964 - val_loss: 0.0927 - val_acc: 0.9911\n",
      "Epoch 183/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0114 - acc: 0.9966 - val_loss: 0.0994 - val_acc: 0.9922\n",
      "Epoch 184/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0123 - acc: 0.9966 - val_loss: 0.0863 - val_acc: 0.9921\n",
      "Epoch 185/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0121 - acc: 0.9962 - val_loss: 0.0906 - val_acc: 0.9915\n",
      "Epoch 186/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0118 - acc: 0.9962 - val_loss: 0.0950 - val_acc: 0.9915\n",
      "Epoch 187/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0106 - acc: 0.9966 - val_loss: 0.0958 - val_acc: 0.9916\n",
      "Epoch 188/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0117 - acc: 0.9960 - val_loss: 0.0917 - val_acc: 0.9916\n",
      "Epoch 189/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000/60000 [==============================] - 5s - loss: 0.0115 - acc: 0.9965 - val_loss: 0.0931 - val_acc: 0.9920\n",
      "Epoch 190/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0119 - acc: 0.9963 - val_loss: 0.1026 - val_acc: 0.9916\n",
      "Epoch 191/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0116 - acc: 0.9962 - val_loss: 0.0944 - val_acc: 0.9919\n",
      "Epoch 192/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0103 - acc: 0.9967 - val_loss: 0.0930 - val_acc: 0.9912\n",
      "Epoch 193/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0118 - acc: 0.9961 - val_loss: 0.0950 - val_acc: 0.9923\n",
      "Epoch 194/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0120 - acc: 0.9962 - val_loss: 0.0952 - val_acc: 0.9920\n",
      "Epoch 195/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0119 - acc: 0.9961 - val_loss: 0.0960 - val_acc: 0.9914\n",
      "Epoch 196/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0123 - acc: 0.9962 - val_loss: 0.0900 - val_acc: 0.9922\n",
      "Epoch 197/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0121 - acc: 0.9962 - val_loss: 0.0939 - val_acc: 0.9911\n",
      "Epoch 198/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0115 - acc: 0.9965 - val_loss: 0.0900 - val_acc: 0.9922\n",
      "Epoch 199/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0115 - acc: 0.9964 - val_loss: 0.0948 - val_acc: 0.9914\n",
      "Epoch 200/200\n",
      "60000/60000 [==============================] - 5s - loss: 0.0115 - acc: 0.9966 - val_loss: 0.0989 - val_acc: 0.9913\n",
      "Test loss: 0.098869990336\n",
      "Test accuracy: 0.9913\n"
     ]
    }
   ],
   "source": [
    "'''Trains a simple convnet on the MNIST dataset.\n",
    "\n",
    "NOTE: This implementation uses new AlphaDropout\n",
    "\n",
    "Gets to 99.25% test accuracy after 56 epochs\n",
    "'''\n",
    "\n",
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras.layers.noise import AlphaDropout\n",
    "\n",
    "from keras import backend as K\n",
    "\n",
    "batch_size = 128\n",
    "num_classes = 10\n",
    "epochs = 200\n",
    "\n",
    "# input image dimensions\n",
    "img_rows, img_cols = 28, 28\n",
    "\n",
    "# the data, shuffled and split between train and test sets\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)\n",
    "    x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
    "    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# convert class vectors to binary class matrices\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Conv2D(32, kernel_size=(3, 3),\n",
    "                 activation='selu',\n",
    "                 input_shape=input_shape))\n",
    "model.add(Conv2D(64, (3, 3), activation='selu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(AlphaDropout(0.25))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(128, activation='selu'))\n",
    "model.add(AlphaDropout(0.5))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))\n",
    "\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}