ann.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Numpy &nbsp;&nbsp;&nbsp;&nbsp;: Numerical library used for manipulating vectors like dot product, vector product, sum etc\n",
    "<br>\n",
    "Matplotlib : 2D Ploting library to visulize graphs\n",
    "             Here used to plot image of number with 28x28 dimensional pixel values\n",
    "<br>\n",
    "Pandas &nbsp;&nbsp;&nbsp;: Used for working with tables efficiently\n",
    "             Here used to read .csv file of mnist database\n",
    "<br>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "Applies activation function after output from every layer\n",
    "To find derivative of activators, include \"derivative=True\" in argument\n",
    "</pre>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class activator:\n",
    "\n",
    "    def sigmoid(z, derivative=False):\n",
    "        if derivative==True:\n",
    "            return (activator.sigmoid(z=z, derivative=False) * (1 - activator.sigmoid(z=z, derivative=False)))\n",
    "        return (1.0 / (1.0 + np.exp(-z)))\n",
    "\n",
    "    def softmax(z, derivative=False):\n",
    "        if derivative==True:\n",
    "            return (activator.softmax(z=z, derivative=False) * (1 - activator.softmax(z=z, derivative=False)))\n",
    "        return (np.exp(z) / np.sum(np.exp(z)))\n",
    "\n",
    "    def tanh(z, derivative=False):\n",
    "        if derivative==True:\n",
    "            return (activator.tanh(z=z, derivative=False) * (1 - activator.tanh(z=z, derivative=False)))\n",
    "        return (np.tanh(z))\n",
    "\n",
    "    def relu(z, derivative=False):\n",
    "        if derivative==True:\n",
    "            der_z = np.zeros(z.shape)\n",
    "            for i in range(len(z.shape)):\n",
    "                for j in range(len(z[i])):\n",
    "                    if(z[i, j]>0):\n",
    "                        der_z[i, j] = 1\n",
    "            return der_z\n",
    "        return (np.maximum(z, 0))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "\n",
    "                              O\n",
    "                              O\n",
    "                              O\n",
    "        O                     O\n",
    "        O                     O\n",
    "        O                     O                     O\n",
    "        O                     O                     O\n",
    "        O                     O                     O\n",
    "        O                     O                     O\n",
    "        O                     O                     O\n",
    "        O                     O                     O\n",
    "        O                     O                     O\n",
    "        O                     O                     O\n",
    "        O                     O                     O\n",
    "        O                     O                     |\n",
    "        O                     O                     |\n",
    "        |                     O                     |\n",
    "        |                     O                     |\n",
    "        |                     O                     |\n",
    "        |                     |                     |\n",
    "        |                     |                     |\n",
    "        |                     |                     |\n",
    "        |                     |                     |\n",
    "        |                     |                     |\n",
    "      Input                Hidden                Output\n",
    "      Layer                Layer                  Layer\n",
    " (inp_size, no_inp)  (hid_size, inp_size)  (out_size, hid_size)\n",
    "    (784, 1000)          (1500, 784)            (10, 1500)\n",
    "</pre>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "Network : A class which creates a network as given above and initialize weights and biases randomly\n",
    "          Further, netowrk is trained my method gradient descent, grad_descn\n",
    "</pre>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class network(object):\n",
    "\n",
    "#   error : List of errors predicted after iterating over all epochs for ploting error vs epochs graph\n",
    "#   error.append(train_error)\n",
    "    error = []\n",
    "\n",
    "    def __init__(self, size):\n",
    "        self.num_layers = len(size)\n",
    "#       Initialize biases randomly        \n",
    "        self.biases  = [np.zeros([y, 1]) for y in size[1:]]\n",
    "#       Initialize weights randomly    \n",
    "        self.weights = [np.random.randn(y, x)*0.01 for x, y in zip(size[:-1], size[1:])]\n",
    "\n",
    "#   Iterates forward to generate results predicted by network\n",
    "    def train_feed_forward(self, size, input, activators, mini_batch_size):\n",
    "#       $self.z : List of numpy array of outputs of every neuron\n",
    "        self.z = [np.zeros([y, mini_batch_size]) for y in size[:]]\n",
    "#       i : Used as index for using actiator function from list \"activators\"\n",
    "        i=0\n",
    "#       $self.z[0] : Same as input values\n",
    "        self.z[0] = input\n",
    "#       Total iteration of loop is same as number of layers present in model\n",
    "        for bias, weight in zip(self.biases, self.weights):\n",
    "#           input : Placeholder to calculate prediction by model and self.z\n",
    "#             y   =     m   *   x   +  c\n",
    "#           input = (weight * input) + bias\n",
    "            input = (np.dot(weight, input) + bias)\n",
    "            self.z[i+1] = input\n",
    "#           input = activator(input)\n",
    "            if(activators[i]==\"sigmoid\"):\n",
    "                input = activator.sigmoid(z=input, derivative=False)\n",
    "            elif(activators[i]==\"softmax\"):\n",
    "                input = activator.softmax(z=input, derivative=False)\n",
    "            elif(activators[i]==\"tanh\"):\n",
    "                input = activator.tanh(z=input, derivative=False)\n",
    "            elif(activators[i]==\"relu\"):\n",
    "                input = activator.relu(z=input, derivative=False)\n",
    "            i=i+1\n",
    "#       returns output from last layer\n",
    "        return input\n",
    " \n",
    "    \n",
    "#   Mean squared error\n",
    "#   (predicted - expected)**2\n",
    "    def loss(self, Y, Y_hat, derivative=False):\n",
    "        if derivative==True:\n",
    "            return (Y_hat-Y)\n",
    "        return ((Y_hat - Y) ** 2)\n",
    "    \n",
    "\n",
    "#   Implementation of Gradient Descent Algorithm\n",
    "    def grad_descn(self, size, expected_value, training_data, activators, alpha, mini_batch_size, drop_prob, epochs):\n",
    "#       Result : Numpy Array of One Hot Encoded expected value\n",
    "#                Just like demultiplexers\n",
    "#\n",
    "#                0   1   2   3   4   5   6   7   8   9\n",
    "#\n",
    "#                1   0   0   0   0   0   0   0   0   0\n",
    "#                0   1   0   0   0   0   0   0   0   0\n",
    "#                0   0   1   0   0   0   0   0   0   0\n",
    "#                0   0   0   1   0   0   0   0   0   0\n",
    "#                0   0   0   0   1   0   0   0   0   0\n",
    "#                0   0   0   0   0   1   0   0   0   0\n",
    "#                0   0   0   0   0   0   1   0   0   0\n",
    "#                0   0   0   0   0   0   0   1   0   0\n",
    "#                0   0   0   0   0   0   0   0   1   0\n",
    "#                0   0   0   0   0   0   0   0   0   1\n",
    "#\n",
    "        result = np.zeros([size[-1], len(training_data.T)])\n",
    "        for i in range(len(training_data)):\n",
    "            result[expected_value[0, i], i]=True\n",
    "\n",
    "#       Training the network for $epochs number of times\n",
    "#       Iterates again and again through the same data\n",
    "        for epoch_no in range(epochs):\n",
    "            print(epoch_no)\n",
    "#           $nabla_b : List of Numpy Array of Gradients of every Biases\n",
    "#           $nabla_w : List of Numpy Array of Gradients of every Weights\n",
    "            nabla_b = [np.zeros([y, 1]) for y in size[1:]]\n",
    "            nabla_w = [np.zeros([y, x]) for x, y in zip(size[:-1], size[1:])]\n",
    "#           Splits data into mini batches for training\n",
    "#           Weights and Biases updates after every iterarion over mini batch\n",
    "            for k in range(0, len(training_data), mini_batch_size):\n",
    "                mini_batch = training_data[:, k:k+mini_batch_size]\n",
    "                y = result[:, k:k+mini_batch_size]\n",
    "#               delta_nabla   : Error to be corrected in prediction\n",
    "#                               List of List of Numpy Arrays of delta_nabla_b and delta_nabla_w\n",
    "#                               [delta_nabla_b, delta_nabla_w]\n",
    "#               delta_nabla_b : List of Numpy Arrays of Gradient of biases to be corrected\n",
    "#               delta_nabla_b : List of Numpy Arrays of Gradient of weights to be corrected\n",
    "                delta_nabla = self.find_nabla(size=size, activators=activators, mini_batch=mini_batch, mini_batch_size=mini_batch_size, y=y, alpha=alpha)\n",
    "#               self.biases  = self.biases - (learning_rate * error)\n",
    "#                              for every neurons\n",
    "#               self.weights = self.weights - (learning_rate * error)\n",
    "#                              for every neuron\n",
    "                self.biases  = [b-((alpha/mini_batch_size)*n_b) for b, n_b in zip(self.biases, delta_nabla[0])]\n",
    "                self.weights = [w-((alpha/mini_batch_size)*n_w) for w, n_w in zip(self.weights, delta_nabla[1])]\n",
    "\n",
    "#               y_hat : Result predicted by model for current mini batch to calculate error\n",
    "                y_hat = test_feed_forward(size=size, input=mini_batch, activators=activators)\n",
    "#               train_error : Sum of errors of predictions of current mini batch\n",
    "                train_error = np.sum((1/mini_batch_size)*self.loss(Y=y, Y_hat=y_hat))\n",
    "#               print(train_error)\n",
    "                self.error.append(train_error)\n",
    "\n",
    "#   On every iteration of mini batch, control is parsed to find gradient of current mini batch and back propogation\n",
    "    def find_nabla(self, size, activators, mini_batch, mini_batch_size, y, alpha):\n",
    "#       y_hat : Result predicted by model for back propogation\n",
    "        y_hat = self.train_feed_forward(size=size, input=mini_batch, activators=activators, mini_batch_size=mini_batch_size)\n",
    "#       cost : Mean of loss over current mini batch\n",
    "        cost = (1/mini_batch_size)*self.loss(Y=y, Y_hat=y_hat)\n",
    "#       error : Sum of cost of outputs\n",
    "#       delta_nabla_b : List of Numpy Array of error of gradient of biases  of every neurons \n",
    "#       delta_nabla_w : List of Numpy Array of error of gradient of weights of every neurons \n",
    "        delta_nabla_b = [np.zeros([y, 1]) for y in size[1:]]\n",
    "        delta_nabla_w = [np.zeros([y, x]) for x, y in zip(size[:-1], size[1:])]\n",
    "#       Iteration over every elements of mini batch\n",
    "#       delta : Numpy Array of Error of prediction in last layer of every elements in mini batch\n",
    "        if activators[-1] == \"sigmoid\":\n",
    "            delta = self.loss(Y=y, Y_hat=y_hat, derivative=True) * activator.sigmoid(z=y_hat, derivative=True)\n",
    "        elif activators[-1] == \"softmax\":\n",
    "            delta = self.loss(Y=y, Y_hat=y_hat, derivative=True) * activator.softmax(z=y_hat, derivative=True)\n",
    "        elif activators[-1] == \"tanh\":\n",
    "            delta = self.loss(Y=y, Y_hat=y_hat, derivative=True) * activator.tanh(z=y_hat, derivative=True)\n",
    "        elif activators[-1] == \"relu\":\n",
    "            delta = self.loss(Y=y, Y_hat=y_hat, derivative=True) * activator.relu(z=y_hat, derivative=True)\n",
    "        delta_nabla_b[-1] += np.sum(delta)\n",
    "        delta_nabla_w[-1] += np.dot(delta, self.z[-2].T)\n",
    "#       Back Propogation Algorithm : Iteration over all the layers from back\n",
    "        for layer_no in range(-1, -self.num_layers+2, -1):\n",
    "#           delta : Numpy Array of Error of prediction in current layer of every elements in mini batch\n",
    "            if activators[layer_no] == \"sigmoid\":\n",
    "                delta = np.dot(self.weights[layer_no].T, delta) * activator.sigmoid(z=self.z[layer_no-1], derivative=True)\n",
    "            elif activators[layer_no] == \"softmax\":\n",
    "                delta = np.dot(self.weights[layer_no].T, delta) * activator.softmax(z=self.z[layer_no-1], derivative=True)\n",
    "            elif activators[layer_no] == \"tanh\":    \n",
    "                delta = np.dot(self.weights[layer_no].T, delta) * activator.tanh(z=self.z[layer_no-1], derivative=True)\n",
    "            elif activators[layer_no] == \"relu\":\n",
    "                delta = np.dot(self.weights[layer_no].T, delta) * activator.relu(z=self.z[layer_no-1], derivative=True)\n",
    "            delta_nabla_b[layer_no-1] += np.sum(delta)\n",
    "            delta_nabla_w[layer_no-1] += np.dot(delta, self.z[layer_no-2].T)\n",
    "            drop_out = np.random.rand(delta_nabla_w[layer_no]) < drop_prob\n",
    "            delta_nabla_w = np.multiply(delta_nabla_w, drop_out)\n",
    "            delta_nabla_w /= drop_prob\n",
    "        delta_nabla = [delta_nabla_b, delta_nabla_w]\n",
    "        return delta_nabla\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#   Feed Forward Network for test data\n",
    "def test_feed_forward(size, input, activators):\n",
    "       i=0\n",
    "       for bias, weight in zip(mnist.biases, mnist.weights):\n",
    "           input = (np.dot(weight, input) + bias)\n",
    "           if activators[i]==\"sigmoid\":\n",
    "               input = activator.sigmoid(z=input, derivative=False)\n",
    "           elif activators[i]==\"softmax\":\n",
    "               input = activator.softmax(z=input, derivative=False)\n",
    "           elif activators[i]==\"tanh\":\n",
    "               input = activator.tanh(z=input, derivative=False)\n",
    "           elif activators[i]==\"relu\":\n",
    "               input = activator.relu(z=input, derivative=False)\n",
    "           i=i+1\n",
    "       return input\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#   train_data_frame : Pandas dataframe of mnist dataset in csv file\n",
    "train_data_frame = pd.read_csv('/home/pushpull/mount/intHdd/Project/ml/mnist_train.csv', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#   Conversion of Pandas dataframe into numpy arrays\n",
    "train_dataset = np.array(train_data_frame)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "It is good practice to first shuffle the data randomly to avoid fitting the model for some particular output\n",
    "</pre>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 785)\n",
      "(1, 60000)\n",
      "(784, 60000)\n"
     ]
    }
   ],
   "source": [
    "#   Shuffling the data using \"random\" library\n",
    "np.random.shuffle(train_dataset)\n",
    "\n",
    "train_lable = np.array([train_dataset[:, 0]])\n",
    "train_data = np.array(train_dataset[:, 1:785]).T\n",
    "\n",
    "print(train_dataset.shape)\n",
    "print(train_lable.shape)\n",
    "print(train_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f66dd097588>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAADdpJREFUeJzt3X+IHPUZx/HP02gE04CJpcdpbGPrpRADmnJo0aOmtJFUheg/sSFCisETrGCgxIpFq0illGrtPwZODCbaWn8kh8FImzTG2EotJlJ/RJsflYRePHP1IjQBMdU8/WMn5dTb72x2Znf27nm/4LjdeXZ2nmzyyczOd2e/5u4CEM8Xqm4AQDUIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoE5p58bMjI8TAi3m7tbI4wrt+c1skZntNrN9ZnZbkecC0F7W7Gf7zWyKpD2SFkoakvSKpKXu/lZiHfb8QIu1Y89/kaR97v6Oux+T9HtJiws8H4A2KhL+syX9a8z9oWzZp5hZv5ntMLMdBbYFoGQtP+Hn7gOSBiQO+4FOUmTPf1DSOWPuz8qWAZgAioT/FUk9ZnaumU2V9ANJG8tpC0CrNX3Y7+4fm9nNkv4oaYqkNe6+q7TOALRU00N9TW2M9/xAy7XlQz4AJi7CDwRF+IGgCD8QFOEHgiL8QFBtvZ4faKc777yzbu3uu+9Orjs0NJSs9/X1JesHDhxI1jsBe34gKMIPBEX4gaAIPxAU4QeCIvxAUAz1oWNNnz49Wb/nnnuS9RUrVtStHT9+PLlud3d3oTpDfQA6FuEHgiL8QFCEHwiK8ANBEX4gKMIPBMU4PyrT1dWVrG/evDlZP//888ts51PuuOOOZH3Xron/LfXs+YGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gqEKz9JrZfklHJH0i6WN37815PLP04v8WL16crK9fv75l2x4eHk7W58yZk6x/+OGHZbZTqkZn6S3jQz7fcff3S3geAG3EYT8QVNHwu6TNZrbTzPrLaAhAexQ97O9z94Nm9mVJW8zsH+7+4tgHZP8p8B8D0GEK7fnd/WD2e0TSoKSLxnnMgLv35p0MBNBeTYffzKaZ2fQTtyVdLunNshoD0FpFDvu7JA2a2Ynn+Z27/6GUrgC0XNPhd/d3JF1QYi+YhC677LK6tcHBweS6RT6DIkmbNm2qW8uboruTx/HLwlAfEBThB4Ii/EBQhB8IivADQRF+ICi+uhuFnHnmmcl6ahrtvKG8vHre12s/8MADdWsRhvLysOcHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAY50dS3jj+008/naxfcsklTW97dHQ0WX/qqaeSdcby09jzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQjPMHl/pqbSl9Pb5UbBx/+/btyXre9fr79u1rettgzw+ERfiBoAg/EBThB4Ii/EBQhB8IivADQeWO85vZGklXSRpx93nZspmSnpA0W9J+SUvc/YPWtYlWWbZsWbJ+6aWXFnr+o0eP1q2tWrUque7OnTsLbRtpjez5H5G06DPLbpO01d17JG3N7gOYQHLD7+4vSjr8mcWLJa3Nbq+VdHXJfQFosWbf83e5+3B2+z1JXSX1A6BNCn+2393dzOpOqmZm/ZL6i24HQLma3fMfMrNuScp+j9R7oLsPuHuvu/c2uS0ALdBs+DdKWp7dXi7pmXLaAdAuueE3s8cl/VXSN8xsyMxWSPqFpIVmtlfS97L7ACYQy5sDvdSNJc4NoDVWr16drPf3p0/HFP33sXDhwrq1bdu2FXpujM/drZHH8Qk/ICjCDwRF+IGgCD8QFOEHgiL8QFB8dfckMDAwULd23XXXtXTbN954Y7L+0ksvtXT7aB57fiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IinH+CeCGG25I1lNj+VOnTi207ZtuuilZf/TRR5P1Y8eOFdo+Woc9PxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ExVd3d4AzzjgjWX/55ZeT9Z6enqa3vXv37mR97ty5TT+3JJ111ll1a1OmTEmu+9FHHyXrIyN1J4oKja/uBpBE+IGgCD8QFOEHgiL8QFCEHwiK8ANB5V7Pb2ZrJF0lacTd52XL7pJ0g6R/Zw+73d2fa1WTk93KlSuT9fPOOy9ZT31WY9OmTcl1V61alazPmTMnWc+73n/FihV1a6effnpy3dHR0WS9r68vWd+zZ0+yHl0je/5HJC0aZ/mv3f3C7IfgAxNMbvjd/UVJh9vQC4A2KvKe/2Yze93M1pjZjNI6AtAWzYZ/taSvS7pQ0rCk++o90Mz6zWyHme1oclsAWqCp8Lv7IXf/xN2PS3pI0kWJxw64e6+79zbbJIDyNRV+M+sec/caSW+W0w6AdmlkqO9xSQskfcnMhiT9TNICM7tQkkvaLyk9TzOAjsP1/G2wdOnSZP2xxx5L1vP+joaGhurWrrzyyuS6M2akz9Vu2LAhWZ85c2aynmKWvuw87899/fXXJ+vr1q076Z4mA67nB5BE+IGgCD8QFOEHgiL8QFCEHwiKob4S5F32umXLlmR91qxZyXre39GiReNddFnzwQcfJNd9/vnnk/Vp06Yl60UUHep77bXXkvUFCxbUrR05ciS57kTGUB+AJMIPBEX4gaAIPxAU4QeCIvxAUIQfCCr3en7UpKaavvfee5Pr5o3j5413Dw4OJusXX3xx3dqtt96aXHf69OnJet5Ye9402Q8++GDdWt7Xhud9xuDJJ59M1ifzWH4Z2PMDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCM8zfo3XffrVvr7u6uW5Pyx8rz5E3RnZI3Vp7X2wsvvJCsL1myJFk/7bTT6taWLVuWXDfvz536O0E+9vxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EFTuOL+ZnSNpnaQuSS5pwN1/Y2YzJT0habak/ZKWuHv6S+InqT179iTrqevtGzFv3rxC9ZS86/HzxvFHR0eT9dT1/D09Pcl18z6DsHfv3mQdaY3s+T+W9GN3nyvpW5J+ZGZzJd0maau790jamt0HMEHkht/dh9391ez2EUlvSzpb0mJJa7OHrZV0dauaBFC+k3rPb2azJc2X9DdJXe4+nJXeU+1tAYAJouHP9pvZFyWtl7TS3f8z9nvn3N3rzcNnZv2S+os2CqBcDe35zexU1YL/W3ffkC0+ZGbdWb1b0rhnjtx9wN173b23jIYBlCM3/FbbxT8s6W13v39MaaOk5dnt5ZKeKb89AK2SO0W3mfVJ+rOkNyQdzxbfrtr7/iclfUXSAdWG+g7nPNeknKL7ggsuSNa3b9+erBf9+uwitm3blqw/99xzyfq1116brM+fP79u7ZRT0u86BwYGkvVbbrklWT927FiyPlk1OkV37nt+d/+LpHpP9t2TaQpA5+ATfkBQhB8IivADQRF+ICjCDwRF+IGgcsf5S93YJB3nBzpJo+P87PmBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiCo3PCb2Tlmts3M3jKzXWZ2S7b8LjM7aGZ/z36uaH27AMqSO2mHmXVL6nb3V81suqSdkq6WtETSUXf/VcMbY9IOoOUanbTjlAaeaFjScHb7iJm9LensYu0BqNpJvec3s9mS5kv6W7boZjN73czWmNmMOuv0m9kOM9tRqFMApWp4rj4z+6Kk7ZJ+7u4bzKxL0vuSXNI9qr01uD7nOTjsB1qs0cP+hsJvZqdKelbSH939/nHqsyU96+7zcp6H8AMtVtpEnWZmkh6W9PbY4GcnAk+4RtKbJ9skgOo0cra/T9KfJb0h6Xi2+HZJSyVdqNph/35JN2YnB1PPxZ4faLFSD/vLQviB1ivtsB/A5ET4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IKvcLPEv2vqQDY+5/KVvWiTq1t07tS6K3ZpXZ21cbfWBbr+f/3MbNdrh7b2UNJHRqb53al0RvzaqqNw77gaAIPxBU1eEfqHj7KZ3aW6f2JdFbsyrprdL3/ACqU/WeH0BFKgm/mS0ys91mts/Mbquih3rMbL+ZvZHNPFzpFGPZNGgjZvbmmGUzzWyLme3Nfo87TVpFvXXEzM2JmaUrfe06bcbrth/2m9kUSXskLZQ0JOkVSUvd/a22NlKHme2X1OvulY8Jm9m3JR2VtO7EbEhm9ktJh939F9l/nDPc/Scd0ttdOsmZm1vUW72ZpX+oCl+7Mme8LkMVe/6LJO1z93fc/Zik30taXEEfHc/dX5R0+DOLF0tam91eq9o/nrar01tHcPdhd381u31E0omZpSt97RJ9VaKK8J8t6V9j7g+ps6b8dkmbzWynmfVX3cw4usbMjPSepK4qmxlH7szN7fSZmaU75rVrZsbrsnHC7/P63P2bkr4v6UfZ4W1H8tp7tk4arlkt6euqTeM2LOm+KpvJZpZeL2mlu/9nbK3K126cvip53aoI/0FJ54y5Pytb1hHc/WD2e0TSoGpvUzrJoROTpGa/Ryru5//c/ZC7f+LuxyU9pApfu2xm6fWSfuvuG7LFlb924/VV1etWRfhfkdRjZuea2VRJP5C0sYI+PsfMpmUnYmRm0yRdrs6bfXijpOXZ7eWSnqmwl0/plJmb680srYpfu46b8drd2/4j6QrVzvj/U9JPq+ihTl9fk/Ra9rOr6t4kPa7aYeB/VTs3skLSmZK2Stor6U+SZnZQb4+qNpvz66oFrbui3vpUO6R/XdLfs58rqn7tEn1V8rrxCT8gKE74AUERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8I6n/DEpC6XT5MJgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(train_dataset[0, 1:785].reshape(28, 28), 'gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#   neuron_layer : Dictionary of number of neurons in model and activation functions for every layer\n",
    "neuron_layer = {\"size_layers\": [784, 2800, 10], \"activations\": [\"tanh\", \"sigmoid\"] }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#   Initializing network with random weights and biases\n",
    "mnist = network(neuron_layer[\"size_layers\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n"
     ]
    }
   ],
   "source": [
    "#   Training network using Gradient Descent Algorithm\n",
    "mnist.grad_descn(size=neuron_layer[\"size_layers\"], expected_value=train_lable, training_data=train_data, activators=neuron_layer[\"activations\"], alpha=0.01, mini_batch_size=2000, drop_prob=0.9, epochs=40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_frame = pd.read_csv('/home/pushpull/mount/intHdd/Project/ml/mnist_test.csv', header=None)\n",
    "\n",
    "test_dataset = np.array(test_data_frame)\n",
    "\n",
    "test_lable = np.array([test_dataset[:, 0]]).T\n",
    "test_data = np.array(test_dataset[:, 1:785])\n",
    "\n",
    "result = test_feed_forward(size=neuron_layer[\"size_layers\"], input=test_data.T, activators=neuron_layer[\"activations\"])\n",
    "\n",
    "no_trues = 0\n",
    "\n",
    "for i in range(len(test_data)):\n",
    "    max_ans = result[0, i]\n",
    "    max_ind = 0\n",
    "    for j in range(10):\n",
    "        if(result[j, i]>max_ans):\n",
    "            max_ind = j\n",
    "            max_ans = result[j, i]\n",
    "    if(test_lable[i]==max_ind):\n",
    "        no_trues+=1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f66dde728d0>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAADMdJREFUeJzt3V+oJvV9x/H3tya5MbnQhi7LulvTIIUg1JRFXZWSkhqsBDSwf+JF2VLp5iJCd9eLSnpRoRSkmJVeBTYo2ZTUuIuKEkqTVEptcXdxFevfJlrZ7O5hdSsGYq5S9duLM1tP9JyZs8/M88xzzvf9gsN5nvk9M/NlOJ8zM89vZn6RmUiq5zfGLkDSOAy/VJThl4oy/FJRhl8qyvBLRRl+qSjDLxVl+KWiPjbLlUWElxNKU5aZsZrP9drzR8RNEfGTiHgtIu7qsyxJsxWTXtsfERcBPwVuBM4ATwO3ZebLLfO455embBZ7/quB1zLz9cz8FfB94JYey5M0Q33Cvwk4veT9mWbar4mIPRFxIiJO9FiXpIFN/Qu/zDwIHAQP+6V50mfPvwBsXvL+smaapDWgT/ifBq6IiM9ExCeArwKPD1OWpGmb+LA/M9+NiDuAHwIXAQ9k5kuDVSZpqibu6ptoZZ7zS1M3k4t8JK1dhl8qyvBLRRl+qSjDLxVl+KWiDL9UlOGXijL8UlGGXyrK8EtFGX6pKMMvFTXTR3drefv27WttP3DgQGv7tm3bVmw7duzYRDVp/XPPLxVl+KWiDL9UlOGXijL8UlGGXyrK8EtF2c8/B7r6+aVpcM8vFWX4paIMv1SU4ZeKMvxSUYZfKsrwS0X16uePiJPAO8B7wLuZuXWIotabzZs392rfv39/a7v37GsSQ1zk84eZ+dYAy5E0Qx72S0X1DX8CP4qIZyJizxAFSZqNvof9N2TmQkT8FvDjiPivzHxy6Qeafwr+Y5DmTK89f2YuNL/PAY8CVy/zmYOZudUvA6X5MnH4I+LiiPjU+dfAl4AXhypM0nT1OezfADwaEeeX84+Z+c+DVCVp6iYOf2a+DvzegLWsW9u3b+81/8LCwkCVSB+wq08qyvBLRRl+qSjDLxVl+KWiDL9UlI/unoEdO3b0mv/o0aMDVSJ9wD2/VJThl4oy/FJRhl8qyvBLRRl+qSjDLxUVmTm7lUXMbmUz1PXo7VOnTrW2nz59urV9y5YtF1yT6srMWM3n3PNLRRl+qSjDLxVl+KWiDL9UlOGXijL8UlHezz+AvXv39pr/yJEjA1Uyf6699toV2/pev3DNNde0trdtV4c1d88vlWX4paIMv1SU4ZeKMvxSUYZfKsrwS0V19vNHxAPAl4FzmXllM+1S4CHgcuAksDMzfz69Mudb1/38XY4fPz5QJbPX1o8PcPjw4RXb+m63Lvv371+xbdu2ba3zVrgOYDV7/u8AN31o2l3AE5l5BfBE817SGtIZ/sx8Enj7Q5NvAQ41rw8Btw5cl6Qpm/Scf0Nmnm1evwFsGKgeSTPS+9r+zMy2Z/NFxB5gT9/1SBrWpHv+NyNiI0Dz+9xKH8zMg5m5NTO3TrguSVMwafgfB3Y3r3cDjw1TjqRZ6Qx/RDwIHAV+NyLORMTtwD3AjRHxKvBHzXtJa0jnOX9m3rZC0xcHrkVrUFtfOvTry+9a9tGjR1vb264xOHDgQOu81113XWv7euAVflJRhl8qyvBLRRl+qSjDLxVl+KWifHS3WnXdsrtjx46Jl71r167W9rauutVouy23T93rhXt+qSjDLxVl+KWiDL9UlOGXijL8UlGGXyrKfn616rqttktbX37ffvxp2rlzZ2v7PNe+Wu75paIMv1SU4ZeKMvxSUYZfKsrwS0UZfqko+/kHcPr06V7zb9q0aaBKLlzXo7W77nvv8/hsjcs9v1SU4ZeKMvxSUYZfKsrwS0UZfqkowy8VFZnZ/oGIB4AvA+cy88pm2t3AnwP/03zsG5n5T50ri2hf2RrV1Vd+6tSpXsuPiF7zt+nqh+/q5++63/++++674JqG8tRTT63Ydtlll7XOu2XLlqHLmZnMXNUfzGr2/N8Bblpm+n2ZeVXz0xl8SfOlM/yZ+STw9gxqkTRDfc7574iI5yPigYi4ZLCKJM3EpOH/FvBZ4CrgLPDNlT4YEXsi4kREnJhwXZKmYKLwZ+abmfleZr4PfBu4uuWzBzNza2ZunbRIScObKPwRsXHJ268ALw5TjqRZ6bylNyIeBL4AfDoizgB/DXwhIq4CEjgJfG2KNUqags7wZ+Zty0y+fwq1rFld9/N33fO+bdu21vZ9+/a1to/Zl76wsDDauruerd+2XfuOR7AeeIWfVJThl4oy/FJRhl8qyvBLRRl+qajOW3oHXdk6vaW3S1eX1EMPPdRr+UeOHFmx7cCBA63zdnVDdum69bXPY827ttu999478bKvv/761va+j2Mf05C39Epahwy/VJThl4oy/FJRhl8qyvBLRRl+qSj7+edA1y27Xe1djw6fpl27dk087/bt21vbux4b3qXtlt5jx471WvY8s59fUivDLxVl+KWiDL9UlOGXijL8UlGGXyrKfv41oKsff+/evSu2reVHVHc9a6DrGoO1fE9+H/bzS2pl+KWiDL9UlOGXijL8UlGGXyrK8EtFdfbzR8Rm4LvABiCBg5n59xFxKfAQcDlwEtiZmT/vWJb9/HPm1KlTre19nxXQNm7A8ePHW+c9fPhwr3VXNWQ//7vAnZn5OeBa4OsR8TngLuCJzLwCeKJ5L2mN6Ax/Zp7NzGeb1+8ArwCbgFuAQ83HDgG3TqtIScO7oHP+iLgc+DxwHNiQmWebpjdYPC2QtEZ8bLUfjIhPAg8DezPzFxEfnFZkZq50Ph8Re4A9fQuVNKxV7fkj4uMsBv97mflIM/nNiNjYtG8Ezi03b2YezMytmbl1iIIlDaMz/LG4i78feCUzl351+ziwu3m9G3hs+PIkTctqDvuvB/4EeCEinmumfQO4BzgcEbcDPwPax1PWXGob3hv63xLc1lXY1Y3Y1Q25nh+/PQud4c/M/wBW6jf84rDlSJoVr/CTijL8UlGGXyrK8EtFGX6pKMMvFbXqy3u1Pt1555295u8zjHbXo7kXFhYmXra6ueeXijL8UlGGXyrK8EtFGX6pKMMvFWX4paIcoltaZxyiW1Irwy8VZfilogy/VJThl4oy/FJRhl8qyvBLRRl+qSjDLxVl+KWiDL9UlOGXijL8UlGGXyqqM/wRsTki/jUiXo6IlyLiL5rpd0fEQkQ81/zcPP1yJQ2l82EeEbER2JiZz0bEp4BngFuBncAvM/PeVa/Mh3lIU7fah3l0jtiTmWeBs83rdyLiFWBTv/Ikje2Czvkj4nLg88DxZtIdEfF8RDwQEZesMM+eiDgRESd6VSppUKt+hl9EfBL4N+BvM/ORiNgAvAUk8Dcsnhr8WccyPOyXpmy1h/2rCn9EfBz4AfDDzDywTPvlwA8y88qO5Rh+acoGe4BnRARwP/DK0uA3XwSe9xXgxQstUtJ4VvNt/w3AvwMvAO83k78B3AZcxeJh/0nga82Xg23Lcs8vTdmgh/1DMfzS9PncfkmtDL9UlOGXijL8UlGGXyrK8EtFGX6pKMMvFWX4paIMv1SU4ZeKMvxSUYZfKsrwS0V1PsBzYG8BP1vy/tPNtHk0r7XNa11gbZMasrbfXu0HZ3o//0dWHnEiM7eOVkCLea1tXusCa5vUWLV52C8VZfilosYO/8GR199mXmub17rA2iY1Sm2jnvNLGs/Ye35JIxkl/BFxU0T8JCJei4i7xqhhJRFxMiJeaEYeHnWIsWYYtHMR8eKSaZdGxI8j4tXm97LDpI1U21yM3NwysvSo227eRrye+WF/RFwE/BS4ETgDPA3clpkvz7SQFUTESWBrZo7eJxwRfwD8Evju+dGQIuLvgLcz857mH+clmfmXc1Lb3VzgyM1Tqm2lkaX/lBG33ZAjXg9hjD3/1cBrmfl6Zv4K+D5wywh1zL3MfBJ4+0OTbwEONa8PsfjHM3Mr1DYXMvNsZj7bvH4HOD+y9KjbrqWuUYwR/k3A6SXvzzBfQ34n8KOIeCYi9oxdzDI2LBkZ6Q1gw5jFLKNz5OZZ+tDI0nOz7SYZ8XpofuH3UTdk5u8Dfwx8vTm8nUu5eM42T9013wI+y+IwbmeBb45ZTDOy9MPA3sz8xdK2MbfdMnWNst3GCP8CsHnJ+8uaaXMhMxea3+eAR1k8TZknb54fJLX5fW7kev5fZr6Zme9l5vvAtxlx2zUjSz8MfC8zH2kmj77tlqtrrO02RvifBq6IiM9ExCeArwKPj1DHR0TExc0XMUTExcCXmL/Rhx8HdjevdwOPjVjLr5mXkZtXGlmakbfd3I14nZkz/wFuZvEb//8G/mqMGlao63eA/2x+Xhq7NuBBFg8D/5fF70ZuB34TeAJ4FfgX4NI5qu0fWBzN+XkWg7ZxpNpuYPGQ/nnguebn5rG3XUtdo2w3r/CTivILP6kowy8VZfilogy/VJThl4oy/FJRhl8qyvBLRf0fENo7F94MSlkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(test_dataset[22, 1:785].reshape(28, 28), 'gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000, 1)\n"
     ]
    }
   ],
   "source": [
    "print(test_lable.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8]\n"
     ]
    }
   ],
   "source": [
    "print(test_lable[110])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10, 10000)\n"
     ]
    }
   ],
   "source": [
    "print(result.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8491\n"
     ]
    }
   ],
   "source": [
    "print(no_trues)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "84.91\n"
     ]
    }
   ],
   "source": [
    "print(100.0*(no_trues/len(test_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f66ddde1e80>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAADWxJREFUeJzt3X+oVHUax/HPU2l/pP0y9iLlZkltRGbWTZaQpdB+bCyYBaERuKzsNakoWGIl/9CKhYqtZQkKrESNtlq0iyKylbJsCVt4i9afqW1oaVfdMLJAyOrZP+YYt7rzPePMmTnn3uf9gsudOc+ccx5GP/ecM9+Z+Zq7C0A8J5XdAIByEH4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0Gd0smdmRlvJwTazN2tkce1dOQ3s5vMbKeZfWhmC1rZFoDOsmbf229mJ0vaJel6SfskbZI02923J9bhyA+0WSeO/FMkfejuH7n715JeljSjhe0B6KBWwn+upE8G3N+XLfsBM+sxsz4z62thXwAK1vYX/Nx9iaQlEqf9QJW0cuTfL2ncgPvnZcsADAGthH+TpIvM7AIzGylplqQ1xbQFoN2aPu1392/M7B5Jr0k6WdJSd99WWGcA2qrpob6mdsY1P9B2HXmTD4Chi/ADQRF+ICjCDwRF+IGgCD8QVEc/z4/h5/TTT0/WFy1aVLd22223Jde97777kvXVq1cn60jjyA8ERfiBoAg/EBThB4Ii/EBQhB8IiqE+tGTlypXJ+rRp05re9sUXX9z0usjHkR8IivADQRF+ICjCDwRF+IGgCD8QFOEHgmKcH0lz585N1qdPn56sr1q1qm7t7bffTq7LR3bbiyM/EBThB4Ii/EBQhB8IivADQRF+ICjCDwTV0iy9ZrZH0peSvpX0jbt35zyeWXorprs7+U+mjRs3JuvHjh1L1q+++uq6tQ8++CC5LprT6Cy9RbzJ5zp3/6yA7QDoIE77gaBaDb9Let3M3jWzniIaAtAZrZ72T3X3/Wb2M0lvmNkH7v7mwAdkfxT4wwBUTEtHfnffn/0+JKlX0pRBHrPE3bvzXgwE0FlNh9/MTjOz0cdvS7pB0taiGgPQXq2c9ndJ6jWz49v5m7v/o5CuALRdS+P8J7wzxvk77vzzz0/W169fn6xfeOGFyfqkSZOS9a1bORnstEbH+RnqA4Ii/EBQhB8IivADQRF+ICjCDwTFV3cPAyNGjKhbmzVrVnLdCRMmJOupr96WGMobyjjyA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQfKR3GJg3b17d2tNPP51cd/v27cn6lCk/+XKmHzh69Giyjs7jI70Akgg/EBThB4Ii/EBQhB8IivADQRF+ICg+zz8EjBkzJlm/884769Y+/fTT5Lp5n/dnHH/44sgPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0HljvOb2VJJv5F0yN0vy5adLekVSeMl7ZF0u7t/3r42Y3v44YeT9WuuuaZubfXq1cl1t23b1lRPGPoaOfIvk3TTj5YtkLTB3S+StCG7D2AIyQ2/u78p6fCPFs+QtDy7vVzSLQX3BaDNmr3m73L3/uz2AUldBfUDoENafm+/u3vqu/nMrEdST6v7AVCsZo/8B81srCRlvw/Ve6C7L3H3bnfvbnJfANqg2fCvkTQnuz1HUvolZQCVkxt+M3tJ0r8l/cLM9pnZXEmPSrrezHZLmp7dBzCE5F7zu/vsOqVpBfeCOiZOnNj0uuvWrSuwEwwnvMMPCIrwA0ERfiAowg8ERfiBoAg/EBRTdFdAV1f6oxEHDhxI1nt7e+vWbr311qZ6KsqVV15ZtzZp0qTkurt27UrWt2zZkqwfOXIkWR+umKIbQBLhB4Ii/EBQhB8IivADQRF+ICjCDwTFFN0V8NxzzyXree/F2LlzZ5Ht/EB3d/oLmF544YVkffz48XVrI0eObKal761fvz5Zv/HGG1va/nDHkR8IivADQRF+ICjCDwRF+IGgCD8QFOEHgmKcvwLOPPPM0vY9a9asZP2pp55K1seMGZOspz6Tn7ft6667LlmfOXNmsj59+vS6tbz3CETAkR8IivADQRF+ICjCDwRF+IGgCD8QFOEHgsod5zezpZJ+I+mQu1+WLVss6feS/pc97EF3Zy7oCsobx8/7PP5JJ6WPD3nfRbBo0aK6tf7+/uS648aNS9bzzJ8/v26Ncf7GjvzLJN00yPK/uPsV2Q/BB4aY3PC7+5uSDnegFwAd1Mo1/z1mttnMlprZWYV1BKAjmg3/M5ImSLpCUr+kJ+o90Mx6zKzPzPqa3BeANmgq/O5+0N2/dffvJD0raUrisUvcvdvd098ECaCjmgq/mY0dcHempK3FtAOgUxoZ6ntJ0rWSzjGzfZIWSbrWzK6Q5JL2SJrXxh4BtEFu+N199iCLn29DL2jSGWecUbeW95n5vDnsH3vssWT98ccfT9bLtHbt2rJbqDTe4QcERfiBoAg/EBThB4Ii/EBQhB8Iiq/uroBly5Yl61OnTk3W77rrrqb3nfeR3DKH8syspfq6dXzYNIUjPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ExTh/BaxcuTJZv/fee5P1iRMnNr3vjz/+uOl1JWn06NHJ+qmnntr0ti+//PJk/ejRo8l66qu/Dx482FRPwwlHfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IinH+Cvjiiy+S9d7e3mS9lXH+Bx54IFm/6qqrkvXJkycn661Os53y0EMPJet9fcwQl8KRHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCMndPP8BsnKQVkrokuaQl7v5XMztb0iuSxkvaI+l2d/88Z1vpnWFQI0aMSNbnz59ft/bII48k1x01alRTPR2X9935ef+/Unbv3p2s572/4dixY03veyhz9/Q/SqaRI/83kv7g7pdK+qWku83sUkkLJG1w94skbcjuAxgicsPv7v3u/l52+0tJOySdK2mGpOXZw5ZLuqVdTQIo3gld85vZeEmTJb0jqcvd+7PSAdUuCwAMEQ2/t9/MRklaJel+dz8y8FrP3b3e9byZ9UjqabVRAMVq6MhvZiNUC/6L7v5qtvigmY3N6mMlHRpsXXdf4u7d7t5dRMMAipEbfqsd4p+XtMPdnxxQWiNpTnZ7jqTVxbcHoF0aGeqbKuktSVskfZctflC16/6/S/q5pL2qDfUdztkWQ30ddskllyTrCxcuTNbvuOOOZL2Vob69e/cm1128eHGyvmLFimQ9qkaH+nKv+d19o6R6G5t2Ik0BqA7e4QcERfiBoAg/EBThB4Ii/EBQhB8IKnecv9CdMc4PtF2RH+kFMAwRfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAULnhN7NxZvZPM9tuZtvM7L5s+WIz229m72c/N7e/XQBFyZ20w8zGShrr7u+Z2WhJ70q6RdLtkr5y9z83vDMm7QDartFJO05pYEP9kvqz21+a2Q5J57bWHoCyndA1v5mNlzRZ0jvZonvMbLOZLTWzs+qs02NmfWbW11KnAArV8Fx9ZjZK0r8k/cndXzWzLkmfSXJJj6h2afC7nG1w2g+0WaOn/Q2F38xGSFor6TV3f3KQ+nhJa939spztEH6gzQqbqNPMTNLzknYMDH72QuBxMyVtPdEmAZSnkVf7p0p6S9IWSd9lix+UNFvSFaqd9u+RNC97cTC1LY78QJsVetpfFMIPtF9hp/0AhifCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAULlf4FmwzyTtHXD/nGxZFVW1t6r2JdFbs4rs7fxGH9jRz/P/ZOdmfe7eXVoDCVXtrap9SfTWrLJ647QfCIrwA0GVHf4lJe8/paq9VbUvid6aVUpvpV7zAyhP2Ud+ACUpJfxmdpOZ7TSzD81sQRk91GNme8xsSzbzcKlTjGXToB0ys60Dlp1tZm+Y2e7s96DTpJXUWyVmbk7MLF3qc1e1Ga87ftpvZidL2iXpekn7JG2SNNvdt3e0kTrMbI+kbncvfUzYzH4l6StJK47PhmRmj0s67O6PZn84z3L3P1akt8U6wZmb29RbvZmlf6sSn7siZ7wuQhlH/imSPnT3j9z9a0kvS5pRQh+V5+5vSjr8o8UzJC3Pbi9X7T9Px9XprRLcvd/d38tufynp+MzSpT53ib5KUUb4z5X0yYD7+1StKb9d0utm9q6Z9ZTdzCC6BsyMdEBSV5nNDCJ35uZO+tHM0pV57pqZ8bpovOD3U1Pd/UpJv5Z0d3Z6W0leu2ar0nDNM5ImqDaNW7+kJ8psJptZepWk+939yMBamc/dIH2V8ryVEf79ksYNuH9etqwS3H1/9vuQpF7VLlOq5ODxSVKz34dK7ud77n7Q3b919+8kPasSn7tsZulVkl5091ezxaU/d4P1VdbzVkb4N0m6yMwuMLORkmZJWlNCHz9hZqdlL8TIzE6TdIOqN/vwGklzsttzJK0usZcfqMrMzfVmllbJz13lZrx2947/SLpZtVf8/ytpYRk91OnrQkn/yX62ld2bpJdUOw08ptprI3MljZG0QdJuSeslnV2h3l5QbTbnzaoFbWxJvU1V7ZR+s6T3s5+by37uEn2V8rzxDj8gKF7wA4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8Q1P8B45BRQFqZ3u8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(test_data[91, 0:785].reshape(28, 28), 'gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f66dddb6390>]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4VeW59/HvnZ15IEwZmOcEURQQcVagDjhUrfWcg0MdWmtr5dTO6ntsr56et6ettrZ9K6eWVmuH40FbrXJaxyoOOBKUUYjMEKYkjIGQ+X7/2Au7RQIBdrI2e/8+15Urez1rrZ0764Jf1n7Ws9Zj7o6IiKSGtLALEBGRrqPQFxFJIQp9EZEUotAXEUkhCn0RkRSi0BcRSSEKfRGRFKLQFxFJIQp9EZEUkh52Afvr3bu3Dx48OOwyRESOKfPmzat196JDbZdwoT948GAqKirCLkNE5JhiZms7sp26d0REUohCX0QkhSj0RURSiEJfRCSFKPRFRFKIQl9EJIUo9EVEUkjCjdM/UvVNLTzw8sqjfyOz6LePN5FmRiQt+pWeZqSZkR4JvqcZJd2yGdmngNJu2ZjZx99bRCRkSRP6e5ta+cXsFUf1HvGaLrhbdjojS7sxsk8B5aUFjCwtoKykgILsjPj8ABGRI5Q0od8rP4vVP7gk7u8bO3F8a5vT6k5bG7S0tUWXg6/mNmfD9r1Ubt7F0s11VG6u44l3N7C7sQWANIOfTR3LZSf1jXuNIiIdlTSh31liu2nSIxZzwCIf27Zf9xwmDOn54bK7U7V9L5Wb6/jx85Xc8+wyLjqhlIyILqWISDiUPp3IzBjQM5fzRpXwjQvKqdq+l1nzN4ZdloikMIV+F/nEccUc16cb019eQWtbnC4eiIgcJoV+FzEzpk0azqqaPTyzeFPY5YhIilLod6EpJ5QyrCiP+19aQZvO9kUkBAr9LhRJM26bNJxlm+t4cVl12OWISApS6Hexy07qy4CeOdz/0vKPDAcVEekKCv0ulh5J40sTh7OgaidzVtSGXY6IpBiFfgiuHNePPoXZ/OKlo7uDWETkcCn0Q5CVHuEL5wzlndXbeHvV1rDLEZEUotAPydQJA+mdn8n9R/m8IBGRw6HQD0l2RoTPnz2U15bXMn/9jrDLEZEU0aHQN7MpZlZpZivM7M4DrP+imS0ys/lmNsfMRgXtGWb2u2DdUjO7K96/wLHs2tMG0T03g/vVty8iXeSQoW9mEWA6cBEwCrh6X6jHeMTdR7v7GOAe4L6g/Z+ALHcfDZwMfMHMBsep9mNeflY6nz1zCH9fuoX3N+4KuxwRSQEdOdOfAKxw91Xu3gTMBC6P3cDdYxMrD9g3AN2BPDNLB3KAJkDpFuOG0weTn5XO9Jd1ti8ina8jod8PWB+zXBW0fYSZ3WZmK4me6X85aP4zsAfYBKwDfuzu246q4iRTmJvB9acP4ulFm1i7dU/Y5YhIkovbhVx3n+7uw4A7gLuD5glAK9AXGAJ83cyG7r+vmd1iZhVmVlFTUxOvko4ZnxrbD3eYt3Z72KWISJLrSOhvAAbELPcP2tozE7gieH0N8Ky7N7t7NfA6MH7/Hdx9hruPd/fxRUVFHas8iQzunUdmJI3KLXVhlyIiSa4joT8XGGFmQ8wsE5gKzIrdwMxGxCxeAiwPXq8DJgfb5AGnAcuOtuhkkxFJY1hxPpWbFfoi0rkOOV2iu7eY2TTgOaJzBD7k7kvM7HtAhbvPAqaZ2XlAM7AduCHYfTrwWzNbAhjwW3df2Bm/yLGuvCSfd1brcoeIdK4OzZHr7k8DT+/X9p2Y17e3s99uosM25RDKS7vx5PyN7NzbTGFORtjliEiS0h25CaK8NB+A5erXF5FOpNBPEOWl3QBYpn59EelECv0E0bcwm4KsdD7Qmb6IdCKFfoIwM8pKC3SmLyKdSqGfQMpKCvhgS52mURSRTqPQTyAjSwvYUd9MdV1j2KWISJJS6CeQspICAN2kJSKdRqGfQEaWKvRFpHMp9BNIj7xMiguy9AweEek0Cv0EU15aoDN9Eek0Cv0EU15SwPLqOlrbNIJHROJPoZ9gykoLaGhuY922+rBLEZEkpNBPMLqYKyKdSaGfYEYUF2Cm0BeRzqHQTzA5mREG9cylcovmjxeR+FPoJ6CyEo3gEZHOodBPQCNLC1iztZ6G5tawSxGRJKPQT0BlpQW0tjkra3aHXYqIJBmFfgLSCB4R6SwK/QQ0qFcemZE0PY5BROJOoZ+AMiJpDCvO15m+iMRdh0LfzKaYWaWZrTCzOw+w/otmtsjM5pvZHDMbFbPuRDN708yWBNtkx/MXSFblJfl8oNAXkTg7ZOibWQSYDlwEjAKujg31wCPuPtrdxwD3APcF+6YDfwS+6O7HAxOB5viVn7zKS7uxcWcDO/fqcIlI/HTkTH8CsMLdV7l7EzATuDx2A3ePvZMoD9j3tLALgIXuviDYbqu7axxiB5SX5gOwXP36IhJHHQn9fsD6mOWqoO0jzOw2M1tJ9Ez/y0FzGeBm9pyZvWtm3zraglNFeWk3AE2ULiJxFbcLue4+3d2HAXcAdwfN6cBZwLXB90+Z2Sf239fMbjGzCjOrqKmpiVdJx7S+hdkUZKXzgc70RSSOOhL6G4ABMcv9g7b2zASuCF5XAa+6e6271wNPA+P238HdZ7j7eHcfX1RU1LHKk5yZUVZaoDN9EYmrjoT+XGCEmQ0xs0xgKjArdgMzGxGzeAmwPHj9HDDazHKDi7rnAu8ffdmpoaykgA+21OGuCVVEJD4OGfru3gJMIxrgS4HH3H2JmX3PzC4LNpsWDMmcD3wNuCHYdzvRkTxzgfnAu+7+t074PZLSyNICdtQ3U13XGHYpIpIk0juykbs/TbRrJrbtOzGvbz/Ivn8kOmxTDlNZyT8ex1DSTbc3iMjR0x25Caxcz+ARkThT6CewnnmZFBVk6WKuiMSNQj/BjSwt0LBNEYkbhX6C2zeCp7VNI3hE5Ogp9BNceWkBjS1trNtWH3YpIpIEFPoJrvzDETyaKF1Ejp5CP8GVlRRgBpWbNXWiiBw9hX6Cy8mMMKhnLpVbdKYvIkevQzdnSbjKSgpYtqmO7Xua2N3Y8o+vhhbqgu/lpfmcPKhn2KWKSIJT6B8DRvbpxvPvb2Hsf7zQ7jY9cjOouPt8ImnWhZWJyLFGoX8MuO60geRkRMjOSCMvK52CrHTys9PJz0qnIDudt1Zt4+4nFzN//Q5OHtQj7HJFJIEp9I8BxQXZ3DpxWLvri/Kz+c5Ti3m5slqhLyIHpQu5SaAwN4OTB/VgdmV12KWISIJT6CeJSSOLWbxhF9W7GsIuRUQSmEI/SUwqLwbg5UpNNyki7VPoJ4mRpQX0KcxWF4+IHJRCP0mYGRPLi3lteS1NLW1hlyMiCUqhn0QmlRexu7GFirXbwi5FRBKUQj+JnDm8N5mRNPXri0i7FPpJJC8rnVOH9uSlZerXF5EDU+gnmYnlxayo3s16PX9fRA6gQ6FvZlPMrNLMVpjZnQdY/0UzW2Rm881sjpmN2m/9QDPbbWbfiFfhcmCTyosAeFmjeETkAA4Z+mYWAaYDFwGjgKv3D3XgEXcf7e5jgHuA+/Zbfx/wTBzqlUMYWpTP4F656uIRkQPqyJn+BGCFu69y9yZgJnB57AbuHvuw9zzgwwldzewKYDWw5OjLlY6YWF7MGyu30tDcGnYpIpJgOhL6/YD1MctVQdtHmNltZraS6Jn+l4O2fOAO4N8P9gPM7BYzqzCzipoajTw5WpNHFtPY0sabq7aGXYqIJJi4Xch19+nuPoxoyN8dNH8X+Km7H3SuP3ef4e7j3X18UVFRvEpKWROG9CQnI8JsdfGIyH46EvobgAExy/2DtvbMBK4IXp8K3GNma4CvAP/HzKYdQZ1yGLIzIpw5vBcvLavG3Q+9g4ikjI6E/lxghJkNMbNMYCowK3YDMxsRs3gJsBzA3c9298HuPhj4GfCf7n5/XCqXg5o0spiq7XtZWbMn7FJEJIEcchIVd28Jzs6fAyLAQ+6+xMy+B1S4+yxgmpmdBzQD24EbOrNoObSJwVM3Zy+rZnhxfsjViEiisET7+D9+/HivqKgIu4ykcOFPX6VXfiaPfP60sEsRkU5mZvPcffyhttMduUls0shi5q7ZRl1Dc9iliEiCUOgnsUnlRTS3Oq+vqA27FBFJEAr9JDZuUA8KstOZvUz3PohIlEI/iWVE0jinrIjZlRq6KSJRCv0kN6m8mOq6RpZs3HXojUUk6Sn0k9y5ZdE7nPUANhEBhX7SKyrI4szhvXjglZUs2bgz7HJEJGQK/RRw3z+PoTAng5t/V0H1roawyxGRECn0U0BJt2x+c8N4du5t5ubfV7C3SY9cFklVCv0UcXzfQn4+dSyLNuzk63+aT1ubRvOIpCKFfgo5f1QJ/+ei43h60WZ+8kJl2OWISAgO+cA1SS43nz2ElTW7mT57JUN75/Ppk/uHXZKIdCGd6acYM+M/rjiBM4b14s4nFvLO6m1hlyQiXUihn4IyImn88tqTGdAjly/8oYK1W/XMfZFUodBPUYW5GTx44yk48NmH57Jzr57EKZIKFPopbEjvPB647mTWbq3nR88uC7scEekCCv0Ud9rQXlx76kAenbue1bXq5hFJdgp9YdrkEWSlp/Hj5zWMUyTZKfSFooIsPnfWEP62cBOLqvR8HpFkptAXAD5/zlB65GZwz3Pq2xdJZh0KfTObYmaVZrbCzO48wPovmtkiM5tvZnPMbFTQfr6ZzQvWzTOzyfH+BSQ+umVncNuk4by2vJY3NL2iSNI6ZOibWQSYDlwEjAKu3hfqMR5x99HuPga4B7gvaK8FPunuo4EbgD/ErXKJu+tOG0Tfwmx+9OwyzbQlkqQ6cqY/AVjh7qvcvQmYCVweu4G7x07LlAd40P6eu28M2pcAOWaWdfRlS2fIzojwlfPLWFC1k2cXbw67HBHpBB0J/X7A+pjlqqDtI8zsNjNbSfRM/8sHeJ9PA++6e+ORFCpd48qx/RhenM+9z1fS0toWdjkiEmdxu5Dr7tPdfRhwB3B37DozOx74EfCFA+1rZreYWYWZVdTU1MSrJDkC6ZE0vnlhOatq9vD4u1VhlyMicdaR0N8ADIhZ7h+0tWcmcMW+BTPrD/wFuN7dVx5oB3ef4e7j3X18UVFRB0qSznTBqBLGDOjOT19YTkOzJlwRSSYdCf25wAgzG2JmmcBUYFbsBmY2ImbxEmB50N4d+Btwp7u/Hp+SpbOZGXdMGcnmXQ38/s01YZcjInF0yNB39xZgGvAcsBR4zN2XmNn3zOyyYLNpZrbEzOYDXyM6Uodgv+HAd4LhnPPNrDj+v4bE2+nDenFuWRHTZ6/Uw9hEkogl2tC88ePHe0VFRdhlCLB4w04u/cUcbps0jG9eODLsckTkIMxsnruPP9R2uiNX2nVCv0I+eVJfHpyzmiff26AJ1UWSgKZLlIP61oXlLKzawVcenU9+VjoXjy7lynH9mTC4J2lpFnZ5InKY1L0jh9TW5ry9ehtPvFvF04s2saeplf49crhybD+uHNefwb3zwi5RJOV1tHtHoS+HZW9TK88t2czj71YxZ0Ut7nDm8F786jPjyc/SB0eRsKhPXzpFTmaEK8b24w+fO5U37/wE37ywnDdXbuU7Ty0OuzQR6QCdmskRKy3M5rZJw2lqaePnLy7nnBFFXDH2Y0/oEJEEojN9OWr/Onk44wf14O4nF7Nua33Y5YjIQSj05ailR9L42dQxpBn868z3aNaD2kQSlkJf4qJ/j1x++OkTWbB+B/e98EHY5YhIOxT6EjcXj+7D1RMG8MArK3lds2+JJCSFvsTVty8dxdDeeXz10fls3a2pE0QSjUJf4io3M51fXD2OHfXN3PH4Qk27KJJgFPoSd6P6duOui0fy96XV/P7NtWGXIyIxFPrSKW48YzCTRxbz/aeXsnTTrkPvICJdQqEvncLMuPeqE+mek8HVv36Lu59cxJsrt9Lapu4ekTDpjlzpNL3ys3j4pglMf3kFj8/bwB/fWkdRQRYXn1DKpSf15eSBPfSkTpEupgeuSZeob2rhpWXV/HXBJmZXVtPY0kZpt2wuHt2H608fpCd1ihwlPWVTEtbuxhZeXLqFvy7cxCuVNXTLyeAvXzqDAT1zwy5N5Jilp2xKwsrPSufyMf349fXjefr2s2hqaeWmh+eys15z8Yp0NoW+hGp4cQEzrh/Puq313PKHChpbNCWjSGdS6EvoThvai3v/6UTeXr2Nb/15IW0a4SPSaToU+mY2xcwqzWyFmd15gPVfNLNFZjbfzOaY2aiYdXcF+1Wa2YXxLF6Sx+Vj+vHNC8t5av5GfvJCZdjliCStQw7ZNLMIMB04H6gC5prZLHd/P2azR9z9gWD7y4D7gClB+E8Fjgf6An83szJ312d4+ZgvTRxG1fa9TJ+9kn7dc7nm1IFhlySSdDpypj8BWOHuq9y9CZgJXB67gbvH3nKZB+z7fH45MNPdG919NbAieD+RjzEz/uPy45lUXsS3n1rM7MrqsEsSSTodCf1+wPqY5aqg7SPM7DYzWwncA3z5cPYV2Sc9ksb914xjZGkBt/33uyzesDPskkSSStwu5Lr7dHcfBtwB3H04+5rZLWZWYWYVNTU18SpJjlF5Wek8dOMp9MjN5KaH5/La8hoamtUjKBIPHXkMwwZgQMxy/6CtPTOBXx7Ovu4+A5gB0ZuzOlCTJLmSbtn89qZT+OdfvclnHnyHrPQ0JgzpyZnDe3PW8N6M6tNNj3AQOQIdCf25wAgzG0I0sKcC18RuYGYj3H15sHgJsO/1LOARM7uP6IXcEcA78Shckl9ZSQFz7pjMO6u3Mmf5VuasqOGHzywDoEduBmcM7825I4q4Ymw/MtM1+likIw4Z+u7eYmbTgOeACPCQuy8xs+8BFe4+C5hmZucBzcB24IZg3yVm9hjwPtAC3KaRO3I48rPSmTyyhMkjSwCo3tXA6ytrP/wj8LeFm3j+/S1Mv3YsWemRkKsVSXx69o4cs9ydP7y1lu88tYSJ5UU8cN3JZGco+CU16dk7kvTMjOtPH8wPrxzNKx/U8PnfV7C3SR8kRQ5GoS/HvKkTBnLvVScxZ0UtNz38DnsaW8IuSSRhKfQlKVx1cn9+9i9jeGf1Nm787TvsVvCLHJBCX5LG5WP68Yurx/Huuh185sG32blXj2oW2Z9CX5LKJSf2Yfo141i8YSefefBtdtQ3hV2SSEJR6EvSmXJCKQ9cdzLLNtVx5S/f4N7nlvHs4k1Uba8n0UariXQ1TYwuSekTx5Xw4I3j+cHTy3jglVW0Bs/o75mXyQn9Chndrxuj+xVyyuCe9MrPCrlaka6j0JekdfaIIs6+vYiG5laWbtrF4g07WVi1k0UbdvLAilpa25yCrHSmXzuOc8qKwi5XpEso9CXpZWdEGDuwB2MH9viwraG5lSUbd/Jvf1nMTQ/P5bufHMVnTh8cXpEiXUR9+pKSsjMinDyoJ3++9QwmlhXx7aeW8N1ZS2hpbQu7NJFOpdCXlJaflc6M68dz81lDePiNNdz8+wrqGjTUU5KXQl9SXiTNuPvSUfznp0YzZ3ktn/7lG6zfVh92WSKdQqEvErjm1IH87rMT2LyzgSumv868tdvCLkkk7hT6IjHOHN6bJ750JvnZ6Vw9421+8MxS/vJeFfPX79AdvpIUNHpHZD/Di/N58ktn8pVH5/PrV1fRFnM/V+/8TIb0zmNo73yGFuVx2Zi+9CnMCa9YkcOk5+mLHERTSxvrtu1hVc0eVtXuYXXNHlbV7mZ17R5qdzdRmJPBvVedyAXHl4ZdqqS4jj5PX2f6IgeRmZ7G8OIChhcXfGzdqprdfHnme9zyh3nceMZg7rxopCZxkYSnPn2RIzS0KJ/Hbz2DzwXDPa/8rzdYWbM77LJEDkqhL3IUstIjfPvSUTx4w3g27dzLJ38xh8fnVYVdlki7FPoicfCJ40p45vZzGN2vkK//aQFfe3S+JnKRhKQ+fZE4KS3M5pHPn8YvXlrO/3txOfPWbefcsiL6FObQt3s2fQpz6FOYTWlhNhkRnW9JODoU+mY2Bfg5EAF+4+4/3G/914CbgRagBvisu68N1t0DXEL0U8ULwO2eaEOGROIkkmZ85bwyTh/ai+8/vZS/vLeBuoaPnvGbQVF+FsOL8/nmheUfeRCcSGc75JBNM4sAHwDnA1XAXOBqd38/ZptJwNvuXm9mtwIT3f1fzOwM4F7gnGDTOcBd7v5yez9PQzYl2exubGHTjr1s3Nnwke+vLa9lS10DN50xhG9cWEZupj54y5GL55DNCcAKd18VvPFM4HLgw9B399kx278FXLdvFZANZAIGZABbOvILiCSL/Kx0RpQUMKLko8M+6xqa+dGzy3jo9dU8//5m/vNTo/Vcf+l0HelY7Aesj1muCtra8zngGQB3fxOYDWwKvp5z96X772Bmt5hZhZlV1NTUdLR2kWNaQXYG//eK0Tz2hdPJjKRx/UPv8PXHFrB9j+b1lc4T16tJZnYdMJ5olw5mNhw4DuhP9A/FZDM7e//93H2Gu4939/FFRTrTkdQyYUhPnr79bKZNGs5T8zdw/k9f4X8XbNR8vtIpOtK9swEYELPcP2j7CDM7D/g34Fx3bwyaPwW85e67g22eAU4HXjuaokWSTXZGhG9cWM7Fo/tw5xML+df/eY9fv7aKgT1z6Z2fRa+8THrlZ9E7/x/f+3XPIV2jgOQwdST05wIjzGwI0bCfClwTu4GZjQV+BUxx9+qYVeuAz5vZD4j26Z8L/CwehYsko1F9u/HErWfw8BtreHbxZpZs3EXt7saPjQAC6N8jh7suOo6LR5diZiFUK8eiDj1wzcwuJhrWEeAhd/++mX0PqHD3WWb2d2A00X57gHXuflkw8ue/iI7eceBZd//awX6WRu+IfFxjSytbdzexdXcTtXsa2bKzgYffWMOyzXWcMrgH3750FCf27x52mRKijo7e0VM2RY5RrW3OYxXr+cnzldTubuLKcf341oUjKS3MDrs0CUFHQ18dgiLHqEiacfWEgcz+xkS+eO4w/rpgE5N+/DI///ty9ja1hl2eJCid6YskifXb6vnhM8v426JN9CnM5pwRRXTPzaAwN4PCnAy652RGl3My6JWfqclfkoyepy+SYgb0zGX6teO4YfU27nuhkpc/qGZ7fTNNLW0H3P70ob346vllTBjSs4srlTDpTF8kyTU0t7Kjvpkde5vYWd/Mjr3NrKzZzUNz1lC7u5Gzhvfmq+eP4ORBCv9jmS7kishB7W1q5Y9vreWBV1aydU8T55QV8dXzRugBcMcohb6IdEh9Uwu/f3Mtv3plJdvrm5lYXsSt5w5jYK9c8rPSyctMJy1N9wEkOoW+iByW3Y0t/O6NNfz6tVXsqG/+sN0s+tC4btkZFGSnk5+VzqBeedx4xmBG9y8MsWKJpdAXkSNS19DMa8tr2bm3mbqGZnY3tLCroYW6hpbocmMLi6p2UtfYwmlDe3LLOUOZWFasTwMh0+gdETkiBdkZXDy6z0G3qWto5tG563lozmo++3AFw4vz+fzZQ7h8TD+yMyJdVKkcCZ3pi8gRa25t428LNzHj1VW8v2kXvfOzuPGMQVxwfGn0ekBWtDsook8BnU7dOyLSZdydN1ZuZcarq3jlg4/PiZGdkfaRPwIThvTk2lMHMbw4P4Rqk5NCX0RCsXxLHcs217GnsYXdjS3saWxlT9O+1y1s29PEW6u20tzqnDa0J9edNogLRpWSma6nwhwN9emLSCgONDXk/mp3N/JYxXoeeXsd0x55j975WUw9ZQBXnzqQft31eIjOpDN9EQlNa5vz6gc1/PGttbxUWR2ddKOsiKFF+eRlRsjJTCcvK0JuZnqwHKFXXhbH9+2m0UL70Zm+iCS8SJoxaWQxk0YWU7W9nv95Zx2zFmzkndXbqG9upb1z0v49crhyXH+uGtefgb1yu7boY5zO9EUkIbk7Dc1t7GlqoT64LlDf1MKa2nqenL+BOStqcY/OMXzVyf25eHQf8rNS9zxWF3JFJKlt3LGXv7y3gcfnVbGqdg85GREuOqGU80aVUJiTQXZGhNzM6FdOZrSLKCcjkrTDRxX6IpIS3J131+3gz/Oq+OvCjQecTzjW4F65XHpiXy4b05eyQ1xwPpYo9EUk5TQ0t7Kiejd7Gluob25lb1Mr9U2t7G1qob6plT1Nrby7djtvrKylzaG8pIBPntSHT57Ul0G98sIu/6go9EVE2lFd18Azizbzvws2UrF2OwAn9S/kkyf1ZWRpN3Iy04LuoWiXUE5GtIsoI2KYJWb3UFxD38ymAD8HIsBv3P2H+63/GnAz0ALUAJ9197XBuoHAb4ABgAMXu/ua9n6WQl9EutKGHXv528KN/O+CTSzasPOg22ZG0jhlSA+mHF/KBceXUtItcSahj1vom1kE+AA4H6gC5gJXu/v7MdtMAt5293ozuxWY6O7/Eqx7Gfi+u79gZvlAm7vXt/fzFPoiEpb12+rZvKuBvU2t7A26h2K/b9vTxOzKalbV7AFg3MDuTDmhlCnH9wl96Gg8Q/904LvufmGwfBeAu/+gne3HAve7+5lmNgqY4e5ndbRwhb6IJLoV1XU8u3gzzy7ZzOINuwA4rk83Jo8sokduJjmZEbLTo11C2RnRrqLsjAhF+VkM6Nk5fxzieXNWP2B9zHIVcOpBtv8c8EzwugzYYWZPAEOAvwN3untrB36uiEhCGl5cwLTJBUybPIL12+p5bslmnluymf96eWW7N5TtM6woj/OOK+ETx5UwbmB30iNd+8yhuN7JYGbXAeOBc2Pe/2xgLLAOeBS4EXhwv/1uAW4BGDhwYDxLEhHpVAN65nLz2UO5+eyhNLW0sbe5lYbgK/q6jb1NrTS0tLKmdg8vLq3moddX86tXV9E9N4PJ5cV84rgSzinrTUF2RqfX25HQ30D0Iuw+/YO2jzCz84B/A85198aguQqY7+6rgm2eBE5jv9B39xnADIh27xzm7yAikhAy09PITE+jMKed8C6Hm84cQl1DM69+UMuLS7fwUmU1T7y3gYyIceHxpdx/zbhOrbEjoT8XGGFmQ4iG/VTgmtgNgn78XwFT3L16v30AjX+DAAAE6UlEQVS7m1mRu9cAkwF12ItISivIzuCSE/twyYl9aGlt4911O3hx6ZYuuVv4kKHv7i1mNg14juiQzYfcfYmZfQ+ocPdZwL1APvCnYAzrOne/zN1bzewbwIsWXTEP+HVn/TIiIsea9EgaE4b0ZMKQnl3y83RzlohIEujo6B1NVSMikkIU+iIiKUShLyKSQhT6IiIpRKEvIpJCFPoiIilEoS8ikkISbpy+mdUAa4/iLXoDtXEqJ95U25FRbUdGtR2ZY7W2Qe5edKg3SLjQP1pmVtGRGxTCoNqOjGo7MqrtyCR7bereERFJIQp9EZEUkoyhPyPsAg5CtR0Z1XZkVNuRSerakq5PX0RE2peMZ/oiItKOpAl9M5tiZpVmtsLM7gy7nlhmtsbMFpnZfDML/bnRZvaQmVWb2eKYtp5m9oKZLQ++90iQur5rZhuCYzffzC7u6rqCOgaY2Wwze9/MlpjZ7UF7Ihy39moL/diZWbaZvWNmC4La/j1oH2Jmbwf/Xx81s8wEqu1hM1sdc9zGdHVtMTVGzOw9M/trsHz0x83dj/kvopO7rASGApnAAmBU2HXF1LcG6B12HTH1nAOMAxbHtN1DdNJ6gDuBHyVIXd8FvpEAx6wPMC54XQB8AIxKkOPWXm2hHzvAgPzgdQbwNtEpUx8DpgbtDwC3JlBtDwNXhf1vLqjra8AjwF+D5aM+bslypj8BWOHuq9y9CZgJXB5yTQnL3V8Ftu3XfDnwu+D174ArurQo2q0rIbj7Jnd/N3hdBywF+pEYx6292kLnUbuDxYzgy4lOnfrnoD2s49ZebQnBzPoDlwC/CZaNOBy3ZAn9fsD6mOUqEuQffcCB581snpndEnYx7Shx903B681ASZjF7GeamS0Mun+6vPtkf2Y2GBhL9MwwoY7bfrVBAhy7oItiPlANvED0U/kOd28JNgnt/+v+tbn7vuP2/eC4/dTMssKoDfgZ8C2gLVjuRRyOW7KEfqI7y93HARcBt5nZOWEXdDAe/eyYKGc8vwSGAWOATcBPwizGzPKBx4GvuPuu2HVhH7cD1JYQx87dW919DNCf6KfykWHUcSD712ZmJwB3Ea3xFKAncEdX12VmlwLV7j4v3u+dLKG/ARgQs9w/aEsI7r4h+F4N/IXoP/xEs8XM+gAE36tDrgcAd98S/MdsA35NiMfOzDKIhup/u/sTQXNCHLcD1ZZIxy6oZwcwGzgd6G5m6cGq0P+/xtQ2Jeguc3dvBH5LOMftTOAyM1tDtLt6MvBz4nDckiX05wIjgivbmcBUYFbINQFgZnlmVrDvNXABsPjge4ViFnBD8PoG4KkQa/nQvkANfIqQjl3Qn/ogsNTd74tZFfpxa6+2RDh2ZlZkZt2D1znA+USvOcwGrgo2C+u4Hai2ZTF/xI1on3mXHzd3v8vd+7v7YKJ59pK7X0s8jlvYV6fjeJX7YqKjFlYC/xZ2PTF1DSU6mmgBsCQRagP+h+jH/Wai/YKfI9pf+CKwHPg70DNB6voDsAhYSDRg+4R0zM4i2nWzEJgffF2cIMetvdpCP3bAicB7QQ2Lge8E7UOBd4AVwJ+ArASq7aXguC0G/kgwwiesL2Ai/xi9c9THTXfkioikkGTp3hERkQ5Q6IuIpBCFvohIClHoi4ikEIW+iEgKUeiLiKQQhb6ISApR6IuIpJD/D5s3qbxiZX4GAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(mnist.error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}