ANN_pre_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math,struct,pickle\n",
    "from pathlib import Path\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tanh(x):\n",
    "    return np.tanh(x)\n",
    "def softmax(x):\n",
    "    exp=np.exp(x-x.max())\n",
    "    return exp/exp.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dimensions=[28*28,10]\n",
    "activation=[tanh,softmax]\n",
    "distribution=[\n",
    "    {'b':[0,0]},\n",
    "    {'b':[0,0],'w':[-math.sqrt(6/(dimensions[0]+dimensions[1])),math.sqrt(6/(dimensions[0]+dimensions[1]))]},\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_parameters_b(layer):\n",
    "    dist=distribution[layer]['b']\n",
    "    return np.random.rand(dimensions[layer])*(dist[1]-dist[0])+dist[0]\n",
    "def init_parameters_w(layer):\n",
    "    dist=distribution[layer]['w']\n",
    "    return np.random.rand(dimensions[layer-1],dimensions[layer])*(dist[1]-dist[0])+dist[0]\n",
    "def init_parameters():\n",
    "    parameter=[]\n",
    "    for i in range(len(distribution)):\n",
    "        layer_parameter={}\n",
    "        for j in distribution[i].keys():\n",
    "            if j=='b':\n",
    "                layer_parameter['b']=init_parameters_b(i)\n",
    "                continue\n",
    "            if j=='w':\n",
    "                layer_parameter['w']=init_parameters_w(i)\n",
    "                continue\n",
    "        parameter.append(layer_parameter)\n",
    "    return parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters=init_parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(img,parameters):\n",
    "    l0_in=img+parameters[0]['b']\n",
    "    l0_out=activation[0](l0_in)\n",
    "    l1_in=np.dot(l0_out,parameters[1]['w'])+parameters[1]['b']\n",
    "    l1_out=activation[1](l1_in)\n",
    "    return l1_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict(np.random.rand(784),parameters).argmax()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_img=[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01171875, 0.0703125, 0.0703125, 0.0703125, 0.4921875, 0.53125, 0.68359375, 0.1015625, 0.6484375, 0.99609375, 0.96484375, 0.49609375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1171875, 0.140625, 0.3671875, 0.6015625, 0.6640625, 0.98828125, 0.98828125, 0.98828125, 0.98828125, 0.98828125, 0.87890625, 0.671875, 0.98828125, 0.9453125, 0.76171875, 0.25, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.19140625, 0.9296875, 0.98828125, 0.98828125, 0.98828125, 0.98828125, 0.98828125, 0.98828125, 0.98828125, 0.98828125, 0.98046875, 0.36328125, 0.3203125, 0.3203125, 0.21875, 0.15234375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0703125, 0.85546875, 0.98828125, 0.98828125, 0.98828125, 0.98828125, 0.98828125, 0.7734375, 0.7109375, 0.96484375, 0.94140625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3125, 0.609375, 0.41796875, 0.98828125, 0.98828125, 0.80078125, 0.04296875, 0.0, 0.16796875, 0.6015625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0546875, 0.00390625, 0.6015625, 0.98828125, 0.3515625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.54296875, 0.98828125, 0.7421875, 0.0078125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.04296875, 0.7421875, 0.98828125, 0.2734375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.13671875, 0.94140625, 0.87890625, 0.625, 0.421875, 0.00390625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.31640625, 0.9375, 0.98828125, 0.98828125, 0.46484375, 0.09765625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.17578125, 0.7265625, 0.98828125, 0.98828125, 0.5859375, 0.10546875, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0625, 0.36328125, 0.984375, 0.98828125, 0.73046875, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.97265625, 0.98828125, 0.97265625, 0.25, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1796875, 0.5078125, 0.71484375, 0.98828125, 0.98828125, 0.80859375, 0.0078125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.15234375, 0.578125, 0.89453125, 0.98828125, 0.98828125, 0.98828125, 0.9765625, 0.7109375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.09375, 0.4453125, 0.86328125, 0.98828125, 0.98828125, 0.98828125, 0.98828125, 0.78515625, 0.3046875, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.08984375, 0.2578125, 0.83203125, 0.98828125, 0.98828125, 0.98828125, 0.98828125, 0.7734375, 0.31640625, 0.0078125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0703125, 0.66796875, 0.85546875, 0.98828125, 0.98828125, 0.98828125, 0.98828125, 0.76171875, 0.3125, 0.03515625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.21484375, 0.671875, 0.8828125, 0.98828125, 0.98828125, 0.98828125, 0.98828125, 0.953125, 0.51953125, 0.04296875, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.53125, 0.98828125, 0.98828125, 0.98828125, 0.828125, 0.52734375, 0.515625, 0.0625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1f23d3f6e10>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADgpJREFUeJzt3X+MVfWZx/HPs1j+kKI4aQRCYSnEYJW4082IjSWrxkzVDQZHrekkJjQapn8wiU02ZA3/VNNgyCrslmiamaZYSFpKE3VB0iw0otLGZuKIWC0srTFsO3IDNTjywx9kmGf/mEMzxbnfe+fec++5zPN+JeT+eM6558kNnznn3O+592vuLgDx/EPRDQAoBuEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxDUZc3cmJlxOSHQYO5u1SxX157fzO40syNm9q6ZPVrPawFoLqv12n4zmybpj5I6JQ1Jel1St7sfSqzDnh9osGbs+ZdJetfd33P3c5J+IWllHa8HoInqCf88SX8Z93goe+7vmFmPmQ2a2WAd2wKQs3o+8Jvo0OJzh/Xu3i+pX+KwH2gl9ez5hyTNH/f4y5KO1dcOgGapJ/yvS7rGzL5iZtMlfVvSrnzaAtBoNR/2u/uImfVK2iNpmqQt7v6H3DoD0FA1D/XVtDHO+YGGa8pFPgAuXYQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8EVfMU3ZJkZkclnZZ0XtKIu3fk0RTyM23atGT9yiuvbOj2e3t7y9Yuv/zy5LpLlixJ1tesWZOsP/XUU2Vr3d3dyXU//fTTZH3Dhg3J+uOPP56st4K6wp+5zd0/yOF1ADQRh/1AUPWG3yXtNbM3zKwnj4YANEe9h/3fcPdjZna1pF+b2f+6+/7xC2R/FPjDALSYuvb87n4suz0h6QVJyyZYpt/dO/gwEGgtNYffzGaY2cwL9yV9U9I7eTUGoLHqOeyfLekFM7vwOj939//JpSsADVdz+N39PUn/lGMvU9aCBQuS9enTpyfrN998c7K+fPnysrVZs2Yl173vvvuS9SINDQ0l65s3b07Wu7q6ytZOnz6dXPett95K1l999dVk/VLAUB8QFOEHgiL8QFCEHwiK8ANBEX4gKHP35m3MrHkba6L29vZkfd++fcl6o79W26pGR0eT9YceeihZP3PmTM3bLpVKyfqHH36YrB85cqTmbTeau1s1y7HnB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgGOfPQVtbW7I+MDCQrC9atCjPdnJVqffh4eFk/bbbbitbO3fuXHLdqNc/1ItxfgBJhB8IivADQRF+ICjCDwRF+IGgCD8QVB6z9IZ38uTJZH3t2rXJ+ooVK5L1N998M1mv9BPWKQcPHkzWOzs7k/WzZ88m69dff33Z2iOPPJJcF43Fnh8IivADQRF+ICjCDwRF+IGgCD8QFOEHgqr4fX4z2yJphaQT7r40e65N0g5JCyUdlfSAu6d/6FxT9/v89briiiuS9UrTSff19ZWtPfzww8l1H3zwwWR9+/btyTpaT57f5/+ppDsveu5RSS+5+zWSXsoeA7iEVAy/u++XdPElbCslbc3ub5V0T859AWiwWs/5Z7t7SZKy26vzawlAMzT82n4z65HU0+jtAJicWvf8x81sriRltyfKLeju/e7e4e4dNW4LQAPUGv5dklZl91dJ2plPOwCapWL4zWy7pN9JWmJmQ2b2sKQNkjrN7E+SOrPHAC4hFc/53b27TOn2nHsJ69SpU3Wt/9FHH9W87urVq5P1HTt2JOujo6M1bxvF4go/ICjCDwRF+IGgCD8QFOEHgiL8QFBM0T0FzJgxo2ztxRdfTK57yy23JOt33XVXsr53795kHc3HFN0Akgg/EBThB4Ii/EBQhB8IivADQRF+ICjG+ae4xYsXJ+sHDhxI1oeHh5P1l19+OVkfHBwsW3vmmWeS6zbz/+ZUwjg/gCTCDwRF+IGgCD8QFOEHgiL8QFCEHwiKcf7gurq6kvVnn302WZ85c2bN2163bl2yvm3btmS9VCrVvO2pjHF+AEmEHwiK8ANBEX4gKMIPBEX4gaAIPxBUxXF+M9siaYWkE+6+NHvuMUmrJf01W2ydu/+q4sYY57/kLF26NFnftGlTsn777bXP5N7X15esr1+/Pll///33a972pSzPcf6fSrpzguf/093bs38Vgw+gtVQMv7vvl3SyCb0AaKJ6zvl7zez3ZrbFzK7KrSMATVFr+H8kabGkdkklSRvLLWhmPWY2aGblf8wNQNPVFH53P+7u5919VNKPJS1LLNvv7h3u3lFrkwDyV1P4zWzuuIddkt7Jpx0AzXJZpQXMbLukWyV9ycyGJH1f0q1m1i7JJR2V9N0G9gigAfg+P+oya9asZP3uu+8uW6v0WwFm6eHqffv2JeudnZ3J+lTF9/kBJBF+ICjCDwRF+IGgCD8QFOEHgmKoD4X57LPPkvXLLktfhjIyMpKs33HHHWVrr7zySnLdSxlDfQCSCD8QFOEHgiL8QFCEHwiK8ANBEX4gqIrf50dsN9xwQ7J+//33J+s33nhj2VqlcfxKDh06lKzv37+/rtef6tjzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQjPNPcUuWLEnWe3t7k/V77703WZ8zZ86ke6rW+fPnk/VSqZSsj46O5tnOlMOeHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCqjjOb2bzJW2TNEfSqKR+d/+hmbVJ2iFpoaSjkh5w9w8b12pclcbSu7u7y9YqjeMvXLiwlpZyMTg4mKyvX78+Wd+1a1ee7YRTzZ5/RNK/uftXJX1d0hozu07So5JecvdrJL2UPQZwiagYfncvufuB7P5pSYclzZO0UtLWbLGtku5pVJMA8jepc34zWyjpa5IGJM1295I09gdC0tV5Nwegcaq+tt/MvijpOUnfc/dTZlVNByYz65HUU1t7ABqlqj2/mX1BY8H/mbs/nz193MzmZvW5kk5MtK6797t7h7t35NEwgHxUDL+N7eJ/Iumwu28aV9olaVV2f5Wknfm3B6BRKk7RbWbLJf1G0tsaG+qTpHUaO+//paQFkv4s6VvufrLCa4Wconv27NnJ+nXXXZesP/3008n6tddeO+me8jIwMJCsP/nkk2VrO3em9xd8Jbc21U7RXfGc391/K6nci90+maYAtA6u8AOCIvxAUIQfCIrwA0ERfiAowg8ExU93V6mtra1sra+vL7lue3t7sr5o0aKaesrDa6+9lqxv3LgxWd+zZ0+y/sknn0y6JzQHe34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCCrMOP9NN92UrK9duzZZX7ZsWdnavHnzauopLx9//HHZ2ubNm5PrPvHEE8n62bNna+oJrY89PxAU4QeCIvxAUIQfCIrwA0ERfiAowg8EFWacv6urq656PQ4dOpSs7969O1kfGRlJ1lPfuR8eHk6ui7jY8wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUObu6QXM5kvaJmmOpFFJ/e7+QzN7TNJqSX/NFl3n7r+q8FrpjQGom7tbNctVE/65kua6+wEzmynpDUn3SHpA0hl3f6rapgg/0HjVhr/iFX7uXpJUyu6fNrPDkor96RoAdZvUOb+ZLZT0NUkD2VO9ZvZ7M9tiZleVWafHzAbNbLCuTgHkquJh/98WNPuipFclrXf3581stqQPJLmkH2js1OChCq/BYT/QYLmd80uSmX1B0m5Je9x90wT1hZJ2u/vSCq9D+IEGqzb8FQ/7zcwk/UTS4fHBzz4IvKBL0juTbRJAcar5tH+5pN9IeltjQ32StE5St6R2jR32H5X03ezDwdRrsecHGizXw/68EH6g8XI77AcwNRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCavYU3R9I+r9xj7+UPdeKWrW3Vu1Lorda5dnbP1a7YFO/z/+5jZsNuntHYQ0ktGpvrdqXRG+1Kqo3DvuBoAg/EFTR4e8vePsprdpbq/Yl0VutCumt0HN+AMUpes8PoCCFhN/M7jSzI2b2rpk9WkQP5ZjZUTN728wOFj3FWDYN2gkze2fcc21m9msz+1N2O+E0aQX19piZvZ+9dwfN7F8L6m2+mb1sZofN7A9m9kj2fKHvXaKvQt63ph/2m9k0SX+U1ClpSNLrkrrd/VBTGynDzI5K6nD3wseEzexfJJ2RtO3CbEhm9h+STrr7huwP51Xu/u8t0ttjmuTMzQ3qrdzM0t9Rge9dnjNe56GIPf8ySe+6+3vufk7SLyStLKCPlufu+yWdvOjplZK2Zve3auw/T9OV6a0luHvJ3Q9k909LujCzdKHvXaKvQhQR/nmS/jLu8ZBaa8pvl7TXzN4ws56im5nA7AszI2W3Vxfcz8UqztzcTBfNLN0y710tM17nrYjwTzSbSCsNOXzD3f9Z0l2S1mSHt6jOjyQt1tg0biVJG4tsJptZ+jlJ33P3U0X2Mt4EfRXyvhUR/iFJ88c9/rKkYwX0MSF3P5bdnpD0gsZOU1rJ8QuTpGa3Jwru52/c/bi7n3f3UUk/VoHvXTaz9HOSfubuz2dPF/7eTdRXUe9bEeF/XdI1ZvYVM5su6duSdhXQx+eY2YzsgxiZ2QxJ31TrzT68S9Kq7P4qSTsL7OXvtMrMzeVmllbB712rzXhdyEU+2VDGf0maJmmLu69vehMTMLNFGtvbS2PfePx5kb2Z2XZJt2rsW1/HJX1f0n9L+qWkBZL+LOlb7t70D97K9HarJjlzc4N6Kzez9IAKfO/ynPE6l364wg+IiSv8gKAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8E9f/Ex0YKZYOZcwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "im = np.reshape(test_img,(28,28))\n",
    "plt.imshow(im , cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "path='model_02.pkl'\n",
    "with open(path,'rb') as f:\n",
    "    parameters=pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict(np.array(test_img),parameters).argmax()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'b': array([-1.22268644e-02, -1.69337185e-02, -1.55384097e-02, -3.15686490e-02,\n",
       "          1.10621898e-02,  5.53598326e-03, -6.35943805e-03, -5.84399455e-03,\n",
       "          8.42867360e-03,  5.93783115e-03, -1.56796413e-02, -5.56046295e-03,\n",
       "         -1.58257714e-02, -2.08147441e-02,  4.14084064e-03, -3.30956987e-03,\n",
       "         -1.44388093e-02, -1.70634684e-02,  1.99394249e-04, -8.62252737e-03,\n",
       "          2.74250602e-02, -1.00596961e-02, -3.61814817e-03,  2.21515384e-03,\n",
       "          1.08541903e-02,  1.01123336e-02, -6.22160272e-03,  2.33535257e-02,\n",
       "         -1.44424552e-02,  6.65463041e-03, -1.68420346e-02,  7.61633816e-04,\n",
       "          4.15787745e-03,  5.26976593e-02,  1.05732735e-02, -7.68558429e-03,\n",
       "         -3.49612494e-02,  5.01457301e-02, -2.15991903e-02,  2.41617243e-02,\n",
       "          9.82514398e-03, -1.22803316e-03,  2.24658303e-02, -1.66578564e-02,\n",
       "          1.14147401e-02, -2.75883366e-02,  1.08087783e-02,  2.57514564e-02,\n",
       "         -2.37001587e-03,  4.03441970e-02, -1.53690019e-02,  3.20923099e-02,\n",
       "         -1.53115285e-02,  1.80925914e-02,  1.47231549e-02,  7.50672206e-03,\n",
       "         -3.17228159e-02,  8.83694087e-03, -3.60282717e-02, -3.47996845e-02,\n",
       "          2.84934299e-02,  1.10098105e-02, -9.80658948e-03, -9.13058644e-03,\n",
       "         -1.08974956e-02,  1.17240803e-02, -8.01785662e-03,  2.57139097e-02,\n",
       "         -4.78369556e-02, -3.31494929e-02, -2.04158147e-03, -4.47920292e-02,\n",
       "         -2.87328217e-02,  1.75568763e-02, -8.00177244e-03, -2.11882832e-02,\n",
       "          2.38258624e-02, -2.01836592e-02,  1.33275166e-02,  2.49533179e-02,\n",
       "         -3.46555390e-02,  1.01372354e-02, -1.80305171e-02, -2.21948497e-03,\n",
       "          4.15862033e-03,  1.92581323e-02, -8.21417572e-03,  2.01718593e-02,\n",
       "         -1.54302325e-02, -2.27069047e-02, -2.15096940e-02, -2.42818986e-02,\n",
       "          2.69154957e-02, -7.99277088e-02, -1.10801423e-01, -8.67917718e-02,\n",
       "         -1.44727466e-01, -2.19170437e-01, -1.29590089e-01, -4.01536356e-02,\n",
       "         -9.47115623e-02, -2.61724631e-02,  2.45285868e-03,  3.48997261e-04,\n",
       "         -7.05706853e-04, -6.05218585e-03, -1.52986847e-02, -1.41163693e-02,\n",
       "          7.29833814e-03,  9.53646405e-04,  5.26814554e-03,  1.02208958e-02,\n",
       "         -5.79355099e-03,  8.09656341e-04,  1.84189471e-03,  2.48031139e-02,\n",
       "          1.72354463e-02, -4.75629015e-02, -7.58360266e-03, -7.01669044e-02,\n",
       "         -1.37665737e-01, -1.71161779e-01, -1.87889514e-01, -1.26953024e-01,\n",
       "         -1.61781093e-01, -2.43987400e-01, -1.54979842e-01, -2.11324446e-01,\n",
       "         -1.12310103e-01, -6.53433340e-02, -6.95706783e-02, -2.26893529e-02,\n",
       "          4.23122214e-03, -4.11015105e-02, -1.57484630e-01, -7.81927202e-02,\n",
       "          1.73304268e-02, -6.04166177e-03,  5.50195821e-03, -7.99048974e-03,\n",
       "          2.32094882e-03,  7.87762776e-03,  5.02679513e-03,  7.11787081e-04,\n",
       "         -3.44040664e-02, -7.52547708e-02, -8.38658104e-02, -9.45158733e-02,\n",
       "         -7.41882382e-02, -3.03271675e-02, -1.03862434e-01, -1.77992423e-01,\n",
       "         -1.10451811e-01, -1.48097800e-01, -1.10971227e-01, -2.12041553e-01,\n",
       "         -1.84888105e-01, -1.87777288e-01, -1.68363779e-01, -1.34908918e-01,\n",
       "         -7.51270295e-02, -3.89287026e-02, -1.76490476e-01, -1.56022348e-01,\n",
       "         -2.05405825e-02,  1.35411699e-02,  1.38939821e-02,  1.10939091e-03,\n",
       "          1.42206066e-02, -3.57383712e-03,  4.12501090e-03, -1.20023732e-02,\n",
       "         -6.75636411e-02, -1.27330046e-01, -1.40162019e-01, -8.53870166e-02,\n",
       "         -6.67225414e-02, -1.85106726e-02, -6.35079466e-02, -9.49834064e-02,\n",
       "         -1.76343638e-02, -2.01113168e-01, -3.15217475e-01,  6.30042821e-02,\n",
       "         -2.12457267e-01, -1.14423871e-01, -2.43303572e-01, -5.73313824e-02,\n",
       "         -8.54364037e-02, -1.37387247e-02, -8.23364657e-02, -1.46469122e-01,\n",
       "         -7.81592843e-02, -7.68741145e-03,  9.35315954e-03,  2.39751398e-02,\n",
       "         -2.37456053e-02,  8.49326210e-03,  3.18910276e-02, -4.12502010e-02,\n",
       "         -1.53702396e-01, -1.44118530e-01, -1.71657678e-01, -9.93919216e-02,\n",
       "         -1.21161216e-01, -1.15974646e-01, -1.58447015e-01, -2.51065905e-01,\n",
       "         -3.67033182e-01,  3.48047961e-02,  2.19305063e-02,  9.88726025e-02,\n",
       "          3.76474339e-02, -2.07093232e-01, -1.21655644e-01, -2.12563969e-01,\n",
       "         -1.53544877e-01, -6.75341526e-02, -3.91669960e-02, -1.20411979e-01,\n",
       "         -1.07524604e-01, -3.49188415e-02,  3.08096345e-02,  2.82244099e-02,\n",
       "          1.87227719e-03,  2.52176686e-02, -8.34886287e-03, -1.41428168e-02,\n",
       "         -1.94960993e-01, -1.86037326e-01, -8.31963458e-02, -1.04662629e-01,\n",
       "         -1.09771569e-01, -3.44537295e-02, -1.03130754e-01, -2.42546529e-01,\n",
       "         -1.48160153e-01,  3.49570490e-02, -1.86860092e-01, -1.92438264e-01,\n",
       "         -9.16941285e-02, -2.45343232e-01, -2.57197736e-01, -2.69781473e-01,\n",
       "         -2.08707431e-01, -1.53521118e-01, -1.23259644e-01, -2.49639138e-01,\n",
       "         -1.84321796e-01, -8.98204944e-02, -3.94065921e-03,  3.04305510e-03,\n",
       "         -2.43202149e-02, -1.31083123e-02,  8.43330005e-03, -5.53174560e-02,\n",
       "         -1.72953254e-01, -1.13506313e-01, -2.36906402e-02, -2.35699544e-03,\n",
       "         -1.44486621e-02,  1.21344558e-02, -1.82424161e-03, -8.88149482e-02,\n",
       "         -2.40158723e-01, -2.35905352e-01, -6.69450500e-01, -5.80441678e-01,\n",
       "         -3.29341355e-01, -2.83979047e-01, -3.21949215e-01, -2.54472500e-01,\n",
       "         -1.22814353e-01, -1.44139806e-01, -9.13440710e-02, -2.16099359e-01,\n",
       "         -2.44268199e-01, -1.25016315e-01,  8.48517665e-03, -2.73813382e-02,\n",
       "         -1.07794779e-04,  1.72252967e-02, -7.90939972e-03, -6.22181876e-02,\n",
       "         -6.88823266e-02, -3.27178185e-02,  1.20520758e-02,  1.84578168e-02,\n",
       "         -6.69921849e-02, -1.90574288e-01, -2.94906496e-01, -2.23125668e-01,\n",
       "         -2.38136695e-01, -2.83676914e-01, -6.68635165e-01, -8.40915666e-01,\n",
       "         -6.73818637e-01, -3.04597926e-01, -1.26694134e-01, -2.73171774e-01,\n",
       "         -1.16843081e-01, -9.72890703e-02, -1.27224229e-01, -2.23067406e-01,\n",
       "         -3.26543964e-01, -2.77547728e-01, -9.12095896e-03,  1.05457407e-02,\n",
       "         -1.48237868e-02,  2.16070092e-03, -1.50465160e-02, -6.35368522e-02,\n",
       "         -2.27514134e-02, -7.05256820e-02, -1.00132914e-01, -2.79171211e-01,\n",
       "         -5.13122336e-01, -3.75722223e-01, -1.76302782e-01, -8.86932436e-02,\n",
       "         -1.99840458e-02, -3.12939523e-01, -5.49250145e-01, -5.97428793e-01,\n",
       "         -6.49224806e-01, -4.44085725e-01, -3.51878717e-01, -2.92710754e-01,\n",
       "         -4.59664431e-01, -3.87971310e-01, -2.57723639e-01, -1.38318566e-01,\n",
       "         -1.50609898e-01, -6.17016832e-02, -9.86031994e-03,  3.42224621e-02,\n",
       "          1.66658133e-03, -3.30453976e-02, -6.17895479e-03, -1.96754107e-02,\n",
       "         -4.23051918e-02, -2.20929662e-01, -4.02760300e-01, -4.29542902e-01,\n",
       "         -4.43720871e-01, -2.73026843e-01, -3.69963030e-01, -3.86620114e-01,\n",
       "         -4.75481437e-01, -2.92541277e-01, -5.41063133e-01, -5.09610624e-01,\n",
       "         -4.28605013e-01, -4.10597401e-01, -1.92457011e-01, -1.12285501e-01,\n",
       "         -1.89432486e-01, -2.63282181e-01, -4.12577244e-01, -3.66085363e-01,\n",
       "         -2.91663524e-01, -6.66641015e-02,  1.63004284e-03, -2.31795893e-03,\n",
       "         -2.78637823e-02, -2.25536029e-02,  5.29765783e-03, -1.55219067e-02,\n",
       "         -8.50665814e-02, -2.71426598e-01, -4.25355031e-01, -4.40105315e-01,\n",
       "         -3.12702732e-01, -2.13580132e-01, -1.67508630e-01, -4.69596534e-01,\n",
       "         -3.75420006e-01, -3.37845414e-01, -3.21085010e-01, -3.89578599e-01,\n",
       "         -2.20229909e-01, -3.18552947e-01, -2.27103277e-01, -2.20452398e-01,\n",
       "         -2.17558625e-01, -1.85352923e-01, -9.86300748e-02, -3.39664130e-01,\n",
       "         -2.52259243e-01, -3.49533823e-02, -2.41803306e-02, -1.12562604e-02,\n",
       "         -1.36240722e-02,  2.68177648e-02,  1.31959603e-02,  3.01506126e-02,\n",
       "         -6.56231213e-02, -2.17469497e-01, -2.98380271e-01, -2.29906969e-01,\n",
       "         -2.69818482e-01, -2.54428980e-01, -2.97020988e-01, -2.37130173e-01,\n",
       "         -2.87150279e-01, -7.69185834e-02, -1.92331814e-01, -9.42780710e-02,\n",
       "         -2.78675158e-01, -2.80364741e-01, -1.87055879e-01, -1.92364548e-01,\n",
       "         -2.32137059e-01, -1.08292845e-01, -1.20692706e-01, -1.96658899e-01,\n",
       "         -1.04086018e-01, -1.98079857e-02, -7.80007316e-03, -5.13595795e-03,\n",
       "          8.36515841e-03, -9.54429244e-03,  3.90468971e-02, -4.50447137e-02,\n",
       "         -4.58927896e-02, -2.29042577e-01, -2.44104772e-01, -2.61095878e-01,\n",
       "         -3.27711808e-01, -2.25661372e-01, -1.61788898e-01, -2.97462923e-01,\n",
       "         -1.31632813e-01, -2.21145464e-01, -3.73007230e-01, -2.69581221e-01,\n",
       "         -2.79297402e-01, -3.19309241e-01, -2.16301890e-01, -1.83451042e-01,\n",
       "         -1.84772612e-01, -1.64635829e-01, -1.83568509e-01, -1.77195628e-01,\n",
       "         -1.06770492e-01, -1.08576558e-01, -5.23872379e-02, -2.91378343e-02,\n",
       "          5.92598226e-03,  8.11879687e-03, -1.40053514e-02, -5.59475634e-03,\n",
       "         -1.05735168e-01, -2.00900618e-01, -3.39018851e-01, -3.29749977e-01,\n",
       "         -4.38490876e-01, -2.99599899e-01, -1.75965683e-01, -8.95142373e-02,\n",
       "          3.89158929e-02, -3.39129992e-01, -4.73877445e-01, -2.41716875e-01,\n",
       "         -2.13476300e-01, -3.94031562e-01, -2.70555064e-01, -1.87622565e-01,\n",
       "         -1.98081001e-01, -2.82577340e-01, -2.15859303e-01, -1.23251724e-01,\n",
       "         -1.69374237e-01, -1.55065743e-01, -5.03200933e-02,  3.21914799e-02,\n",
       "          1.09210532e-02, -2.07722793e-02,  1.27766606e-02, -6.15728590e-02,\n",
       "         -1.66441957e-01, -2.25421198e-01, -2.33192807e-01, -2.98226284e-01,\n",
       "         -5.21575560e-01, -5.71481427e-01, -4.32793359e-01, -2.67803953e-01,\n",
       "         -2.17069597e-01, -4.93463612e-01, -4.84429823e-01, -2.57379064e-01,\n",
       "         -1.49082698e-01, -2.41954766e-01, -2.13170518e-01, -1.39979312e-01,\n",
       "         -1.42071975e-01, -6.92747000e-02, -1.62633985e-01, -7.51408490e-02,\n",
       "         -1.07118281e-01, -1.00746271e-01, -2.25087079e-02, -1.35342794e-02,\n",
       "         -3.63728147e-03,  1.95986245e-02,  2.45602032e-02, -5.28443142e-02,\n",
       "         -2.04052887e-01, -2.96661332e-01, -1.67588652e-01, -2.08367068e-01,\n",
       "         -1.87013984e-01, -1.80277025e-01, -4.55763215e-01, -1.51330492e-01,\n",
       "         -5.33445341e-01, -4.26931718e-01, -3.68864924e-01, -8.47182818e-02,\n",
       "         -2.88196737e-01, -2.07865287e-01, -1.45185672e-01, -7.01546305e-02,\n",
       "         -1.16709495e-02, -2.08378927e-01, -1.95098045e-01, -2.09146246e-01,\n",
       "         -2.63609428e-01, -9.28347033e-02, -1.37160510e-02, -4.36503355e-03,\n",
       "         -2.87629533e-02,  7.49413668e-03,  5.67512376e-03, -9.02641914e-02,\n",
       "         -1.20289024e-01, -1.42797407e-01, -1.06463881e-01, -1.56660915e-01,\n",
       "         -2.47968376e-01, -2.85034555e-01, -1.73900650e-01, -2.94735360e-01,\n",
       "         -1.41531259e-01, -2.73910033e-01, -3.39727978e-01, -8.03252051e-02,\n",
       "         -1.02089255e-01, -1.62019369e-01, -8.09343889e-02, -9.01358305e-02,\n",
       "         -8.99351307e-02, -2.15086118e-01, -1.38735207e-01, -7.58265342e-02,\n",
       "         -8.68047172e-02, -7.34877618e-02, -2.01200149e-02, -1.36053060e-02,\n",
       "          7.91710076e-03,  1.39837430e-02,  1.64057822e-02, -3.64241052e-02,\n",
       "         -1.84386574e-01, -1.06917864e-01, -2.59905349e-01, -2.17530512e-01,\n",
       "         -1.87078879e-01, -2.34035386e-01, -2.77515774e-01, -4.04869531e-01,\n",
       "         -2.13300556e-01, -2.24947395e-01, -3.15840061e-01, -7.70943371e-02,\n",
       "         -2.67802739e-01, -1.47284654e-01, -1.51310282e-01, -8.14579388e-02,\n",
       "         -1.24634212e-01, -8.16140113e-02, -1.20413244e-01, -4.60831703e-02,\n",
       "         -6.23793094e-02, -1.53225855e-02,  6.98508579e-03,  1.78891673e-03,\n",
       "          1.37863638e-03,  2.93005843e-04, -1.77360794e-02, -1.90664121e-02,\n",
       "         -1.39989187e-01, -1.62313160e-01, -1.61060677e-01, -1.08275769e-01,\n",
       "         -1.24983336e-01, -1.18766417e-01, -2.35212614e-01, -3.20493880e-01,\n",
       "         -2.32436684e-01, -5.62999583e-02, -2.12606050e-01, -1.38022994e-01,\n",
       "         -1.72298216e-01, -9.95642190e-02, -1.70824165e-01, -1.09902532e-01,\n",
       "         -3.43779240e-02, -1.13899672e-01, -8.53724494e-02, -5.59262223e-02,\n",
       "         -3.92097790e-02, -5.95219650e-03,  4.20466374e-03,  6.51565618e-03,\n",
       "          1.45827662e-02, -2.48913214e-03,  1.16688625e-02, -2.78870153e-03,\n",
       "         -1.11414901e-01, -1.92004399e-01, -1.57487879e-01, -1.08451270e-01,\n",
       "         -1.32432937e-01, -1.82714334e-01, -2.04570023e-01, -1.01889571e-01,\n",
       "         -6.91079766e-02, -1.08182834e-03, -2.78767952e-02, -7.99285637e-02,\n",
       "         -1.69668998e-01, -1.79870083e-01, -2.14096211e-01, -1.79845293e-01,\n",
       "         -1.48217354e-01, -7.60177288e-02, -1.25912866e-01, -1.06655130e-01,\n",
       "         -4.07573900e-02, -9.95440237e-03, -1.85522735e-02, -4.83758212e-04,\n",
       "         -1.87080311e-02, -3.70287818e-03, -2.11299896e-02, -2.98011852e-02,\n",
       "         -4.76841504e-02, -1.82013833e-01, -2.61216575e-01, -1.55968177e-01,\n",
       "         -9.65055018e-02, -1.78451289e-01, -1.38053023e-01, -1.04428226e-01,\n",
       "          4.99404977e-02,  2.40630765e-01,  1.67018544e-02,  5.91195007e-02,\n",
       "         -9.49863341e-02, -1.42102279e-01, -1.80285759e-01, -1.64555397e-01,\n",
       "         -1.68261535e-01, -1.22376557e-01, -1.12365832e-01, -2.28170170e-02,\n",
       "         -2.79019121e-02,  5.94000775e-03, -1.83717799e-02, -7.57018522e-03,\n",
       "         -1.45564382e-02, -4.49766609e-05, -5.56187825e-03,  2.38635404e-03,\n",
       "         -5.74899444e-02, -1.49673253e-01, -2.09541284e-01, -1.85923994e-01,\n",
       "         -7.36931260e-02, -5.50624929e-02, -1.00670539e-01, -8.23911576e-02,\n",
       "         -8.59719177e-02, -1.35583152e-01, -1.23570162e-01, -8.60931599e-02,\n",
       "         -5.67039592e-02,  7.09535631e-03, -5.51935634e-02, -1.23500747e-01,\n",
       "         -1.11114073e-01, -2.62364511e-02, -2.07041015e-02, -2.66246219e-02,\n",
       "         -4.75992166e-03,  8.42124269e-03,  1.85347985e-02, -3.72606050e-02,\n",
       "         -2.05333030e-02,  9.78144822e-03,  1.37291744e-02, -6.07043370e-03,\n",
       "         -3.28211260e-02, -5.06378093e-02, -4.01424519e-02, -1.38809479e-01,\n",
       "         -1.23208786e-01, -1.16435591e-01, -1.53569009e-01, -1.85469943e-01,\n",
       "         -1.17631694e-01, -7.44899192e-02, -6.48336179e-02, -7.19160225e-02,\n",
       "         -9.24632794e-02, -4.91341506e-02, -8.48304743e-02, -5.93439775e-02,\n",
       "         -8.62027210e-02, -3.67767418e-02, -3.21588718e-03, -1.20862829e-02,\n",
       "          6.58541398e-03, -7.12433284e-03,  4.98478134e-03, -1.46439224e-02,\n",
       "         -1.86283844e-02, -2.96034680e-03,  4.81032989e-03,  4.37350114e-03,\n",
       "         -6.02080072e-03, -2.74059884e-02,  6.84022071e-03, -4.75969929e-02,\n",
       "         -3.06459505e-02, -6.06866755e-02, -8.51643953e-02, -9.67014895e-02,\n",
       "         -8.27342087e-02, -9.89252974e-02, -1.84632945e-01, -9.83921718e-02,\n",
       "         -9.44570577e-02, -7.83795556e-02, -6.48276731e-02, -3.19482712e-02,\n",
       "         -3.26097261e-02,  1.20533521e-02,  2.13054216e-02,  3.83909066e-03,\n",
       "         -3.17931422e-02, -3.93746571e-03, -3.11557256e-02, -2.92565563e-02,\n",
       "          1.74182616e-02, -9.29867681e-03, -7.26465547e-04,  4.03957913e-02,\n",
       "          3.92190945e-02,  3.44294697e-03, -7.78731639e-03, -5.15889658e-03,\n",
       "         -4.40789673e-03, -9.34247013e-04, -1.32463128e-02, -2.06281348e-02,\n",
       "         -3.47260929e-02,  7.94829280e-03, -8.51018995e-03,  2.11435687e-02,\n",
       "         -2.34050780e-02, -1.39743571e-04, -1.39631449e-02, -1.60058933e-02,\n",
       "         -1.00456277e-02, -2.14583523e-02, -7.43237077e-03, -3.85340322e-03,\n",
       "         -4.90339799e-03, -2.50748895e-02,  2.88008228e-02, -2.20376321e-03])},\n",
       " {'w': array([[-0.05954934,  0.03032614, -0.00032207, ...,  0.06142349,\n",
       "          -0.04571915,  0.06348648],\n",
       "         [ 0.07681583,  0.00797561, -0.00266409, ...,  0.02930079,\n",
       "           0.03400993,  0.04864621],\n",
       "         [ 0.00330505,  0.01478545,  0.08203109, ..., -0.01547735,\n",
       "          -0.03037345, -0.02761256],\n",
       "         ...,\n",
       "         [ 0.06173073,  0.06865266,  0.07350748, ...,  0.03291034,\n",
       "          -0.04128883,  0.07230146],\n",
       "         [-0.0475267 , -0.01876552,  0.027709  , ..., -0.03261928,\n",
       "          -0.00434977, -0.08390339],\n",
       "         [-0.00782083,  0.05744823,  0.05240999, ...,  0.00289999,\n",
       "          -0.04355425, -0.00459715]]),\n",
       "  'b': array([-0.258677  , -0.08531378, -0.09109682, -0.03821053, -0.02365061,\n",
       "          0.36632011,  0.06790924,  0.01340604,  0.0680387 , -0.01872535])},\n",
       " {}]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_path=Path('./MNIST')\n",
    "train_img_path=dataset_path/'train-images.idx3-ubyte'\n",
    "train_lab_path=dataset_path/'train-labels.idx1-ubyte'\n",
    "test_img_path=dataset_path/'t10k-images.idx3-ubyte'\n",
    "test_lab_path=dataset_path/'t10k-labels.idx1-ubyte'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_num=50000\n",
    "valid_num=10000\n",
    "test_num=10000\n",
    "\n",
    "with open(train_img_path,'rb') as f:\n",
    "    struct.unpack('>4i',f.read(16))\n",
    "    tmp_img=np.fromfile(f,dtype=np.uint8).reshape(-1,28*28)/255\n",
    "    train_img=tmp_img[:train_num]\n",
    "    valid_img=tmp_img[train_num:]\n",
    "    \n",
    "with open(test_img_path,'rb') as f:\n",
    "    struct.unpack('>4i',f.read(16))\n",
    "    test_img=np.fromfile(f,dtype=np.uint8).reshape(-1,28*28)/255\n",
    "\n",
    "with open(train_lab_path,'rb') as f:\n",
    "    struct.unpack('>2i',f.read(8))\n",
    "    tmp_lab=np.fromfile(f,dtype=np.uint8)\n",
    "    train_lab=tmp_lab[:train_num]\n",
    "    valid_lab=tmp_lab[train_num:]\n",
    "    \n",
    "with open(test_lab_path,'rb') as f:\n",
    "    struct.unpack('>2i',f.read(8))\n",
    "    test_lab=np.fromfile(f,dtype=np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_train(index):\n",
    "    plt.imshow(train_img[index].reshape(28,28),cmap='gray')\n",
    "    print('label : {}'.format(train_lab[index]))\n",
    "def show_valid(index):\n",
    "    plt.imshow(valid_img[index].reshape(28,28),cmap='gray')\n",
    "    print('label : {}'.format(valid_lab[index]))\n",
    "def show_test(index):\n",
    "    plt.imshow(test_img[index].reshape(28,28),cmap='gray')\n",
    "    print('label : {}'.format(test_lab[index]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "label : 3\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADW1JREFUeJzt3W+sVPWdx/HPBxYisdVoCBQFFraadY0P7HpjNqHZuEEbd0MCfYCBB4q0lEYxsUbNEjGpyUqsm213G4IkNCUgobRN0Aq1bFuN0aobIpKKtmypErZFbrhFmtQ+EYHvPriH5oJ3fnPv/DsD3/crITNzvmfO+Wa4nzln5jczP0eEAOQzoe4GANSD8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSOqverkz23ycEOiyiPBY1mvryG/7dtu/sf2u7dXtbAtAb7nVz/bbnijpoKTbJB2R9IakpRHx68J9OPIDXdaLI//Nkt6NiEMRcVLS9yUtbGN7AHqonfBfLen3I24fqZadw/ZK23tt721jXwA6rJ03/EY7tfjEaX1EbJS0UeK0H+gn7Rz5j0iaNeL2TElH22sHQK+0E/43JF1re67tyZKWSNrZmbYAdFvLp/0Rccr2fZJ+KmmipE0R8auOdQagq1oe6mtpZ7zmB7quJx/yAXDhIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiCplqfoliTbhyV9KOm0pFMRMdCJpi42l1xySbF+1VVXFesrVqwo1qdMmTLuns66++67i/W33nqrWN+1a1exvm7duoa1kydPFu+L7mor/JV/iojjHdgOgB7itB9Iqt3wh6Sf2X7T9spONASgN9o97Z8XEUdtT5P0c9v/GxGvjFyhelLgiQHoM20d+SPiaHU5JOlZSTePss7GiBjgzUCgv7QcftuX2v702euSviDpnU41BqC72jntny7pWdtnt/O9iPjvjnQFoOscEb3bmd27nfXQ5MmTi/X169cX68uXL+9kOx1VPbk31Ozv5+WXX25Ye/zxx4v3femll4p1jC4iyv9pFYb6gKQIP5AU4QeSIvxAUoQfSIrwA0kx1NcB06dPL9bff//9HnXSee0O9ZWcPn26WF+0aFGxvnv37pb3fTFjqA9AEeEHkiL8QFKEH0iK8ANJEX4gKcIPJNWJX+9Nb2hoqFi///77i/U1a9YU66+//nqxftdddxXr3XTvvfcW62vXrm1YmzhxYvG+CxYsKNZfeOGFYv3jjz8u1rPjyA8kRfiBpAg/kBThB5Ii/EBShB9IivADSfF9fnTVk08+2bD24IMPtrXtefPmFet79uxpa/sXKr7PD6CI8ANJEX4gKcIPJEX4gaQIP5AU4QeSajrOb3uTpAWShiLihmrZlZJ+IGmOpMOS7oiIPzbdGeP86UybNq1h7b333ived8qUKcV6s6nNt27dWqxfrDo5zr9Z0u3nLVst6cWIuFbSi9VtABeQpuGPiFcknThv8UJJW6rrWySVp1YB0Hdafc0/PSIGJam6bHxuB6Avdf03/GyvlLSy2/sBMD6tHvmP2Z4hSdVlw1+wjIiNETEQEQMt7gtAF7Qa/p2SllXXl0l6rjPtAOiVpuG3vV3S/0j6W9tHbH9Z0jck3Wb7t5Juq24DuIDwfX501aRJkxrWTpw4fxDpXM3G+bdv316s33nnncX6xYrv8wMoIvxAUoQfSIrwA0kRfiApwg8kxRTdKJo8eXKxPn/+/GJ99erGX/hsNpTXTLOhQpRx5AeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpBjnT+7WW28t1hcuXFis33PPPZ1s5xz79u0r1tevX9+1fWfAkR9IivADSRF+ICnCDyRF+IGkCD+QFOEHkmKc/wKwatWqYv2BBx5oedszZ84s1ks/vS1J7fz0+1NPPVWsb968uVg/ePBgy/sGR34gLcIPJEX4gaQIP5AU4QeSIvxAUoQfSKrpOL/tTZIWSBqKiBuqZY9J+oqkP1SrPRIRP+lWk9lddtllxfrcuXO7tu8JE8rHhzNnzrS87SeeeKJYHxwcbHnbaG4sR/7Nkm4fZfl/RsSN1T+CD1xgmoY/Il6RxNQowEWmndf899neb3uT7Ss61hGAnmg1/BskfVbSjZIGJX2z0Yq2V9rea3tvi/sC0AUthT8ijkXE6Yg4I+k7km4urLsxIgYiYqDVJgF0Xkvhtz1jxM0vSnqnM+0A6JWxDPVtl3SLpKm2j0j6uqRbbN8oKSQdlvTVLvYIoAvczvexx70zu3c7u4gsXbq0WF++fHnX9j1//vxivZ2/nz179hTr8+bNa3nbmUWEx7Ien/ADkiL8QFKEH0iK8ANJEX4gKcIPJMVQH4qaTcH98MMPF+uzZ89uWDt9+nTxvosWLSrWd+/eXaxnxVAfgCLCDyRF+IGkCD+QFOEHkiL8QFKEH0iKKbpRtGHDhmL90KFDxfrzzz/fsDZx4sTifS+//PJiHe3hyA8kRfiBpAg/kBThB5Ii/EBShB9IivADSTHOj7ZMnTq17hbQIo78QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5BU03F+27MkPS3pM5LOSNoYEd+2faWkH0iaI+mwpDsi4o/daxV1aPZ9/sWLF7e87Y8++qhYP3z4cMvbRnNjOfKfkvRgRPydpH+QtMr29ZJWS3oxIq6V9GJ1G8AFomn4I2IwIvZV1z+UdEDS1ZIWStpSrbZFUnl6FQB9ZVyv+W3PkfQ5SXskTY+IQWn4CULStE43B6B7xvzZftufkrRD0tci4k/2mKYDk+2Vkla21h6AbhnTkd/2JA0Hf1tEPFMtPmZ7RlWfIWlotPtGxMaIGIiIgU40DKAzmobfw4f470o6EBHfGlHaKWlZdX2ZpOc63x6Abmk6Rbftz0v6haS3NTzUJ0mPaPh1/w8lzZb0O0mLI+JEk20xRXcXTJjQ+Dn8+uuvL953x44dxfo111xTrDf7+zl16lTDWrNhwl27dhXrGN1Yp+hu+po/Il6V1Ghj88fTFID+wSf8gKQIP5AU4QeSIvxAUoQfSIrwA0nx090XgOuuu65YX7JkScPao48+2ul2zvHaa68V62vWrGlYe/XVVzvdDsaBIz+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJMU4fx9Yt25dsb5ixYpifdKkSZ1s5xwPPfRQsb5169Zi/fjx451sBx3EkR9IivADSRF+ICnCDyRF+IGkCD+QFOEHkmKcvw/cdNNNxXqzcfwPPvigYW3btm3F+27evLlY379/f7GOCxdHfiApwg8kRfiBpAg/kBThB5Ii/EBShB9Iys3mV7c9S9LTkj4j6YykjRHxbduPSfqKpD9Uqz4SET9psq3yzgC0LSI8lvXGEv4ZkmZExD7bn5b0pqRFku6Q9OeI+I+xNkX4ge4ba/ibfsIvIgYlDVbXP7R9QNLV7bUHoG7jes1ve46kz0naUy26z/Z+25tsX9HgPitt77W9t61OAXRU09P+v6xof0rSy5LWRsQztqdLOi4pJP2bhl8afKnJNjjtB7qsY6/5Jcn2JEk/lvTTiPjWKPU5kn4cETc02Q7hB7psrOFvetpv25K+K+nAyOBXbwSe9UVJ74y3SQD1Gcu7/Z+X9AtJb2t4qE+SHpG0VNKNGj7tPyzpq9Wbg6VtceQHuqyjp/2dQviB7uvYaT+AixPhB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gqV5P0X1c0v+NuD21WtaP+rW3fu1LordWdbK3vx7rij39Pv8ndm7vjYiB2hoo6Nfe+rUvid5aVVdvnPYDSRF+IKm6w7+x5v2X9Gtv/dqXRG+tqqW3Wl/zA6hP3Ud+ADWpJfy2b7f9G9vv2l5dRw+N2D5s+23bv6x7irFqGrQh2++MWHal7Z/b/m11Oeo0aTX19pjt96vH7pe2/6Wm3mbZfsn2Adu/sn1/tbzWx67QVy2PW89P+21PlHRQ0m2Sjkh6Q9LSiPh1TxtpwPZhSQMRUfuYsO1/lPRnSU+fnQ3J9r9LOhER36ieOK+IiH/tk94e0zhnbu5Sb41mlr5bNT52nZzxuhPqOPLfLOndiDgUESclfV/Swhr66HsR8YqkE+ctXihpS3V9i4b/eHquQW99ISIGI2Jfdf1DSWdnlq71sSv0VYs6wn+1pN+PuH1E/TXld0j6me03ba+su5lRTD87M1J1Oa3mfs7XdObmXjpvZum+eexamfG60+oI/2izifTTkMO8iPh7Sf8saVV1eoux2SDpsxqexm1Q0jfrbKaaWXqHpK9FxJ/q7GWkUfqq5XGrI/xHJM0acXumpKM19DGqiDhaXQ5JelbDL1P6ybGzk6RWl0M19/MXEXEsIk5HxBlJ31GNj101s/QOSdsi4plqce2P3Wh91fW41RH+NyRda3uu7cmSlkjaWUMfn2D70uqNGNm+VNIX1H+zD++UtKy6vkzSczX2co5+mbm50czSqvmx67cZr2v5kE81lPFfkiZK2hQRa3vexChs/42Gj/bS8Dcev1dnb7a3S7pFw9/6Oibp65J+JOmHkmZL+p2kxRHR8zfeGvR2i8Y5c3OXems0s/Qe1fjYdXLG6470wyf8gJz4hB+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaT+H4pZ/hIg1cAjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_train(np.random.randint(train_num))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "label : 2\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADXRJREFUeJzt3W+oXPWdx/HPJ/8QbB8o1RhTs6lRdNc8sEuIi41rRBPdIMRAWhoEs2zx+qCKkRU26IMIa6Au2+7qk0JqQlJo7RaMaxBto7KYimtMDMVEXeulZGs2IVGiJBWlmnz3wT3ZvcY7v5k7c2bOJN/3C8L8+c6Z82XI554z8zvn/BwRApDPlKYbANAMwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+IKlpg1yZbQ4nBPosItzJ63ra8tu+xfY7tkdtr+3lvQAMlrs9tt/2VEm/k7RE0gFJuyStioi3Csuw5Qf6bBBb/oWSRiPi9xHxJ0m/kLS8h/cDMEC9hH+2pPfGPT5QPfcFtkds77a9u4d1AahZLz/4TbRr8aXd+ojYIGmDxG4/MEx62fIfkHTJuMdfl3Swt3YADEov4d8l6XLb37A9Q9J3JW2rpy0A/db1bn9EfG77bkm/ljRV0qaIeLO2zgD0VddDfV2tjO/8QN8N5CAfAGcuwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkhropbtx9nnrrZbXa5UkXXnllS1rdvnks+3btxfr69atK9ZfffXVYj07tvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBRX70VP3nyzfLX20jh/r44ePVqsr1ixomXt5ZdfrrudocHVewEUEX4gKcIPJEX4gaQIP5AU4QeSIvxAUj2N89veL+m4pBOSPo+IBW1ezzj/WWbJkiXF+tq1a1vWrr/++uKy7c73b+f9999vWVu0aFFx2dHR0Z7W3aROx/nruJjHDRHxQQ3vA2CA2O0Hkuo1/CFpu+3XbY/U0RCAweh1t/9bEXHQ9oWSnrf9XxGxY/wLqj8K/GEAhkxPW/6IOFjdHpH0lKSFE7xmQ0QsaPdjIIDB6jr8ts+1/dVT9yUtlbSvrsYA9Fcvu/0zJT1VDcdMk/TziPhVLV0B6DvO50dj2o3z33vvvcX68uXLu173NddcU6zv3r276/duGufzAygi/EBShB9IivADSRF+ICnCDyTFFN1ozEsvvVSsr1y5ckCd5MSWH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkmobftubbB+xvW/cc+fbft72u9Xtef1tE0DdOtnyb5Z0y2nPrZX0YkRcLunF6jGAM0jb8EfEDklHT3t6uaQt1f0tkm6ruS8Afdbtd/6ZEXFIkqrbC+trCcAg9H2uPtsjkkb6vR4Ak9Ptlv+w7VmSVN0eafXCiNgQEQsiYkGX6wLQB92Gf5uk1dX91ZKerqcdAIPSyVDfE5L+U9IVtg/Y/p6kH0haYvtdSUuqxwDOIG2/80fEqhalG2vuBcmcc845xfqNN/JfrJ84wg9IivADSRF+ICnCDyRF+IGkCD+QVN8P70Xvpkwp/42+9tprW9aWLVtWXPaqq64q1l977bVifeHChcV6ybRp5f9+V1xxRdfvLUl79+5tWRsdHe3pvc8GbPmBpAg/kBThB5Ii/EBShB9IivADSRF+ICnG+YfAkiVLivUHH3ywWL/uuuvqbOcLbr311r69N5rFlh9IivADSRF+ICnCDyRF+IGkCD+QFOEHknJEDG5l9uBWNkA333xzsb52bXkS49L5+FL7894xefv37y/WFy9eXKy/99579TVTs4hwJ69jyw8kRfiBpAg/kBThB5Ii/EBShB9IivADSbUdQLa9SdKtko5ExPzquYck3Snp/eplD0TEs/1qchiMjIy0rD322GPFZadPn153O1/w7LOtP/qtW7cWl/3444+L9UsvvbRYX79+fbHei+PHjxfr27ZtK9Zvv/32lrW5c+cWl50xY0axfjboZMu/WdItEzz/LxFxdfXvrA4+cDZqG/6I2CHp6AB6ATBAvXznv9v2G7Y32T6vto4ADES34f+xpHmSrpZ0SNIPW73Q9ojt3bZ3d7kuAH3QVfgj4nBEnIiIk5J+IqnlbI0RsSEiFkTEgm6bBFC/rsJve9a4hysk7aunHQCD0slQ3xOSFkv6mu0DktZJWmz7akkhab+ku/rYI4A+4Hz+yrx584r1d955p2XN7uj06a4tW7asWH/hhRda1k6cOFFcdurUqcX6I488Uqzfd999xXpJu2MMbrrppmJ9165dxXrpOgh33HFHcdl2x0d8+OGHxXqTOJ8fQBHhB5Ii/EBShB9IivADSRF+ICmG+iqXXXZZsV4a6uvVM888U6yvXLmyWP/ss89a1tqdTvzwww8X6/fff3+x3ot2lyzfuXNn39Z9NmOoD0AR4QeSIvxAUoQfSIrwA0kRfiApwg8kxdzPA/DRRx8V63fdVb4cQmkcX5Jmz57dsnbPPfcUl+3nOL4kbd68uWWt3Sm56C+2/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOP8A7Bnz55ivd1xADfccEOxvmnTppa1OXPmFJft1caNG4v10nEGJ0+erLsdTAJbfiApwg8kRfiBpAg/kBThB5Ii/EBShB9Iqu11+21fIumnki6SdFLShoh41Pb5kv5N0lxJ+yV9JyKK8xYP83X7L7roomK9dO75xRdf3NO633jjjWJ9/vz5xfqUKf37G/74448X62vWrCnWP/nkkzrbQQfqvG7/55L+PiL+XNJfSfq+7b+QtFbSixFxuaQXq8cAzhBtwx8RhyJiT3X/uKS3Jc2WtFzSluplWyTd1q8mAdRvUvuLtudK+qaknZJmRsQhaewPhKQL624OQP90fGy/7a9IelLSmog4Znf0tUK2RySNdNcegH7paMtve7rGgv+ziNhaPX3Y9qyqPkvSkYmWjYgNEbEgIhbU0TCAerQNv8c28RslvR0RPxpX2iZpdXV/taSn628PQL90MtS3SNJvJO3V2FCfJD2gse/9v5Q0R9IfJH07Io62ea+hHeprZ+nSpS1rzz333AA7mZx2pwuvWrWqWN+xY0ex/umnn066J/RXp0N9bb/zR8TLklq92Y2TaQrA8OAIPyApwg8kRfiBpAg/kBThB5Ii/EBSbcf5a13ZGTzOP21a61HRO++8s7jsunXrivULLrigWH/llVeK9e3bt7esPfroo8Vljx07VqzjzFPnKb0AzkKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4/zAWYZxfgBFhB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5BU2/DbvsT2f9h+2/abtu+tnn/I9v/Y/m31b1n/2wVQl7YX87A9S9KsiNhj+6uSXpd0m6TvSPpjRPxzxyvjYh5A33V6MY/W09D8/xsdknSoun/c9tuSZvfWHoCmTeo7v+25kr4paWf11N2237C9yfZ5LZYZsb3b9u6eOgVQq46v4Wf7K5JekrQ+IrbaninpA0kh6R819tXg79q8B7v9QJ91utvfUfhtT5f0jKRfR8SPJqjPlfRMRMxv8z6EH+iz2i7gaduSNkp6e3zwqx8CT1khad9kmwTQnE5+7V8k6TeS9ko6WT39gKRVkq7W2G7/fkl3VT8Olt6LLT/QZ7Xu9teF8AP9x3X7ARQRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkmp7Ac+afSDpv8c9/lr13DAa1t6GtS+J3rpVZ29/1ukLB3o+/5dWbu+OiAWNNVAwrL0Na18SvXWrqd7Y7QeSIvxAUk2Hf0PD6y8Z1t6GtS+J3rrVSG+NfucH0Jymt/wAGtJI+G3fYvsd26O21zbRQyu299veW8083OgUY9U0aEds7xv33Pm2n7f9bnU74TRpDfU2FDM3F2aWbvSzG7YZrwe+2297qqTfSVoi6YCkXZJWRcRbA22kBdv7JS2IiMbHhG3/taQ/SvrpqdmQbP+TpKMR8YPqD+d5EfEPQ9LbQ5rkzM196q3VzNJ/qwY/uzpnvK5DE1v+hZJGI+L3EfEnSb+QtLyBPoZeROyQdPS0p5dL2lLd36Kx/zwD16K3oRARhyJiT3X/uKRTM0s3+tkV+mpEE+GfLem9cY8PaLim/A5J222/bnuk6WYmMPPUzEjV7YUN93O6tjM3D9JpM0sPzWfXzYzXdWsi/BPNJjJMQw7fioi/lPQ3kr5f7d6iMz+WNE9j07gdkvTDJpupZpZ+UtKaiDjWZC/jTdBXI59bE+E/IOmScY+/LulgA31MKCIOVrdHJD2lsa8pw+TwqUlSq9sjDffzfyLicESciIiTkn6iBj+7ambpJyX9LCK2Vk83/tlN1FdTn1sT4d8l6XLb37A9Q9J3JW1roI8vsX1u9UOMbJ8raamGb/bhbZJWV/dXS3q6wV6+YFhmbm41s7Qa/uyGbcbrRg7yqYYy/lXSVEmbImL9wJuYgO1LNba1l8bOePx5k73ZfkLSYo2d9XVY0jpJ/y7pl5LmSPqDpG9HxMB/eGvR22JNcubmPvXWambpnWrws6tzxuta+uEIPyAnjvADkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5DU/wIVzAJ80BxYDgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_valid(np.random.randint(valid_num))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "label : 6\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADYJJREFUeJzt3XuMVOUZx/Hf46WagBKJsCUgxVtqq0ml2RDvoSpqq3FVooH4B6a1a4yYEk28J5A0JqSpl0YTkzUSMFFRIyrRpl6wKZRUZNVGQSpe2MLKBjSYICoo8vSPPTQr7rwze+bMnNl9vp+EzMx55j3nyYTfnjNzzsxr7i4A8RxUdgMAykH4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8EdUgzN2ZmXE4INJi7Wy3Pq2vPb2YXmdn7Zvahmd1Wz7oANJflvbbfzA6WtFHSDEm9ktZKmu3u7yXGsOcHGqwZe/5pkj5094/d/RtJSyV11LE+AE1UT/gnStoy4HFvtux7zKzTzLrNrLuObQEoWD0f+A12aPGDw3p375LUJXHYD7SSevb8vZKOGfB4kqSt9bUDoFnqCf9aSSea2bFm9iNJsyQtL6YtAI2W+7Df3fea2VxJL0k6WNIid19fWGcAGir3qb5cG+M9P9BwTbnIB8DwRfiBoAg/EBThB4Ii/EBQhB8Iqqnf50fzjRs3LlnfuHFjsr506dJk/frrrx9yT2gN7PmBoAg/EBThB4Ii/EBQhB8IivADQXGqb4S75pprkvUxY8Yk62+88UaB3aCVsOcHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaA4zz/CTZ48uewW0KLY8wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUHWd5zezHklfSPpO0l53by+iKQzNYYcdVrHW0dGRHPv1118n69V+2hvDVxEX+fzK3T8rYD0AmojDfiCoesPvkl42szfNrLOIhgA0R72H/We6+1YzGy/pFTP7j7uvHPiE7I8CfxiAFlPXnt/dt2a32yU9K2naIM/pcvd2PgwEWkvu8JvZKDM7Yv99SRdIWldUYwAaq57D/jZJz5rZ/vU87u5/K6QrAA2XO/zu/rGkXxTYC3K64oorKtYmTZqUHHvnnXcm66tXr87VE1ofp/qAoAg/EBThB4Ii/EBQhB8IivADQfHT3SPAGWeckXvs5s2bC+wEwwl7fiAowg8ERfiBoAg/EBThB4Ii/EBQhB8Iyty9eRsza97GRpC2trZkff369RVrmzZtSo49/fTTk/W9e/cm62g97m61PI89PxAU4QeCIvxAUIQfCIrwA0ERfiAowg8Exff5h4GTTjopWR87dmzF2n333Zccy3n8uNjzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQVc/zm9kiSZdI2u7up2TLxkp6UtIUST2SrnL3zxvXZmxnn3127rG7d+8usBOMJLXs+RdLuuiAZbdJWuHuJ0pakT0GMIxUDb+7r5S044DFHZKWZPeXSLqs4L4ANFje9/xt7t4nSdnt+OJaAtAMDb+238w6JXU2ejsAhibvnn+bmU2QpOx2e6UnunuXu7e7e3vObQFogLzhXy5pTnZ/jqTni2kHQLNUDb+ZPSHpX5J+ama9ZvY7SQslzTCzDyTNyB4DGEaqvud399kVSucV3Asq6OjoyD127dq1BXaCkYQr/ICgCD8QFOEHgiL8QFCEHwiK8ANB8dPdLaC9PX3x4wknnJCsf/555W9Tb9iwIVdPrcAsPdP04Ycfnnvde/bsSdb37duXe93DBXt+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8/wtYPr06cn6mDFjkvVly5ZVrH366ad5WirMkUceWbF29dVXJ8fOnDkzWT/33HNz9SRJK1euTNZvvfXWZH3NmjW5t90q2PMDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCc5x8B3n777dK2ffLJJyfrr732WsXauHHjim6nZuecc06yXu03FjjPD2DYIvxAUIQfCIrwA0ERfiAowg8ERfiBoKqe5zezRZIukbTd3U/Jli2Q9HtJ+78sfoe7/7VRTaI8F154YbK+ePHiZD11Ln/Lli3JsTfeeGOyfv755yfrc+fOTdZTXnzxxdxjh4ta9vyLJV00yPL73P3U7B/BB4aZquF395WSdjShFwBNVM97/rlm9o6ZLTKzowrrCEBT5A3/Q5KOl3SqpD5J91R6opl1mlm3mXXn3BaABsgVfnff5u7fufs+SQ9LmpZ4bpe7t7t7+psSAJoqV/jNbMKAh5dLWldMOwCapZZTfU9Imi7paDPrlTRf0nQzO1WSS+qRdF0DewTQAFXD7+6zB1n8SAN6QU4rVqzIPXbUqFHJ+vz585P1tra2ZH316tUVaxdffHFy7FdffZWsz5s3L1lPee6555L1vr6+3OseLrjCDwiK8ANBEX4gKMIPBEX4gaAIPxAUP909AkybVvECS73++uvJsXfddVeyftpppyXr999/f7K+YMGCirWdO3cmxy5cuDBZrza1+caNGyvWbrrppuTYPXv2JOsjAXt+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8/wjwIwZMyrWurq6kmNHjx5d17Y3bdqUrO/atati7fbbb0+OrfbT3bt3707WZ82aVbHW09OTHBsBe34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCMrcvXkbM2vexoaRKVOmJOuvvvpqsn7cccdVrFX7vv1HH32UrD/wwAPJ+qpVq5L11HUEU6dOTY7t7e1N1m+++eZk/emnn07WRyp3t1qex54fCIrwA0ERfiAowg8ERfiBoAg/EBThB4Kqep7fzI6R9KikH0vaJ6nL3f9iZmMlPSlpiqQeSVe5++dV1sV5/hyuvfbaZL3ad/ZTvvzyy2S92hTe1ZhVPuW8efPm5NiZM2cm693d3bl6GumKPM+/V9LN7v4zSadJusHMfi7pNkkr3P1ESSuyxwCGiarhd/c+d38ru/+FpA2SJkrqkLQke9oSSZc1qkkAxRvSe34zmyJpqqQ1ktrcvU/q/wMhaXzRzQFonJp/w8/MRkt6RtI8d9+Zei93wLhOSZ352gPQKDXt+c3sUPUH/zF3X5Yt3mZmE7L6BEnbBxvr7l3u3u7u7UU0DKAYVcNv/bv4RyRtcPd7B5SWS5qT3Z8j6fni2wPQKLWc6jtL0ipJ76r/VJ8k3aH+9/1PSZosabOkK919R5V1caovh4MOSv+NvuWWWyrWLr300rq2PXHixGT9k08+SdZTX6t98MEHk2O//fbbZB2Dq/VUX9X3/O7+T0mVVnbeUJoC0Dq4wg8IivADQRF+ICjCDwRF+IGgCD8QFD/dDYww/HQ3gCTCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8Iqmr4zewYM/u7mW0ws/Vm9ods+QIz+8TM/p39+03j2wVQlKqTdpjZBEkT3P0tMztC0puSLpN0laRd7v7nmjfGpB1Aw9U6acchNayoT1Jfdv8LM9sgaWJ97QEo25De85vZFElTJa3JFs01s3fMbJGZHVVhTKeZdZtZd12dAihUzXP1mdloSf+QdLe7LzOzNkmfSXJJf1T/W4PfVlkHh/1Ag9V62F9T+M3sUEkvSHrJ3e8dpD5F0gvufkqV9RB+oMEKm6jTzEzSI5I2DAx+9kHgfpdLWjfUJgGUp5ZP+8+StErSu5L2ZYvvkDRb0qnqP+zvkXRd9uFgal3s+YEGK/SwvyiEH2i8wg77AYxMhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaCq/oBnwT6T9N8Bj4/OlrWiVu2tVfuS6C2vInv7Sa1PbOr3+X+wcbNud28vrYGEVu2tVfuS6C2vsnrjsB8IivADQZUd/q6St5/Sqr21al8SveVVSm+lvucHUJ6y9/wASlJK+M3sIjN738w+NLPbyuihEjPrMbN3s5mHS51iLJsGbbuZrRuwbKyZvWJmH2S3g06TVlJvLTFzc2Jm6VJfu1ab8brph/1mdrCkjZJmSOqVtFbSbHd/r6mNVGBmPZLa3b30c8Jmdo6kXZIe3T8bkpn9SdIOd1+Y/eE8yt1vbZHeFmiIMzc3qLdKM0tfoxJfuyJnvC5CGXv+aZI+dPeP3f0bSUsldZTQR8tz95WSdhywuEPSkuz+EvX/52m6Cr21BHfvc/e3svtfSNo/s3Spr12ir1KUEf6JkrYMeNyr1pry2yW9bGZvmlln2c0Mom3/zEjZ7fiS+zlQ1Zmbm+mAmaVb5rXLM+N10coI/2CzibTSKYcz3f2Xkn4t6Ybs8Ba1eUjS8eqfxq1P0j1lNpPNLP2MpHnuvrPMXgYapK9SXrcywt8r6ZgBjydJ2lpCH4Ny963Z7XZJz6r/bUor2bZ/ktTsdnvJ/fyfu29z9+/cfZ+kh1Xia5fNLP2MpMfcfVm2uPTXbrC+ynrdygj/WkknmtmxZvYjSbMkLS+hjx8ws1HZBzEys1GSLlDrzT68XNKc7P4cSc+X2Mv3tMrMzZVmllbJr12rzXhdykU+2amM+yUdLGmRu9/d9CYGYWbHqX9vL/V/4/HxMnszsyckTVf/t762SZov6TlJT0maLGmzpCvdvekfvFXobbqGOHNzg3qrNLP0GpX42hU543Uh/XCFHxATV/gBQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwjqfysY9H6+KoCSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_test(np.random.randint(test_num))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def d_softmax(data):\n",
    "    sm=softmax(data)\n",
    "    return np.diag(sm)-np.outer(sm,sm)\n",
    "\n",
    "# def d_tanh(data):\n",
    "#     return np.diag(1/(np.cosh(data))**2)\n",
    "def d_tanh(data):\n",
    "    return 1/(np.cosh(data))**2\n",
    "\n",
    "differential={softmax:d_softmax,tanh:d_tanh}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.41997434, 0.07065082, 0.00986604, 0.00134095])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d_tanh([1,2,3,4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.03103085, -0.00279373, -0.00759413, -0.02064299],\n",
       "       [-0.00279373,  0.07955019, -0.02064299, -0.05611347],\n",
       "       [-0.00759413, -0.02064299,  0.18076935, -0.15253222],\n",
       "       [-0.02064299, -0.05611347, -0.15253222,  0.22928869]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d_softmax(np.array([1,2,3,4]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-4.73181237e-06  1.00658877e-06  1.95246895e-06  1.77275604e-06]\n",
      "[ 2.20056632e-06 -4.73398460e-06  1.19438817e-06  1.33903151e-06]\n",
      "[ 1.06526890e-06  1.15213585e-06 -3.20049125e-06  9.83086788e-07]\n",
      "[ 1.71278433e-06  1.55024184e-06  1.20785061e-06 -4.47087678e-06]\n"
     ]
    }
   ],
   "source": [
    "h=0.0001\n",
    "func=softmax\n",
    "input_len=4\n",
    "for i in range(input_len):\n",
    "    test_input=np.random.rand(input_len)\n",
    "    derivative=differential[func](test_input)\n",
    "    value1=func(test_input)\n",
    "    test_input[i]+=h\n",
    "    value2=func(test_input)\n",
    "#     print((value2-value1)/h)\n",
    "#     print(derivative[i])\n",
    "    print(derivative[i]-(value2-value1)/h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.972907651166423e-07\n",
      "3.6578738538128874e-07\n",
      "1.1782731657028478e-07\n",
      "3.4406706139744614e-07\n"
     ]
    }
   ],
   "source": [
    "h=0.000001\n",
    "func=tanh\n",
    "input_len=4\n",
    "for i in range(input_len):\n",
    "    test_input=np.random.rand(input_len)\n",
    "    derivative=differential[func](test_input)\n",
    "    value1=func(test_input)\n",
    "    test_input[i]+=h\n",
    "    value2=func(test_input)\n",
    "#     print((value2-value1)/h)\n",
    "#     print(derivative[i])\n",
    "    print(derivative[i]-((value2-value1)/h)[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "onehot=np.identity(dimensions[-1])\n",
    "\n",
    "def sqr_loss(img,lab,parameters):\n",
    "    y_pred=predict(img,parameters)\n",
    "    y=onehot[lab]\n",
    "    diff=y-y_pred\n",
    "    return np.dot(diff,diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.10263648423433012"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sqr_loss(train_img[0],train_lab[0],parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grad_parameters(img,lab,parameters):\n",
    "    l0_in=img+parameters[0]['b']\n",
    "    l0_out=activation[0](l0_in)\n",
    "    l1_in=np.dot(l0_out,parameters[1]['w'])+parameters[1]['b']\n",
    "    l1_out=activation[1](l1_in)\n",
    "    \n",
    "    diff=onehot[lab]-l1_out\n",
    "    act1=np.dot(differential[activation[1]](l1_in),diff)\n",
    "    \n",
    "    grad_b1=-2*act1\n",
    "    grad_w1=-2*np.outer(l0_out,act1)\n",
    "    grad_b0=-2*differential[activation[0]](l0_in)*np.dot(parameters[1]['w'],act1)\n",
    "    \n",
    "    return {'w1':grad_w1,'b1':grad_b1,'b0':grad_b0}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'w1': array([[-0., -0., -0., ...,  0., -0., -0.],\n",
       "        [-0., -0., -0., ...,  0., -0., -0.],\n",
       "        [-0., -0., -0., ...,  0., -0., -0.],\n",
       "        ...,\n",
       "        [-0., -0., -0., ...,  0., -0., -0.],\n",
       "        [-0., -0., -0., ...,  0., -0., -0.],\n",
       "        [-0., -0., -0., ...,  0., -0., -0.]]),\n",
       " 'b1': array([-0.01032757, -0.01100189, -0.00625271, -0.01436477, -0.04113696,\n",
       "        -0.01429591, -0.01368695,  0.1332406 , -0.00883714, -0.0133367 ]),\n",
       " 'b0': array([-5.98547213e-003,  1.08860687e-002,  8.80746548e-003,\n",
       "        -7.09257601e-003, -9.36767912e-003, -7.92815278e-003,\n",
       "        -1.43994727e-003, -4.92165675e-003, -8.98305274e-004,\n",
       "        -1.81751739e-003, -1.94410104e-003,  1.08356455e-002,\n",
       "         1.51099470e-003, -1.21007911e-003, -9.68122364e-003,\n",
       "         1.25622739e-003,  7.69433354e-003,  8.32541700e-003,\n",
       "         5.89558171e-003,  1.32192981e-002, -2.68537925e-003,\n",
       "        -7.81582981e-004, -5.29590228e-003, -6.04824607e-003,\n",
       "        -9.44469443e-003,  1.39601454e-004, -4.96457378e-004,\n",
       "         1.18984319e-002, -2.39202555e-003, -9.17028688e-003,\n",
       "         7.41317262e-003, -2.72627873e-003, -1.21583530e-002,\n",
       "        -6.89568728e-003,  6.88860213e-003, -5.31797565e-003,\n",
       "        -6.62492831e-003, -8.92713730e-003,  4.43476127e-004,\n",
       "         9.69611148e-003, -6.33626464e-003, -1.26024759e-003,\n",
       "         2.45546776e-003,  7.73292352e-003,  6.14519271e-003,\n",
       "        -3.38754278e-003,  1.11681922e-002, -5.48075450e-004,\n",
       "        -7.44619682e-004, -1.01015652e-002,  9.13111197e-003,\n",
       "        -5.14899014e-004, -4.03417925e-003, -8.46347076e-003,\n",
       "         8.02778948e-004, -1.08994321e-002, -5.40508287e-003,\n",
       "        -9.57236001e-003, -4.25299035e-004,  2.33381035e-003,\n",
       "        -2.59048919e-004,  7.35002601e-003, -2.30659931e-003,\n",
       "        -1.33259119e-002,  2.38080978e-003, -1.62776309e-002,\n",
       "        -3.56388622e-003,  1.99775993e-003,  4.61783075e-003,\n",
       "        -2.46417596e-003,  4.22443588e-003, -9.13376564e-003,\n",
       "         1.29369189e-004,  8.52470027e-003, -4.62619224e-003,\n",
       "         3.74615640e-003, -2.25638084e-003, -1.52046084e-003,\n",
       "         3.04992573e-003,  3.15357958e-003,  1.58891619e-004,\n",
       "        -8.11943673e-003, -3.98017531e-003, -6.96176704e-003,\n",
       "        -1.45162889e-002, -3.61331214e-003,  7.27178707e-003,\n",
       "        -7.71497907e-003, -6.17848522e-003,  9.79495899e-003,\n",
       "        -6.31419957e-003, -4.80878371e-003,  7.82832762e-003,\n",
       "         6.77528597e-003,  8.46763143e-003, -5.73694706e-003,\n",
       "         5.91815354e-003, -2.26764620e-004, -3.80849747e-003,\n",
       "         8.91603749e-003,  7.25727745e-003, -7.48382392e-003,\n",
       "        -4.26776549e-003,  6.35595735e-003, -1.17809214e-002,\n",
       "        -4.26942353e-003, -1.34167559e-002, -5.13452239e-003,\n",
       "         5.20733827e-003,  3.47893895e-003, -6.57597035e-003,\n",
       "        -8.54483107e-003, -1.13665863e-002,  6.53837289e-003,\n",
       "        -1.11372726e-003,  1.20997988e-002,  7.26909789e-003,\n",
       "         9.43024003e-003,  1.36011227e-003, -5.60757327e-003,\n",
       "         5.33428354e-003, -1.64936968e-003, -7.58198965e-003,\n",
       "        -3.18070036e-003,  5.39065590e-003, -1.32033377e-002,\n",
       "        -7.63616264e-003, -8.18909596e-004, -7.35260756e-003,\n",
       "         6.52457573e-003,  4.49386907e-003, -3.30549917e-003,\n",
       "        -7.28237927e-003, -8.20310177e-003, -5.12994049e-003,\n",
       "        -2.00781614e-003, -1.24589693e-004, -8.02181806e-003,\n",
       "        -1.86702243e-003, -8.51378865e-003, -1.06999202e-002,\n",
       "        -1.02981461e-002, -3.97129058e-003, -5.38826796e-003,\n",
       "        -5.06541803e-003,  1.12880462e-003,  5.73207324e-003,\n",
       "         1.08648252e-002, -1.03304826e-002, -1.32418858e-002,\n",
       "        -2.10611191e-003, -1.13370322e-002,  4.43009679e-003,\n",
       "        -5.74277887e-003,  6.64063480e-003,  5.88393110e-003,\n",
       "        -6.56193587e-003, -1.02623099e-002,  6.32607948e-003,\n",
       "         4.54800622e-003, -1.90079661e-060,  1.31160055e-204,\n",
       "         6.70422944e-036,  1.33012242e-003, -9.09421244e-003,\n",
       "        -8.50319604e-003,  2.30037605e-004,  1.04214161e-002,\n",
       "         5.76436855e-003, -8.76347757e-003, -2.52672675e-003,\n",
       "         1.86202849e-003,  6.10565725e-056, -1.68898966e-072,\n",
       "         1.10776364e-002,  1.20225042e-002,  4.01516313e-003,\n",
       "        -4.04188245e-003,  1.10593209e-002,  5.35400183e-003,\n",
       "         7.08438195e-003, -5.04897837e-003, -3.47367305e-003,\n",
       "         8.38930796e-003, -7.46997121e-003,  1.08273970e-002,\n",
       "         5.40723434e-007,  8.89965627e-004, -1.80045789e-107,\n",
       "        -2.33241928e-158,  4.93908996e-036,  3.28775050e-003,\n",
       "        -8.43452406e-003,  1.47746153e-003, -1.08336810e-002,\n",
       "         6.93874490e-003, -6.96655729e-003,  1.17444964e-002,\n",
       "         7.05296890e-003, -4.03522986e-003,  1.38164250e-111,\n",
       "        -5.63243761e-145,  8.73039530e-003, -2.22103862e-004,\n",
       "        -1.18895534e-002, -1.76249283e-003, -1.55310120e-002,\n",
       "         1.48255393e-002,  1.08180746e-002,  5.39408321e-004,\n",
       "         7.43848851e-005, -4.39265645e-003,  7.88417590e-004,\n",
       "        -2.41390227e-003, -1.21295428e-002,  6.75901933e-004,\n",
       "         4.13835510e-135,  8.13096837e-185, -2.19397656e-037,\n",
       "        -1.02457744e-003,  6.94085944e-003,  1.09584125e-002,\n",
       "        -1.46550596e-002,  1.00778745e-002,  6.70818882e-003,\n",
       "        -1.88704932e-003, -1.10198007e-002, -7.26651180e-003,\n",
       "        -1.07013878e-194, -6.82259395e-144,  7.93092605e-003,\n",
       "         4.63730647e-003,  7.95577314e-003,  3.13947495e-003,\n",
       "        -1.37518038e-003,  1.21561186e-002,  1.75756348e-003,\n",
       "         3.64624286e-004, -6.25077894e-003,  6.09136173e-003,\n",
       "        -2.87859764e-003,  1.19238855e-002,  1.26306340e-002,\n",
       "         1.27410626e-025, -7.53567704e-223, -4.41395926e-143,\n",
       "        -2.89005442e-003, -9.70259641e-004,  2.92909412e-003,\n",
       "         9.48610701e-003,  5.31927946e-004,  5.71775724e-003,\n",
       "        -2.21311096e-003, -9.97887560e-003, -3.28574960e-003,\n",
       "        -1.94081978e-003,  7.91118331e-195,  3.55991099e-145,\n",
       "        -1.73484435e-003,  7.67401497e-004,  8.91567435e-003,\n",
       "        -8.91065090e-003,  1.16048372e-002, -9.29700198e-003,\n",
       "         9.10047736e-003,  7.23988646e-003, -1.22615347e-002,\n",
       "         7.30255883e-004, -3.11629446e-004, -2.12275188e-004,\n",
       "        -1.41909610e-002, -1.89509877e-161, -6.83893682e-224,\n",
       "         1.23187591e-110, -6.52548221e-003, -6.30824377e-003,\n",
       "        -4.85214768e-003, -1.56834684e-004, -6.10154819e-003,\n",
       "         8.07599278e-003, -7.94494842e-003, -1.10403309e-002,\n",
       "         4.23759347e-003,  4.32360231e-042,  4.01918739e-216,\n",
       "        -7.62982184e-144, -1.42021855e-002, -1.19346693e-002,\n",
       "         5.09820490e-003, -2.37083406e-003,  1.30839618e-003,\n",
       "        -8.28767835e-003,  7.01728234e-003,  9.23724674e-003,\n",
       "        -2.80166478e-003, -1.01363702e-003,  8.37532814e-003,\n",
       "         4.78951620e-003,  7.03392780e-003,  1.32385064e-174,\n",
       "        -7.51028896e-225,  3.99049869e-051,  1.05165448e-002,\n",
       "        -9.19371167e-003,  4.82463177e-003,  4.20392883e-003,\n",
       "        -4.22638516e-003, -5.15117603e-003, -9.12187754e-003,\n",
       "         6.86642323e-003, -5.52029471e-003, -1.57091199e-106,\n",
       "        -2.02462179e-223,  3.49427815e-144, -4.11992720e-003,\n",
       "        -4.55540100e-003,  4.87428426e-003, -3.45270744e-003,\n",
       "        -1.90493370e-003,  6.48406031e-003, -8.98945057e-003,\n",
       "         9.27590389e-003,  7.74215063e-003, -7.27888397e-003,\n",
       "         5.17234308e-004, -4.56422477e-003,  2.27636414e-022,\n",
       "         9.48903951e-203,  2.45002647e-223,  9.33219732e-028,\n",
       "        -5.77502142e-003,  9.78012779e-003, -1.05819212e-002,\n",
       "         1.99075802e-003, -1.08929687e-002, -4.24978044e-003,\n",
       "         4.22522359e-003,  3.35801361e-004,  8.39185016e-003,\n",
       "         1.45614171e-140, -6.77613209e-223,  1.08620752e-106,\n",
       "        -3.23924341e-003,  2.96637653e-003, -2.03837060e-003,\n",
       "         5.80082994e-003, -4.69154838e-003, -6.28325649e-003,\n",
       "         4.56865862e-003, -8.07425731e-003,  1.24309564e-003,\n",
       "        -5.27628969e-003, -9.16035397e-003, -7.52611599e-003,\n",
       "         1.13145239e-144,  5.77353811e-223,  8.50048788e-190,\n",
       "         3.83770225e-016, -3.25287567e-003, -1.43630994e-003,\n",
       "         6.02453774e-003,  1.17842733e-003,  4.01877175e-003,\n",
       "         7.61139118e-003, -9.06062693e-003,  9.60641726e-003,\n",
       "         4.42109869e-003,  2.41410690e-140,  1.00009671e-222,\n",
       "         4.37465771e-061, -3.89117905e-003,  1.18429025e-002,\n",
       "        -5.63207450e-003, -7.62349313e-003, -5.76738937e-003,\n",
       "         1.61556016e-002, -1.55661844e-002,  8.82356802e-003,\n",
       "        -1.07069978e-002,  1.13709693e-014,  1.19962760e-077,\n",
       "         1.18363486e-156, -1.56040798e-218,  4.54609030e-223,\n",
       "         2.85933216e-081, -1.07713790e-002, -6.34127914e-003,\n",
       "         9.61505208e-003, -1.11652450e-002,  4.32155668e-003,\n",
       "         8.49952718e-003, -7.75733347e-003,  6.96288456e-003,\n",
       "        -8.38295575e-003,  1.14236207e-002,  2.13921630e-140,\n",
       "         1.24831987e-222,  9.23536857e-076, -8.19426921e-003,\n",
       "         5.90296320e-004,  7.76905380e-003,  6.80800836e-043,\n",
       "         3.39263887e-045,  4.27913669e-103,  3.67777776e-127,\n",
       "        -3.64024068e-134,  1.50401038e-211,  2.08754084e-213,\n",
       "        -7.87714389e-206,  1.25000600e-157,  1.19108878e-211,\n",
       "         7.60093333e-221,  8.58008092e-037, -1.30464411e-002,\n",
       "        -1.00765785e-002, -6.38300987e-004,  6.57976347e-003,\n",
       "        -1.34851518e-003, -7.46270134e-003,  6.36990484e-003,\n",
       "         4.54394521e-003,  4.95971677e-003,  6.12920087e-003,\n",
       "         2.35604081e-132, -8.37111516e-222, -4.63988356e-208,\n",
       "         8.39456372e-182, -3.96237519e-182, -3.45994625e-182,\n",
       "         1.54109936e-222,  5.57857120e-223, -3.08935297e-219,\n",
       "         9.05154445e-212,  4.49809538e-175,  1.09100879e-126,\n",
       "         2.55803371e-081,  1.08759494e-026, -3.99143616e-007,\n",
       "         7.23401146e-205,  1.32269322e-219, -4.44971691e-003,\n",
       "         7.02535892e-003, -7.04880945e-003, -1.27581123e-002,\n",
       "        -9.70056498e-003,  9.70999628e-004, -3.71704251e-003,\n",
       "         1.59801270e-005,  1.63602211e-003,  6.56653097e-003,\n",
       "        -8.87340220e-003, -2.18952673e-003,  1.36263576e-106,\n",
       "        -1.22621493e-156, -1.36433569e-156, -4.33009993e-156,\n",
       "        -1.95259018e-156, -5.28549091e-156,  3.04120830e-088,\n",
       "        -4.96625815e-051,  1.32484939e-003,  6.92485753e-003,\n",
       "         8.66992499e-003, -8.40195042e-003, -7.52501237e-003,\n",
       "        -5.34690228e-091, -3.40424423e-225,  1.98543660e-193,\n",
       "        -1.18408419e-002, -8.70277691e-003, -1.05573257e-002,\n",
       "         9.31733038e-003,  1.20558016e-002, -6.41659234e-004,\n",
       "         2.78047600e-003,  3.85700433e-004, -8.52211192e-003,\n",
       "        -4.19058903e-003,  4.92186723e-003,  1.13094569e-002,\n",
       "         5.30824131e-003, -3.64392306e-003,  4.58078869e-003,\n",
       "         7.08844227e-003, -7.36380849e-003, -6.87944462e-004,\n",
       "        -7.16445686e-003, -4.75751749e-003,  1.36374278e-002,\n",
       "        -2.98333055e-005,  8.68507655e-003,  1.16372692e-002,\n",
       "        -8.24017528e-003, -3.56956934e-149,  6.77093474e-223,\n",
       "        -7.51638281e-122,  9.81864350e-003, -3.71507872e-003,\n",
       "         1.09526725e-002,  7.92509105e-003, -9.87909046e-003,\n",
       "         2.35111943e-003,  1.03656146e-002, -6.42968611e-003,\n",
       "         1.35808039e-002, -3.13183591e-003, -1.26430420e-003,\n",
       "        -1.11759330e-003,  5.54464856e-003, -2.93113581e-003,\n",
       "        -8.49883920e-003, -1.56113569e-002,  2.21736628e-004,\n",
       "        -6.12208869e-004,  6.69502036e-004,  1.64798100e-002,\n",
       "         5.47868759e-003, -1.18220248e-003,  8.57552243e-003,\n",
       "         3.54332960e-003,  2.28018404e-003, -5.91882430e-150,\n",
       "         3.94690239e-223,  8.26844263e-052,  1.23009674e-002,\n",
       "         2.01808214e-003, -7.55832162e-003,  4.15245790e-003,\n",
       "        -7.27980943e-003,  5.58509097e-003, -8.31431450e-004,\n",
       "         1.03266787e-002, -3.48509911e-003, -1.05205374e-002,\n",
       "         8.58952185e-003,  6.35310432e-003,  8.74049198e-003,\n",
       "         1.12624811e-002,  3.59088069e-003,  6.31249132e-003,\n",
       "        -8.19223864e-003, -5.73165029e-003,  6.07697990e-003,\n",
       "         8.20065879e-003,  3.68598186e-003,  7.83374607e-003,\n",
       "        -4.65381407e-003,  6.50883434e-003, -6.73706713e-003,\n",
       "        -5.17573102e-149,  7.03217837e-223,  1.22094609e-051,\n",
       "        -2.18873571e-004, -2.63317541e-003, -3.13007919e-003,\n",
       "        -3.20220775e-003, -3.35036788e-003,  3.11725183e-003,\n",
       "         5.77736077e-003, -3.37819705e-003,  2.25637157e-003,\n",
       "         1.04622258e-002,  1.17290140e-002, -6.37144767e-003,\n",
       "        -2.66443606e-004,  2.62269725e-003, -5.62676464e-003,\n",
       "        -7.28428319e-003, -8.94193479e-003, -2.54369495e-003,\n",
       "         3.76101873e-003, -2.37981637e-003,  6.54358262e-003,\n",
       "        -3.48332548e-003,  4.13846432e-003, -8.44357859e-003,\n",
       "        -1.57681364e-003,  1.73916804e-149,  7.35921465e-224,\n",
       "         4.40661188e-084, -5.23969873e-003,  6.66222582e-003,\n",
       "         2.31333493e-003, -8.13279776e-003, -1.06147628e-002,\n",
       "        -6.58040989e-004, -9.12187874e-003, -3.67940697e-003,\n",
       "        -2.86414196e-004,  1.44792347e-003, -4.67771126e-003,\n",
       "         2.36793509e-003, -1.15687118e-002,  2.78493396e-003,\n",
       "         2.60068178e-004,  1.08897302e-002, -1.73358813e-002,\n",
       "         5.86924886e-003,  8.59415138e-003, -1.50517087e-002,\n",
       "         8.42757891e-003, -2.90174528e-003,  7.98510804e-003,\n",
       "        -6.49166249e-003,  1.33605445e-003, -5.81454704e-149,\n",
       "        -2.45290523e-223, -6.94071671e-086, -3.70272686e-004,\n",
       "        -7.98346740e-003, -8.47684612e-004,  1.59742408e-002,\n",
       "         1.11562974e-002,  5.57438858e-003,  2.27252390e-003,\n",
       "         7.76836026e-003,  5.12140706e-004,  4.18984166e-003,\n",
       "        -6.21598430e-003, -2.32337101e-003,  8.53858349e-003,\n",
       "         4.69070035e-003,  8.80220490e-003,  7.87975905e-004,\n",
       "        -1.02148212e-002, -6.45056426e-003,  8.75341711e-003,\n",
       "        -8.35146583e-003,  5.46985459e-003, -2.34541045e-003,\n",
       "         1.27223547e-003,  6.98059021e-004,  8.11040628e-003,\n",
       "         4.43302371e-149, -9.54946986e-223,  4.26392868e-136,\n",
       "        -1.29204571e-002,  3.65587064e-003, -1.00455447e-002,\n",
       "         1.76150449e-003, -1.20150354e-004,  7.10366165e-003,\n",
       "        -2.38372115e-003,  4.01033432e-003, -9.43312901e-003,\n",
       "         7.62031251e-003, -4.71610901e-003, -8.13372421e-003,\n",
       "        -1.22175607e-003, -3.29565599e-003,  1.39724452e-003,\n",
       "         5.55263913e-003,  1.05582172e-002, -2.64533011e-003,\n",
       "         8.83400801e-004,  1.17379738e-002,  6.79849708e-003,\n",
       "        -7.86285087e-003, -7.06098688e-003,  2.12424383e-003,\n",
       "        -2.76026460e-003, -9.18376898e-149,  9.45509231e-224,\n",
       "        -3.99418605e-135,  1.63466735e-002, -5.12472329e-003,\n",
       "         2.61458904e-003, -3.50390623e-003,  1.32994275e-002,\n",
       "         5.76889672e-003,  1.77991863e-003, -2.92558934e-003,\n",
       "        -4.86453141e-003,  2.64102850e-003, -5.47929108e-004,\n",
       "        -1.33075780e-002,  5.71896483e-003, -1.16354555e-002,\n",
       "         2.16486932e-003,  5.99111085e-003,  1.25601948e-002,\n",
       "         2.75201532e-004,  7.49268155e-003,  1.00860116e-002,\n",
       "         3.62738282e-003, -6.80296470e-003, -4.69049703e-003,\n",
       "        -6.76722667e-003, -1.43205149e-002,  2.33940500e-085,\n",
       "         6.00241472e-223, -2.35325019e-135, -1.16876487e-002,\n",
       "         6.88374959e-003, -7.54103504e-003, -3.25792388e-004,\n",
       "        -5.56270249e-003,  2.99099582e-003,  4.92994712e-003,\n",
       "         9.03801011e-003,  2.35103804e-003,  4.06482687e-003,\n",
       "        -1.50485217e-002,  6.60716040e-003, -5.48553237e-003,\n",
       "        -9.19316209e-004,  2.73865155e-003, -1.40794800e-002,\n",
       "        -1.04120303e-004,  2.96492209e-004,  7.03127912e-003,\n",
       "        -8.06882624e-003, -3.21567413e-003, -6.43912203e-003,\n",
       "         3.04115001e-003, -7.68336456e-003, -9.52808836e-003,\n",
       "         5.57367620e-003,  1.29465789e-003,  9.36682621e-003,\n",
       "         1.68017446e-003,  8.86319837e-003, -2.77498253e-003,\n",
       "         8.44487462e-004, -2.10112292e-003,  2.32930337e-003,\n",
       "        -1.55591258e-003, -1.36204857e-002, -8.54021485e-003,\n",
       "        -1.40873576e-002,  3.52482469e-004, -1.22582756e-002,\n",
       "        -8.15575478e-003, -5.17900258e-003,  1.12621992e-002,\n",
       "         4.51110817e-003, -1.64267286e-003, -6.92033183e-004,\n",
       "        -7.49378084e-004,  1.98811721e-003,  5.48399539e-003,\n",
       "         6.18226834e-003, -3.64977291e-003, -9.41143843e-003,\n",
       "         8.78305281e-003, -9.74322443e-003,  2.43633121e-003,\n",
       "        -4.04685182e-003, -5.08467557e-003,  5.68790062e-003,\n",
       "         1.67779591e-002,  2.14040425e-003,  7.17948222e-003,\n",
       "        -3.07055539e-003,  6.71195101e-003, -3.63111969e-003,\n",
       "        -1.00185077e-002, -4.54622399e-003,  9.43730221e-003,\n",
       "         2.36080423e-003,  2.39830961e-003,  4.76354710e-003,\n",
       "         1.01584312e-002, -1.49427242e-002, -4.35368258e-003,\n",
       "         1.04063398e-002, -1.22420116e-002,  6.47755776e-004,\n",
       "         2.19277081e-003,  8.99028600e-003,  2.66759493e-003,\n",
       "        -1.40226228e-004, -1.43803354e-003, -8.43144444e-004,\n",
       "         3.78149295e-003, -3.03230804e-003, -9.83029072e-003,\n",
       "         2.48293667e-003, -2.73464167e-003, -9.43536278e-003,\n",
       "         1.44633142e-002, -1.00672606e-003,  9.51960721e-004,\n",
       "        -1.07158490e-002])}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grad_parameters(train_img[2],train_lab[2],init_parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-1.32931432103231e-08\n",
      "-2.409304682982061e-07\n",
      "6.717242480025476e-07\n",
      "1.9010109349182347e-07\n",
      "-1.1090499531414555e-07\n",
      "-4.4586077828367987e-07\n",
      "-8.876067625340767e-08\n",
      "-4.425811561204265e-09\n",
      "-1.4665282977235905e-07\n",
      "-2.5267951572066627e-07\n"
     ]
    }
   ],
   "source": [
    "# b1 \n",
    "h=0.00001\n",
    "for i in range(10):\n",
    "    img_i=np.random.randint(train_num)\n",
    "    test_parameters=init_parameters()\n",
    "    derivative=grad_parameters(train_img[img_i],train_lab[img_i],test_parameters)['b1']\n",
    "    value1=sqr_loss(train_img[img_i],train_lab[img_i],test_parameters)\n",
    "    test_parameters[1]['b'][i]+=h\n",
    "    value2=sqr_loss(train_img[img_i],train_lab[img_i],test_parameters)\n",
    "    print(derivative[i]-(value2-value1)/h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.626917369220944e-06"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# w1\n",
    "grad_list=[]\n",
    "h=0.0001\n",
    "for i in range(784):\n",
    "    for j in range(10):\n",
    "        img_i=np.random.randint(train_num)\n",
    "        test_parameters=init_parameters()\n",
    "        derivative=grad_parameters(train_img[img_i],train_lab[img_i],test_parameters)['w1']\n",
    "        value1=sqr_loss(train_img[img_i],train_lab[img_i],test_parameters)\n",
    "        test_parameters[1]['w'][i][j]+=h\n",
    "        value2=sqr_loss(train_img[img_i],train_lab[img_i],test_parameters)\n",
    "        grad_list.append(derivative[i][j]-(value2-value1)/h)\n",
    "np.abs(grad_list).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.187988961728198e-06"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# b0 \n",
    "grad_list=[]\n",
    "h=0.001\n",
    "for i in range(784):\n",
    "    img_i=np.random.randint(train_num)\n",
    "    test_parameters=init_parameters()\n",
    "    derivative=grad_parameters(train_img[img_i],train_lab[img_i],test_parameters)['b0']\n",
    "    value1=sqr_loss(train_img[img_i],train_lab[img_i],test_parameters)\n",
    "    test_parameters[0]['b'][i]+=h\n",
    "    value2=sqr_loss(train_img[img_i],train_lab[img_i],test_parameters)\n",
    "    grad_list.append(derivative[i]-(value2-value1)/h)\n",
    "np.abs(grad_list).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def valid_loss(parameters):\n",
    "    loss_accu=0\n",
    "    for img_i in range(valid_num):\n",
    "        loss_accu+=sqr_loss(valid_img[img_i],valid_lab[img_i],parameters)\n",
    "    return loss_accu/(valid_num/10000)\n",
    "def valid_accuracy(parameters):\n",
    "    correct=[predict(valid_img[img_i],parameters).argmax()==valid_lab[img_i] for img_i in range(valid_num)]\n",
    "    return correct.count(True)/len(correct)\n",
    "def train_loss(parameters):\n",
    "    loss_accu=0\n",
    "    for img_i in range(train_num):\n",
    "        loss_accu+=sqr_loss(train_img[img_i],train_lab[img_i],parameters)\n",
    "    return loss_accu/(train_num/10000)\n",
    "def train_accuracy(parameters):\n",
    "    correct=[predict(train_img[img_i],parameters).argmax()==train_lab[img_i] for img_i in range(train_num)]\n",
    "    return correct.count(True)/len(correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1503.6989970176978"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_loss(parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation accuracy : 0.1438\n"
     ]
    }
   ],
   "source": [
    "valid_accuracy(init_parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size=100\n",
    "def train_batch(current_batch,parameters):\n",
    "    grad_accu=grad_parameters(train_img[current_batch*batch_size+0],train_lab[current_batch*batch_size+0],parameters)\n",
    "    for img_i in range(1,batch_size):\n",
    "        grad_tmp=grad_parameters(train_img[current_batch*batch_size+img_i],train_lab[current_batch*batch_size+img_i],parameters)\n",
    "        for key in grad_accu.keys():\n",
    "            grad_accu[key]+=grad_tmp[key]\n",
    "    for key in grad_accu.keys():\n",
    "        grad_accu[key]/=batch_size\n",
    "    return grad_accu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'w1': array([[ 1.03347004e-05, -6.25449278e-05, -1.76394549e-05, ...,\n",
       "          2.13970188e-05, -1.12676075e-04, -6.77088021e-06],\n",
       "        [ 1.43124922e-05, -8.66182629e-05, -2.44288226e-05, ...,\n",
       "          2.96326603e-05, -1.56044722e-04, -9.37696954e-06],\n",
       "        [ 1.31333662e-05, -7.94822695e-05, -2.24162688e-05, ...,\n",
       "          2.71913915e-05, -1.43189071e-04, -8.60445355e-06],\n",
       "        ...,\n",
       "        [ 2.11910490e-05, -1.28246837e-04, -3.61692687e-05, ...,\n",
       "          4.38740610e-05, -2.31039521e-04, -1.38835234e-05],\n",
       "        [-2.43382458e-05,  1.47293465e-04,  4.15409616e-05, ...,\n",
       "         -5.03900342e-05,  2.65352444e-04,  1.59454403e-05],\n",
       "        [ 1.86281043e-06, -1.12736064e-05, -3.17947880e-06, ...,\n",
       "          3.85677266e-06, -2.03096519e-05, -1.22043851e-06]]),\n",
       " 'b1': array([-8.45287478e-04,  5.11562430e-03,  1.44275208e-03,  1.38193281e-03,\n",
       "        -4.61754164e-03, -1.05960103e-02,  9.89089639e-05, -1.75008771e-03,\n",
       "         9.21591063e-03,  5.53798374e-04]),\n",
       " 'b0': array([-2.49378522e-04,  7.87627838e-05,  2.22748909e-04,  1.42547439e-03,\n",
       "        -2.78053397e-04,  3.01622146e-04, -1.03210472e-03,  3.02157686e-04,\n",
       "        -5.22200099e-04,  5.78244113e-04, -3.40685808e-04, -6.62068398e-04,\n",
       "        -6.78071246e-04, -1.83565492e-04, -5.59414002e-04,  8.20306083e-04,\n",
       "         3.04324637e-04, -7.15209993e-05,  8.80717785e-05,  1.73999091e-03,\n",
       "        -2.73954210e-04,  7.71976207e-05,  1.00808015e-03, -6.90924164e-04,\n",
       "        -1.02490893e-03,  1.07627559e-03, -2.50678519e-04, -5.57217094e-04,\n",
       "         5.20146592e-04,  8.20333511e-04,  1.08169515e-03, -1.94017300e-04,\n",
       "         4.36621966e-04, -4.83430671e-04,  2.59706556e-04, -4.57513245e-04,\n",
       "         1.07504792e-03,  6.41981900e-05,  6.30327662e-04, -1.21994766e-04,\n",
       "         1.10292154e-03, -8.38219117e-04,  3.83053912e-04,  9.14964904e-04,\n",
       "         4.91837979e-04,  1.80044516e-04, -1.22867028e-04, -1.07562241e-03,\n",
       "        -5.81282834e-04, -1.81429510e-04,  2.30304441e-04,  5.07470762e-04,\n",
       "         5.10480888e-04, -6.84936289e-04, -1.04217680e-03, -4.27818316e-05,\n",
       "         4.11321299e-04, -5.68905105e-04,  1.15400143e-03,  1.07637516e-03,\n",
       "        -5.83838268e-04,  6.16931368e-04, -7.33638079e-04,  9.04379441e-05,\n",
       "         3.19852269e-04,  1.08359990e-03,  1.57287497e-03, -4.92406381e-04,\n",
       "         2.28546339e-03,  1.45491492e-03,  1.54088065e-03,  3.32951046e-03,\n",
       "         1.13437914e-03, -6.71185190e-04, -1.59464979e-05,  5.31386632e-04,\n",
       "         8.39275927e-04, -1.17380942e-04, -3.66535330e-04, -1.20617992e-03,\n",
       "        -9.39369473e-05, -2.13173574e-07,  8.58109306e-04,  1.12594900e-03,\n",
       "        -3.57346362e-04, -8.80206560e-04,  6.79328191e-04,  2.15215038e-04,\n",
       "        -9.33130079e-04,  7.97136848e-04,  8.39010402e-04,  5.51470296e-04,\n",
       "        -9.00886369e-04,  8.13125756e-04,  3.40657952e-03,  2.97387931e-03,\n",
       "         2.13316175e-03,  3.42874408e-03,  2.77562981e-03,  1.19594320e-03,\n",
       "         1.05699214e-03,  2.14422693e-03, -1.46774048e-03, -2.31076871e-03,\n",
       "        -1.24873354e-03, -1.78085328e-03,  1.82766136e-04,  1.00241695e-04,\n",
       "        -4.36791032e-04, -3.31010197e-04,  5.19829176e-05, -6.62429081e-04,\n",
       "        -1.07690216e-03, -1.15047061e-03, -1.11612984e-03, -1.38754469e-04,\n",
       "        -1.13885355e-03,  1.36127349e-03,  1.72620157e-03,  8.78328475e-04,\n",
       "         3.84785729e-03,  5.66486992e-03,  5.44594576e-03,  4.44140760e-03,\n",
       "         4.61690986e-03,  7.15075211e-03,  8.06827184e-03,  3.85139880e-03,\n",
       "         1.59483452e-03,  1.00801929e-03, -1.76305174e-03,  1.18486968e-04,\n",
       "         1.75891520e-03, -8.14865166e-04, -3.47237167e-03, -3.16439725e-03,\n",
       "        -2.67216292e-03, -8.86438797e-04,  1.30275351e-04, -4.66911677e-04,\n",
       "         1.69285492e-03,  6.94275848e-04,  4.87655144e-04,  5.09543913e-04,\n",
       "         2.54473376e-03,  1.70757057e-03,  3.00020556e-03,  1.23280292e-03,\n",
       "         4.48011515e-04,  1.74898856e-03, -2.71029307e-03,  3.09497887e-03,\n",
       "         3.67972210e-03,  4.24248670e-03,  4.68701202e-03,  4.72425359e-03,\n",
       "         3.33259760e-04, -4.24001621e-04, -3.30540609e-03,  8.33049316e-04,\n",
       "        -9.02341854e-04, -3.71699792e-04, -2.92520761e-03, -5.97390709e-03,\n",
       "        -4.65581241e-03, -2.73428328e-03, -1.55744931e-03, -4.09213152e-04,\n",
       "        -3.70342354e-04,  8.08621489e-04, -5.40903635e-04, -1.52990634e-04,\n",
       "         2.96220702e-03,  1.51827155e-03,  1.48488675e-03, -3.76004516e-03,\n",
       "        -2.97183350e-03, -4.19902429e-03, -1.08606008e-03, -7.63823440e-04,\n",
       "         1.93760035e-03,  3.57167210e-03,  1.45575743e-03,  4.18529505e-03,\n",
       "         2.77843965e-03, -7.56743603e-04,  2.34331590e-03, -7.30649773e-04,\n",
       "        -2.42063991e-04,  3.58643167e-04, -1.07484157e-03, -2.19073988e-03,\n",
       "        -5.39497668e-03, -1.66410151e-03, -2.22831530e-03, -9.03113276e-04,\n",
       "         5.38314587e-04, -5.91095367e-04, -1.41272754e-03,  1.05598727e-03,\n",
       "         4.51133575e-03,  1.38700755e-03, -1.58829743e-03, -3.33182707e-03,\n",
       "        -4.15316352e-03, -5.32070977e-03, -2.15318702e-03, -1.02971822e-03,\n",
       "        -1.97959705e-03,  1.87441741e-03,  2.52054221e-03,  4.81815388e-03,\n",
       "        -5.16102051e-03, -6.25432079e-03, -2.48886893e-03, -1.82138733e-03,\n",
       "         2.76754309e-04,  1.40498993e-03, -1.83846866e-03, -3.08834587e-03,\n",
       "        -4.98207928e-03, -4.46019174e-03, -2.32325150e-03, -4.09616094e-04,\n",
       "         5.48725185e-04, -9.34828785e-04,  1.10146926e-03,  6.42098727e-04,\n",
       "         5.40079501e-03,  4.41007136e-03,  3.21660375e-03,  1.42878264e-03,\n",
       "        -3.66931588e-03, -7.46049353e-04, -2.42488485e-03, -2.28497750e-03,\n",
       "         4.03054317e-05,  1.95280544e-03, -1.64993344e-03, -8.82897285e-05,\n",
       "         7.86699680e-04, -1.27771225e-03, -5.38610979e-04,  1.33194492e-04,\n",
       "        -1.17015863e-03,  8.23653622e-04, -1.57972668e-03, -4.24569795e-03,\n",
       "        -6.68524663e-03, -5.67649556e-03, -1.13556169e-03,  6.25119124e-04,\n",
       "         8.12830570e-04, -5.37403579e-05, -1.37830259e-04,  1.46906851e-03,\n",
       "         5.02657956e-03,  6.22632991e-03,  2.79235477e-03,  8.27953853e-04,\n",
       "        -2.05599752e-04, -2.04713826e-03, -5.18723623e-04, -2.14887114e-03,\n",
       "        -2.28603038e-03,  2.67667222e-04, -3.19991739e-04,  3.42876299e-03,\n",
       "         5.73266520e-04,  4.98612682e-04,  6.84024662e-04,  6.65158287e-04,\n",
       "         2.29105325e-04, -4.15861544e-03, -5.50856877e-03, -7.02509101e-03,\n",
       "        -9.55623809e-03, -1.17358054e-02, -7.98612253e-04,  1.14546547e-03,\n",
       "        -1.64713035e-04, -3.49333448e-04,  6.15627687e-04,  1.74206833e-03,\n",
       "         3.45770301e-03, -2.85572290e-04,  1.03848400e-04, -7.28469692e-04,\n",
       "        -3.21052022e-03,  4.25279020e-04,  2.99827136e-04, -1.20820437e-03,\n",
       "        -1.80177222e-03, -7.80429878e-04,  1.76377673e-04,  1.90528494e-03,\n",
       "         1.52020654e-03,  3.02988360e-03,  2.32933592e-03,  2.66334070e-03,\n",
       "         3.85723112e-03, -1.36044838e-03, -2.77069082e-03, -6.94336667e-03,\n",
       "        -1.56350172e-02, -1.28004081e-02, -4.54290914e-03, -1.32629891e-03,\n",
       "         2.63918373e-04,  8.50162609e-04,  4.46037719e-04,  1.97239861e-03,\n",
       "         1.33482842e-03,  1.37393827e-03, -3.39133115e-03, -3.41834920e-04,\n",
       "        -6.03981787e-03,  4.82061289e-04, -1.45662474e-03, -8.18963347e-04,\n",
       "        -2.19314906e-03, -1.17782847e-03,  3.64172200e-05,  5.53417848e-04,\n",
       "        -1.84602818e-03,  8.33848193e-04,  1.54939593e-03,  5.14679171e-03,\n",
       "         1.18410369e-02,  8.16558361e-03,  1.13071460e-02,  6.26161718e-03,\n",
       "         4.75269699e-03, -2.06098308e-03, -5.46922825e-04, -7.12689001e-04,\n",
       "        -4.01990829e-04,  1.07009367e-03,  8.16082602e-04,  2.26573893e-04,\n",
       "         1.63124458e-03,  1.55794618e-03, -6.23730205e-03, -3.75331468e-03,\n",
       "        -4.14337724e-03, -2.43849679e-03, -5.82946455e-03, -1.90116951e-03,\n",
       "        -1.70055142e-03, -9.86145785e-04,  1.17305608e-03,  1.34594907e-03,\n",
       "        -6.58906875e-04,  3.99742646e-03,  5.30624292e-03,  6.26253422e-03,\n",
       "         5.45973289e-03,  8.30357008e-03,  1.41623934e-02,  1.78384497e-02,\n",
       "         1.39293183e-02,  4.95271555e-03,  7.86445386e-04,  1.18238413e-03,\n",
       "        -1.07854721e-04, -4.23158682e-04, -6.82435180e-04,  6.83671318e-04,\n",
       "        -1.69159828e-03, -2.72683458e-03, -5.59622188e-03, -6.22909815e-03,\n",
       "        -6.44128989e-03, -1.47453953e-03, -4.99467557e-03, -1.16544112e-03,\n",
       "         4.71816864e-03,  1.07767620e-02,  6.27881962e-03,  4.39277051e-03,\n",
       "         2.43078595e-03,  1.09070179e-03,  1.63924981e-03,  4.84923518e-04,\n",
       "        -2.04630167e-03, -2.46415656e-04,  3.05699458e-03,  1.00431794e-02,\n",
       "         1.01670975e-02,  1.98738691e-03,  5.11998587e-04,  7.04499866e-04,\n",
       "        -1.01613743e-03, -2.51017261e-04, -2.14597084e-06, -6.77745952e-04,\n",
       "        -4.43500273e-03, -4.92233535e-03, -6.12315033e-03, -5.84725211e-03,\n",
       "        -6.35065157e-03, -2.72697308e-03, -5.54934424e-03,  1.66533364e-03,\n",
       "         8.90918077e-03,  8.52492436e-03,  4.78168708e-03,  4.96185811e-03,\n",
       "         1.53147667e-03,  7.98896411e-04,  1.41577351e-03, -1.59800834e-03,\n",
       "        -4.12603165e-03, -2.32789599e-03, -3.51429280e-03,  9.18328220e-05,\n",
       "         3.51117081e-03,  3.25253982e-03,  7.11087973e-04, -1.60077980e-03,\n",
       "        -7.45929880e-05,  9.20317808e-04, -8.03703880e-04, -3.93708035e-04,\n",
       "        -4.83001243e-04, -3.52985049e-04, -1.64528512e-03, -1.65077291e-03,\n",
       "        -4.86573994e-03, -5.02705375e-03, -8.70083224e-04,  1.15524730e-03,\n",
       "         4.39763231e-03,  1.91091186e-03,  2.83089576e-03,  1.92675271e-03,\n",
       "         1.02594524e-03,  1.84584100e-03, -5.40482298e-03, -5.12544092e-03,\n",
       "        -3.95413110e-03, -2.98927299e-03,  7.56527891e-05, -3.56428207e-03,\n",
       "         4.57545467e-04,  2.68733459e-03,  1.15481738e-03,  1.03752617e-03,\n",
       "        -5.72507046e-04,  3.15364228e-04,  1.06281457e-03,  6.89156450e-04,\n",
       "         4.21503081e-04,  7.82481631e-04,  2.02974550e-03,  3.24849388e-03,\n",
       "        -2.13385133e-03,  2.63220852e-04,  2.74123927e-03,  4.40028060e-03,\n",
       "         6.19032761e-04,  2.75474105e-03,  2.64023908e-03,  6.87430056e-04,\n",
       "         3.83472922e-03, -3.47837376e-03, -5.59499917e-03, -6.93628062e-03,\n",
       "        -6.03351213e-03, -5.19213285e-03, -5.07301400e-03, -1.12497937e-03,\n",
       "         9.78652146e-05, -4.98655016e-04,  5.65710763e-04,  2.51058565e-04,\n",
       "         3.02522959e-04, -2.34548851e-04,  3.34400284e-04,  3.13561676e-04,\n",
       "         5.84925638e-04, -4.21628316e-03, -3.69770044e-03, -4.89188269e-04,\n",
       "        -1.70680576e-06,  9.19054957e-04,  4.51219221e-03,  3.40642774e-03,\n",
       "         2.02070375e-03,  3.74926295e-03,  1.32193274e-03, -8.73968109e-04,\n",
       "        -1.60729512e-03, -6.53170952e-03, -9.37995643e-03, -1.76303882e-03,\n",
       "        -3.92061282e-03, -3.04852655e-03,  4.64319061e-04, -2.63494473e-04,\n",
       "        -3.38586489e-03, -9.91294807e-04,  1.53637860e-03, -2.94541130e-04,\n",
       "         8.42240417e-04, -1.03933897e-03, -5.89970355e-04, -1.03376443e-03,\n",
       "        -2.82456140e-03, -2.84033123e-03, -3.61300423e-03, -6.14154071e-04,\n",
       "         1.15539352e-03,  7.46533090e-03,  3.75636122e-03,  6.54758296e-03,\n",
       "        -1.00787854e-03, -7.59702016e-04, -1.92635490e-04,  2.56126206e-04,\n",
       "        -5.01309649e-03, -6.03089195e-03, -3.17821429e-03, -1.03633802e-03,\n",
       "        -8.22611883e-04, -1.33967799e-03,  2.66328404e-03,  1.66603677e-04,\n",
       "        -3.86784470e-03, -4.21842250e-04,  7.45042393e-04,  3.88043592e-05,\n",
       "         1.46442449e-03,  9.39298655e-04, -8.85214712e-04, -2.66228042e-03,\n",
       "        -4.78480380e-03, -4.17338569e-03, -1.14530796e-03,  3.71113521e-04,\n",
       "         1.56702639e-03,  3.38785251e-04,  1.47578533e-03, -9.36652747e-04,\n",
       "        -3.42555215e-04, -7.43485959e-04, -1.40169232e-03, -2.53241265e-03,\n",
       "         4.34654122e-04,  1.51963195e-03,  3.78731961e-04,  1.98900047e-03,\n",
       "         9.88674393e-04, -1.08034276e-03,  1.35434278e-03,  1.21392214e-03,\n",
       "        -1.79741695e-03,  6.10424255e-04,  1.23413454e-03, -1.55035180e-04,\n",
       "        -1.55923265e-03,  2.66876136e-04,  9.12745331e-05, -2.16179151e-03,\n",
       "        -3.48555664e-03,  1.55915286e-04,  2.58058715e-03,  3.34344096e-03,\n",
       "         1.06843509e-03,  1.58412338e-03,  1.99958126e-03, -5.88860408e-05,\n",
       "        -3.44378448e-04, -1.49464576e-03, -3.67618400e-04,  9.12686854e-04,\n",
       "        -2.84651353e-04,  2.03215634e-03,  2.71762000e-03,  2.74259031e-03,\n",
       "         5.21183051e-03,  1.97772748e-03, -1.54937250e-03,  1.46273128e-03,\n",
       "        -2.58525480e-04,  2.28210021e-04, -2.16891198e-04, -8.94358828e-04,\n",
       "         1.11640097e-03, -1.06086995e-03,  7.75982933e-04, -1.54865912e-03,\n",
       "        -4.36061898e-03,  2.21693967e-03,  4.19675240e-03,  3.02908922e-03,\n",
       "        -2.85888237e-03, -3.57929193e-04, -6.87390961e-04, -7.89805706e-04,\n",
       "        -2.91461320e-03, -2.79911726e-04, -8.13550590e-04,  1.34508375e-03,\n",
       "         2.47141853e-04,  7.87189028e-04,  2.42733614e-03,  1.08589591e-03,\n",
       "        -1.94805755e-03, -1.06719965e-03, -1.63122575e-03, -1.91585216e-03,\n",
       "         1.07688963e-06, -2.61395499e-04, -1.29338780e-03, -1.07945806e-05,\n",
       "         1.41360267e-04, -3.88763692e-04,  3.03557990e-04,  9.23626463e-04,\n",
       "        -3.86193010e-03,  1.01780732e-03,  4.68433255e-03,  4.60716713e-03,\n",
       "        -4.70301881e-04, -1.13100908e-03,  1.23803261e-03, -1.49093371e-03,\n",
       "        -3.80156480e-03, -1.43207299e-03, -2.06576050e-04,  1.57710348e-03,\n",
       "         4.45274234e-04,  3.57282743e-03,  3.87459819e-04, -2.01430894e-03,\n",
       "        -1.48753100e-03, -4.71895429e-04, -1.26983229e-03, -2.93900470e-03,\n",
       "        -6.96202576e-04,  1.51641679e-03,  8.02202421e-05,  1.26209226e-03,\n",
       "         8.82297459e-04,  1.45204866e-04,  1.96377748e-04,  8.67852960e-04,\n",
       "        -7.94908857e-04, -2.58542217e-03, -2.54133915e-03,  3.53577971e-04,\n",
       "        -1.58222365e-04,  3.50994633e-04, -2.24643727e-03,  1.06226561e-03,\n",
       "         5.19049324e-03,  4.04539438e-03, -1.41401387e-03,  1.28204554e-03,\n",
       "         1.27036567e-03,  2.57365481e-03,  5.14077648e-03,  2.60426121e-03,\n",
       "        -1.76123864e-04, -1.86468769e-03, -2.69585597e-03, -2.20665271e-03,\n",
       "         4.24993903e-05, -3.67022569e-04,  6.38460597e-04, -4.99338920e-04,\n",
       "         9.68033421e-04, -3.74390926e-04, -4.46012617e-04, -1.87681823e-03,\n",
       "        -1.66942305e-03, -2.69118591e-03, -4.08500560e-03, -2.59955373e-03,\n",
       "        -5.87935727e-04, -4.85759955e-03, -5.01601944e-03, -2.43303981e-03,\n",
       "        -1.31627183e-03, -2.94705494e-03, -1.96237967e-03, -7.02390155e-04,\n",
       "         2.06600201e-03, -1.87895980e-04,  5.12162797e-04,  3.98910548e-03,\n",
       "         1.16240784e-03, -1.74996287e-03, -3.04335827e-03, -1.65598378e-03,\n",
       "        -4.91594933e-04,  5.11925423e-05, -7.18072992e-04,  1.72520734e-04,\n",
       "        -4.40710058e-04, -1.83047515e-04, -3.84108483e-04, -8.20369389e-04,\n",
       "        -7.57659287e-04, -2.67090574e-03, -3.12830291e-05,  2.44031988e-04,\n",
       "        -4.43732347e-03, -4.90727167e-03, -3.95300001e-03, -4.96220748e-03,\n",
       "        -1.66643204e-03, -3.90693980e-03,  1.07467934e-04, -3.50446319e-03,\n",
       "        -1.81156079e-03, -1.09283993e-03,  6.38339382e-04,  7.99867641e-04,\n",
       "        -3.42239095e-04, -1.78575512e-03, -2.01354245e-03, -1.35372247e-03,\n",
       "        -1.41350169e-03,  7.59921758e-04, -7.13162271e-05, -3.26621895e-04,\n",
       "        -3.33376059e-04,  1.43402523e-03,  8.35166520e-04, -7.39096877e-04,\n",
       "         5.43281180e-04,  1.22737385e-03,  3.52988937e-04,  1.25354671e-03,\n",
       "        -8.63991669e-05, -2.09484017e-03, -1.25328533e-03, -4.01637782e-04,\n",
       "         6.21410747e-04, -8.81920900e-04,  8.16074493e-04, -2.90795123e-04,\n",
       "        -1.36087182e-03,  1.75750114e-03,  5.57161879e-04,  7.74480414e-04,\n",
       "         5.84679273e-04, -4.67498871e-04, -6.54332472e-04,  4.50650914e-06,\n",
       "         8.16644865e-04,  5.16478817e-04,  6.03930038e-04,  5.44844828e-04,\n",
       "        -7.33412292e-04, -3.90081878e-04, -7.04877547e-05, -6.35416963e-04,\n",
       "         6.12120370e-04, -9.69294623e-04, -1.52980160e-03,  1.34762411e-03,\n",
       "        -2.01177743e-04,  8.28422069e-04, -2.30743351e-05, -2.31008764e-04,\n",
       "         8.58061808e-04,  6.17454690e-04,  6.04800796e-05, -1.15101738e-04,\n",
       "         1.36635033e-03, -5.49411715e-04,  3.12853377e-04,  1.60138382e-03,\n",
       "         2.93483301e-05,  9.88199805e-04,  5.59812498e-04, -9.60604507e-05,\n",
       "         5.69170106e-04, -4.21510670e-04, -4.41805263e-04, -4.34087978e-04])}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_batch(0,parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "def combine_parameters(parameters,grad,learn_rate):\n",
    "    parameter_tmp=copy.deepcopy(parameters)\n",
    "    parameter_tmp[0]['b']-=learn_rate*grad['b0']\n",
    "    parameter_tmp[1]['b']-=learn_rate*grad['b1']\n",
    "    parameter_tmp[1]['w']-=learn_rate*grad['w1']\n",
    "    return parameter_tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'b': array([-1.19774858e-02, -1.70124813e-02, -1.57611586e-02, -3.29941234e-02,\n",
       "          1.13402431e-02,  5.23436111e-03, -5.32733333e-03, -6.14615223e-03,\n",
       "          8.95087370e-03,  5.35958704e-03, -1.53389555e-02, -4.89839456e-03,\n",
       "         -1.51477001e-02, -2.06311786e-02,  4.70025465e-03, -4.12987595e-03,\n",
       "         -1.47431339e-02, -1.69919474e-02,  1.11322471e-04, -1.03625183e-02,\n",
       "          2.76990144e-02, -1.01368938e-02, -4.62622831e-03,  2.90607800e-03,\n",
       "          1.18790992e-02,  9.03605805e-03, -5.97092420e-03,  2.39107428e-02,\n",
       "         -1.49626018e-02,  5.83429690e-03, -1.79237298e-02,  9.55651116e-04,\n",
       "          3.72125548e-03,  5.31810900e-02,  1.03135669e-02, -7.22807104e-03,\n",
       "         -3.60362974e-02,  5.00815319e-02, -2.22295179e-02,  2.42837191e-02,\n",
       "          8.72222244e-03, -3.89814042e-04,  2.20827764e-02, -1.75728213e-02,\n",
       "          1.09229021e-02, -2.77683812e-02,  1.09316453e-02,  2.68270788e-02,\n",
       "         -1.78873303e-03,  4.05256266e-02, -1.55993063e-02,  3.15848391e-02,\n",
       "         -1.58220094e-02,  1.87775277e-02,  1.57653317e-02,  7.54950389e-03,\n",
       "         -3.21341372e-02,  9.40584597e-03, -3.71822731e-02, -3.58760596e-02,\n",
       "          2.90772682e-02,  1.03928791e-02, -9.07295140e-03, -9.22102439e-03,\n",
       "         -1.12173478e-02,  1.06404804e-02, -9.59073159e-03,  2.62063161e-02,\n",
       "         -5.01224190e-02, -3.46044078e-02, -3.58246213e-03, -4.81215396e-02,\n",
       "         -2.98672009e-02,  1.82280615e-02, -7.98582594e-03, -2.17196698e-02,\n",
       "          2.29865865e-02, -2.00662782e-02,  1.36940519e-02,  2.61594978e-02,\n",
       "         -3.45616021e-02,  1.01374486e-02, -1.88886264e-02, -3.34543397e-03,\n",
       "          4.51596669e-03,  2.01383389e-02, -8.89350391e-03,  1.99566443e-02,\n",
       "         -1.44971024e-02, -2.35040416e-02, -2.23487044e-02, -2.48333689e-02,\n",
       "          2.78163821e-02, -8.07408346e-02, -1.14208002e-01, -8.97656511e-02,\n",
       "         -1.46860627e-01, -2.22599181e-01, -1.32365718e-01, -4.13495788e-02,\n",
       "         -9.57685544e-02, -2.83166900e-02,  3.92059916e-03,  2.65976597e-03,\n",
       "          5.43026685e-04, -4.27133257e-03, -1.54814509e-02, -1.42166110e-02,\n",
       "          7.73512918e-03,  1.28465660e-03,  5.21616262e-03,  1.08833249e-02,\n",
       "         -4.71664883e-03,  1.96012695e-03,  2.95802455e-03,  2.49418683e-02,\n",
       "          1.83742998e-02, -4.89241750e-02, -9.30980423e-03, -7.10452329e-02,\n",
       "         -1.41513594e-01, -1.76826649e-01, -1.93335459e-01, -1.31394432e-01,\n",
       "         -1.66398003e-01, -2.51138152e-01, -1.63048114e-01, -2.15175845e-01,\n",
       "         -1.13904937e-01, -6.63513533e-02, -6.78076266e-02, -2.28078399e-02,\n",
       "          2.47230694e-03, -4.02866454e-02, -1.54012258e-01, -7.50283229e-02,\n",
       "          2.00025898e-02, -5.15522297e-03,  5.37168286e-03, -7.52357807e-03,\n",
       "          6.28093906e-04,  7.18335191e-03,  4.53913999e-03,  2.02243168e-04,\n",
       "         -3.69488001e-02, -7.69623414e-02, -8.68660159e-02, -9.57486762e-02,\n",
       "         -7.46362497e-02, -3.20761561e-02, -1.01152141e-01, -1.81087402e-01,\n",
       "         -1.14131533e-01, -1.52340287e-01, -1.15658239e-01, -2.16765807e-01,\n",
       "         -1.85221365e-01, -1.87353286e-01, -1.65058373e-01, -1.35741967e-01,\n",
       "         -7.42246876e-02, -3.85570028e-02, -1.73565269e-01, -1.50048441e-01,\n",
       "         -1.58847701e-02,  1.62754531e-02,  1.54514314e-02,  1.51860406e-03,\n",
       "          1.45909489e-02, -4.38245861e-03,  4.66591454e-03, -1.18493825e-02,\n",
       "         -7.05258481e-02, -1.28848317e-01, -1.41646905e-01, -8.16269715e-02,\n",
       "         -6.37507079e-02, -1.43116483e-02, -6.24218865e-02, -9.42195829e-02,\n",
       "         -1.95719641e-02, -2.04684840e-01, -3.16673232e-01,  5.88189871e-02,\n",
       "         -2.15235707e-01, -1.13667127e-01, -2.45646888e-01, -5.66007326e-02,\n",
       "         -8.51943397e-02, -1.40973679e-02, -8.12616241e-02, -1.44278382e-01,\n",
       "         -7.27643076e-02, -6.02330994e-03,  1.15814748e-02,  2.48782531e-02,\n",
       "         -2.42839199e-02,  9.08435747e-03,  3.33037552e-02, -4.23061882e-02,\n",
       "         -1.58213732e-01, -1.45505538e-01, -1.70069381e-01, -9.60600946e-02,\n",
       "         -1.17008052e-01, -1.10653936e-01, -1.56293828e-01, -2.50036187e-01,\n",
       "         -3.65053585e-01,  3.29303787e-02,  1.94099641e-02,  9.40544487e-02,\n",
       "          4.28084544e-02, -2.00838911e-01, -1.19166775e-01, -2.10742582e-01,\n",
       "         -1.53821631e-01, -6.89391425e-02, -3.73285273e-02, -1.17323633e-01,\n",
       "         -1.02542524e-01, -3.04586497e-02,  3.31328860e-02,  2.86340260e-02,\n",
       "          1.32355201e-03,  2.61524974e-02, -9.45033212e-03, -1.47849156e-02,\n",
       "         -2.00361788e-01, -1.90447397e-01, -8.64129495e-02, -1.06091412e-01,\n",
       "         -1.06102253e-01, -3.37076801e-02, -1.00705869e-01, -2.40261551e-01,\n",
       "         -1.48200458e-01,  3.30042436e-02, -1.85210159e-01, -1.92349974e-01,\n",
       "         -9.24808282e-02, -2.44065520e-01, -2.56659125e-01, -2.69914667e-01,\n",
       "         -2.07537272e-01, -1.54344772e-01, -1.21679917e-01, -2.45393440e-01,\n",
       "         -1.77636550e-01, -8.41439988e-02, -2.80509752e-03,  2.41793597e-03,\n",
       "         -2.51330455e-02, -1.30545719e-02,  8.57113031e-03, -5.67865245e-02,\n",
       "         -1.77979833e-01, -1.19732643e-01, -2.64829950e-02, -3.18494929e-03,\n",
       "         -1.42430623e-02,  1.41815941e-02, -1.30551798e-03, -8.66660770e-02,\n",
       "         -2.37872692e-01, -2.36173019e-01, -6.69130508e-01, -5.83870441e-01,\n",
       "         -3.29914622e-01, -2.84477659e-01, -3.22633240e-01, -2.55137658e-01,\n",
       "         -1.23043459e-01, -1.39981190e-01, -8.58355022e-02, -2.09074268e-01,\n",
       "         -2.34711961e-01, -1.13280510e-01,  9.28378891e-03, -2.85268037e-02,\n",
       "          5.69182562e-05,  1.75746302e-02, -8.52502741e-03, -6.39602559e-02,\n",
       "         -7.23400296e-02, -3.24322462e-02,  1.19482274e-02,  1.91862865e-02,\n",
       "         -6.37816647e-02, -1.90999567e-01, -2.95206323e-01, -2.21917463e-01,\n",
       "         -2.36334922e-01, -2.82896484e-01, -6.68811543e-01, -8.42820950e-01,\n",
       "         -6.75338843e-01, -3.07627810e-01, -1.29023470e-01, -2.75835115e-01,\n",
       "         -1.20700312e-01, -9.59286220e-02, -1.24453538e-01, -2.16124039e-01,\n",
       "         -3.10908947e-01, -2.64747320e-01, -4.57804982e-03,  1.18720396e-02,\n",
       "         -1.50877052e-02,  1.31053831e-03, -1.54925537e-02, -6.55092508e-02,\n",
       "         -2.40862418e-02, -7.18996202e-02, -9.67415831e-02, -2.78829376e-01,\n",
       "         -5.07082518e-01, -3.76204284e-01, -1.74846157e-01, -8.78742802e-02,\n",
       "         -1.77908967e-02, -3.11761694e-01, -5.49286562e-01, -5.97982211e-01,\n",
       "         -6.47378777e-01, -4.44919573e-01, -3.53428113e-01, -2.97857545e-01,\n",
       "         -4.71505468e-01, -3.96136894e-01, -2.69030785e-01, -1.44580183e-01,\n",
       "         -1.55362595e-01, -5.96407001e-02, -9.31339712e-03,  3.49351511e-02,\n",
       "          2.06857216e-03, -3.41154913e-02, -6.99503739e-03, -1.99019846e-02,\n",
       "         -4.39364364e-02, -2.22487608e-01, -3.96522997e-01, -4.25789587e-01,\n",
       "         -4.39577493e-01, -2.70588347e-01, -3.64133565e-01, -3.84718944e-01,\n",
       "         -4.73780886e-01, -2.91555131e-01, -5.42236189e-01, -5.10956573e-01,\n",
       "         -4.27946106e-01, -4.14594828e-01, -1.97763254e-01, -1.18548036e-01,\n",
       "         -1.94892219e-01, -2.71585752e-01, -4.26739637e-01, -3.83923812e-01,\n",
       "         -3.05592842e-01, -7.16168171e-02,  8.43597454e-04, -3.50034307e-03,\n",
       "         -2.77559276e-02, -2.21304443e-02,  5.98009301e-03, -1.62055780e-02,\n",
       "         -8.33749831e-02, -2.68699763e-01, -4.19758809e-01, -4.33876217e-01,\n",
       "         -3.06261442e-01, -2.12105592e-01, -1.62513954e-01, -4.68431092e-01,\n",
       "         -3.80138175e-01, -3.48622176e-01, -3.27363830e-01, -3.93971370e-01,\n",
       "         -2.22660695e-01, -3.19643649e-01, -2.28742527e-01, -2.20937322e-01,\n",
       "         -2.15512324e-01, -1.85106508e-01, -1.01687069e-01, -3.49707310e-01,\n",
       "         -2.62426341e-01, -3.69407692e-02, -2.46923292e-02, -1.19607603e-02,\n",
       "         -1.26079347e-02,  2.70687820e-02,  1.31981063e-02,  3.08283586e-02,\n",
       "         -6.11881186e-02, -2.12547161e-01, -2.92257120e-01, -2.24059717e-01,\n",
       "         -2.63467831e-01, -2.51702007e-01, -2.91471644e-01, -2.38795506e-01,\n",
       "         -2.96059460e-01, -8.54435078e-02, -1.97113501e-01, -9.92399291e-02,\n",
       "         -2.80206634e-01, -2.81163637e-01, -1.88471653e-01, -1.90766539e-01,\n",
       "         -2.28011028e-01, -1.05964949e-01, -1.17178413e-01, -1.96750732e-01,\n",
       "         -1.07597188e-01, -2.30605255e-02, -8.51116113e-03, -3.53517816e-03,\n",
       "          8.43975140e-03, -1.04646103e-02,  3.98506010e-02, -4.46510057e-02,\n",
       "         -4.54097884e-02, -2.28689592e-01, -2.42459486e-01, -2.59445105e-01,\n",
       "         -3.22846068e-01, -2.20634319e-01, -1.60918815e-01, -2.98618171e-01,\n",
       "         -1.36030446e-01, -2.23056376e-01, -3.75838126e-01, -2.71507973e-01,\n",
       "         -2.80323347e-01, -3.21155082e-01, -2.10897067e-01, -1.78325601e-01,\n",
       "         -1.80818481e-01, -1.61646556e-01, -1.83644162e-01, -1.73631346e-01,\n",
       "         -1.07228038e-01, -1.11263893e-01, -5.35420553e-02, -3.01753605e-02,\n",
       "          6.49848930e-03,  7.80343264e-03, -1.50681659e-02, -6.28391279e-03,\n",
       "         -1.06156671e-01, -2.01683100e-01, -3.41048597e-01, -3.32998471e-01,\n",
       "         -4.36357024e-01, -2.99863120e-01, -1.78706923e-01, -9.39145179e-02,\n",
       "          3.82968601e-02, -3.41884733e-01, -4.76517684e-01, -2.42404305e-01,\n",
       "         -2.17311029e-01, -3.90553188e-01, -2.64960065e-01, -1.80686284e-01,\n",
       "         -1.92047489e-01, -2.77385207e-01, -2.10786289e-01, -1.22126744e-01,\n",
       "         -1.69472102e-01, -1.54567088e-01, -5.08858040e-02,  3.19404214e-02,\n",
       "          1.06185303e-02, -2.05377304e-02,  1.24422603e-02, -6.18864207e-02,\n",
       "         -1.67026882e-01, -2.21204915e-01, -2.29495107e-01, -2.97737096e-01,\n",
       "         -5.21573853e-01, -5.72400481e-01, -4.37305552e-01, -2.71210380e-01,\n",
       "         -2.19090300e-01, -4.97212875e-01, -4.85751756e-01, -2.56505096e-01,\n",
       "         -1.47475403e-01, -2.35423057e-01, -2.03790561e-01, -1.38216274e-01,\n",
       "         -1.38151362e-01, -6.62261735e-02, -1.63098304e-01, -7.48773545e-02,\n",
       "         -1.03732416e-01, -9.97549759e-02, -2.40450865e-02, -1.32397382e-02,\n",
       "         -4.47952189e-03,  2.06379635e-02,  2.51501736e-02, -5.18105497e-02,\n",
       "         -2.01228326e-01, -2.93821000e-01, -1.63975647e-01, -2.07752913e-01,\n",
       "         -1.88169377e-01, -1.87742356e-01, -4.59519576e-01, -1.57878075e-01,\n",
       "         -5.32437462e-01, -4.26172016e-01, -3.68672289e-01, -8.49744080e-02,\n",
       "         -2.83183640e-01, -2.01834395e-01, -1.42007458e-01, -6.91182925e-02,\n",
       "         -1.08483376e-02, -2.07039249e-01, -1.97761329e-01, -2.09312849e-01,\n",
       "         -2.59741583e-01, -9.24128611e-02, -1.44610934e-02, -4.40383790e-03,\n",
       "         -3.02273778e-02,  6.55483803e-03,  6.56033847e-03, -8.76019110e-02,\n",
       "         -1.15504220e-01, -1.38624021e-01, -1.05318573e-01, -1.57032029e-01,\n",
       "         -2.49535402e-01, -2.85373340e-01, -1.75376435e-01, -2.93798707e-01,\n",
       "         -1.41188704e-01, -2.73166547e-01, -3.38326286e-01, -7.77927925e-02,\n",
       "         -1.02523909e-01, -1.63539001e-01, -8.13131209e-02, -9.21248310e-02,\n",
       "         -9.09238051e-02, -2.14005775e-01, -1.40089550e-01, -7.70404564e-02,\n",
       "         -8.50073003e-02, -7.40981861e-02, -2.13541495e-02, -1.34502708e-02,\n",
       "          9.47633341e-03,  1.37168669e-02,  1.63145077e-02, -3.42623137e-02,\n",
       "         -1.80901017e-01, -1.07073779e-01, -2.62485936e-01, -2.20873953e-01,\n",
       "         -1.88147314e-01, -2.35619510e-01, -2.79515355e-01, -4.04810645e-01,\n",
       "         -2.12956178e-01, -2.23452750e-01, -3.15472443e-01, -7.80070240e-02,\n",
       "         -2.67518088e-01, -1.49316811e-01, -1.54027902e-01, -8.42005291e-02,\n",
       "         -1.29846043e-01, -8.35917388e-02, -1.18863872e-01, -4.75459016e-02,\n",
       "         -6.21207840e-02, -1.55507955e-02,  7.20197699e-03,  2.68327556e-03,\n",
       "          2.62235410e-04,  1.35387580e-03, -1.85120623e-02, -1.75177530e-02,\n",
       "         -1.35628568e-01, -1.64530099e-01, -1.65257430e-01, -1.11304858e-01,\n",
       "         -1.22124453e-01, -1.18408487e-01, -2.34525223e-01, -3.19704074e-01,\n",
       "         -2.29522071e-01, -5.60200466e-02, -2.11792500e-01, -1.39368078e-01,\n",
       "         -1.72545358e-01, -1.00351408e-01, -1.73251501e-01, -1.10988428e-01,\n",
       "         -3.24298664e-02, -1.12832472e-01, -8.37412237e-02, -5.40103702e-02,\n",
       "         -3.92108558e-02, -5.69080100e-03,  5.49805154e-03,  6.52645077e-03,\n",
       "          1.44414059e-02, -2.10036845e-03,  1.13653045e-02, -3.71232799e-03,\n",
       "         -1.07552971e-01, -1.93022206e-01, -1.62172212e-01, -1.13058437e-01,\n",
       "         -1.31962635e-01, -1.81583325e-01, -2.05808055e-01, -1.00398637e-01,\n",
       "         -6.53064118e-02,  3.50244646e-04, -2.76702192e-02, -8.15056672e-02,\n",
       "         -1.70114272e-01, -1.83442910e-01, -2.14483671e-01, -1.77830984e-01,\n",
       "         -1.46729823e-01, -7.55458333e-02, -1.24643033e-01, -1.03716125e-01,\n",
       "         -4.00611874e-02, -1.14708192e-02, -1.86324937e-02, -1.74585047e-03,\n",
       "         -1.95903285e-02, -3.84808305e-03, -2.13263673e-02, -3.06690382e-02,\n",
       "         -4.68892415e-02, -1.79428411e-01, -2.58675236e-01, -1.56321755e-01,\n",
       "         -9.63472794e-02, -1.78802284e-01, -1.35806585e-01, -1.05490492e-01,\n",
       "          4.47500044e-02,  2.36585370e-01,  1.81158682e-02,  5.78374552e-02,\n",
       "         -9.62566998e-02, -1.44675934e-01, -1.85426535e-01, -1.67159658e-01,\n",
       "         -1.68085411e-01, -1.20511869e-01, -1.09669976e-01, -2.06103643e-02,\n",
       "         -2.79444115e-02,  6.30703032e-03, -1.90102405e-02, -7.07084631e-03,\n",
       "         -1.55244716e-02,  3.29414265e-04, -5.11586563e-03,  4.26317227e-03,\n",
       "         -5.58205214e-02, -1.46982067e-01, -2.05456278e-01, -1.83324440e-01,\n",
       "         -7.31051903e-02, -5.02048934e-02, -9.56545201e-02, -7.99581178e-02,\n",
       "         -8.46556458e-02, -1.32636097e-01, -1.21607782e-01, -8.53907697e-02,\n",
       "         -5.87699612e-02,  7.28325229e-03, -5.57057262e-02, -1.27489852e-01,\n",
       "         -1.12276481e-01, -2.44864882e-02, -1.76607433e-02, -2.49686381e-02,\n",
       "         -4.26832672e-03,  8.37005015e-03,  1.92528715e-02, -3.74331258e-02,\n",
       "         -2.00925930e-02,  9.96449574e-03,  1.41132829e-02, -5.25006431e-03,\n",
       "         -3.20634667e-02, -4.79669035e-02, -4.01111688e-02, -1.39053511e-01,\n",
       "         -1.18771462e-01, -1.11528319e-01, -1.49616009e-01, -1.80507736e-01,\n",
       "         -1.15965262e-01, -7.05829794e-02, -6.49410859e-02, -6.84115593e-02,\n",
       "         -9.06517186e-02, -4.80413107e-02, -8.54688137e-02, -6.01438451e-02,\n",
       "         -8.58604819e-02, -3.49909867e-02, -1.20234473e-03, -1.07325604e-02,\n",
       "          7.99891567e-03, -7.88425460e-03,  5.05609757e-03, -1.43173006e-02,\n",
       "         -1.82950083e-02, -4.39437203e-03,  3.97516337e-03,  5.11259801e-03,\n",
       "         -6.56408191e-03, -2.86333623e-02,  6.48723178e-03, -4.88505396e-02,\n",
       "         -3.05595513e-02, -5.85918353e-02, -8.39111099e-02, -9.62998517e-02,\n",
       "         -8.33556195e-02, -9.80433765e-02, -1.85449019e-01, -9.81013767e-02,\n",
       "         -9.30961859e-02, -8.01370567e-02, -6.53848350e-02, -3.27227516e-02,\n",
       "         -3.31944054e-02,  1.25208510e-02,  2.19597541e-02,  3.83458415e-03,\n",
       "         -3.26097871e-02, -4.45394453e-03, -3.17596556e-02, -2.98014011e-02,\n",
       "          1.81516739e-02, -8.90859493e-03, -6.55977792e-04,  4.10312082e-02,\n",
       "          3.86069742e-02,  4.41224160e-03, -6.25751479e-03, -6.50652068e-03,\n",
       "         -4.20671898e-03, -1.76266908e-03, -1.32232384e-02, -2.03971261e-02,\n",
       "         -3.55841547e-02,  7.33083811e-03, -8.57067003e-03,  2.12586704e-02,\n",
       "         -2.47714283e-02,  4.09668144e-04, -1.42759983e-02, -1.76072771e-02,\n",
       "         -1.00749760e-02, -2.24465521e-02, -7.99218327e-03, -3.75734277e-03,\n",
       "         -5.47256810e-03, -2.46533788e-02,  2.92426281e-02, -1.76967523e-03])},\n",
       " {'w': array([[-0.05955968,  0.03038869, -0.00030443, ...,  0.06140209,\n",
       "          -0.04560647,  0.06349325],\n",
       "         [ 0.07680152,  0.00806223, -0.00263966, ...,  0.02927116,\n",
       "           0.03416597,  0.04865559],\n",
       "         [ 0.00329191,  0.01486493,  0.0820535 , ..., -0.01550454,\n",
       "          -0.03023026, -0.02760395],\n",
       "         ...,\n",
       "         [ 0.06170954,  0.06878091,  0.07354365, ...,  0.03286647,\n",
       "          -0.04105779,  0.07231534],\n",
       "         [-0.04750236, -0.01891281,  0.02766746, ..., -0.03256889,\n",
       "          -0.00461513, -0.08391934],\n",
       "         [-0.00782269,  0.0574595 ,  0.05241317, ...,  0.00289614,\n",
       "          -0.04353394, -0.00459593]]),\n",
       "  'b': array([-0.25783171, -0.0904294 , -0.09253957, -0.03959246, -0.01903307,\n",
       "          0.37691612,  0.06781033,  0.01515613,  0.05882279, -0.01927915])},\n",
       " {}]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combine_parameters(parameters,train_batch(0,parameters),1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters=init_parameters()\n",
    "current_epoch=0\n",
    "train_loss_list=[]\n",
    "valid_loss_list=[]\n",
    "train_accu_list=[]\n",
    "valid_accu_list=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation accuracy : 0.0935\n"
     ]
    }
   ],
   "source": [
    "valid_accuracy(parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm_notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5dc9d3a9b5c14f4ebad7ca6b4ec02484",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_rate=10**-0.6\n",
    "epoch_num=10\n",
    "for epoch in tqdm_notebook(range(epoch_num)):\n",
    "    for i in range(train_num//batch_size):\n",
    "#         if i%100==99:\n",
    "#             print('running batch {}/{}'.format(i+1,train_num//batch_size))\n",
    "        grad_tmp=train_batch(i,parameters)\n",
    "        parameters=combine_parameters(parameters,grad_tmp,learn_rate)\n",
    "    current_epoch+=1\n",
    "    train_loss_list.append(train_loss(parameters))\n",
    "    train_accu_list.append(train_accuracy(parameters))\n",
    "    valid_loss_list.append(valid_loss(parameters))\n",
    "    valid_accu_list.append(valid_accuracy(parameters))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X10HPV97/H3V7JlWfKTZMmSLXll2fhJssHAAsa+JgQank4ChAMNBIhvQurbFFp6c3oSepoTepN7GtKm7b0hSVNu4gskDZCnBp8EQhyn1FzAYBts/PwgG9vyo2T5+Vna3/1jZqWVtKtdrVZazerzOmfOzP7mt7O/Hcvz2Zn5zYw55xARkaEnL9sNEBGR7FAAiIgMUQoAEZEhSgEgIjJEKQBERIYoBYCIyBClABARGaIUACIiQ1TSADCzpWZ2xMw2xpTdZ2abzCxiZuEu9f/azHaa2TYzuzWm/Da/bKeZPZHZryEiIr1lya4ENrMbgNPA8865OX7ZbCAC/CvwV865NX55HfACcC0wCfg9MMNf1HbgY0AjsBp4wDm3uafPLisrc1OmTEnri4mIDFVr165tds6VJ6s3LFkF59xKM5vSpWwLgJl1rX4X8KJz7gKw28x24oUBwE7n3C7/fS/6dXsMgClTprBmzZpkTRQRkRhmtieVepk+B1AF7It53eiXJSrvxsyWmNkaM1vT1NSU4eaJiEhUpgOg2y4B4Hoo717o3DPOubBzLlxennQPRkRE0pT0EFAvNQKTY15XAwf86UTlIiKSBZneA1gG3G9mI8ysFpgOvIt30ne6mdWaWQFwv19XRESyJOkegJm9ANwIlJlZI/Ak0AI8DZQDvzGzdc65W51zm8zsp3gnd1uBR51zbf5yHgNeA/KBpc65Tf3xhUREJDVJu4FmUzgcduoFJCLSO2a21jkXTlZPVwKLiAxRCgCRGK2traxevZqnn36abdu2Zbs5Iv0q072ARALl/PnzvPvuu6xcuZKVK1fy9ttvc/r0aQAefPBBfvzjH2e5hSL9RwEgQ8qpU6d466232jf47777LhcvXsTMmDt3LosXL+aGG27gu9/9Lhs3bky+QJEAUwBITmtubuaNN97gjTfeYOXKlbz//vtEIhHy8/MJh8M8/vjjLFq0iIULF1JaWtr+vnfffZfvfOc7tLW1kZ+fn8VvINJ/FACSM5xz7Nq1i7feeos333yTN954g82bvdtNFRYWMn/+fL7yla+waNEi5s+fz6hRoxIuq76+ngsXLrBr1y6mT58+UF9BZEApACSwLl68yHvvvde+wX/zzTc5fPgwAGPGjGHhwoU8/PDDLFq0iHA4zIgRI1Jedn19PQCbNm1SAEjOUgBIYBw9epS33nqrfYO/evVqzp8/D0BtbS0f+9jHWLhwIQsWLKC+vr5Ph25mz54NeAFw9913Z6T9IoONAkAGJecc27dv580332zf4G/duhWAYcOGcdVVV/GFL3yhfYM/ceLEjH7+6NGjCYVC7YeQRHKRAkAGhePHj7N69WpWrVrFqlWreOeddzh69CgAJSUlLFiwgM985jMsXLiQcDhMUVFRv7epvr6eTZt0xxLJXQoAGXCtra1s2rSp08Z+y5YtgPeQodmzZ3PnnXeyYMECFixYwKxZs8jLG/hrFuvr6/nDH/6gnkCSsxQA0u8OHDjAO++8076xX716NWfPngWgrKyM+fPn8+lPf5r58+dzzTXXMHbs2Cy32FNXV6eeQJLTFACSUadOneK9995jzZo17Rv9ffu8h8ENHz6cK6+8ks9//vNcd911zJ8/n9ra2niPFh0U1BNIcp0CQNJ25swZ1q1bx5o1a9qHbdu2Eb3DbG1tLQsXLmT+/Plcd911zJs3j8LCwiy3OnV1dXWAegJJ7lIASErOnz/P+vXrO23sN2/eTCQSAaCqqopwOMyDDz5IOBzm6quvJuiP9Bw1ahQ1NTU6ESw5SwEg3Zw/f55Nmzaxdu3a9o39hg0baG1tBaC8vJxrrrmGe+65p31jP2nSpCy3un/U1dWpK6jkLAXAENfS0sL69et5//33WbduHevWrWPLli3tG/vS0lLC4TBf+tKXCIfDhMNhqqurB+1x+0xTTyDJZQqAIcI5x549e9o38tEN/t69e9vrTJo0iXnz5vGJT3yCefPmcfXVVw/qk7QDIXpPoIaGBmbMmJHt5ohklAIgB124cIGtW7e2b+yjw/HjxwGvr/3MmTNZuHAhjz76KPPmzWPevHlMmDAhyy0ffKIngjdv3qwAkJyjAAiwtrY2Ghoa2LhxY/uwYcMGduzYQVtbGwAjR47k8ssv51Of+hRXXnkl8+bNY86cORQXF2e59cGgnkCSyxQAAeCco7GxsX0DH93Yb9mypf1maGbG1KlTmTt3Lvfeey/19fVcccUVzJgxQ8eu+0A9gSSXKQAGkUgkwr59+9i2bRtbt25l06ZN7Rv7kydPtterqqpizpw53HTTTcyZM4c5c+Ywe/Zs/arvJ7onkOSqpAFgZkuBjwNHnHNz/LJS4CVgCvAh8MfOuWNmNhb4MRDyl/0t59z/9d+zGPiKv9j/6Zx7LrNfJThOnz7N9u3b2bp1a/vGftu2bWzfvp1z58611yspKWHu3Lk89NBDzJ07lzlz5lBfX09JSUkWWz/01NXVsWLFClpbWxk2TL+ZJHek8tf8LPAd4PmYsieAFc65p8zsCf/1l4FHgc3OuU+YWTmwzcz+DRgFPAmEAQesNbNlzrljmfsqg0skEqGxsbHbRn7btm00Nja218vLy2PKlCnMmjWLm266iVmzZjFz5kxmzpxJRUXFkO6BM1jEPh1MJ4IllyQNAOfcSjOb0qX4LuBGf/o54HW8AHDAaPO2WqOAFqAVuBVY7pxrATCz5cBtwAt9/QLZdPbsWXbv3s2uXbu6Dbt37+70a37MmDHMnDmTj370o8ycObN9Q3/ZZZcF6vYIQ1HsPYEUAJJL0t2frXDOHQRwzh00s2j/we8Ay4ADwGjgU865iJlVAfti3t8IVKX52QMmEolw6NCh9o16Q0NDp438oUOHOtUvLi5m2rRpzJgxg1tvvZUZM2a0b+grKyv1az6gok8H27x5M5/85Cez3BqRzMn0Ac1bgXXATcA0YLmZvQHE2/K5eAswsyXAEoBQKJTh5nWIRCI0Nzezf/9+Ghsb28fR6X379rFnz572XjZ+26iurmbq1KncfvvtTJs2jalTp7YPZWVl2sjnIPUEklyVbgAcNrOJ/q//icARv/yzwFPOux3kTjPbDczC+8V/Y8z7q/EOG3XjnHsGeAYgHA7HDYlkIpEI+/fv77ZRj53ev38/Fy9e7PS+vLw8Jk6cSHV1NfX19Xz84x/vtIGvqanp1YPFJXeoJ5DkonQDYBmwGHjKH7/sl+8FbgbeMLMKYCawC9gJ/J2ZRbuv3AL8dbqNTubQoUPd9h5GjBhBdXU11dXVXH/99VRXV1NVVdVeVlVVRUVFhXp5SFz19fX8/ve/V08gySmpdAN9Ae/Xe5mZNeL15nkK+KmZPYK30b/Pr/514Fkz24B32OfLzrlmfzlfB1b79b4WPSHcHyoqKvj+97/P5MmT2zfypaWlOjwjaaurq+PixYvqCSQ5JZVeQA8kmHVznLoH8H7dx1vOUmBpr1qXpvz8fP7b5z4Hw4cPxMfJEKCeQJKLBv5J2wPh1CkYNQouvxwefhi+9S1YvhyOHEn+XpE4oj2BdB5AckluHsy8eBG++EVYvx7+8Af48Y875lVWwhVXdB5mzNDegvQo2hNID4eRXJKbATB+PHzjGx2vm5u9MFi/Hj74wBv/8z/DpUve/BEjoK6uezCUlman/TIoqSeQ5JrcDICuysrg5pu9IerSJdi6tSMY1q+HV16BZ5/t/L5p0+IPlZWgk8pDinoCSa4Zun/Fw4fD3Lne8NBDHeWHDnXsKezYAQ0N8Oab8OKL4D8AHYCiIpg6tXMoXHaZNw6FdEgpB9XX16snkOSUoRsAiVRWesOtt3Yuv3gRPvzQC4TYYccOeO01iLlimPx8qKmB6mqYOLHzUFnZMV1aqr2IAIl9OIwCQHKBAiBVBQXeyeJ4//EjETh4sHMw7NoF+/fDe+95806fjr/M2EDoOowfD+PGQUmJN9ZVyJl1/jycONExHD/eMX3qFNx5p7eX54vtCaR7AkkuUABkQl4eVFV5ww03xK9z+rQXBImGHTtg5Upo6eH6uMLCzoEQHRK9LiqCkSO9obCw8/Tw4cHZ+3DOO2dz9qw3nDnTMR1viM4/eTL+xj06XLjQ8+du2AA//GH7y1GjRjFlyhSdCJacoQAYKKNGwfTp3tCTCxe88xAHD8KxY95w/HjHEPu6qQm2b+947T8HOCV5ed1Doev0sGFeSOTldQy9ed3WBq2t3sY73rineZcueb/Qoxv13ny32HU+dmzHUF7unaeJvh43rvP82LJ774U9e7otsq6uTl1BJWcoAAabESO88wc1Nb17n3PeL99oQBw75m04z5+Hc+c6xrHTPZUdO+ZtdCMRb9mRSMeQyuu2Nu9cyPDhXpAkGhcVxS8fNswLoaIibygu7phONMTWKSz0QihdU6fC2rXditUTSHKJ/oJzhZn3i3fUKJg8OdutCb5QCH71Ky/QYoIk2hOooaGBmTNnZrGBIn2Xm7eCEOmrUMg7HNfU1Kk4ek8gHQaSXKAAEIknejvxvXs7Fc+aNQvQPYEkNygAROJJEADqCSS5RAEgEk+CAADdE0hyhwJAJJ6SEq9XUZwAqKurY9u2bbS2tmahYSKZowAQicfM2wtIsAcQ7QkkEmQKAJFEeggA0IlgCT4FgEgiNTVxAyDaE0hdQSXoFAAiiYRC3mNEz53rVKyeQJIrFAAiiUR7AjU2dpulnkCSCxQAIokk6QqqnkASdEkDwMyWmtkRM9sYU1ZqZsvNbIc/LomZd6OZrTOzTWb2nzHlt5nZNjPbaWZPZP6riGRYDwFQV1ennkASeKnsATwL3Nal7AlghXNuOrDCf42ZjQO+B9zpnKsH7vPL84HvArcDdcADZlaXiS8g0m+qqrzuoOoJJDkqaQA451YCXZ9SchfwnD/9HHC3P/1p4JfOub3+e4/45dcCO51zu5xzF4EX/WWIDF4FBd6T2eIEQOzTwUSCKt1zABXOuYMA/niCXz4DKDGz181srZl9xi+vAvbFvL/RLxMZ3BJcC1BcXMyUKVPUFVQCLdPPAxgGXA3cDIwE3jazVUC8Zw+6eAswsyXAEoBQ9BisSLaEQrBuXdxZ6gkkQZfuHsBhM5sI4I+jh3oagd86584455qBlcAVfnnsU0qqgQPxFuyce8Y5F3bOhcvLy9NsnkiGRPcAXPffK+oJJEGXbgAsAxb704uBl/3pl4FFZjbMzIqA64AtwGpgupnVmlkBcL+/DJHBLRTyHpPZ3NxtVvSeQDt37sxCw0T6LpVuoC8AbwMzzazRzB4BngI+ZmY7gI/5r3HObQF+C3wAvAv8wDm30TnXCjwGvIYXCD91zmnfWQa/JF1BQbeEkOBKeg7AOfdAglk3J6j/D8A/xCl/BXilV60TybbYALj66k6zYnsC3XPPPQPdMpE+05XAIj3pYQ+guLiY2tpanQiWwFIAiPSktBSKimDPnriz6+rqdAhIAksBINKTHh4MA+oJJMGmABBJJkkAqCeQBJUCQCSZJAEA6gkkwaQAEEkmFILDh73rAbqIPh1MJ4IliBQAIsn08GAY9QSSIFMAiCTTQ1dQ0D2BJLgUACLJJAmAuro69QSSQFIAiCRTXe2Ne9gDuHTpknoCSeAoAESSGTECKiuT9gTSYSAJGgWASCp66Ao6e/ZszExdQSVwFAAiqeghAIqKipgyZYr2ACRwFAAiqejhwTCgnkASTAoAkVTU1MC5c3D0aNzZ0XsCXbp0aYAbJpI+BYBIKlLoCnrp0iUaGhoGsFEifaMAEElFCheDgXoCSbAoAERSkSQAoj2BFAASJAoAkVSMHw8jR/bYE6i2tlZdQSVQFAAiqUjyYBjwzgNoD0CCRAEgkqokAaCeQBI0CgCRVKUQALonkARJ0gAws6VmdsTMNsaUlZrZcjPb4Y9LurznGjNrM7N7Y8oW+/V3mNnizH4NkQEQCsHBg3DhQtzZdXV1gJ4OJsGRyh7As8BtXcqeAFY456YDK/zXAJhZPvBN4LWYslLgSeA64Frgya6hITLoRXsC7d8fd7Z6AknQJA0A59xKoKVL8V3Ac/70c8DdMfP+HPgFcCSm7FZguXOuxTl3DFhO91ARGdySdAWN9gRSAEhQpHsOoMI5dxDAH08AMLMq4JPA97vUrwL2xbxu9MtEgiMaAHv2JKxSX1+vQ0ASGJk+Cfy/gC8759q6lFucunHvqmVmS8xsjZmtaWpqynDzRPogyYNhoOPpYOoJJEGQbgAcNrOJAP44ergnDLxoZh8C9wLfM7O78X7xT455fzVwIN6CnXPPOOfCzrlweXl5ms0T6QeFhVBRoZ5AkjPSDYBlQLQnz2LgZQDnXK1zbopzbgrwc+DPnHO/wjshfIuZlfgnf28h5iSxSGCk0BUUdE8gCYZhySqY2QvAjUCZmTXi9eZ5CvipmT0C7AXu62kZzrkWM/s6sNov+ppzruuJZZHBLxSCHjbus2bN0tPBJDCSBoBz7oEEs25O8r7/2uX1UmBpyi0TGYxCIXj1Ve/BMNb91JZ6AkmQ6Epgkd4IheDsWWhJvAOrp4NJUCgARHojybUA4AXA9u3b1RNIBj0FgEhvpBAA0aeDqSeQDHYKAJHeSHEPANQTSAa/pCeBRSRGeTmMGNFjAER7Av3oRz/iwoULVFZWUlFRQWVlJaWlpeTl6XeXDA4KAJHeSOHBMEVFRSxatIhly5axbNmyTvOGDRvGhAkTqKysbB+i4dB1esyYMVicnkYimaIAEOmtJAEA8Prrr3PixAkOHz7MoUOHOHToUPt0bNn69es5fPgwra2t3ZZRUFBAeXl5ysO4ceO0dyG9ogAQ6a1QCF7r+UJ2M2PcuHGMGzeOmTNn9lg3EonQ0tLSKRgOHTpEU1MTR44coampiaamJhoaGmhqauLUqVNxl5Ofn09ZWVmnUCgrK2P8+PGdxrHTxcXF2ssYwhQAIr0VfTDMxYtQUNDnxeXl5bVvmKMnkHty/vx5mpub24MhOsSGRVNTE+vWrePo0aO0tLTgXNx7L1JQUNAtFGLHJSUllJaWdhuPGDGiz99bsk8BINJboZB3JfD+/VBbO+AfX1hYSHV1NdXRu5Mm0dbWxvHjx2lubqa5uZmjR492GsdOb9iwgebmZlpaWohEIgmXOXLkyE6BEC8kxo0bx9ixY7uNtdcxeCgARHortitoFgKgt/Lz8xk/fjzjx49PejgqKhKJcOLECY4dO0ZLSwstLS3t0/HGDQ0N7a/Pnj2btD1jx45tH6LB0DUsxowZw+jRoxOOCzKw9zXUKQBEequmxhsnOREcZHl5eZSUlFBSUsLUqVN79d4LFy5w7Ngxjh8/zvHjxzlx4gQnTpzocXr37t3t0ydPnkx4yCpWQUFBt1DoOj1q1KhuQ3FxcdzykSNHDrmT6AoAkd5K4cEwQ9mIESPau7KmIxKJcOrUKU6ePJnSOHb6yJEjNDQ0cPLkSU6fPs3p06dT/lwzo6ioqFNQFBcXU1RU1GkcryzeODqMHDmyfZyfn5/WOukvCgCR3ho5EiZMUAD0k7y8vPZDQn0ViUQ4d+5cexicPn2aM2fOdHodb96pU6c4e/YsZ86c4ezZszQ3N3d6febMmbTu9TR8+PD2MIgNhtjp6HjGjBl86Utf6vM66IkCQCQdKVwLINmXl5fX/qu9oqIio8u+dOkSZ8+e7RYMseNz585x7ty59umu49jpY8eOceDAgfbX+/fvVwCIDEqhEGzdmu1WSBYNHz48Y3sq2TK0zniIZEp0DyCFk5Uig5UCQCQdoRCcPg3HjmW7JSJpUwCIpCOF20KLDHYKAJF0KAAkBygARNKhAJAcoAAQSUcKD4YRGewUACLpyMuDyZMVABJoSQPAzJaa2REz2xhTVmpmy81shz8u8csfNLMP/OEtM7si5j23mdk2M9tpZk/0z9cRGUC6GEwCLpU9gGeB27qUPQGscM5NB1b4rwF2Ax9xzl0OfB14BsDM8oHvArcDdcADZlbX59aLZJMCQAIuaQA451YCLV2K7wKe86efA+72677lnIt2jF4FRG9Yfi2w0zm3yzl3EXjRX4ZIcIVCcOAApHFPGJHBIN1zABXOuYMA/nhCnDqPAK/601XAvph5jX5ZN2a2xMzWmNmapqamNJsnMgBiHwwjEkD9chLYzD6KFwBfjhbFqRb3Gnrn3DPOubBzLlxeXt4fzRPJDHUFlYBLNwAOm9lEAH98JDrDzC4HfgDc5Zw76hc3ApNj3l8NHEjzs0UGBwWABFy6AbAMWOxPLwZeBjCzEPBL4GHn3PaY+quB6WZWa2YFwP3+MkSCa7L/m0YBIAGV9HbQZvYCcCNQZmaNwJPAU8BPzewRYC9wn1/9q8B44Hv+Q59b/cM5rWb2GPAakA8sdc5tyvSXERlQRUVQVqYAkMBKGgDOuQcSzLo5Tt3PA59PsJxXgFd61TqRwU5dQSXAdCWwSF8oACTAFAAifREKwZ49ejCMBJICQKQvog+GOXEi2y0R6TUFgEhfqCuoBJgCQKQvFAASYAoAkb5QAEiAKQBE+qKiAgoKFAASSAoAkb7Qg2EkwBQAIn0V7QoqEjAKAJG+0sVgElAKAJG+0oNhJKAUACJ9FQpBJOKFgEiAKABE+kpdQSWgFAAifaUAkIBSAIj0lR4MIwGlABDpq+JiGD9eASCBowAQyQR1BZUAUgCIZIICQAJIASCSCQoACSAFgEgmhEJw8qQeDCOBogAQyQR1BZUAUgCIZIICQAIoaQCY2VIzO2JmG2PKSs1suZnt8MclfrmZ2bfNbKeZfWBmV8W8Z7Fff4eZLe6fryOSJQoACaBU9gCeBW7rUvYEsMI5Nx1Y4b8GuB2Y7g9LgH8BLzCAJ4HrgGuBJ6OhIZITKith+HAFgARK0gBwzq0EWroU3wU8508/B9wdU/6886wCxpnZROBWYLlzrsU5dwxYTvdQEQmuvDyorlYASKCkew6gwjl3EMAfT/DLq4B9MfUa/bJE5SK5Q11BJWAyfRLY4pS5Hsq7L8BsiZmtMbM1TU1NGW2cSL9SAEjApBsAh/1DO/jjI355IzA5pl41cKCH8m6cc88458LOuXB5eXmazRPJglAI9u+H1tZst0QkJekGwDIg2pNnMfByTPln/N5A84ET/iGi14BbzKzEP/l7i18mkjtCIWhrg4MHs90SkZQMS1bBzF4AbgTKzKwRrzfPU8BPzewRYC9wn1/9FeAOYCdwFvgsgHOuxcy+Dqz2633NOdf1xLJIsMV2BZ08uee6IoNA0gBwzj2QYNbNceo64NEEy1kKLO1V60SCJDYAFi7MbltEUqArgUUyRQ+GkYBRAIhkyujRUFICe/ZkuyUiKVEAiGRSTY32ACQwFAAimaRrASRAFAAimaQAkABRAIhkUijkPRRGD4aRAFAAiGRStCvovn091xMZBBQAIpmk5wJIgCgARDJJASABogAQyaTKShg2TAEggaAAEMmk/Hw9GEYCQwEgkmnqCioBoQAQyTQFgASEAkAk00IhaGz0ng0gMogpAEQyTQ+GkYBQAIhkmrqCSkAoAEQyTQEgAaEAEMk0PRhGAiLpIyFFpJfGjIFx4+AHP4DiYvjUp6CsLNutEulGewAi/eHb34bCQnjsMZg4Ee66C37+czh/PtstE2mnABDpDw8/DB98AOvWweOPw+rVcN993q0iliyBN96ASCTbrZQhTgEg0p+uuAK+9S3v9tCvvQZ33gk/+QnccANMmwZf+Qps25btVsoQ1acAMLPHzWyjmW0ys7/0y+aZ2SozW2dma8zsWr/czOzbZrbTzD4ws6sy8QVEAiE/H265BZ5/Hg4dgh/9CGbMgG98A2bNgmuvhaefhqambLdUhpC0A8DM5gB/AlwLXAF83MymA38P/A/n3Dzgq/5rgNuB6f6wBPiXPrRbJLhGjYKHHvL2CBob4R//ES5dgr/4C5g0CT7xCXjpJTh9OtstlRzXlz2A2cAq59xZ51wr8J/AJwEHjPHrjAUO+NN3Ac87zypgnJlN7MPniwTfxInwxS/C++/Dhg0d0/ffD2PHwlVXeSeSf/IT+PBDcC7bLZYcYi7NPygzmw28DFwPnANWAGuA7wGvAYYXMAucc3vM7NfAU865/+e/fwXwZefcmkSfEQ6H3Zo1CWeL5Ka2Nli5Ev7jP+Ctt2DVKjhzxps3aRIsWNAxXHklFBRkt70y6JjZWudcOFm9tK8DcM5tMbNvAsuB08B6oBX4AvDfnXO/MLM/Bn4I/BFeIHRbTJyGL8E7REQoekWlyFCSnw8f/ag3ALS2wsaNXhhEh5//3Js3YgRcc01HIFx/PUyYkL22S6CkvQfQbUFmfwc0At8AxjnnnJkZcMI5N8bM/hV43Tn3gl9/G3Cjcy7hHbO0ByCSwMGD8PbbHYGwdi1cvOjNu+wyLwzCYa8X0uWXexemyZDR73sA/odMcM4dMbMQcA/e4aA/Bz4CvA7cBOzwqy8DHjOzF4Hr8IJBt0sUScfEiXDPPd4A3gVm773XEQi//a3X4ygqFOoIg+j4ssu8vQ0Zsvp6K4hfmNl44BLwqHPumJn9CfC/zWwYcB7/cA7wCnAHsBM4C3y2j58tIlGFhR2HgcA7WXzwIKxf712QFh2/8krHcwqKimDOnM6hoL2FISVjh4D6gw4BiWTY+fOweXNHKESHlpaOOtG9hTlzYPZs7zqFWbNg9OjstVt6ZUAOAYlIwBQWel1Lr4q5DtM5OHCg897C+vXw6qveCeio6movEKKhEJ2eMAEsXh8PGewUACJDnRlUVXnDHXd0lF+6BDt3wtatsGVLx/DDH3Z0SwUoKekeCrNnQ02NzjEMcjoEJCK945x3BfOWLd3D4ciRjnoFBd79jqZP7z5UVUGebkXWX3QISET6h5n30JvJk737G8VqaekIhe3bYccOb/jd7zrfCruw0OuFFBsK0deTJumQ0gBRAIhI5pSWdu6NFBWJwP79HYFg40XhAAAHMUlEQVQQDYctW+A3v+m4hgG83kmXXeYN06bB1Kkd41AIhg8f2O+UwxQAItL/8vI69hpuuqnzvLY27/GZ0XCIDps2wa9/3Tkc8vO9EJg6tXMwRKfVhbVXFAAikl35+VBb6w1dDylFIl4PpYYG2LWrY2hogF/9qvvts0tKOsKgthamTOkY19R4h56knQJARAavvDyv+2l1NXzkI93nnzoFu3d3D4j33/cCInbvAbwnskUDIXaorfX2LEaM6P/vNIgoAEQkuEaP7riCuatIxLsa+sMPvZD48MOO4Z134Gc/63ydA3gnoKOBUFPjhUJNTcd0cXG/f6WBpAAQkdyUl9dxfcPChd3nt7V5h5fiBcSbb8KLL3bcNiNq/PjOwdA1IMrLA9WDSQEgIkNTfn7HielFi7rPjwbE3r2wZ0/HeM8e7yT173/f/althYWdwyG6/MmTO14XFQ3M90uBAkBEJJ7YgIi3B+EcHD/ePRyi07/5jff8565KS+MHQ3Soqhqwh/woAERE0mHm9ToqKYF58+LXuXjRu/5h3z5v2Lu3Y3rfPu/W3bE34osut6LCO+n94ov9+hUUACIi/aWgoKOLayJnznQOhegwAE92UwCIiGRTcXHHLbcHmO7GJCIyRCkARESGKAWAiMgQpQAQERmiFAAiIkOUAkBEZIhSAIiIDFEKABGRIWpQPxTezJqAPX1YRBnQnKHm9Ae1r2/Uvr5R+/pmMLevxjlXnqzSoA6AvjKzNc65cLbbkYja1zdqX9+ofX0z2NuXCh0CEhEZohQAIiJDVK4HwDPZbkASal/fqH19o/b1zWBvX1I5fQ5AREQSy/U9ABERSSDwAWBmt5nZNjPbaWZPxJk/wsxe8ue/Y2ZTBrBtk83sP8xsi5ltMrPH49S50cxOmNk6f/jqQLUvpg0fmtkG//PXxJlvZvZtfx1+YGZXDWDbZsasm3VmdtLM/rJLnQFdh2a21MyOmNnGmLJSM1tuZjv8cUmC9y726+wws8UD2L5/MLOt/r/fv5vZuATv7fFvoR/b97dmtj/m3/COBO/t8f97P7bvpZi2fWhm6xK8t9/XX0Y55wI7APlAAzAVKADWA3Vd6vwZ8H1/+n7gpQFs30TgKn96NLA9TvtuBH6d5fX4IVDWw/w7gFcBA+YD72Tx3/sQXh/nrK1D4AbgKmBjTNnfA0/4008A34zzvlJglz8u8adLBqh9twDD/OlvxmtfKn8L/di+vwX+KoV//x7/v/dX+7rM/0fgq9laf5kcgr4HcC2w0zm3yzl3EXgRuKtLnbuA5/zpnwM3m5kNROOccwedc+/506eALUDVQHx2ht0FPO88q4BxZjYxC+24GWhwzvXl4sA+c86tBLo8yLXT39lzwN1x3norsNw51+KcOwYsB24biPY5537nnGv1X64CqjP9ualKsP5Skcr/9z7rqX3+tuOPgRcy/bnZEPQAqAL2xbxupPsGtr2O/x/gBDB+QFoXwz/0dCXwTpzZ15vZejN71czqB7RhHgf8zszWmtmSOPNTWc8D4X4S/8fL9jqscM4dBC/4gXgPdB0s6/FzeHt08ST7W+hPj/mHqJYmOIQ2GNbfIuCwc25HgvnZXH+9FvQAiPdLvmu3plTq9CszGwX8AvhL59zJLrPfwzukcQXwNPCrgWybb6Fz7irgduBRM7uhy/zBsA4LgDuBn8WZPRjWYSoGw3r8G6AV+LcEVZL9LfSXfwGmAfOAg3iHWbrK+voDHqDnX//ZWn9pCXoANAKTY15XAwcS1TGzYcBY0tv9TIuZDcfb+P+bc+6XXec750465077068Aw82sbKDa53/uAX98BPh3vF3tWKms5/52O/Cec+5w1xmDYR0Ch6OHxfzxkTh1sroe/ZPOHwcedP4B665S+FvoF865w865NudcBPg/CT432+tvGHAP8FKiOtlaf+kKegCsBqabWa3/C/F+YFmXOsuAaG+Le4E/JPrjzzT/eOEPgS3OuX9KUKcyek7CzK7F+zc5OhDt8z+z2MxGR6fxThZu7FJtGfAZvzfQfOBE9HDHAEr4yyvb69AX+3e2GHg5Tp3XgFvMrMQ/xHGLX9bvzOw24MvAnc65swnqpPK30F/tiz2n9MkEn5vK//f+9EfAVudcY7yZ2Vx/acv2Wei+Dng9VLbj9Q74G7/sa3h/6ACFeIcNdgLvAlMHsG3/BW8X9QNgnT/cAfwp8Kd+nceATXg9GlYBCwZ4/U31P3u9347oOoxtowHf9dfxBiA8wG0swtugj40py9o6xAuig8AlvF+lj+CdV1oB7PDHpX7dMPCDmPd+zv9b3Al8dgDbtxPv+Hn07zDaM24S8EpPfwsD1L4f+X9bH+Bt1Cd2bZ//utv/94Fon1/+bPRvLqbugK+/TA66ElhEZIgK+iEgERFJkwJARGSIUgCIiAxRCgARkSFKASAiMkQpAEREhigFgIjIEKUAEBEZov4/JDV8hhTtPhgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "lower=-20\n",
    "plt.plot(valid_loss_list[lower:], color='black', label='validation loss')\n",
    "plt.plot(train_loss_list[lower:], color='red', label='train loss')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xt8VPWd//HXJ4FwvwQCikm4eKtgTbFGNEGq1dVyLd5IrW217bZqV3f7W9dWbXd7cetiW3W7F1tXXay6XS1StRFBq269BIISFQUEK4lIQrgEQeQSCEk+vz/OSTKEhAzJTCYzvJ+PxzzmzDnfM+dzJpPvZ873fM/3mLsjIiKSlugARESkZ1BCEBERQAlBRERCSggiIgIoIYiISEgJQUREACUEEREJKSGIiAighCAiIqFeiQ7gSGRlZfnYsWMTHYaISFJ54403trn7iI7KJVVCGDt2LGVlZYkOQ0QkqZjZh9GUU5ORiIgASggiIhJSQhAREUAJQUREQkoIIiICKCGIiEhICUFERIAkuw5BROSoUVsL774L77wD69bB7bfHfZNKCCIiieQOH34YVPyRj/ffh8bGoEy/fnDTTZCZGddQlBBERLrLJ5/AypUHV/wrV8KuXS1ljj8e8vLgS18KnvPygnnp6XEPTwlBRCRe1q+HP/4R/u//gsp//fqWZUOGBJX9VVe1VPynngqDBiUqWiUEEZGYcQ9+8T/5JDz1FKxYEcw/+WQ4+2y45pqWyj8nB8wSG28rSggiIl3R0ABLlgQJ4Kmn4IMPgop+8mS4806YPRtOPDHRUUZFCUFE5EjV1sILLwQJoLgYtm2DjAy48EL4wQ9g1iw45phER3nElBBERKKxYwc880zQHPTss7B3b3AeYMYMuPhimDo1oe3/saCEICLSpLERamqgsjJ4VFUFz2++CS+/DPX1MGoUXH11kATOOy84MkgRSggicnRwDyr7pko+ssJvet64EerqDl4vIwNOOim4DuDii+HMMyEtNQd5UEIQkdTx8cdQUQHl5cFz0/T69UGlv3//weV794bsbMjNhYKCoOdPbm7Lc24uZGWlbAJoTQlBRJJHQ0NQsbdV6VdUwPbtB5fPygou6srPh0suObSyHznyqKnsoxFVQjCzqcC/AenAA+5+R6vlY4B5wAhgO/BVd6+KWD4YWAM86e43hPPOAH4L9AMWAd91d+/qDolIktu9u6Wij6zsm37pHzjQUrZXLxg7tqXSP+GEYLrpMXhwovYiKXWYEMwsHbgHuBCoApabWbG7vxtR7E7gYXd/yMzOB+YCX4tY/s/Ay63e+jfANcAygoQwFVjc2R0RkSTR2AibNrVd4VdUwNatB5fPzIRx42DiRLjsspbK/oQTgl/7vdTQESvRfJKTgHXuXgFgZo8Bs4HIhDAB+Ptw+s/AU00LwiOBY4Bngfxw3ihgsLuXhq8fBi5GCUEktezdC4sWwauvtlT6H3wA+/a1lElLg9Gjg0p+9uyDK/zjj4/7gG7SIpqEkA1URryuAs5qVeZt4DKCZqVLgEFmNhzYAdxFcLRwQav3rIp4XRXOE5FkV1sLixfD/Pnw9NNBUhgwILha95RTgn77kZX+6NEp1XUzmUWTENoabKN1W/9NwH+a2deBV4CNQD3wN8Aid6+0g8fsiOY9g4Jm1xA0LTF69OgowhWRbrdvX3CxVlMS2L0bRowIBm4rKoLPfa5bRuuUrokmIVQBuRGvc4DqyALuXg1cCmBmA4HL3H2nmRUAU8zsb4CBQIaZ7SY4ksg53HtGvPd9wH0A+fn5Ouks0lPs3w/PPRckgeLiYAjn4cPhyiuDJHDuuWrfTzLR/LWWAyeZ2TiCX/5XAFdGFjCzLGC7uzcCtxL0OMLdvxJR5utAvrvfEr7eZWZnA68BVwH/0eW9EZH42r8fnn8+SAJ//GMwvv+wYcHY/UVFwZW7vXsnOkrppA4TgrvXm9kNwHME3U7nuftqM7sNKHP3YuA8YK6ZOUGT0fVRbPs7tHQ7XYxOKIv0THv2BMM2zJ8fDOa2c2dwovfyy4MkcP75SgIpwpKp639+fr6XlZUlOgyR1OMOmzfDmjWwdu3Bj8qwT8mQIcHFXUVFcMEFOhGcRMzsDXfP76icGvhEjiZ1dUHXz9aV/tq1QfNPk0GDgh5B550XPJ9+enAk0KdPwkKX+FNCEEll27bB//5vcAvHNWuCZNDQ0LI8Jyeo8K+6Kng+5RQYPz4Y0bOH3c1L4k8JQSTV1NfDn/4E8+YFvX8OHIBPfQpOOw3mzGmp+D/1qaQfv19iSwlBJFX85S/w4IPw0EPB0BAjRsANN8A3vhEkA5EOKCGIJLNdu4LeP/PmwdKlwcVf06cHSWDGDJ34lSOihCCSbNyDsYHmzYPHHw+GhjjlFPjFL+CrXw3a/0U6QQlBJFlUVgbNQb/9bXByeNAg+MpXgqOBs8/WSWDpMiUEkZ6sri64IviBB4IrhN2DrqA//jFcemkwaJxIjCghiPREFRVw//1Bs9DWrcHdvf7xH+HrXw9GCRWJAyUEkZ7iwIFgpND/+q+g22haGsycCddeC1/4gkYLlbhTQhBJtA8/DJqE/vu/g+6iOTnwk5/AX/91MC3STZQQRBKhvj64icy99wbPANOmBa+nT9ew0ZIQ+taJdKeqquBI4IEHgulRo+CHP4RvfQvGjEl0dHKUU0IQibeGhuCcwL33wsKFQU+hiy6Cf//34ByBho6WHkIJQSTefvxjuP12GDkSvv99+Pa31VNIeiQlBJF4e+UVOPNMKCnRUBLSo6UlOgCRlFdeDp/+tJKB9HhKCCLxVFsL1dVqIpKkoIQgEk8ffBA8n3BCYuMQiYISgkg8lZcHzzpCkCSghCASTxUVwbOOECQJKCGIxFPTMNXDhyc6EpEOKSGIxFNFRXB0oHsVSBJQQhCJp/JynT+QpKGEIBIvjY1BLyOdP5AkoYQgEi/V1bB/v44QJGkoIYjEi3oYSZKJKiGY2VQze8/M1pnZLW0sH2NmL5rZO2b2kpnlRMx/w8xWmNlqM7suYp0vm9nKcJ1nzSwrdrsl0gPoGgRJMh0mBDNLB+4BpgETgC+b2YRWxe4EHnb3POA2YG44fxNQ6O4TgbOAW8zsODPrBfwb8PlwnXeAG2KxQyI9RkVFcNvL0aMTHYlIVKI5QpgErHP3CnevAx4DZrcqMwF4MZz+c9Nyd69z9/3h/D4R27PwMcDMDBgMVHd6L0R6ovLy4KY3ut+BJIloEkI2UBnxuiqcF+lt4LJw+hJgkJkNBzCzXDN7J3yPn7t7tbsfAL4DrCRIBBOA/+70Xoj0RBUVai6SpBJNQmjrihpv9fom4Fwzews4F9gI1AO4e2XYLHQicLWZHWNmvQkSwunAcQRNRre2uXGza8yszMzKampqotknkZ6hvFwnlCWpRJMQqoDciNc5tGreCX/1X+rupwM/DOftbF0GWA1MASaG88rd3YH5QGFbG3f3+9w9393zR4wYEd1eiSTaJ5/Atm06QpCkEk1CWA6cZGbjzCwDuAIojixgZllm1vRetwLzwvk5ZtYvnM4EJgPvERxBTDCzphr+QmBNV3dGpMdQl1NJQh3eQtPd683sBuA5IB2Y5+6rzew2oMzdi4HzgLlm5sArwPXh6uOBu8L5Btzp7isBzOynwCtmdgD4EPh6TPdMJJHU5VSSUFT3VHb3RcCiVvN+FDG9AFjQxnrPA3ntvOe9wL1HEqxI0mg6QlBCkCSiK5VF4qG8PBjyesiQREciEjUlBJF4aBr2WiSJKCGIxIOGvZYkpIQgEmv19fDhhzpCkKSjhCASaxs2QEODjhAk6SghiMSarkGQJKWEIBJrTdcgKCFIklFCEIm1igro0weOOy7RkYgcESUEkVgrL4dx4yBN/16SXPSNFYk1DXstSUoJQSSW3DXstSQtJQSRWProo2Doax0hSBJSQhCJJXU5lSSmhCASSxr2WpKYEoJILDUdIYwbl9g4RDpBCUEklsrLYdQo6N8/0ZGIHDElBJFY0rDXksSUEERiScNeSxJTQhCJlX37YONGHSFI0lJCEImV9euDC9N0hCBJSglBJFZ0DYIkOSUEkVjRsNeS5JQQRGKlogIGDIARIxIdiUinKCGIxErToHZmiY5EpFOUEERiRcNeS5JTQhCJBXddlCZJTwlBJBY2bYLaWh0hSFKLKiGY2VQze8/M1pnZLW0sH2NmL5rZO2b2kpnlRMx/w8xWmNlqM7suYp0MM7vPzP5iZmvN7LLY7ZZIN1OXU0kBvToqYGbpwD3AhUAVsNzMit393YhidwIPu/tDZnY+MBf4GrAJKHT3/WY2EFgVrlsN/BDY6u4nm1kaMCy2uybSjTTstaSADhMCMAlY5+4VAGb2GDAbiEwIE4C/D6f/DDwF4O51EWX6cPARyTeBU8JyjcC2TsQv0jNUVEBaGowZk+hIRDotmiajbKAy4nVVOC/S20BTk88lwCAzGw5gZrlm9k74Hj9392ozGxqW/Wcze9PMHjezYzq9FyKJVl4OubmQkZHoSEQ6LZqE0Fanam/1+ibgXDN7CzgX2AjUA7h7pbvnAScCV4cVfy8gB1ji7p8FSgmanQ7duNk1ZlZmZmU1NTXR7JNI91MPI0kB0SSEKiA34nUOUB1ZwN2r3f1Sdz+d4NwA7r6zdRlgNTAF+AjYCzwZLn4c+GxbG3f3+9w9393zR+gKUOmpNOy1pIBoEsJy4CQzG2dmGcAVQHFkATPLCk8MA9wKzAvn55hZv3A6E5gMvOfuDjwNnBeucwEHn5MQSR67d8PWrTpCkKTX4Ulld683sxuA54B0YJ67rzaz24Aydy8mqNjnmpkDrwDXh6uPB+4K5xtwp7uvDJfdDDxiZr8CaoBvxHC/RLpPU5dTHSFIkoumlxHuvghY1GrejyKmFwAL2ljveSCvnff8EPjckQQr0iPpGgRJEbpSWaSrNOy1pAglBJGuqqiAzEwYOrTjsiI9mBKCSFc1DXstkuSUEES6SsNeS4pQQhDpioYGWL9eRwiSEpQQRLqishIOHNARgqQEJQSRrlCXU0khSggiXaFhryWFKCGIdEVFBfTuDTk5iY5EpMuUEES6orwcxo6F9PRERyLSZUoIIl2hYa8lhSghiHSFhr2WFKKEINJZO3bAxx/rCEFShhKCSGeph5GkGCUEkc7SNQiSYpQQRDpLRwiSYpQQRDqrogKOOQYGDEh0JCIxoYQg0lka9lpSjBKCSGdp2GtJMUoIIp1RVxeMdKojBEkhSgginbF+PTQ26ghBUooSgkhnqMuppCAlBJHOUJdTSUFKCCKdUVEB/frBsccmOhKRmFFCEOmMpkHtzBIdiUjMKCGIdIaGvZYUpIQgcqTcdQ2CpKSoEoKZTTWz98xsnZnd0sbyMWb2opm9Y2YvmVlOxPw3zGyFma02s+vaWLfYzFZ1fVdEusnWrbBnj44QJOV0mBDMLB24B5gGTAC+bGYTWhW7E3jY3fOA24C54fxNQKG7TwTOAm4xs+Mi3vtSYHeX90KkOzX1MFJCkBQTzRHCJGCdu1e4ex3wGDC7VZkJwIvh9J+blrt7nbvvD+f3idyemQ0EbgR+1vnwRRKg6RoENRlJiokmIWQDlRGvq8J5kd4GLgunLwEGmdlwADPLNbN3wvf4ubtXh+X+GbgL2NvJ2EUSo7w86F00dmyiIxGJqWgSQlv96rzV65uAc83sLeBcYCNQD+DulWFT0onA1WZ2jJlNBE509yc73LjZNWZWZmZlNTU1UYQrEmcVFZCTA336JDoSkZjqFUWZKiA34nUOUB1ZIPzVfyk0NwVd5u47W5cxs9XAFGAEcIaZrQ9jGGlmL7n7ea037u73AfcB5Ofnt05EIt1Pw15LioomISwHTjKzcQS//K8ArowsYGZZwHZ3bwRuBeaF83OAj9y91swygcnA3e6+APhNWGYssLCtZCDSI1VUwLRpiY5CeqidO3dSVVVFZWUllZWVVFVVsW/fPs4880wKCgrIzm7d4t5zdJgQ3L3ezG4AngPSgXnuvtrMbgPK3L0YOA+Ya2YOvAJcH64+HrgrnG/Ane6+Mg77IdI99u6FTZt0hHCU2rVr1yGVfevpXbt2HbSOmdGrVy8OHDgAQG5uLoWFhRQUFFBQUMDEiRPJyMhIxO4cwtyTpxUmPz/fy8rKEh2GHM1WrYLTToNHH4Urrkh0NCmtvr6el19+mfnz57N27VqOO+44cnNzycnJITc3t3l65MiRpKV1/RrbPXv2dFjZ79x5UEs4ZsYxxxzTZlxN06NGjcLdWbFiBUuXLqW0tJTS0lIqK4O+On379iU/P5+CgoLmRHHMMcd0eX9axfmGu+d3WE4JQeQIFBfD7Nnw+utw5pmJjiblNDQ08MorrzB//nz+8Ic/UFNTw4ABA5g4cSKbN2+mqqqK/fv3H7RORkYG2dnZ7VbIOTk5DBw4kI0bNx62st+xY8ch8YwcOfKwlf1xxx3X6V/3VVVVzcmhtLSUN954o/koYty4cQcdReTl5dGrVzQt/G1TQhCJh3/9V7jxRti2DYYPj2qVn/zkJ9xzzz1MmjSp+R980qRJDBo0KM7BJoeGhgZKSkqak8CWLVvo378/s2bNoqioiGnTptGvXz8A3J1t27YdtmKvqqpqrlgPJysrq8PKvm/fvvHe/Wb79u3jzTffpLS0tPlIYtOmTQD079+fDRs2MDzK71xrSggi8fC3fwuPPAI7dkQ90un48ePZs2cPAwcOZM2aNQCkpaVx2mmnHdRMcMIJJ2A9bPTUffv2sXHjRmpra8nJyWHIkCExibGxsZElS5Ywf/58FixYwObNm+nXrx8zZ86kqKiI6dOn079//06/d01NzUGJYvfu3WRnZzdX9tnZ2c1JpqdydzZs2EBpaSkrV67k9ttv7/R7KSGIxMP06bB5M7z5ZlTFa2pqGDlyJHfccQc333wzO3bs4LXXXmv+Bfjaa681n4QcMWJE8xFEYWEh+fn5na4Uo1FXV9dhM0rra38GDhx42F/VOTk5DB48uM3tNTY2Ulpa2pwEqqur6du3LzNmzKCoqIgZM2YwYMCAuO3v0SzahND5RimRo1FFRXBSOUpLly4F4JxzzgEgMzOTqVOnMnXqVCBoLlm9enVzO/LSpUspLi4GoFevXnzmM5+hoKCAU045pUsnTnfv3n1IZb9ly5ZDyg0dOrS5gs/Pz2+e7tev30FNMpWVlaxatYrNmzfT+kfl4MGDD0kU27dvZ8GCBWzcuJE+ffowffp0ioqKmDlzJgMHDuz0fklsKSGIRKuhAT74IDipHKWSkhL69OlDfn7bP87S09PJy8sjLy+Pa6+9FoBt27axbNmy5qOIefPmsXdv10d4GTx4cHMlPXHixDZ/5R9p5VxXV8emTZvaPcpYsWIFW7ZsISMjg2nTpvGLX/yCWbNm6fxJD6WEIBKt6mqoqzuiaxBKSko488wz6XMEw1xkZWUxc+ZMZs6cCQTdLz/66KMjDjdSv3792m3K6YqMjAzGjBnDmDFj2i2zf/9+Ghsbe3ybvSghiETvCIe93rt3L2VlZdx0001d2myvXr1i3i+9Ox1JMpTE0h3TRKJ1hMNev/7669TX1zefPxDp6ZQQRKJVXg69ekFubsdlCZqLzIzCwsI4ByYSG0oIItGqqIAxY4KkEIWSkhI+/elPk5mZGefARGJDCUEkWkcw7HVDQwNLly5Vc5EkFSUEkWhVVER9/mDlypXs2rVLCUGSihKCSDR27oSPPor6COHVV18FUEKQpKKEIBKNpi6nUR4hlJSUkJuby+jRo+MYlEhsKSGIRKOpy2kURwjuTklJCVOmTIlzUCKxpYQgEo0jOEJYv3491dXVai6SpKOEIBKNigoYMQKiGIOnpKQE0PkDST5KCCLRKC8/ovMHQ4YM4dRTT41zUCKxpYQgEo2KiiPqYTR58uSY3OdXpDvpGyvSkQMHYMOGqI4Qtm3bxpo1a9RcJElJo532BO6we3dwW8YdO2D79pbpptf19ZCR0fLo3bvt6fZeDxsGo0dHPexCl9XVwbp1sHZt8Ni9O4ghM/PgR9O8gQOjviVlh9xh//5gm3v2BM8DBgTDTnRmGxs2BPdCiOIIofUNcUSSiRJCPNXVwapV8MYbQaXSXmX/8cdBhd+e9PSgUj9wIKiYOis9PagUTzgh+LXb9Nw03Znx8nfsaKn0Ix/l5QfHmp5++Nh79YKhQw9OEpGP3r1bKve2nlvPa2w8dBuDBwd3O8vLO/i5o/0+gmGvS0pKyMjI4Mwzz+ywrEhPo4QQKwcOwOrVUFYWJICyMnjnnSApAKSltVR4TZXe2LFt/1pu/Rg0qOWXbUNDsK0DB4L3rqs7eLr16wMHgl/LNTVBxVZRETwWLAiuvI00fHj7yaKxEdasObTij7wNY0YGnHRSUMkWFcEppwSPk08OjgB27To4GbaVIJvmbdsG778fvP7442D7ffsG7zNgwMHPw4a1PT/y+eOPYeXK4G/y6KPwm9+0xD12bJAcmh6nnRbsR3p6sPwIhr1uuiFO3759O/U1EkkkJYTOqK+Hd989uPJ/++2g4gUYMgTOOAO++13Izw8eY8cGSaGr0tODRywqnJ07WxJEZLJ4/XV4/PH2f9EPGwbjx8PMmS2V/imnBPt4uCapwYODx2HurtWmxsagGaipgu4qd6iqCpJD5OOZZ1r2uW9fOPXUIEFUVASvR4067NvW1tZSVlbGjTfeGJs4RbqZEkJH3OEvf4HS0pbKf8UK2LcvWD5oUFD533BDS+V//PGxqfzjbcgQOP304NFafX3QzNWULNLSgkp//HjIyureOGP9WZoF9zTIzYUZM1rm79sXHPVEJolFi4KjoEmTOoxj+fLlHDhwQOcPJGkpIbRl61Z44YXg8fzzwa9JCJofPvtZ+M53Wir/E09Mjsr/SPXq1dJk9Fd/lehoukffvjBxYvCItHVrVEdkTQPa6YY4kqyiSghmNhX4NyAdeMDd72i1fAwwDxgBbAe+6u5V4fwnwvV6A//h7veaWX/gceAEoAF42t1vidE+Hbm9e6GkJKj8n38+aP6BoP3+/PPhH/8RPve5oC08Vs0WkjxGjoyqWElJCaeeeirDhg2Lc0Ai8dFhQjCzdOAe4EKgClhuZsXu/m5EsTuBh939ITM7H5gLfA3YBBS6+34zGwisMrNi4GPgTnf/s5llAC+a2TR3Xxzb3WtHYyO89VZLAliyJGj/790bJk+G22+HCy8MjgaUACQKTTfEufLKKxMdikinRXOEMAlY5+4VAGb2GDAbiEwIE4C/D6f/DDwF4O51EWX6EF4I5+57w3K4e52ZvQnkdH43orB+fUsCePHFoCcLBD1Krr8+SABTpgQ9UkSO0KpVq/jkk090/kCSWjQJIRuojHhdBZzVqszbwGUEzUqXAIPMbLi7f2RmucAzwInA99y9OnJFMxsKzArXPYSZXQNcA3R+bPmzzgp6zgAcdxzMmhUkgAsugGOP7dx7ikTQgHaSCqJJCG1d2umtXt8E/KeZfR14BdgI1AO4eyWQZ2bHAU+Z2QJ33wJgZr2AR4F/bzoCOWRD7vcB9wHk5+e33m50LrkEvvKV4OTo+PGxuyJWJFRSUkJOTo5uiCNJLZqEUAXkRrzOAQ76lR/+6r8UIDxXcJm772xdxsxWA1OABeHs+4D33f1XnQs/Srck7ny1pD5359VXX2XKlCmYfmxIEosmISwHTjKzcQS//K8ADjpzZmZZwHZ3bwRuJehxhJnlAB+5e62ZZQKTgbvDZT8DhgDfitG+tOu6666jsbGRgoICCgsLOfnkk/WPKzHz4YcfsnHjRjUXSdLrMCG4e72Z3QA8R9B9dJ67rzaz24Aydy8GzgPmmpkTNBldH64+HrgrnG8EPYtWhonih8Ba4M2wcv5Pd38gtrsX2L59O3/605+4//77ARg2bBgFBQXNj0mTJjFw4MB4bFqOAjp/IKnC3DvXLJ8I+fn5XlZW1ql1GxsbWbt2LaWlpSxdupTS0lLWrFkDQFpaGnl5eRQWFjYnieOPP15HERKV6667jscee4yPPvqIdHVTlh7IzN5w9/wOyx0tCaEtO3bsYNmyZZSWllJaWsqyZcvYvXs3ACNHjmxODoWFhRqwTNr16U9/mtGjR7No0aJEhyLSpmgTwlE9dEVmZibTpk1j2rRpQHBx0erVqw86ivjjH/8IwKBBg5g9ezZFRUVcdNFF9OnTJ5GhSw+xfft2Vq9erQvSJCUc1QmhtfT0dPLy8sjLy+Paa68FgjtgLV26lKeffponnniC//mf/2Hw4MFcfPHFFBUVceGFF5KRkZHgyCVRlixZAuj8gaSGFByVLbaysrL44he/yP3338/mzZt59tlnufzyyykuLmbmzJmMHDmSb3zjGyxevJi6urqO31BSSklJCb1799YNcSQlHNXnELqirq6OF198kfnz5/Pkk0+yc+dOMjMzueSSSygqKuL888+nd+/eiQ5T4mzy5Mm4e/OtM0V6omjPIegIoZMyMjKYNm0aDz74IFu2bGHhwoXMmjWLBQsWMHXqVI499li+/e1v8/zzz1N/uNtjStKqra1l+fLlTJkyJdGhiMSEEkIM9OnThxkzZvDQQw+xdetWiouLmT59Or///e+56KKLGDVqFHPnziWZjsakY2VlZbohjqQUJYQY69OnD7NmzeKRRx5h69atPPXUU5x99tn84Ac/4Fvf+paOFlJI0wVpuiGOpAolhDjq27cvs2fPpri4mH/6p39i3rx5XHzxxezZsyfRoUkMvPrqq0yYMIHhw4cnOhSRmFBC6AZmxm233ca9997L4sWLueCCC9i2bVuiw5IuaLohjpqLJJUoIXSja6+9lj/84Q+8/fbbTJ48mQ8++CDRIUknrV69mp07dyohSEpRQuhmF198MS+88AI1NTUUFhby1ltvJTok6YSm8wfqYSSpRAkhASZPnsySJUvo3bs35557Li+88EKiQ5IjVFJSQnZ2NmPGjEl0KCIxo4SQIOPHj6e0tJSxY8cyffp0Hn300USHJEegpKSEc845RyPiSkpRQkig7OxsXnnlFQoLC7nyyiu56667Eh2SROHDDz+ksrJS5w8k5SghJNjQoUN59tlnmTNnDjfddBP/8A/mQzMEAAAL20lEQVT/QGNjY6LDksPQDXEkVWm00x6gb9++PPbYY4waNYq7776bTZs28eCDD2qI7R6qpKSEQYMGcdpppyU6FJGYUkLoIdLS0vjVr35FdnY2N998M1u3buWJJ55g8ODBiQ5NWikpKaGwsFB3R5OUoyajHsTM+P73v8/DDz/Myy+/zLnnnsumTZsSHZZE2LFjB6tWrVJ3U0lJSgg90Ne+9jUWLlzI+++/T2FhIe+9916iQ5JQ0zDXOn8gqUhNRj3UF77wBV566SVmzJjB5MmTeeihh5g6dWrSNFPU19dTXV1NVVUVlZWV7Nq1q0vvd/rpp3PGGWfEKLrO0w1xJJUpIfRg+fn5LF26lKlTpzbfne3yyy+nqKiIc845J2HJoaGhgc2bN1NZWUllZWVzpR85vWnTppj2lkpLS2Pu3Ll873vfS2jf/1dffZUzzjiD/v37JywGkXjRHdOSQG1tLYsWLWL+/PksXLiQvXv3cuyxxzYnh8mTJ5OWFvvWv48//pjXXnuN0tJS1q5d21zpV1dX09DQcFDZ/v37k5ubS05ODrm5uYdMDxkypNMV+YEDB7j55pt5/PHHufTSS3nwwQcTcrJ93759DBkyhL/7u7/jl7/8ZbdvX6Szor1jmhJCktmzZ09zcnjmmWeora1l1KhRzJkzh6KiIgoKCjqVHBobG3nvvfcoLS2ltLSUpUuXsmbNGtydtLQ0jj/++ObKva2KPzMzM66/3N2du+++m5tvvpmTTjqJJ554gvHjx8dte20pKSlhypQpPPXUU8yePbtbty3SFUoIR4Hdu3fzzDPPMH/+fBYtWsS+ffvIzs5uTg5nnXVWu8lh165dvP766yxdupTS0lKWLVvGjh07AMjMzKSgoKD5MWnSJAYNGtSdu9aul156iS996Uvs3buXBx98kMsvv7zbtn3HHXdw6623UlNTQ1ZWVrdtV6SrlBCOMrt27WLhwoXMnz+fxYsXs3//fnJzc5kzZw5z5sxh+PDhB/36X7VqFY2NjZgZEyZMoKCggMLCQgoKCjj55JPj0gQVK1VVVcyZM4dly5bxve99j3/5l3+hV6/4nw6bOXMmFRUVvPvuu3HflkgsKSEcxT755BOKi4uZP38+zz33HHV1dc3LBg8ezNlnn9386/+ss85i6NChCYy2c/bv38+NN97Ir3/9az7/+c/z2GOPMXLkyLhtr7GxkeHDhzNnzhzuu+++uG1HJB5imhDMbCrwb0A68IC739Fq+RhgHjAC2A581d2rwvlPhOv1Bv7D3e8N1zkD+C3QD1gEfNc7CEYJ4cjt3LmTp59+mtraWgoKChg/fnzSdF2NxsMPP8y1115LVlYWCxYs4KyzzorLdlauXEleXh4PPfQQV111VVy2IRIv0SYE3P2wD4LKvBw4HsgA3gYmtCrzOHB1OH0+8Eg4nQH0CacHAuuB48LXrwMFgAGLgWkdxXLGGWe4SGtvvvmmjxs3znv37u2/+c1vvLGxMebb+PWvf+2Al5eXx/y9ReINKPMO6ld3j+pK5UnAOnevcPc64DGgdReLCcCL4fSfm5a7e5277w/n9yG8MtrMRgGD3b00DPZh4OIoYhE5xOmnn05ZWRkXXHAB3/nOd/jmN79JbW1tTLdRUlLCqFGjGDduXEzfV6QniSYhZAOVEa+rwnmR3gYuC6cvAQaZ2XAAM8s1s3fC9/i5u1eH61d18J6E619jZmVmVlZTUxNFuHI0GjZsGAsXLuRHP/oRv/3tb2N+z2rdEEeOBtEkhLb+A1q39d8EnGtmbwHnAhuBegB3r3T3POBE4GozOybK9yRc/z53z3f3/BEjRkQRrhyt0tPT+elPf8rTTz/NBx98QH5+Ps8991yX33fDhg1s2LBBA9pJyosmIVQBuRGvc4DqyALuXu3ul7r76cAPw3k7W5cBVgNTwvfMOdx7inTWzJkzKSsrIycnh2nTpvGzn/2sS8NoLFmyBNCAdpL6OuxlZGa9gL8AFxD88l8OXOnuqyPKZAHb3b3RzG4HGtz9R2aWA3zk7rVmlgm8Blzm7ivNbDnwt+G8RQQ9kBYdLhb1MpIjsXfvXq655hp+97vfMXbs2E6PP1RTU8O+ffvYvn17t1zvIBJr0fYy6vDb7e71ZnYD8BxBj6N57r7azG4jOHNdDJwHzDUzB14Brg9XHw/cFc434E53Xxku+w4t3U4Xhw+RmOnfvz+PPPII5513Xpebjj7/+c8rGUjK04VpIiIpLtojhJ47PoGIiHQrJQQREQGUEEREJKSEICIigBKCiIiElBBERARQQhARkZASgoiIAEl2YZqZ1QAfdnL1LGBbDMOJNcXXNYqvaxRf1/T0+Ma4e4ejgyZVQugKMyuL5kq9RFF8XaP4ukbxdU1Pjy9aajISERFACUFEREJHU0K4L9EBdEDxdY3i6xrF1zU9Pb6oHDXnEERE5PCOpiMEERE5jJRLCGY21czeM7N1ZnZLG8v7mNnvw+WvmdnYbowt18z+bGZrzGy1mX23jTLnmdlOM1sRPn7UXfGF219vZivDbR9y8wkL/Hv4+b1jZp/txtg+FfG5rDCzT8zs/7Uq062fn5nNM7OtZrYqYt4wM3vezN4PnzPbWffqsMz7ZnZ1N8b3SzNbG/79njSzoe2se9jvQhzj+4mZbYz4G05vZ93D/q/HMb7fR8S23sxWtLNu3D+/mHP3lHkQ3NGtHDgeyADeBia0KvM3wL3h9BXA77sxvlHAZ8PpQQS3Jm0d33nAwgR+huuBrMMsn05wdzsDzgZeS+DfejNB/+qEfX7A54DPAqsi5v0CuCWcvgX4eRvrDQMqwufMcDqzm+K7COgVTv+8rfii+S7EMb6fADdF8fc/7P96vOJrtfwu4EeJ+vxi/Ui1I4RJwDp3r3D3OuAxYHarMrOBh8LpBcAFZmbdEZy7b3L3N8PpXcAaILs7th1Ds4GHPbAMGGpmoxIQxwVAubt39kLFmHD3V4DtrWZHfsceAi5uY9UvAM+7+3Z33wE8D0ztjvjc/U/uXh++XAbkxHq70Wrn84tGNP/rXXa4+MJ6owh4NNbbTZRUSwjZQGXE6yoOrXCby4T/FDuB4d0SXYSwqep04LU2FheY2dtmttjMTu3WwMCBP5nZG2Z2TRvLo/mMu8MVtP+PmMjPD+AYd98EwY8AYGQbZXrK5/hN2r+feUffhXi6IWzSmtdOk1tP+PymAFvc/f12lify8+uUVEsIbf3Sb92NKpoycWVmA4E/AP/P3T9ptfhNgmaQzwD/ATzVnbEBk939s8A04Hoz+1yr5T3h88sAvgg83sbiRH9+0eoJn+MPgXrgd+0U6ei7EC+/AU4AJgKbCJplWkv45wd8mcMfHSTq8+u0VEsIVUBuxOscoLq9MmbWCxhC5w5ZO8XMehMkg9+5+xOtl7v7J+6+O5xeBPQ2s6zuis/dq8PnrcCTBIfmkaL5jONtGvCmu29pvSDRn19oS1MzWvi8tY0yCf0cw5PYM4GveNjg3VoU34W4cPct7t7g7o3A/e1sN9GfXy/gUuD37ZVJ1OfXFamWEJYDJ5nZuPBX5BVAcasyxUBTj47Lgf9r7x8i1sI2x/8G1rj73e2UObbpnIaZTSL4G33UTfENMLNBTdMEJx9XtSpWDFwV9jY6G9jZ1DzSjdr9ZZbIzy9C5HfsauCPbZR5DrjIzDLDJpGLwnlxZ2ZTgZuBL7r73nbKRPNdiFd8keekLmlnu9H8r8fTXwFr3b2qrYWJ/Py6JNFntWP9IOgF8xeCHgg/DOfdRvDlB+hL0NSwDngdOL4bYzuH4LD2HWBF+JgOXAdcF5a5AVhN0GtiGVDYjfEdH2737TCGps8vMj4D7gk/35VAfjf/ffsTVPBDIuYl7PMjSEybgAMEv1r/muCc1IvA++HzsLBsPvBAxLrfDL+H64BvdGN86wja35u+g0297o4DFh3uu9BN8T0SfrfeIajkR7WOL3x9yP96d8QXzv9t03cuomy3f36xfuhKZRERAVKvyUhERDpJCUFERAAlBBERCSkhiIgIoIQgIiIhJQQREQGUEEREJKSEICIiAPx/M6QEjQtS0NEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(valid_accu_list[lower:], color='black', label='validation accuracy')\n",
    "plt.plot(train_accu_list[lower:], color='red', label='train accuracy')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "rand_batch=np.random.randint(train_num//batch_size)\n",
    "grad_lr=train_batch(rand_batch,parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:5: DeprecationWarning: object of type <class 'float'> cannot be safely interpreted as an integer.\n",
      "  \"\"\"\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65c2b16709fe4e8caa58acfb61353bdb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=20), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr_list=[]\n",
    "lower=-2.75\n",
    "upper=-0.75\n",
    "step=0.1\n",
    "for lr_pow in tqdm_notebook(np.linspace(lower,upper,num=(upper-lower)//step+1)):\n",
    "    learn_rate=10**lr_pow\n",
    "    parameters_tmp=combine_parameters(parameters,grad_lr,learn_rate)\n",
    "    train_loss_tmp=train_loss(parameters_tmp)\n",
    "    lr_list.append([lr_pow,train_loss_tmp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[-2.0, 1071.0215374352226],\n",
       " [-1.8947368421052633, 1070.786844251847],\n",
       " [-1.7894736842105263, 1070.4901468441062],\n",
       " [-1.6842105263157894, 1070.115911661122],\n",
       " [-1.5789473684210527, 1069.6452597476195],\n",
       " [-1.473684210526316, 1069.0556162389337],\n",
       " [-1.368421052631579, 1068.3206135371859],\n",
       " [-1.263157894736842, 1067.4105319461148],\n",
       " [-1.1578947368421053, 1066.2937689759572],\n",
       " [-1.0526315789473686, 1064.9401698902805],\n",
       " [-0.9473684210526316, 1063.3276088929422],\n",
       " [-0.8421052631578947, 1061.454112527371],\n",
       " [-0.736842105263158, 1059.3592726169363],\n",
       " [-0.6315789473684212, 1057.1610460832171],\n",
       " [-0.5263157894736843, 1055.117864794632],\n",
       " [-0.42105263157894735, 1053.7323452520732],\n",
       " [-0.3157894736842106, 1053.923921545092],\n",
       " [-0.2105263157894739, 1057.3179502805692],\n",
       " [-0.10526315789473695, 1066.7377260820094],\n",
       " [0.0, 1087.0599696072613]]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XucjeX+//HXZ4ZxGIc9MaIYY+SUUzQkm5lMoiilVFu7kxJ717dz+4vdYX/b9i58O9jfvdukUMkhpZKsEbaSlORQjBBaQ0JIkWgGc/3+WGv8hoZZw8y615p5Px+Pecxyr3td9+dec5v33Nd9res25xwiIiIxXhcgIiKRQYEgIiKAAkFERIIUCCIiAigQREQkSIEgIiKAAkFERIIUCCIiAigQREQkqILXBRRH7dq1XXJystdliIhEleXLl+92ziUWtV5UBUJycjLLli3zugwRkahiZptDWU9dRiIiAigQREQkSIEgIiKAAkFERIIUCCIiAigQREQkSIEgIiKAAkFEJKKtWbOGxx57jB07dpT6thQIIiIRbMmSJQwfPpyDBw+W+rYUCCIiEczv9xMbG0uDBg1KfVsKBBGRCOb3+2nQoAEVKpT+TEMKBBGRCOb3+2nUqFFYtqVAEBGJYAoEERHh4MGD7NixQ4EgIlLeZWdnA4Gp/8NBgSAiEqH8fj+AzhBERMq7iAwEM7vXzLLMbI2Z3Vdg+d1mtj64fFRwWUUze9nMVpvZWjMbdoI2LzazFWb2uZl9ZGbnlMwuiYiUDdnZ2VSqVIm6deuGZXtFDmw1s1bAHUBHIBeYY2azgfrAlUAb51yOmdUJvuRaoJJzrrWZVQW+NLOpzrns45oeA1zpnFtrZncCjwC3lsROiYiUBX6/n+TkZGJiwtOZE8onHVoAS5xzBwDMbCHQF0gFRjjncgCcczuD6zsg3swqAFUIhMi+Qtp1QI3g45rAtlPdCRGRsiicQ04htC6jLCDNzGoF/+LvBTQAmgJdzexTM1toZh2C678B/AxsB7YATznn9hTS7kDAZ2ZbgZuAEYVt3MwGmdkyM1u2a9euYu2ciEg0i7hAcM6tBUYC84A5wBfAYQJnFwlAJ+BPwHQzMwJdS0eAs4BGwINmllJI0/cDvZxz9YGJwDMn2P4451yqcy41MTGxmLsnIhKd9u7dyw8//BBZgQDgnBvvnGvvnEsD9gAbgK3Amy5gKZAH1AZuAOY45w4Fu5EWE+heOsrMEoG2zrlPg4teAzqXyB6JiJQB4R5hBKGPMqoT/J4EXA1MBd4GMoLLmwJxwG4C3UQZFhBP4Axi3XFN/gDUDL4O4BJg7entiohI2eFFIIQ6fd4MM6sFHALucs79YGYTgAlmlkXgwvEtzjlnZs8R6ALKAgyY6JxbBWBmPmCgc26bmd0RbDePQEDcVrK7JiISvSI2EJxzXQtZlgvcWMjy/QSGnhbWTq8Cj98C3gq5UhGRcsTv91OjRg0SEhLCtk19UllEJALljzAKjNUJDwWCiEgECveQU1AgiIhEHOcc2dnZYZvlNJ8CQUQkwuzcuZMDBw7oDEFEpLzzYoQRKBBERCJO/o1xFAgiIuVc/hmCriGIiJRzfr+fxMREqlWrFtbtKhBERCKMF0NOQYEgIhJxFAgiIsKRI0fYsmWLAkFEpLz79ttvOXTokAJBRKS88+ozCKBAEBGJKAoEEREBAoFgZiQlJYV92woEEZEI4vf7Ofvss4mLiwv7thUIIiIRxKshp6BAEBGJKAoEEREhJyeHbdu2KRBERMq7LVu24JxTIIiIlHdeDjkFBYKISMRQIIiICBAIhIoVK3LWWWd5sn0FgohIhPD7/TRs2JDY2FhPtq9AEBGJEF4OOQUFgohIxFAgiIgI+/fvZ/fu3QoEEZHyzusRRqBAEBGJCFETCGZ2r5llmdkaM7uvwPK7zWx9cPmo4LKKZvayma02s7VmNuwEbZqZ/d3Mvgqud0/J7JKISPTJD4Tk5GTPaqhQ1Apm1gq4A+gI5AJzzGw2UB+4EmjjnMsxszrBl1wLVHLOtTazqsCXZjbVOZd9XNO3Ag2A5s65vAKvFxEpd/x+P1WrViUxMdGzGooMBKAFsMQ5dwDAzBYCfYFUYIRzLgfAObczuL4D4s2sAlCFQIjsK6TdPwI3OOfyjnu9iEi5kz/CyMw8qyGULqMsIM3MagX/4u9F4C/7pkBXM/vUzBaaWYfg+m8APwPbgS3AU865PYW02xi43syWmVmmmTU57b0REYlS2dnZnl4/gBACwTm3FhgJzAPmAF8AhwmcXSQAnYA/AdMtEG0dgSPAWUAj4EEzSymk6UrAL865VOAFYEJh2zezQcHQWLZr165i7p6ISORzznn+GQQI8aKyc268c669cy4N2ANsALYCb7qApUAeUBu4AZjjnDsU7AZaTKB76XhbgRnBx28BbU6w7XHOuVTnXKqXfWsiIqVlz549/PTTT9ERCPkXfM0sCbgamAq8DWQElzcF4oDdBLqJMoKjiOIJnEGsK6TZo68H0oGvTn03RESiVyQMOYXQLioDzDCzWsAh4C7n3A9mNgGYYGZZBC4c3+Kcc2b2HDCRwLUHAyY651YBmJkPGOic2waMACab2f3AfmBgie6ZiEiUiKpAcM51LWRZLnBjIcv3Exh6Wlg7vQo8/hHoHXKlIiJlVKQEgj6pLCLiMb/fzxlnnEGNGjU8rUOBICLisUgYYQQKBBERzykQRESEvLy8iPhQGigQREQ8tX37dnJzcxUIIiLlXSTMcppPgSAi4qFIGXIKCgQREU/pDEFERIDALKf16tWjcuXKXpeiQBAR8VKkDDkFBYKIiKcUCCIiwqFDh/jmm28UCCIi5d0333xDXl6eAkFEpLyLpCGnoEAQEfGMAkFERIBAIMTGxlK/fn2vSwEUCCIinvH7/SQlJVGhQqg3ryxdkVFFKdu4cSMA9erVIz4+3uNqREQCImnIKZSTQLj33nvx+XwAVK9enXr16h39qlu37jH/zv9KSEjAzDyuXETKMr/fz+WXX+51GUeVi0B49NFHue6669ixYwfbt28/+rVs2TK2b9/Ozz///KvXxMXFHRMWZ599NsnJyTRs2JCGDRuSnJxMYmKiQkNETsmBAwf47rvvImIOo3zlIhA6depEp06dTvj8/v37jwmK/K/8ANm4cSMffPABP/744zGvq1KlytGAyA+Jgo/r1atHTIwu04jIr2VnZwORM8IIykkgFKVatWo0adKEJk2anHS9vXv3snnzZrKzs9m8efMxj5cvX87u3buPWb9ixYo0aNCA5ORkUlJSaN68Oc2aNaN58+YkJydHzIUkEQm/SBtyCgqEYqlZsyZt2rShTZs2hT7/888//yoo8h/PnDmTF1988ei6cXFxNGnS5GhA5IdFs2bNqFmzZrh2SUQ8ojOEMi4+Pp5zzz2Xc889t9Dnv//+e9avX8/69etZt24d69atY82aNcycOZMjR44cXa9evXq/CoqWLVtSv359XbMQKSP8fj+VK1embt26XpdylAIhjGrVqkXnzp3p3LnzMctzc3P5+uuvjwZF/vfXXnuNH3744eh6tWvXpn379sd8paSkKCREopDf7yc5OTmi/v+ac87rGkKWmprqli1b5nUZYeOcY/fu3axdu5asrCxWrFjBihUryMrK4tChQwDUqFGDdu3aHRMSzZo1IzY21uPqReRk2rdvT926dY8OiS9NZrbcOZda1Ho6Q4hgZkZiYiKJiYmkpaUdXZ6Tk8OaNWtYsWIFK1euZMWKFYwdO5aDBw8CULVqVdq2bXs0INq1a0fLli2Ji4vzaldE5Dh+v58LL7zQ6zKOoUCIQpUqVTr6yz7f4cOHWb9+/dGziBUrVvDKK6/w3HPPAYEhshdeeCFpaWmkp6dzwQUXUKVKFa92QaRc+/HHH/nxxx8j6oIyKBDKjAoVKtCyZUtatmzJTTfdBEBeXh6bNm1i+fLlLFmyhIULF/L444/jnCMuLo6OHTuSnp5Oeno6F154IdWqVfN4L0TKh0gccgohTm5nZveaWZaZrTGz+wosv9vM1geXjwouq2hmL5vZajNba2bDimj7n2a2//R2QwoTExNDkyZN+N3vfsfo0aNZuXIle/bsYdasWdxzzz3k5uYyYsQIevToQUJCAp06dWLIkCHMnj2bvXv3el2+SJkVqYFQ5BmCmbUC7gA6ArnAHDObDdQHrgTaOOdyzKxO8CXXApWcc63NrCrwpZlNdc5lF9J2KvCbktkVCcVvfvMbLr/88qPzp/z00098/PHHfPjhhyxcuJBnn32WUaNGERMTw3nnnXe0iyktLY0zzjjD4+pFyoaoDQSgBbDEOXcAwMwWAn2BVGCEcy4HwDm3M7i+A+LNrAJQhUCI7Du+UTOLBf4XuCHYnnigevXq9OzZk549ewKB+VWWLFlyNCDGjh3L6NGjiY2NpUuXLvTp04c+ffpwzjnneFy5SPTy+/3UrFmThIQEr0s5RpHDTs2sBTATuBA4CPwHWAZ0DS6/FPgFeMg595mZVQQmARcDVYH7nXPjCmn3XiDGOfesme13zhXZgV3ehp1GgpycHJYuXcrcuXOZOXMmq1evBqBFixZceeWV9OnTh44dO2qYq0gx9O7dm23btrFy5cqwbC/UYachfQ7BzG4H7gL2A18SCIZLgAXAvUAH4DUgBegM3AncCiQAi4DLnHNfF2jvLGA6cJFz7vDJAsHMBgGDAJKSks7fvHlzkfVK6fH7/cyaNYuZM2eycOFCjhw5Qp06dbj88svp06cPl1xyCVWrVvW6TJGIdu6559KsWTPeeuutsGwv1EAI6aKyc268c669cy4N2ANsALYCb7qApUAeUJtAF9Ac59yhYDfSYgLdSwW1A84BNppZNlDVzDaeYNvjnHOpzrnUxMTEUMqVUtSoUSPuuece/vOf/7Br1y6mTJlCRkYGb7zxBldddRW1atWiT58+vPjii3z33XdelysScZxzZGdnR9z1Awhx2KmZ1XHO7TSzJOBqAt1HeUAG8IGZNQXigN3AFiDDzF4l0GXUCRhdsD3n3GygboH29zvn1CkdZRISEujfvz/9+/cnNzeXDz/8kHfeeYeZM2cya9YszIwLLriAPn360LdvX5o3b+51ySKe++677zh48GBEBkKok/XPMLMvgVnAXc65H4AJQIqZZQHTgFtcoP/pOaAakAV8Bkx0zq0CMDNfsLtIypi4uDi6d+/O//3f/5Gdnc3nn3/O448/zuHDh/nzn/9MixYt6NSpE88//7yGtEq5FomznObTXEZS6r799lumTZvGxIkTWbNmDZUrV+aaa65hwIABdOvWTTcRknJl6tSp3HDDDWRlZdGyZcuwbLNEryGInI6zzz6bBx98kNWrV7N06VJuvfVW3n33Xbp3705KSgp/+ctfjo7LFinr8o/1SLp1Zj4FgoSNmdGhQwfGjBnD9u3bmTJlCk2bNmX48OGkpKSQkZHBpEmTOHDggNelipQav99PnTp1iI+P97qUX1EgiCeqVKlC//79mTt3LtnZ2QwfPpzNmzdz8803U7duXe644w4++eQToqlLUyQUfr8/Iq8fgAJBIkBSUhKPPPIIGzZs4IMPPuDqq69mypQpdO7cmRYtWjBy5Ei2b9/udZkiJUKBIBKCmJgY0tPTeemll9ixYwfjx48nMTGRoUOH0rBhQ26//XbWrVvndZkip+zIkSNs2bJFgSBSHNWrV+e2225j0aJFrF+/nsGDBzNlyhTOPfdcrr76apYuXep1iSLFtnXrVg4fPqxAEDlVTZs25Z///CebN2/m4Ycf5v333+eCCy4gIyOD9957T9cZJGpE6iyn+RQIEjXq1KnD8OHD2bJlC08//TRfffUVl156Ke3bt2fatGkcPnzY6xJFTkqBIFLCqlevzgMPPMDXX3/NhAkTOHjwIP3796dZs2bH3FtaJNL4/X5iYmJo0KCB16UUSoEgUSsuLo4BAwbw5Zdf8uabb1K7dm3++Mc/kpyczJNPPsmPP/7odYkix/D7/Zx99tnExcV5XUqhFAgS9WJiYujbty9Llizh/fffp127dvz5z38mKSmJ//7v/2bbtm1elygCRPaQU1AgSBliZlx00UXMmTOHFStW0Lt3b55++mkaNWrEH/7wB3bs2OF1iVLOReq01/kUCFImtWvXjqlTp/LVV18xYMAAxo8fT5MmTRgxYgS//PKL1+VJOZSTk8O2bdsUCCJeady4MWPHjmXNmjVkZGQwbNgwWrRowRtvvKHhqhJWmzdvxjmnQBDxWtOmTZk5cybz5s2jWrVqXHvttaSnp7N8+XKvS5NyItKHnIICQcqZ7t27s3LlSsaOHcu6devo0KEDAwYM0IVnKXUKBJEIVKFCBQYPHsyGDRt46KGHmDx5Mk2bNuVvf/ubPsMgpcbv9xMXF8dZZ0XuTSMVCFJu1axZk1GjRrF27Vp69uzJo48+SvPmzZk2bZquL0iJ8/v9NGzYMKLvEBi5lYmESePGjZkxYwbvv/8+Z5xxBv3796dLly6aQE9KVKR/BgEUCCJHXXTRRSxbtowXX3yRTZs2ccEFF3DzzTezdetWr0uTMkCBIBJlYmNjuf3229mwYQPDhg1j+vTpR2/zmZub63V5EqV++uknvv/+ewWCSDSqXr06TzzxBGvXrqV379489thjdOrUiaysLK9LkygUDSOMQIEgclKNGjXi9ddf56233mLr1q2cf/75jBo1iiNHjnhdmkQRBYJIGXLVVVeRlZVF7969GTJkCOnp6WzcuNHrsiRK5AdCcnKyt4UUQYEgEqI6deowY8YMJk2aRFZWFm3btmXMmDEaoipF8vv9xMfHU7t2ba9LOSkFgkgxmBk33ngjWVlZdOnShTvvvJOePXtqJJKcVP4sp2bmdSknpUAQOQX169dnzpw5jBkzhsWLF9OqVSsmTZqkswUpVDQMOQUFgsgpMzP+8Ic/sGrVKlq1asXNN9/MNddcw65du7wuTSKIc06BIFJeNG7cmIULFzJq1Chmz55Ny5Ytefvtt70uSyLE999/z/79+xUIIuVFbGwsf/rTn1i+fDn169enb9++3HLLLbqvs0TNkFMIMRDM7F4zyzKzNWZ2X4Hld5vZ+uDyUcFlFc3sZTNbbWZrzWzYCdqcHHxtlplNMLOKJbNLIt5p1aoVS5Ys4dFHH2Xy5Mm0bt2aefPmeV2WeCh/eHKZCAQzawXcAXQE2gKXm1kTM+sGXAm0cc61BJ4KvuRaoJJzrjVwPjDYzJILaXoy0BxoDVQBBp7erohEhri4OP7617/yySefUK1aNXr06MF//dd/kZOT43Vp4oEFCxZQo0YNWrRo4XUpRQrlDKEFsMQ5d8A5dxhYCPQF/giMcM7lADjndgbXd0C8mVUg8Is+F9h3fKPOOZ8LApYC9U97b0QiSIcOHVixYgX33Xcfzz33HBdddJFuxFPOOOfIzMykR48eVKwY+Z0goQRCFpBmZrXMrCrQC2gANAW6mtmnZrbQzDoE138D+BnYDmwBnnLO7TlR48GuopuAOSd4fpCZLTOzZRq9IdGmSpUqPPvss7z++uusXr2a888/n48//tjrsiRMVq9ezbfffkuvXr28LiUkRQaCc24tMBKYR+CX9hfAYaACkAB0Av4ETLfApy46AkeAs4BGwINmlnKSTfwb+NA5t+gE2x/nnEt1zqUmJiaGvGMikaRfv34sWbKE+Ph4LrroIsaNG+d1SRIGPp8PgEsvvdTjSkIT0kVl59x451x751wasAfYAGwF3gz2+iwF8oDawA3AHOfcoWA30mIgtbB2zewvQCLwwOnvikhka9WqFZ999hkXX3wxgwcPZtCgQbquUMb5fD7atWtHvXr1vC4lJKGOMqoT/J4EXA1MBd4GMoLLmwJxwG4C3UQZFhBP4AxiXSFtDgR6Av2dc3mnvysikS8hIYF3332XYcOG8cILL9CtWzddVyijfvjhBz7++OOo6S6C0D+HMMPMvgRmAXc5534AJgApZpYFTANuCV4gfg6oRuDaw2fAROfcKgAz85lZ/h2mxwJnAp+Y2edm9liJ7ZVIBIuNjeWJJ57g9ddfZ9WqVbquUEbNmzePI0eORFUgWDTNvZKamuqWLVvmdRkiJSYrK4urrrqKLVu28K9//YtBgwZ5XZKUkFtvvZVZs2axc+dOYmNjPa3FzJY75wrtui9In1QW8VD+dYWMjAxdVyhD8vLyyMzMpGfPnp6HQXEoEEQ8lpCQwOzZsxk6dKiuK5QRK1euZOfOnVHVXQQKBJGIEBsby5NPPsn06dP54osvSE1N5ZNPPvG6LDlFPp8PM6Nnz55el1IsCgSRCHLttdeyZMkSqlSpQnp6uj6vEKV8Ph8dOnQg2j47pUAQiTCtW7fms88+o1u3bgwePJjBgwfrukIU2b17N59++mnUdReBAkEkIp1xxhn4fD6GDBnCuHHjyMjI4Pvvv/e6LAnBe++9h3NOgSAiJSc2NpYRI0Ywbdo0li9fTpcuXdiyZYvXZUkRfD4fiYmJnH/++V6XUmwKBJEId/311/Pee++xbds2OnfuzJo1a7wuSU7gyJEjzJkzh8suu4yYmOj79Rp9FYuUQ+np6SxatIgjR47QpUsXFi9e7HVJUoilS5eyZ8+eqOwuAgWCSNRo06YNH3/8MYmJiXTv3p1Zs2Z5XZIcJzMzk5iYGHr06OF1KadEgSASRRo1asTixYtp1aoVffv2ZeLEiV6XJAX4fD4uvPBCEhISvC7llCgQRKJMYmIiCxYsICMjg9tuu40RI0YQTXOSlVU7duxg+fLlUdtdBAoEkahUvXp13n33Xfr378+wYcN44IEHyMvTLPJemjMncNPHaA6ECl4XICKnJi4ujldffZXExERGjx7Nzp07mThxInFxcV6XVi75fD7q1atH27ZtvS7llCkQRKJYTEwMo0ePpl69egwbNozdu3czY8YMqlWr5nVp5cqhQ4eYO3cu/fr1I3An4eikLiORKGdmDB06lPHjxzN//nwyMjLYtWuX12WVK5988gl79+6N6u4iUCCIlBm33XYbb731FqtXr6ZLly5kZ2d7XVK5kZmZSYUKFejevbvXpZwWBYJIGdKnTx/mz5/Pzp076dy5M6tWrfK6pHLB5/PRpUsXatSo4XUpp0WBIFLG/Pa3v+Wjjz4iJiaGtLQ0Fi1a5HVJZdrWrVtZtWpV1HcXgQJBpExq2bIlH3/8MXXr1qVHjx7MnDnT65LKrMzMTCC6h5vmUyCIlFFJSUl89NFHtG3blquvvpopU6Z4XVKZ5PP5SEpK4txzz/W6lNOmQBApw2rXrs38+fNJT0/nxhtv5KWXXvK6pDIlJyeH+fPn06tXr6gebppPgSBSxlWrVo13332X7t27M2DAAN2WswR99NFH7N+/v0x0F4ECQaRcqFq1Ku+88w69e/dm8ODB/Otf//K6pDIhMzOTuLg4MjIyvC6lRCgQRMqJypUr8+abb3LVVVdx99138/TTT3tdUtTz+Xykp6cTHx/vdSklQoEgUo7ExcUxffp0rrvuOh566CGeeOIJr0uKWn6/n7Vr15aZ7iLQXEYi5U7FihWZPHkycXFxPPzww+Tm5vKXv/ylTFwUDaeyNNw0nwJBpByqUKECL730EhUrVuTxxx8nJyeHJ554QqFQDD6fj8aNG9OkSROvSykxCgSRcio2NpYXX3yRSpUqMWLECHJycnj66acVCiE4ePAgCxYsYODAgWXq/QrpGoKZ3WtmWWa2xszuK7D8bjNbH1w+Krisopm9bGarzWytmQ07QZuNzOxTM9tgZq+ZmSZxFwmzmJgY/v3vf3PPPffw7LPPcvfdd+tGOyFYuHAhBw8eLFPdRRDCGYKZtQLuADoCucAcM5sN1AeuBNo453LMrE7wJdcClZxzrc2sKvClmU11zmUf1/RI4Fnn3DQzGwvcDowpkb0SkZCZGaNHjyYuLo6nnnqKnJwcnn/+eWJiNObkRDIzM6lcuTLp6elel1KiQukyagEscc4dADCzhUBfIBUY4ZzLAXDO7Qyu74B4M6sAVCEQIvsKNmiBc6wM4IbgopeB/0GBIOIJM2PUqFFUqlSJv//97xw6dIjx48cTGxvrdWkRyefzkZGRQZUqVbwupUSF8idAFpBmZrWCf/H3AhoATYGuwW6fhWbWIbj+G8DPwHZgC/CUc27PcW3WAn50zh0O/nsrcPZp7ouInAYz429/+xt//etfefnll7nppps4fPhw0S8sZzZs2MDGjRvLXHcRhHCG4Jxba2YjgXnAfuAL4HDwtQlAJ6ADMN3MUgh0LR0Bzgo+v8jM5jvnvi7QbGFXYVxh2zezQcAgCEzWJSKl69FHHyUuLo6hQ4eSm5vLlClTdJ/mAnw+HwCXXXaZx5WUvJA6CZ1z451z7Z1zacAeYAOBv+rfdAFLgTygNoFuoDnOuUPBbqTFBLqXCtoN/CbYrQSB6xHbTrDtcc65VOdcamJiYnH3T0ROwZAhQ3jmmWeYMWMG/fr1Iycnx+uSIobP56N58+akpKR4XUqJC3WUUZ3g9yTgamAq8DaB6wCYWVMgjsAv+i1AhgXEEziDWFewPeecA94H+gUX3QJownaRCHL//ffz3HPPMWvWLK688koOHDjgdUme+/nnn/nggw/KZHcRhD51xQwz+xKYBdzlnPsBmACkmFkWMA24JfiL/jmgGoFrD58BE51zqwDMzGdmZwXbHAI8YGYbCVxTGF9SOyUiJePOO+/khRdeYO7cuVx22WXs27ev6BeVYQsWLCA3N7dMdhdBiB9Mc851LWRZLnBjIcv3Exh6Wlg7vQo8/prA9QYRiWADBw4kPj6em2++mYsvvpg5c+ZQq1Ytr8vyRGZmJvHx8XTt+qtfiWWCBhqLSJH69+/Pm2++yerVq0lPT2f79u1elxR2zjl8Ph/du3enUqVKXpdTKhQIIhKSK664gszMTLKzs+natSvZ2dlelxRWa9euZfPmzWX2+gEoEESkGLp168b8+fP5/vvv6dKlC+vWrSv6RWVEWR5umk+BICLF0qlTJxYuXMjhw4dJS0tj5cqVXpcUFj6fj9atW9OgQQOvSyk1CgQRKbY2bdrw4YcfUrlyZbp168bixYu9LqlU7du3j0WLFpUriUqCAAALTklEQVTp7iJQIIjIKWratCkfffQRderUoUePHsybN8/rkkrN/PnzOXz4cJnuLgIFgoichqSkJBYtWsQ555zD5Zdfzttvv+11SaUiMzOTGjVq0LlzZ69LKVUKBBE5LWeeeSYffPAB7dq1o1+/frz66qtel1Si8oeb9ujRg4oVK3pdTqlSIIjIaUtISGDevHmkpaVx0003MWZM2ZnJ/osvvmDbtm1l/voBKBBEpIRUr14dn8/HFVdcwZ133snIkSO9Lum07d+/n0GDBlG5cmUFgohIcVSuXJkZM2bQv39/hg4dysMPP0xgirPok5ubyzXXXMOKFSt47bXXOPPMM70uqdSFNJeRiEioKlasyKRJk6hWrRpPPPEE+/bt4x//+EdU3ZIzLy+PAQMGMHfuXMaPH0+fPn28LiksFAgiUuJiY2N5/vnnqVmzJk899RR79+5l3LhxVK5c2evSiuSc48EHH2TKlCk8+eST3HbbbV6XFDbRE9kiElXy79M8fPhwJk2aRMeOHVmzZo3XZRVp5MiRjB49mnvvvZchQ4Z4XU5YKRBEpNSYGY888gizZ89mx44dpKamMmbMmIi9rjBhwgSGDRvGDTfcwDPPPINZYXf7LbsUCCJS6nr16sWqVatIS0vjzjvvpG/fvuzevdvrso7xzjvvcMcdd9CzZ08mTpwYVdc8Skr522MR8UTdunXJzMzkmWeewefz0bZtWxYsWOB1WQB89NFHXH/99Zx//vm88cYbxMXFeV2SJxQIIhI2MTEx3H///Xz66adUr16d7t27M2zYMA4dOuRZTatXr+aKK64gKSmJ2bNnU61aNc9q8ZoCQUTCrl27dixfvpyBAwcyYsQIfvvb37Jx48aw17F582YuvfRSqlatyty5c0lMTAx7DZFEgSAinoiPj2fcuHG8/vrrbNiwgXbt2vHKK6+E7YLzrl276NGjBwcOHOC9996jYcOGYdluJFMgiIin+vXrx6pVq2jfvj233HILv//979m7d2+pbnP//v307t2bLVu2MGvWLFq1alWq24sWCgQR8VyDBg1YsGABw4cPZ/r06Zx33nl88sknpbKtglNSTJ8+nS5dupTKdqKRAkFEIkJsbCyPPPIIixYtAqBr164MHz6cI0eOlNg28vLyuPXWW5k7dy7jxo3jiiuuKLG2ywIFgohElAsvvJDPP/+c6667jscee4xu3bqxZcuW027XOccDDzzA1KlTy92UFKHSXEYiEnFq1qzJ5MmTufTSS7nrrrs455xzSElJISUlhcaNGx/zPSUlhapVqxbZ5ogRI/jHP/7BfffdV+6mpAiVRepHyAuTmprqli1b5nUZIhJGmzZt4oUXXmDjxo18/fXXbNq0iX379h2zTt26dX8VFPnfzzzzTCZMmMDAgQP5/e9/zyuvvFLuPoVsZsudc6lFrqdAEJFo4pxjz549bNq06WhAFPy+devWY4auVq1alV9++YVLLrmEd955p1x+CjnUQFCXkYhEFTOjVq1a1KpVi44dO/7q+V9++YXNmzcfExQxMTE8/vjj5TIMikOBICJlSuXKlWnWrBnNmjXzupSoE1JHmpnda2ZZZrbGzO4rsPxuM1sfXD4quOz3ZvZ5ga88MzuvkDbPM7MlwXWWmdmvo15ERMKmyDMEM2sF3AF0BHKBOWY2G6gPXAm0cc7lmFkdAOfcZGBy8LWtgZnOuc8LaXoU8LhzLtPMegX/fdHp75KIiJyKULqMWgBLnHMHAMxsIdAXSAVGOOdyAJxzOwt5bX9g6gnadUCN4OOawLZi1C0iIiUslC6jLCDNzGqZWVWgF9AAaAp0NbNPzWyhmXUo5LXXc+JAuA/4XzP7BngKGFb88kVEpKQUGQjOubXASGAeMAf4AjhM4OwiAegE/AmYbgXuN2dmFwAHnHNZJ2j6j8D9zrkGwP3A+MJWMrNBwWsMy3bt2hXyjomISPGEdFHZOTfeOdfeOZcG7AE2AFuBN13AUiAPqF3gZb/jxGcHALcAbwYfv07gGkVh2x7nnEt1zqWW97nKRURKU6ijjOoEvycBVxP4Rf82kBFc3hSIA3YH/x0DXAtMO0mz24D04OMMAiEjIiIeCfVzCDPMrBZwCLjLOfeDmU0AJphZFoHRR7e4///xwDRgq3Pu64KNmNmLwFjn3DICI5f+YWYVgF+AQSWwPyIicoqiauoKM9sFbD7Fl9cmeAYTYVRX8aiu4lFdxVNW62ronCuyzz2qAuF0mNmyUObyCDfVVTyqq3hUV/GU97rK15R/IiJyQgoEEREBylcgjPO6gBNQXcWjuopHdRVPua6r3FxDEBGRkytPZwgiInISZSoQzOx/zWydma0ys7fM7DeFrNPsuOm59+VP6W1m/2Nm3xZ4rle46gqul21mq/OnBC+w/Awzm2dmG4LfE8JVl5k1MLP3zWxtcJrzews85/X7dWlw+vWNZja0wPJGwTm2NpjZa2ZWIndFMbNrg+9BnpkVOuLDo+OryLqC64X7+Arl/fLi+Ar1/Qr38VXkz8HMuh13fP1iZlcFn3vJzPwFnvvVbQeK5JwrM19AD6BC8PFIYGQR68cCOwiM0QX4H+Ahr+oCsoHahSwfBQwNPh5a1H6VZF1APaB98HF14CvgXK/fr+DPbhOQQuBT8l8UqGs68Lvg47HAH0uorhZAM+ADIDWE9cN1fIVUlwfHV5F1eXR8hVKXF8dXsX4OwBkEphKqGvz3S0C/06mhTJ0hOOfmOucOB/+5hMA9G07mYmCTc+5UP+xWWnUd70rg5eDjl4GrwlWXc267c25F8PFPwFrg7JLY/unURWDuq43Oua+dc7kEpkm50syMwFQobwTXK8n3a61zbn0xXhKu46u4dR2vtI6vIuvy6PgK5f0K+/FF8X8O/YBMF7w1QUkoU4FwnNuAzCLWKWwCvv8KdlVMKKlT52LU5YC5ZrbczApO5XGmc247BP4DAXXCXBcAZpYMtAM+LbDYq/frbOCbAv/eGlxWC/ixQKDkL/eCF8fXyXh5fBXJo+PrRLw4vor7cyjs+Pp78P161swqFbeAqAsEM5tvgdt5Hv91ZYF1HiYwRffkk7QTB/QhMNNqvjFAY+A8YDvwdJjr+q1zrj1wGXCXmaWFuv1SrgszqwbMAO5zzu0LLvby/bJClrmTLC+xukJsJ+zHVwg8Ob5CbCfsx1dRTRSyLJKOr3pAa+C9AouHAc2BDgS6k4YUp00IfXK7iOGc636y583sFuBy4GIX7Fg7gcuAFc657wq0ffSxmb0AvBvOupxz24Lfd5rZWwROWz8EvjOzes657cEDobC705VaXWZWkcB/1snOufwpy71+v7YSuFFTvvoEZtDdDfzGzCoE/4rLX14idRVDWI+vENsI+/EVCi+OrxCE/fgys+L8HK4D3nLOHSrQ9vbgwxwzmwg8FGpd+aLuDOFkzOxSAqnYJ4R+tV/d3jP4Q8jXl8Dd4sJSl5nFm1n1/McELqzmb/8dAvePIPh9ZhjrMgI3L1rrnHvmuOc8e7+Az4AmFhjxEUfg9PmdYHi8T6B/FUrw/SqmsB1fofDi+AqxrrAfXyHy4vgqzs/hhMdX8D29ilN5v0ri6nikfAEbCfT7fR78GhtcfhbgK7BeVeB7oOZxr58ErAZWBX849cJVF4HRDF8Ev9YADxd4fS3gPwTuGfEf4Iww1tWFwCnxqgLr9fL6/Qr+uxeBUSmbjnu/UoClwXZeByqVUF19CfzlmAN8B7wXIcdXkXV5dHyFUpcXx1eoP8dwH1+F/hwI3L/+xQLrJQPfAjHHvX5B8P3KAl4FqhW3Bn1SWUREgDLWZSQiIqdOgSAiIoACQUREghQIIiICKBBERCRIgSAiIoACQUREghQIIiICwP8D35G6s7+HGE8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "upper=len(lr_list)\n",
    "plt.plot(np.array(lr_list)[:upper,0],np.array(lr_list)[:upper,1], color='black')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}