ML_-_basics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sympy import *\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Linear Regression\n",
    "$\n",
    "\\hat{y} = \\theta_0+\\theta_1 x_1 +\\theta_2 x_2 + \\cdots + \\theta_n x_n = h_\\theta(\\mathbf{x})= \\mathbf{\\theta}^{\\top}\\mathbf{x}\n",
    "$\n",
    "\n",
    "MSE cost function for Linear Regression Model:\n",
    "\n",
    "$\n",
    "\\text{MSE}(\\mathbf{X}, h_\\theta) = \\frac{1}{m}\\sum\\limits_{i=1}^m (\\mathbf{\\theta}^\\top \\mathbf{x}^{(i)}-y^{(i)})^2\n",
    "$\n",
    "\n",
    "#### Normal Equation\n",
    "Normal equation to minimize the cost funtion:\n",
    "\n",
    "$\n",
    "\\hat{\\mathbf{\\theta}}=(\\mathbf{X}^\\top\\mathbf{X})^{-1} \\mathbf{X}^\\top \\mathbf{y}\n",
    "$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# example\n",
    "a = 2\n",
    "b = 4\n",
    "X = 2*np.random.rand(100,1)\n",
    "y = b+a * X + 1*np.random.randn(100,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1ba41abaf48>]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEGCAYAAABiq/5QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de5xV1X338c8PGGBQcKDghUEYiREFUUbHSwFFuYhaEyba9GWemJjExjRNmpgLeTT2iUmaFBvbpOmrT9rHJmnSNK9EYyxJaxvDRaOAoAPDHVGDgAzKRUAQRhiG9fyxzzBnDufMue7b2d/368WLmXOZs86eM9+99m+tvbY55xARkeToE3YDREQkWAp+EZGEUfCLiCSMgl9EJGEU/CIiCdMv7AYUYvjw4a6hoSHsZoiIxMrKlSv3OudGZN4ei+BvaGigpaUl7GaIiMSKmW3LdrtKPSIiCaPgFxFJGAW/iEjCKPhFRBJGwS8ikjAKfhGRhInFdE4Rkaia39rGQ09uZueBdkbW1TJ39jiaG+vDblavFPwiIiWa39rGfY+vo72jE4C2A+3c9/g6gEiHv0o9IiIleujJzSdDv0t7RycPPbk5pBYVRsEvIlKinQfai7o9KhT8IiIlGllXW9TtUaHgFxEp0dzZ46it6dvjttqavsydPS6kFhVGg7siIiXqGsDVrB4RkQRpbqyPfNBnUqlHRCRhFPwiIgmj4BcRSRgFv4hIwij4RUQSRsEvIpIwCn4RkYRR8IuIJIyCX0QkYRT8IiIJo+AXEUkYBb+ISMIo+EVEEkbBLyKSMAp+EZGEUfCLiCSMgl9EJGEU/CIiCaPgFxFJGAW/iEjCKPhFRBLGt+A3sx+a2W4zW5922zAzW2BmL6f+H+rX64uISHZ+9vh/BNyYcdu9wCLn3LuBRanvRUQkQL4Fv3PuGWBfxs1zgB+nvv4x0OzX64uISHZB1/jPcs69DpD6/8xcDzSzu82sxcxa9uzZE1gDRUSqXWQHd51zDzvnmpxzTSNGjAi7OSIiVSPo4N9lZucApP7fHfDri4gkXtDB/2vgztTXdwK/Cvj1RUQSz8/pnD8DngPGmdkOM7sLeBCYZWYvA7NS34uISID6+fWDnXMfyHHXDL9eU0RE8ovs4K6IiPhDwS8ikjAKfhGRhPGtxi8iUqr5rW089ORmdh5oZ2RdLXNnj6O5sT7sZlUNBb+IRMr81jbue3wd7R2dALQdaOe+x9cBKPwrRMEvIpHy0JObT4Z+l/aOTh56cnPgwV+tRx4KfhGJlJ0H2ou63S/VfOShwV0RiZSRdbVF3e6X3o484k7BLyKRMnf2OGpr+va4rbamL3Nnjwu0HVE58vCDgl9EIqW5sZ55t06kvq4WA+rrapl368TAyytROfLwg2r8IhI5zY31odfR584e16PGD+EcefhBwS8ikkXXjkezekREEiQKRx5+UI1fRCRhFPwiIgmj4BcRSRjV+EUkVNW6LEKUKfhFJDTVvCxClKnUIyKhqeZlEaJMwS8ioanmZRGiTMEvIqGp5mURokzBLyKhicqCbEmjwV0RCU01L4sQZQp+EQlVtS6LEGUq9YiIJIyCX0QkYVTqEZGsdEZtZUVpeyr4ReQUOqO2sqK2PVXqEZFT5Dqj9guPrmF+a1tIrYqvqJ2hrOAXkVPkOnO20znue3ydwr9IUTtDWcEvIqfo7cxZraVTvKidoRxK8JvZ58xsg5mtN7OfmdnAMNohItllO6M2ndbSKU7UzlAOfHDXzOqBzwDjnXPtZvYocDvwo6DbIiLZdQ04fuHRNXQ6d8r9hfZUw5zJEsZr53rNQs5QDrK9Yc3q6QfUmlkHMAjYGVI7RCSHrtBJn40ChfdUw5zJEsZr53vN3s5QzvrcX65l8JaXmHHrdWBW0bYGHvzOuTYz+1tgO9AO/NY599ug2yESpijN6e5NOWvp9DaTxe/3GsZrl/OaXc89++Bepmxbw5Rtq5m6dTVnHt4PL70E7353RdsaRqlnKDAHOA84APzCzO5wzv17xuPuBu4GGD16dNDNFPFN1OZ051PqWjphzmQJ47VLes233oKnn+buX3yfKVtXc/6+HQDsHXQGS8dMYmnDpXxr+PCKtzWMUs9M4FXn3B4AM3scmAz0CH7n3MPAwwBNTU2nFhlFYirMnnCQRtbV0pYl9IKYyVLsa1fiCKyg1zx2DJYvhwULYOFCeP55OHGCP6kZyIpRE/jZpbNZ2jCJzSPG4KwP9XW1MHRoUe0oRBizerYDV5vZIDMzYAawKYR2iIQianO6/RLmTJZiXrvrCKztQDuO7iOwYs9VyPqa/frwV+d1wre/DTff7IX4tGnw13/tPeDLX4bf/Y7fLtnIJz/4DX5w5ft48czzcNbH120VRo1/hZk9BqwCjgOtpHr2IkkQZk84SGGutV/Ma1fqCKzrsf/2yDO8a+0KZu1cx7WvrWXgvr3eA8aNg499DGbMgOuug7q6k8+dA7ia/oFtK3NZpmpFTVNTk2tpaQm7GSIVkVnjB683Ou/WibEu9cRlwDrTefc+QbYUNODVB/8o/w/Yvx+eftor3Sxc6A3GApx1FsyaBTNnemE/alQFW10YM1vpnGvKvF2LtIkErBqvOhWHAetcO6ZcR2B9zJjf2nZq+48ehWXLuoO+pQVOnIDTTvN68p/8pBf0F19c8WmYlaIev0gVKLS37VevfMqDi7OGZ31dLUvvnV6Rtpejt6MsOPVchR6PaZ5Ac5+93UH/7LPQ3g59+8LVV3shP2sWXHkl9O9f0XaXSz1+kSpVaG/bz155qQPWQR0p9FbH79oxpZ+lPOqtXUzZ6s2ln/qdtXD4Le9J48fDxz/eXacfMqRibQySgl8k5godnKzEIGax5ZJ8A9ZBTW3Nt2NqHj2Q3256linbVjNl6xoaDrwOwK7Th7H4vMu57Usf8cJ+5MiCXi/q4x0KfpGYK7S3netx2QI7m95653NnjytpaYegprZm7pgGdBylqW0TU7eu5sVRn2fczpf5nnMc6l/L8tET+fHlt/BsQyOv/MG51A8dxG0f6r1clS4O4x0KfpGYK7S3netxBtkHMTMUUi4ptpcb1NTWuTPP5yf//CuueGUVU7e2csWOjQzo7KCjT19WjbyQf7zmg/S/aTb/eLCOQye6B2RLmUsfhxP0FPwiMVdob3vu7HF87pHVp0xddFBQKOUtl5SwtEOpRwp5OQdbtpwckG1evJjmffsA2DSigZ803syShkk8f+7FHOnv7WTqqeWv3j+u7BJNHE7QU/CLxEBvNeNCp4c2N9ZzzyOrs/78QkLJj955Rae27t0Lixd7Yb9gAWzd6t0+ahTMmePNp58+nZv/fmXWefs7D7SXvC5RujicoKfgF4m4QmrGhQZWfRmh5FfvvOSwPXIElizpnmbZ2urdfsYZcP318MUvemF/wQU95tP7Hcy+HcVUkIJfJGIye/eHjx6vWM24nFAK/cSzzk5Ytap7gbOlS71Fz2pqYPJk+MY3vKC//HLolzva/A7m0LdTAXQCl0iEZDvRKJeClxTI8hpRDqWTnINXXunu0S9eDAcOePddeqkX8rNmwdSp3lmzRYjNNiiTTuASiYFsM0JyKXWJ4UrUsX2ze7cX8F29+u3bvdtHj4bbbvPm0s+YAWeeWdbLRHobBEDBLxIhhc78yLfEcJTnkPdw+LC3BEJXr37NGu/2oUNh+nS4916vZ3/++ZFd9yaOFPwiEZJr4HHooBoG9e9X0SWGezsy8K0Ucvy4t6hZV9AvWwYdHRzrW0NL/UWsnX0XF97xPq77wI3eWjjiCwW/SITkGnh84D0TCgreQueQ93ZkAFTuqME52LwZFi3ygv6pp7zLDQI0NvLS//pT/uZYPUvPuZB3agZ673dzX+atfSOaRygZ4jpWoOAXiZByZ4QUOlWxtyODrq+z3VdQO954ozvoFy6EHd51ZGlogD/5E690c/31MGIEH82yqmfUznLNJXZltTQKfpGIKWfgsdCpiqWcXZrzvkOH4JlnuoN+/Xrv9mHDvIHYrguRvOtdBf/MQtcPClMclmbIRcEvUkUKPWLId2TQ61FDRwe88AIsWsTe/3iCujUt9DvRydF+/Xnr8qs488EHvWmWkyZBn94v613u+kFh6m2nNeXBxZEu+2gev0gIwq4NF3VhEue4+K02/nrILi55scW7zOChQzgzNpx9Ps+MuZQlYyaxctR4+tTWFnUJyfmtbVnXD4LCLuISplwXn+kShctpah6/SEQEVRsuZ32fgXve4PnvP8bFG1dwzWtrGXHwTe+Hvutd8IEPwKxZ3LS2Hy8eq+n5okWWOspdPyhM2cpq6aJc9lHwiwQsiNpw0ev7HDwIv/sd/OhbsHAhN27cyI0Aw4fDTTO7Ly/Y0HDyNTa3PJH1tYsN7HLWDwpT+s4zV88/qjsvBX+ChV1uSKoglu3Nu3M5dgxWrOgekF2xwlsLp7YWrr0WPvpRb1D2kkty1ukrtdhZHBY1y6Vr55mr7BPVnVfe4DezTwM/dc7tD6A9EpAoTEVL6o4niGV7T9mJOMcFe7cx9YU1sOTbXp3+8GEv1K+4ovsM2T/8QxgwoKDXyBfYhf5+47CoWT5x23kV0uM/G3jBzFYBPwSedHEYEZZehT0VLQo7nrAEERIj62rp3P4aU7et9i4avm01Iw6nFji74AK4806vdDNtmrc8Qgl6C+xif79xXzsnbjuvgmb1mJkBNwAfBZqAR4EfOOd+72/zPJrVk18hvav0x+T6rZe64mOxch0aR30mR6X4crTz1lvembELF3Loid8weKv357lnUB1LGy7l+bGXMe2TtzP75isr8A56ynw/R44dZ/+RjlMel5Tfb1SUNavHOefM7A3gDeA4MBR4zMwWOOe+VNmmxlOYZYtCeleFLvcbVE0yDpen81NFerhHj8Jzz3XX6V94AU6cgEGDGDxtGutuvYNvdY5iyYCzGTl0EHNnj2O2D5/JbJ+/XJLy+426Qmr8nwHuBPYC3wfmOuc6zKwP8DKQ+OAPu2xRSNmmkOV+yyk3FLvji8Pl6SLnxAlYt6770oLPPAPt7d5iZldeCV/+sle+ufpq6N+ficBPAmhWJZaSlmAV0uMfDtzqnNuWfqNz7oSZ3eJPs+Il7Hp5Ib3n3npaBmUdpZSy44vbYFhotm3r7tEvWgR79ni3X3QR3HVXd53+jDNCa2K5S0lXqyhPXsgb/M65r/Ry36bKNieewi5bFNJ7zvWYStRcS9nxxW0wLDD795+s07NggXcFKoCzz4bZs7vXvhk1Ku+PCip4cn226mprOG1A/qWkq1HYVYB8NI+/AsIuWxTSe/azh13Kji/KvaFAvfOOtyZ9V69+5UqvpHP66XDddfDpT3thP2FCURciCTJ4cn22vvrewpaSrkZhVwHyUfBXQNhli0J6z372sIvd8UW9N+SrEye8q0x1Bf2zz3bX6a++Gv7yL73yzVVXMX/9bu/39e/bGFm3u6jfV5DBE8TRW9w6CmFXAfJR8FdAFMoWhcwS8WuudLE7vqj3hiru1Vd71unfTK17M2EC3H23V7q59loYMuTkU8rdOQYdPH7Ow49jRyHsKkA+oQS/mdXhzRC6GHDAx5xzz4XRlkqJ+wko5Sh2xxf13lDZ3nzTq9N3XTB8yxbv9pEj4ZZbui8YPnJkzh+Rb+eYrwdcTPBEvTcdx45C2FWAfMLq8X8X+I1z7o/NrD8wKKR2SIUUs+OLem+oaO3tsHRpd69+1SrvkoODB3t1+nvu8Xr1F15YcJ2+t51jIT3gQoMnDr3pOHYUolAF6E3gwW9mQ4BrgY8AOOeOAceCboeEJ6jekG892c5OaG3tDvolS7yTqfr189a6+drXvKC/4grvthL0tnMspAdcaPDEoTcd145ClKsAYfT4xwJ7gH81s0uBlcBnnXOH0x9kZncDdwOMHj068EZKbuUGalCDgRW9YPiWLd1Bv3gx7Nvn3TdxIvz5n3fX6U8/vSLt723n+LkC168vJHji0JuOetkkjsII/n7AZcBfOOdWmNl3gXuB/5P+IOfcw8DD4K3VE3grJatKBWoxvaFSdjTF9mQzX+P+K4dz855N3WG/dav3wFGjYM6c7uvInnVWQe+hWL3tHHOt/15KD9iP3nSlj7SiWDaJ+rhIPmEE/w5gh3NuRer7x/CCXyog3wey3A9s0KWBUnc0xfRk57e28bVHXmDilrV8eNsapm5dzYTdqQHZM86A6dNh7lwv6C+4oKj59OXItXOsZA+40r1pv8YMolQ2icO4SD6BB79z7g0ze83MxjnnNgMzgI1Bt6Ma5ftAVuIDG0RpIH3n1MeMzowVZAvZ0eTtyXZ2eidLLVzIuT/4Bcu3rWdA53GO9enHqvoLeeiaD/HSxVfxL//wiZLr9H6pZA+40r3pOIwZlKsa3mNYn+i/AH6amtGzBW+5ZylTvg9kJT6wfg+0Ze6cMkO/S74dzSk9Wee48O1d/I3tYuf0hzj9uSUMeedtAAaeOZYfX/YeljRM4oVRE2jvPxDw1jCKWuh3qWQPuJI/Kw5jBuWqhvcYyqfaObcab11/qaB8H8hKfGD9HmgrdKXHfDua5sZ6Bry5h+Xff4zxG59n2mtrOPvAbgB2nHEmT1wwmWVjLmVpwyT2Dcq+wFnUZ41EUVxn4BSjGt5jNLszUpJ8H8hKfGD9HmgrZCeUc0dz+LC3VHFqQPamtWu5CbwrTM2YDjNn8t6NA1hbOyJvnV6zRkqThBk41fAeFfxVJN8HslIfWD8H2nLtnPqaccK5njua48ehpaV7JcvnnoOODu+asVOmwLx53uybxkZvLRxg7b1P5Hzt+rra2M7SiIoozsCptGp4jwVdejFsuvRi4fye1VOJNuR7brad07xbJ9I8aSRs3tw9xfKpp+DgQa/33tjozbqZNQumToXa7EcxDb0E/9YALjkpEqSyLr0ohQt7fm++3rjf0+LKnTmU2Zua2OcIXxnwOk3f/bkX9m1t3gPHjoXbb/fCfvp0GD68oPYNHVST9VqwQwfVFPR8kWqg4K+gapjfW66yZw4dOkRzWyvNb6R69Rs2eLcPG9Z9EZKZM73gL8ED75nA3MfW0NHZfaRb09d44D0TSvp5InGk4K+gapjfW66iZw51dMDzz3eXb5Yv92r3AwfCNdfAhz/sBX5jI/TpU3b7qqE+G7Swj2Kl8hT8FVQN83vLlXfmkHOwcWN30D/9NLz9tlenv/xy+OIXvTr95Mle+PsgSmeBRl22o9jPPbKalm37+EbzxJBbJ6VS8FdQVOf3BtljyzZzaEz7fr592ivw4R95Yf/6694d558PH/ygF/TXX++VcyRSsh3FOuCny7fTNGaYdqAxpeCvoCjO7w163KG5sZ5+bx/i2X95jItSJ06dt3u7d+fw4d2Lm82cCQ0NFX99qaxcR6sOElXCrDYK/gqKYv3Y73GH+a1tfOeJ9Zy1aTU3vrGBOW9u4pb1rdzS2elNqZw2DWZ+xgv7Sy6pSJ0+KOUeKVVDbTzXUSwkq4RZbRT8FRaV+nFX6PjyR+scrF/Puh/9kqG//m/+e9s6Tut4h07rw/pz3s3ej3yKcXe8z7soyYABpb9OiMo9UqqWGV5d6/9nO9sn7BKmlE7BX4WynQSVqeg/2h07us+QXbQIdu1iIvD7YaP45cUzWNIwieWjJ3Jw4OnU19Wy9LrrynoPYSv3SKlaZng1N9bTsm0fP12+vUf4h13ClPIo+KtQvoXOCvqjPXDAm3HTNftm82bv9jPPPDmXfvILxs4hI055ajWUAMqdoVVNM7y+0TyRpjHDYl+2km4K/irUW7jU5/qjPXrUW+umK+hfeAFOnIDTTvPq9J/4hFennzjx5AJntmsxRHAWUyWUO0PLrxleYY0bRKWEKZWh4I+hfH/8uUKnvq6WpfdO9745cQLWresO+meegSNHvMXMrrgC7r/fm2Z51VXQv3/WdkRxFlOllPve/Ng21TJuIOFT8MdMIX/8uULnKxNPgx/8wAv6RYtgzx7vzosugo99zAv6adO8yw0WIIqzmCql3Pfmx7aplnEDCZ9W54yZKQ8uzt+bx9tB/PP8FhrWPs8Nr6/nhp1rOf21rd6dZ5/dvebNzJlQr9AoVhgll/PufSLr7BoDXtXKopKFVuesEr0OGr7zDixbBgsX0rxwIc0tLd7Uy8GDvZ785z/r9erHj6/YBcOrYa56scIquUT1zHCJHwV/zKT/8Zs7wfhdW5i6bTUzdqyD7/6xF/79+sHVV8MDD3hBf8UVUFP5ZYcLCcBq3DGEVXKp5jEVCZaCP2YeuLiWJf/0K67c0srkbWsY1n4QgLfOH+fNvJk1C6691uvl+yxfAFbrYGS+qZp+7eyqeUxFgqXgD1BJgfDmm7B48cnZNzds2cINwJ4hf8BT72pi/fgrufKu93PTDZcF8h7S5QvAXDuGLzy6Bohv+PdWcvF7Z6dplVIJCv6AFBwI7e2wZEn3NMvWVq9OP2SIt4LlPffArFmMGDeO28y4LYw3k5Kv5pxrx9DpXKx7/r2VXDTzRuIgPitmxVyuQPi7/9nonSw1b553gtTQoXDDDfCd73jlmq9/nd/966+49kuPcd6FH2fK4QnMbx9cscHZcsydPY7amr49bkuvOfc26NgVhnHU3FjPvFsnUl9Xi+HNqJp360SaG+ur6oxdqV7q8RegEjXbk3/4zjHmwOtM3bqaKVtXM3n7Wrj/be++iRPhU5/yplhecw2cfnrakcIxIFp18nw152w943RxDsNcJRfNvJE4UPDnUZGa7Z49fGj7csZveJ6pW1cz6uBu72cNHsGSCVO55Qsf9i4YftZZpzy1kqUDPwYde6s5d93+hUfX0JnlfJFqDEPNvJE4SFzwFxt+JQXvkSPw7LPddfrVq/k6cHDAaSwbcwn/76pbWdLQyBtnnsu82y6BXl6/UqWDsGbYdP3spIShZt5IHCQq+EsJv4KCt7MTVq7sXrZ42TI4dsxb42bKFPjmN2HGDJ7qczbfWvT7k4Ewr4BAqFTpIMxBx2LCsBrm/WvmjURdIoK/t4uSZIZfZvCcUVvDgfaOnk9yjquOvwnf+54X9k895S1jDDBpEnzmM958+qlTYdCgk0+bA8y5YkxRba9U6SDsQcdCwrBa5/2LRE3VB38hFyXZeaCd+a1tfPXXG3qEfNuBdmr6GjV9jDMO7WPytjVM3bqaqdvWMPJgaoGzMWPgttu8oJ8+HUacuj59OSpVOojDoKOmQooEo+qDP99FSQDOqK05tVd97B2uem09U7atZtr2NVyw61UADtYO5tDka+D97/Vm34wd6/vUykqUDuIw6Bj2UYlIUlR98OcLjdqavpjBsaPHuOz1l5iS6tE3tr1I/xPHOdq3hhdGjeeCefNg5kyGNDYypG/PuetxqEvHYdAxDkclItUgtOA3s75AC9DmnLvFr9fJFSY4x+SOPdzrdrDr509w1fZ1DDl2hBMYG84ayw+vmMOzDY201F/E8BF1PZY87pKrPBTVunTUBx3jcFQiUg3C7PF/FtgEDPHzRdLDZMTb+5iybQ3Ttq/hxjfWU7v7DQBeG3oO/3XRtSxpmMRzoyeyf1D3hUhyBc9fzl93ygWou6guXZo4HJWIVINQgt/MRgF/BHwT+LxvL3ToEM1trUz8/Xxs8SLG7toKwNG6oQy4YZY3IDtjBu/9ycvsP9JxytP7GCdPxU83v7UtZ+h3UV26NFE/KhGpBmH1+P8e+BLg79rBt9wCzzxDw4ABtIyeyKMXXceLE6+m+c6bab783JMPO3BkY9anO5e9XPPQk5t7DX1QXVpEoivw4DezW4DdzrmVZnZdL4+7G7gbYPTo0aW92AMPsOSVvXzq1YG85boHZFfM3wB9+pwM9WIHFQsZMFZdWkSiKozVOacA7zWzrcDPgelm9u+ZD3LOPeyca3LONY0odW789On8733De4Q+nLoyZL5VJjP11psfOqgma3lIRCQqAg9+59x9zrlRzrkG4HZgsXPuDr9er5C54b0ts5tNth2FAXdcPZrWr9yg0BeRSKv6efyFlnGKGVTU7JPiZDvPAbT9RMJiLstyuVHT1NTkWlpaSnputiUbamv6qhwTkGzbv6aPgUFHZ/dnT78Tkcozs5XOuabM26v+ClzFlnGksrItmdFxwvUIfYj3FblE4qbqSz2gueFhKuZ8Bp37IBKMRAR/tYvyWkE5l8zI8VgR8V/Vl3qqXVcNve1AO47utYLmt7aF3TQg+wyomj5GTd+eK5rq3AeR4Cj4Y663NeyjINsYy0Pvv5SH/vhSjbuIhESlnpiLwxr2ucZYFPQi4VDwZxHlmnkmrWEvIsVSqSdD1GvmmYpdbkJERD3+DLlq5l/7zw2RPArQWcQiUiwFf4ZctfH9RzpOrtkftats6TwFESmGSj0ZCq2NR2nmjIhIMRT8GbLVzHOJ0swZEZFCqdSTIVvN/PDR4z0uqN5FM2dEJI4SH/y5pm6m18xzrfAZ1ZkzcZqOKiLBS3TwZwZ6rkHbOM2cKfQ9iUhyJTr4e1vuIDMk4zJzppj3JCLJlOjB3Tgsd1CsanxPIlJZiQ7+XIOzcR60rcb3JCKVlejgr8blDqrxPYlIZSW6xh+nQdtCVeN7EpHKqvqLrYuIJFWui60nuscvpdF5AiLxpuCXoug8AZH4S/TgrhQv6pd6FJH8FPxSFJ0nIBJ/Cn4pis4TEIk/Bb8URecJiMSfBnelKDpPQCT+FPxStLgsWCci2anUIyKSMAp+EZGEUfCLiCRM4MFvZuea2VNmtsnMNpjZZ4Nug4hIkoUxuHsc+IJzbpWZDQZWmtkC59zGENoiIpI4gff4nXOvO+dWpb4+BGwCNEVERCQgodb4zawBaARWZLnvbjNrMbOWPXv2BN00EZGqFVrwm9npwC+Be5xzBzPvd8497Jxrcs41jRgxIvgGiohUqVCC38xq8EL/p865x8Nog4hIUgU+uGtmBvwA2OSc+3bQrx80XbRERKImjB7/FOBDwHQzW536d3MI7fBd10VL2g604+i+aMn81rawmyYiCRZ4j985twSwoF83DL1dtES9fhEJi87c9ZEuWiIiUaTg95EuWiIiUaTg95EuWiIiUaT1+H2ki5aISBQp+H2mi5aISJPN0N4AAAaKSURBVNSo1CMikjAKfhGRhFHwi4gkjIJfRCRhFPwiIgljzrmw25CXme0BtpX49OHA3go2p1LUruKoXcVRu4pTre0a45w7ZV37WAR/OcysxTnXFHY7MqldxVG7iqN2FSdp7VKpR0QkYRT8IiIJk4TgfzjsBuSgdhVH7SqO2lWcRLWr6mv8IiLSUxJ6/CIikkbBLyKSMLEOfjO70cw2m9krZnZvlvsHmNkjqftXmFlD2n33pW7fbGazA27X581so5mtNbNFZjYm7b7OtGsR/zrgdn3EzPakvf6fpt13p5m9nPp3Z8Dt+k5am14yswNp9/myvczsh2a228zW57jfzOwfUm1ea2aXpd3n57bK164Pptqz1syWmdmlafdtNbN1qW3VEnC7rjOzt9J+V19Ju6/X37/P7Zqb1qb1qc/TsNR9fm6vc83sKTPbZGYbzOyzWR7j32fMORfLf0Bf4PfAWKA/sAYYn/GYPwf+OfX17cAjqa/Hpx4/ADgv9XP6Btiu64FBqa8/2dWu1Pdvh7i9PgL8Y5bnDgO2pP4fmvp6aFDtynj8XwA/DGB7XQtcBqzPcf/NwP/gXT/6amCF39uqwHZN7no94KaudqW+3woMD2l7XQf8V7m//0q3K+Ox7wEWB7S9zgEuS309GHgpy9+jb5+xOPf4rwRecc5tcc4dA34OzMl4zBzgx6mvHwNmmJmlbv+5c+6oc+5V4JXUzwukXc65p5xzR1LfLgdGVei1y2pXL2YDC5xz+5xz+4EFwI0htesDwM8q9No5OeeeAfb18pA5wL85z3KgzszOwd9tlbddzrllqdeF4D5bhWyvXMr5XFa6XYF8tgCcc68751alvj4EbAIyL9zh22cszsFfD7yW9v0OTt1wJx/jnDsOvAX8QYHP9bNd6e7C26t3GWhmLWa23MyaK9SmYtp1W+qw8jEzO7fI5/rZLlIlsfOAxWk3+7W98snVbj+3VbEyP1sO+K2ZrTSzu0Nozx+a2Roz+x8zm5C6LRLby8wG4YXnL9NuDmR7mVeCbgRWZNzl22cszlfgsiy3Zc5NzfWYQp5bqoJ/tpndATQB09JuHu2c22lmY4HFZrbOOff7gNr1n8DPnHNHzezP8I6Wphf4XD/b1eV24DHnXGfabX5tr3zC+GwVzMyuxwv+qWk3T0ltqzOBBWb2YqpHHIRVeOvGvG1mNwPzgXcTke2FV+ZZ6pxLPzrwfXuZ2el4O5t7nHMHM+/O8pSKfMbi3OPfAZyb9v0oYGeux5hZP+AMvMO+Qp7rZ7sws5nA/cB7nXNHu253zu1M/b8FeBqvJxBIu5xzb6a15V+Aywt9rp/tSnM7GYfiPm6vfHK1289tVRAzuwT4PjDHOfdm1+1p22o38B9UrryZl3PuoHPu7dTX/w3UmNlwIrC9Unr7bPmyvcysBi/0f+qcezzLQ/z7jPkxcBHEP7yjlS14h/5dg0ITMh7zKXoO7j6a+noCPQd3t1C5wd1C2tWIN6D17ozbhwIDUl8PB16mQgNdBbbrnLSv3wcsd92DSa+m2jc09fWwoNqVetw4vME2C2J7pX5mA7kHK/+IngNvz/u9rQps12i8MavJGbefBgxO+3oZcGOA7Tq763eHF6DbU9uuoN+/X+1K3d/VITwtqO2Veu//Bvx9L4/x7TNWsY0bxj+8Ue+X8EL0/tRtX8frRQMMBH6R+kN4Hhib9tz7U8/bDNwUcLsWAruA1al/v07dPhlYl/rwrwPuCrhd84ANqdd/Crgw7bkfS23HV4CPBtmu1PdfBR7MeJ5v2wuv9/c60IHXw7oL+DPgz1L3G/B/U21eBzQFtK3ytev7wP60z1ZL6vaxqe20JvU7vj/gdn067bO1nLQdU7bff1DtSj3mI3iTPdKf5/f2mopXnlmb9ru6OajPmJZsEBFJmDjX+EVEpAQKfhGRhFHwi4gkjIJfRCRhFPwiIgmj4BcpUmplxVfTVnEcmvp+TL7nikSBgl+kSM6514B/Ah5M3fQg8LBzblt4rRIpnObxi5Qgdbr9SuCHwMeBRuetLikSeXFepE0kNM65DjObC/wGuEGhL3GiUo9I6W7CWw7g4rAbIlIMBb9ICcxsEjALb/Gsz6UukCESCwp+kSKlruL2T3hrqG8HHgL+NtxWiRROwS9SvI8D251zC1Lffw+40Mym9fIckcjQrB4RkYRRj19EJGEU/CIiCaPgFxFJGAW/iEjCKPhFRBJGwS8ikjAKfhGRhPn/QeC1OVVk1wEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(1,1)\n",
    "ax.scatter(X, y)\n",
    "_ = ax.set_xlabel('X')\n",
    "_ = ax.set_ylabel('y')\n",
    "x = np.linspace(0,2,100)\n",
    "ax.plot(x, x*a + b, color='r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.82652634],\n",
       "       [2.18471117]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_b = np.c_[np.ones((100, 1)), X] # add x0=1 to each instance\n",
    "theta_best = np.linalg.inv(X_b.T @ X_b) @ X_b.T @ y\n",
    "theta_best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1ba41ba9608>]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEGCAYAAABiq/5QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXhU5dn48e8zk4QkbGEVCIR1QEWKYETZd4EZEVDrUrXihq3WqrX0hZ+2amsrlS72fbsZFbXVuitqEzZZRFTEsEQEhIQtEJA9rJNt5vn9cSbJkMwkM8nMnJnM/bkuLpKZMznPnJzcc5/73Oc5SmuNEEKI+GExewBCCCEiSwK/EELEGQn8QggRZyTwCyFEnJHAL4QQcSbB7AEEon379rpHjx5mD0MIIWLK+vXrj2qtO9R8PCYCf48ePcjNzTV7GEIIEVOUUnt9PS6lHiGEiDMS+IUQIs5I4BdCiDgjgV8IIeKMBH4hhIgzEviFECLOxEQ7pxBCRKuFG4uYv2Q7B4qddElLYfakfkwflG72sOokgV8IIRpo4cYi5r63GWe5C4CiYidz39sMENXBX0o9QgjRQPOXbK8K+pWc5S7mL9lu0ogCI4FfCCEa6ECxM6jHo4UEfiGEaKAuaSlBPR4tJPALIUQDzZ7Uj5RE63mPpSRamT2pn0kjCoyc3BVCiAaqPIErXT1CCBFHpg9Kj/pAX5OUeoQQIs5I4BdCiDgjgV8IIeKMBH4hhIgzEviFECLOSOAXQog4I4FfCCHijAR+IYSIMxL4hRAizkjgF0KIOCOBXwgh4owEfiGEiDMS+IUQIs5I4BdCiDgjgV8IIeKMBH4hhIgzEviFECLOSOAXQog4I4FfCCHijAR+IYSIMxL4hRAizoQt8CulFiilDiulvvF6rK1SaplSKt/zf5twrV8IIYRv4cz4XwYm13hsDrBca20Dlnu+F0IIEUFhC/xa69XA8RoPTwNe8Xz9CjA9XOsXQoiYVVEBH30EN90EJSUh//GRrvFfoLU+COD5v6O/BZVSs5RSuUqp3CNHjkRsgEIIYZrCQnj8cejRA665Bj75BLZvD/lqovbkrtY6S2udqbXO7NChg9nDEUKI8KiogA8+AIfDCPi/+Q0MGADvv298EAwcGPJVJoT8J9btkFKqs9b6oFKqM3A4wusXQojosHcvvPACLFgABw5A587w6KNw113GB0AYRTrwfwjcDszz/P9BhNcvhBDmKS+H7GzIyoLFi43HpkyBv/0Nrr4aEiITksO2FqXU68AYoL1Saj/wOEbAf0spdRdQCHw/XOsXQoiosWdPdXZ/8CB06QKPPWZk9927R3w4YQv8Wuub/Tw1PlzrFEKIqFGZ3T/3HCxZAkrB5Mnwz3+C3R6x7N4X89YshBBNUc3sPj0dfvUruPNOyMgwe3SABH4hhGi88nKj7z4rC5YuNbJ7ux3uvdfI8k3M7n2JrtEIIUQs2b27Orv/7jvo2tXow7/zTujWzezR+SWBXwgRdRZuLGL+ku0cKHbSJS2F2ZP6MX1QutnDMpSXw4cfVmf3FovRkXPPPUaHjtVq9gjrJYFfCBFVFm4sYu57m3GWuwAoKnYy973NAOYG/1274Pnn4aWX4NAhI6N/4gmjM6drV/PG1QAS+IUQUWX+ku1VQb+Ss9zF/CXbIx74P1i3m3V/eZnJn3/EyD0bcVutWBwOo3Y/aVJMZPe+SOAXQkSVA8XOoB4Pi4ICdvz2zwx/+z9MO1vM/lYd+OOIW/hw8GQenjk2espODSSBXwgRVbqkpVDkI8h3SUsJ74rLymDhQqN2v3w5vSwWVvQewn8GTmJ1z8G4LUZ2b8aRR6hJ4BdCRJXZk/qdV+MHSEm0MntSv/CsMD/fqN2//DIcOWL02v/614z4LoPvWravtXhEjzzCRAK/ECKqVGbTYe3qKS2tzu5XrDBq9VOnGrX7iRPBasU6bwWYceQRARL4hRBRZ/qg9PCUU3bsqM7ujx415sl56imj775z5/MWjfiRRwRJ4BdCNG2lpfDee0Z2v2qVkd1PmwazZhnZvcX3bUkicuRhEgn8Qoimafv26uz+2DHo2RN+9zuYObNWdu9P2I48TCaBXwjRdJSUVGf3n3xizJFTmd1PmOA3u483EviFELHv22+N7P6VV4zsvlcvI7u/4w7o1Mns0UUdCfxCiNhUUgLvvmtk96tXG9n99OlGZ864cZLd10ECvxDCVEFPyPbtt0awf+UVOH4ceveGefOM2v0FF0Rs3LFMAr8QwjQBT8hWmd0/9xx8+ikkJsKMGUbtfuxYye6DJFtLCGGauiZkA2DrVnj4YeMetbfeatzR6plnYP9+ePNNGD9egn4DSMYvhDCNr+kPmpWXcsWaFTDycVizxsjur7vOmO9+zBgJ9CEggV8IYRrvCdlsR/Zyc94Srt2ygrSSM2CzGdn9zJnQoYO5A21iJPALIUzzP6Mz+HzeP7lu/SIuL9pKqTWBj/sNp/VD9zPi7uuNe9eKkJPAL4SIvG++gawsrvn3v7mmuJi9Hbrx1Ni7+Hy4g1nXDmFEE7xaNppI4BdCRMa5c/D220ZnzhdfQFISXH893HMP3UeP5jHJ7iNGAr8QIrw2bzb67v/9bzh5Evr1gz/8AW6/HdrXnu9ehJ8EfiFE6J09C2+9ZQT8tWuN7P6664yrakeNktq9ySTwCyF8CvqKWoC8PGPOnH//G06dggsvhD/9CX74Q2jXLjIDj1IN2p5hIoFfCFFLwFfUgpHdv/mmkd1/+SU0awbf/75xVe2IEZLdE+T2jAC5EkIIUYu/K2ofeSuPhRuLjAc2bYL77jOuqr3rLiPDf/ZZOHDAyPhHjpSg71HvFcoRJhm/EKIWfzcUb1Z6jtxf/oFRe1fR9ptNRnZ/ww3GVbWS3fvlb3vWd+P2XSd20atNr5CPRwK/EKIW7ytqAfof2snNmxYzbesqWpY52d2xO22ffRZuuw3atjVxpLGh5vb0ftxbaUUpn+z9hOwd2eQU5FBwvID8B/Lp07ZPSMdjSuBXSj0M3A1oYDNwh9a6xIyxCCFqmz2pH795Yx0Tvl7JDzYtZuB3+ZQkJJHdbzj/uXQKG9IvYveDV5s9zJhR143b953cx6KCRWTnZ/Pxro85V36O5IRkxvYYy0NXPESb5DYhH0/EA79SKh34KXCx1tqplHoLuAl4OdJjEUL4sGED07OycPz7VRLPnWV7+wwen3Av7/cfy6nkFgCk18hU/TGzk8WMdftbp/eN24uKz9Cy1S5s3fP51dpfsPlD4yRvh5SutHJNoEXpIHomDWHWxQPDNl6zSj0JQIpSqhxIBQ6YNA4hBMCZM/D668ZVtevXQ0oKiTfeyCejZ/CjgkScFe6qRSsz1fqY2clixrrrWuewvomcsiynW58ctu5cwt6SYr7dlcDIjJHMnzifZhWZ/G2Zk5JyYzt/d1KHdbwRD/xa6yKl1B+AQsAJLNVaL430OIQwU9T0dK9fb7Rh/uc/RvAfMAD+7/+Mue/T0hgNPN3AsdbVyRLu92rGur3XqXFTpgoo1rnc9tEGzurtaDSdWnRixoUzsNvsTOw1kdbJrQEYPm9FVdCPxHjNKPW0AaYBPYFi4G2l1K1a61drLDcLmAWQkZER6WEKETam93SfPm1k91lZVdk9N91kdOZceWWtzhzvUkUwGtrJEgpmrHtf8RGc1o04Lbk4retxq2LQimauvjw57knsNjuDOg/Comp30Ud6vGaUeiYAu7XWRwCUUu8Bw4DzAr/WOgvIAsjMzNSRHqQQ4WJKJqz1+dn92bPwve/BX/9qZPetW4d8lYF2soRDsOtuyBGY1potR7aQk59Ddn42+1LWAG4sugXJrsGkuC8nxTWYjLRO/HL0uJCOt7HMCPyFwJVKqVSMUs94INeEcQhhiohmd6dOGYE+Kws2boTUVLjxRmPOnCFDwtp3X1cnS7gFs+5gjsDOlZ9jxe4VVe2WhScLARh4wUCus93PV9u6Q7kNhbXOdTZmvKFgRo3/S6XUO8AGoALYiCezFyIehD270xq++soI9q+/bkyHPHAg/P3v8IMfhCW798W7kyXS5zKCWXd9R2A7j++syupX7VlFqauU5onNmdh7Io+NfIwptil0bdUVaPi5m0hvK6V19FdRMjMzdW6uHBSIpqFmhglGdvf0tQMa94d+8mR1dr9pk5Hd33STkd1ffnnYr6qNmhPWQeo5JxvvKKgpp8SyhRLrV3S6YBs7ju0AoF+7fthtdhw2ByMyRtAsoZk5Aw6CUmq91jqz5uNy5a4QERbS7E5rWLfOCPZvvHF+dn/LLdCqVYhH75vpJ6wD4O+DqUtaCnuL9+G05uK05lJiyUMrJ0on0tsyjL9Mvh+HzUHvtr3NfgshI4FfCBM0tFOmysmT8NprRt/9119TkZLKov6jeeHiiRy98HvMvvJCpvsI+uHKyhtzwjoSRwo1P5j2F5/hwffe4NVtezjQbAVFKVsBsLo70Nw1mhTX5SS7B+I82JyMpAH0bhsdH16hIoFfiFihtTHtcWV273TCoEFs+n9Pc3e5jaOWZGO5kyU+s+1wZuUNPWEdqSOF+Uu2c6b8OE7rBpyWXEqs63GrMxTmWxjdfSQj0+eyclNXLO4MFNUlsUhddxBpEviFiHbFxfDqq0bA37wZmjc3yjj33guZmdw/bwVHawRYXwErFG2kdZVLGnLCOpytrW7tZtN3m8jekc1652uUJu8ApbHoNFJcV5DiupwU96WsmnkjAD03nF/rr9SQbqtoP98hgV+IaKS1ccvCrCzjJidOJwwebJR2br4ZWrasWjTQbNvfcr4Cti91ZecNbUcMdWvryZKTfLzrY3Lyc1hUsIiDZw6iUDS39iO57GZSXJkk6T4or1uRDJ+3gtmT+oWs2yoWzndI4Bcimpw4UZ3df/MNtGhh3Lbwnnvgsst8viTQgOVvOYURrOoLSnVl55/NGVe1TDBZbmODrdaab49+S3Z+Ntn52awpXEOFu4K05DSu6n0VDpuDyX0m8/mOciMYa1etn1EZmK+7LJ131xc1upfezKkqAiWBXwizaQ2ff24E+7fegpISI8j7yO59CTTbnj2pHw+/ualWOUNDQEGpvuy8ISesG3KkcK78HKv2rKrqrd9TvAeAAR0H8MjQR3DYHAztNpQES3V4mz7I+N+YHbP2+3CWu1j57RGevnZAo0s0Zk5VESgJ/EKY5cQJ4xaFWVmwZYsR4GfONLL7wYPPW7SumnGg7aHTB6Xz0JubfA4lkKAUjgvPAh377hO7ycnPIacghxW7V1BSUUJqYirje45nzvA5TLFNIaN13XN6VX4w1ezbr3Sg2Nn4bivMnaoiUBL4hYgkreGzz4xg//bbRnY/ZAg8/7xxsVWLFrVeEkjNONCAld6IoBSuaQV8jb3MVcaawjVVWf23R78FoE/bPswaPAtHXwejuo8iOSE56PWFOzCbOVVFoCTwCxEJx4/Dv/5lBPxt24zs/o47YNYsuPTS8xatmd2fLa0IWc24MUEp3NMKHDx9sOpOVMt2LuN02WmSrEmM6j6Key+7F4fNga2drdHrCXdgNnOqikDJlA1ChIvWsGZNdXZfWmpk9/fea0yU1rx5rZf4ms7BHwXsnucIeljR0mrocrtYV7SuqoSz4eAGALq26sqUPlNw2ByM7zWeFkm1j4IaK1q2Qbj5m7JBAr8QoXbsWHXtfts2Y9qE224zavcDB9b50uHzVgTcXpmellLVTeMtmoPasXPHWLJzCTn5OSwuWMwx5zEsysKwbsOw97Hj6OtgQMcBqDDPKxQvZK4eIcJJa/j0UyPYv/OOkd1feSW8+KLf7N6XQDs/QjHFcCRorck7lFc1jfHa/WtxazftU9tXTXh2Ve+raJMS+huKC/8k8AvRGMeOVdfuv/3WyO7vvjug7N4Xfyce26QmkpqU0Ogphr3VdWTQmKOG06WnWb57eVWwP3DauKX2ZZ0v47GRj2G32cnskonVYvVa18aoPEJpqiTwCxEsrWH16ursvqwMhg6Fl16CG24wpkNuIH8nHh+f2j+gYBhoD3ldRwZAUEcNWmu2H9tu1Orzc1i9dzXl7nJaNWt13kVUnVp0qvXaaDtCCVY0l9XqIoFfiEAdPQqvvGK0Xm7fDmlp8KMfGdn9JZeEZBWN7QgJtFWxriODyq99PVc5Dme5k0/2flKV1e86sQuAiztczENXPoTD5mBYt2EkWhPrHG8sXOXqTyx/aEngF6IuWsMnnxjZ/bvvGtn98OEwdy58//uNyu79acxFRIG2Kjbk6tLCk3v5x1f/IKcgh+W7luOscJKSkMK4nuP4+dCfM8U2hR5pPYIab2PnDzJTLH9oSeAXwpcjR4zsPisL8vOrs/tZs6B/f7NH51egRwz1HRkUFTvRVFBq2YbTkovT+hXllkLuy4GeaT25a9Bd2G12Tp208b8fF/LM105eTdvF7EmJQQW9xs4fZKa6PrQqJ36L1vFL4BeiktawapUxR85770F5uZHdP/aYkd2nhO6S+3DWhgM5YvB3ZHD36DZsPLyKzRve54zagFZnQSeQqvtzS/+ZzBnzA/q264tSioUbi3jig8aVOho7f5CZ/H1oQfSXfSTwC3H4cHV2X1BgZPf33WfU7sOQ3UeqNhzI/D7PLN7GnlNfY22+iYqWX3Pn0jwA2qRcQIfyUWjnIHq1HMqcyYNqjS0UpY7Gzh9kJl8fnt6iuewjgV/EJ7cbVq40gv377xvZ/YgR8PjjcN11Ic3ua4pEbbiuD5fRF6YaF1HtyaEgYTFHko9gcVu4suWV3DH4t9htdgZeMLDei6hCNQtlY+YPMpN3Wc1f5h+tH14S+ONYrLaiNcrhw0bb5fPPw86d0KYN3H+/Ubu/6KKIDCES0/Z6f7hoNOVqDyd1Lj/8aDZnP9qKW7tpl9KOyX0mV11E1S61XVDrCNVkZ7EwqZk/lWU1f1dcR+uHV72BXyn1E+A1rfWJCIxHREg0tKJF7IPH7YYVK4zsfuFCI7sfNQqeeAKuvx6Sg5/hsTEiMW3v/uLjOC15OK25OC25uCxHAUhy9Wbu6Lk4bA6GpA+puoiqIeoL2IH+fmNhUrP6xNqHVyAZfyfgK6XUBmABsETHwgQ/ok5mt6JF5IPn0KHq7H7XLmjbFn7yEyO7v/DC0KyjAcIVJPKP5ZOdn01Ofg77UlahKUfpFFLcg0gp+wHJrsvontaVp8bVnt+nIeoK2MH+fkMxD76ZYu3DK6BJ2pRR7LsKuAPIBN4CXtRa7wzv8AwySVv9AsmuvJfx91tv6IyPwfJ3aOxv4rGAud2wfHl1dl9RAWPGGCdqr7024tm9P6E42imtKOWTvZ9UzVlfcLwAgIvaX4St1Ug25vdElV2IwriIKiXRytPXDghLMKr5fs6VVXDiXHmt5Rr9+xVBadQkbVprrZT6DvgOqADaAO8opZZprX8R2qHGJjPr5YFkV4FO9xupmmTI69zffVed3e/eDe3awYMPGgG/X/Qdbjc0w913cl/VnPUf7/qYc+XnSE5IZmyPsTx4xYPYbXZ6tekFRG6f9LX/+ROtJzvjTSA1/p8CtwNHgReA2VrrcqWUBcgH4j7wm10vD6Rs42uZmhpTbgg2yISkzu12w7JlRnb/4YfV2f3vfgczZkCzZg14J9Glwl3BF/u+qCrhbD5s7FfdW3dn5sCZ2G12xvYcS2pi7SuII1U+CWTfqhStJzvjTSAZf3vgWq31Xu8HtdZupdTV4RlWbDG7Xh5I9lxXpqWgURlhQz74GlXnPniwOrvfs8fI7h96yMju+/YNevzR5vDZwywuWEx2fjZLdy6luKSYBEsCIzNGMn/ifKb0mcLFHS6OmjnrGzuVdFMVzV1z9QZ+rfWv6nhuW2iHE5si0Z5Xl0CyZ3/LhKLm2pAPvqBPhrlc52f3LheMGwfz5sH06TGd3bu1mw0HN1TV6r8q+gqN5oLmFzDjwhk4bA4m9JpA6+TWQf3cSAUef/tWWkoizZvVP5V0U2R2FaA+0scfApFoz6tLINlzONvNGvLBF3BQOnCgOrvfuxc6dICf/cyY8z6Gs/vikmKW7VxGTkEOi/IXcejsIRSKIelDeHLMkzj6Ori006VYlKVBPz+SgcffvvXENYFNJd0UmV0FqI8E/hAwu4c3kOw5nO1mwX7w1RuUXC5YutTI7j/6yPh+/HiYPx+mTYOkpEaPOdK01mw5sqUqq/+s8DNc2kWb5DZM6jMJh83BpN6T6NC8Q9VrGpOxRzLwRKKVMZrLJr6YXQWoj9xzN0RibccMJV8dQ3W1Dvpr5RyozvBB6g544QUoLISOHeGOO4zsvk+fsL6HcDhbdpaVe1ZWzVlfeLIQgIEXDKy67eAVXa8gwVI7/wp2m9bUc062z5bdSLXrhlJjt4UZwtauHKSouueuUioNo0PoEoyJ+O7UWn9hxlhCJdYvQGmMYDM+76zH4nYxavcGfpC3hHEF60C7jez+D3+Iyex+14ldVYF+5e6VlLpKaZ7YnAm9JvDYyMeYYptC11Zd6/059WXs9SUawRyFRXvSEu1lE1/MrgLUx6xSz1+AxVrr65VSSUDo72YhIiqYD74uaSm4Cvdxw+Zl3Ji3lPTTRziSmsbro27kthd/A717h3m0oVNaUcqnhZ9W3XZw+zHjDlZ92/Xlx5k/xtHXwciMkTRLCO7kc12lgkDq94EGnmg/CQnRXzbxJdqv5I144FdKtQJGATMBtNZlQFmkxyFM4HLB4sW8s/h/6bj6Y6zazeoeg/jN+Lv5/KKh/Pr7g6F39NeFi04VnXcR1ZmyMzSzNmNMjzHcf/n9TLFNoU/bxpWm6srYA8mAAw08sZBNm9080VDRXAUwI+PvBRwBXlJKDQTWAw9qrc96L6SUmgXMAsjIyIj4IIV/QQfUfftgwQJ48UXYt4/OHTuyY+aPebT9leRa2tAlLYVfh+FkYKgyWZfbxdr9a6tOzOYdMuas79aqG7cOuBW7zc64nuNontQ8ZOOvK2N/OMD56wMJPLGQTUd72SQWmRH4E4DBwANa6y+VUn8B5gC/9F5Ia50FZIFxcjfioxQ+BRxQKypg0SKjMycnx7jK9qqr4M9/hqlT6ZuUxNtBrDPYzD3YTLbmOu4d0wFLah45+Tks2bmE487jWJWV4RnD+f2E3zOlzxQu6XhJ2C6iqitj9zf/e0My4HBk06E+0orGskm0nxepjxmBfz+wX2v9pef7dzACvwiB+nbIxu6w9QbUffuMzP7FF2H/fujUCebMgbvugl69GvR+GpK5B5PJLtxYxJz38jhZkY8z4SsOOHP5fPEOUJqOzTsyte9UHDYHE3tPJC05Lej30FD+MvZQZsChzqbDdc4gmsomsXBepD4RD/xa6++UUvuUUv201tuB8cDWSI+jKapvhwzFDusrcFrdLi7KXQVX/8nI8rWGSZPgL3+BqVMhMTHo91H54WRRCleNluNAatCBZLKnSk+xbOcy7vvvCxyzrsOVcByAJLeN1hU3k5E6nE2P/KjBF1GFSygz4FBn07FwzqCxmsJ7NKur5wHgNU9Hzy6M6Z5FI9W3Q4Zih/UOqF1OHebGvGXc8PVSOp85Bp07G9n93XdDz54Neg81P5xqBv1K9dWgfWWyyYkWbhpmYebbv+T9bf/llHszKBdKNyfFNYgU9+WkuAZjpQ0Ap08RdUG/Uigz4FD+rFg4Z9BYTeE9mhL4tdabMOb1FyFU3w4Zih32F+N7s3T+Aq5fn83oXRsAWNP7Mgqf/D1XPPDDoLP7mgKd6bG+GnRlIJu3OI/dp9dhSdmIM3kjP125D4BEdw9auWaQ4s6kmfsiFLXvRBXtXSPRKFY7cILRFN6jTNnQhNS3QzZqh927F154gWkLFjDtwAGOtGrH34bewMqR0/jhTaPCnjF6q68Gvad4T9VFVHnuFZQ0KyHVmsr4ruPZuGMGuuRSEnTHRq1D+BYPHThN4T1K4G9C6tshg95hy8shOxueew6WLDEemzwZ/v53OjgcPJCQwAMhfg/+PpysSuHW2mcNusxVxprCNVUXUW07akwa26dtH+697F7sNjujuo8iOSGZHnOy/a47PS0lZrs0okU0duCEWlN4jzJXTxMTkq6ePXuM+XIWLDDmvk9PN7py7rwTundv9Bjqe20g87IcPH2w6iKqZTuXcbrsNEnWJEZ3H43D5mCKbQp929WevbOuwL8nxuawEaI+UTVXT1Nmdn9vfSfq/D5fXm7MhJmVZcyMqRTY7cbNTex2SAhsV2ls55C/bGrqwE58se+LqouoNn63EYD0luncfMnN2G12xvcaT4ukFnX+/DapiT7vBdsmtXHnJoSIJZLxh1AsziLI7t3V2f1331Vn93fdBQ24YjqUsxIeO3eMpTuXkp2fzeKCxRxzHsOiLAzrNgyHzYHdZmdAxwFBXUS1cGMRs9/Jo9xVvd8nWhXzrx8Yvb8jIRpIMv4IiJn+3vJy4y5Wldm9xQIOB8yaZdTwA8zufWlM55DWmrxDeVVZ/dr9a3FrN+1T2zPFNqVqzvo2KW0aPL6mUJ+NNLOPYkXoSeAPoajv7921qzq7P3QIunaFJ54wavfduoVkFcF2Dp0uPc3y3curunAOnD4AwGWdL+OxkY9ht9nJ7JKJ1VK73bKhoukq0Gjnq3T38JubyN17nKemDzB5dKKhJPCHUFT295aVse4vL0HW8wwpWI9LWTg8agKdX/ypkd1bQxdQof7OIa01O47tqMrqV+9dTbm7nFbNWjGx18SqE7OdWnQK6bhEw/g6itXAa2sLyezeVj5AY5QE/hCKqv7enTvhhRcoyXqBIcePsr9VB/404hbeGjCRk+0u4OkuA5ge4qAPvkspP52QQXLLzTyQM4+cghx2ndgFQP8O/Xnoyodw2BwM6zaMRKucYI02/o5WNURfCVMETAJ/CJlePy4rgw8+MGr3H38MVivr+l3BgnH3s7rnYNyV5ZIQnnfwV/8d1LPC6KsvyOGOJctxVjhJSUhhXM9xPDL0Eew2Oz3SejR6/eHU2Np2U6iN+zuKhSgqYYqgSVdPU1BQYNTuX3oJDh+GjAy2OW5gTutM8rTv9sZQ3HvVu/6rqaDUso3yxPU0b72ZfaeNO1H1TOuJw+bA0dfB6O6jSUmMjcvaG9uhFZ+UHmYAABQASURBVJMdXj4s3FjEw29u8nn/3kjfP1YET7p6mpqyMli40Mjuly83avVTp8I997Cw4yXM/WBrnXPehOK8w28Xf8ER92c4E3NxWjei1VnQCbjOfI8/XvVHHDYHfdv1Dduc9eHU2A6tmOnwqsf0Qenk7j3Oa2sLzwv+sTZFgTifBP5Yk58Pzz8PL78MR44YV9I+9RTccQd06QLA/Hkr6gz6Df2jdWs3uQdyqzpwcstyIQmsui3NXcNJcWWS7L4UK6n8bGhsXwXb2A6tqO/wCsJT0weQ2b1tzJetRDUJ/LGgtNTI7p97DlauNLL7a64x+u4nTqzVmVNXcEkP8o/2hPPEeRdRHTl3BIuycEX6FWRY76T87KUk6p4oqrP6WJql0J/GdmiFq8PLrPMG0gLbtEjgj2Y7dlRn90ePQo8e8NRTLLp8Ck9tKObAKiddNn1S64/fX9AJpCarteabw9+QnZ9NTn4On+/7HJd20S6lHZP7TMZuszOp9yTapbbzW8duCiWAxnZohaPDqync+UlEBwn80aa0FN57z6jdr1plXEU7dSrcey9MnMjCvIP1/vEHG3TOlJ1hxe4VVSWc/af2AzCo0yDmjpiL3WZnSPqQWhdRmd7FFEaNfW/h2DZN5byBMJ909USL7duNYP/KK3DsmHEHq3vugZkzjTtbeQQ6F059JYH8Y/lV7Zar9qyizFVGy6SWTOw9EXsfO1NsU+jSsktY33IsM6Pk0nNOts/umlB0aImmSbp6olFJSXV2/8knRnY/fbpRux8/3phDp4ZATxrWrMmWVJSwdOfSqjnr84/nA3BR+4t4YMgD2G12RmSMIMmaFNRbaAq96sEyq+QSlVeGi5gkgd8M27YZtftXXoHjx6FXL3j6aSO771T3VAXB/PHvO7mvKqv/eNfHnCs/R3JCMmN7jOWnV/wUu81Orza9Gvw2AgmATfGDwaySS1RdGS5imgT+SCkpgXffNTpzPv3UyO5nzDDKOX6ye1/q+uOvcFecN2f95sNGEO7eujszB87EbrMztudYUhNTQ/KW6guATfVkZH1HXeH6sGvK51REZEngD7etW6uz+xMn2N+2C6+Onsma4Q7uvn5o0H+0Nf/427cuYVj//byx82XuWLqE4pJiEiwJjMgYwTMTnsFus3Nxh4vDchFVfQHQ3wfDI2/lnfdeYk1dR13h/rCTtkoRChL4w8HprM7u16yBxESKxkzisY7DWJV+CVpZwEWDAoJbu+l6wUGuuvJTcgpy+KroK3I3ajq16MSMC2fgsDmY0GsCrZNbh+vdVamv7OTvg8GldUxn/nUddUnnjYgFEvhDaetW40Ttv/4FJ05Anz7wzDNw++3csOCbWkEy0IDw2rpt/HrZGxwo+ZzShPWUcwKFYkj6EJ4c8ySOvg4u7XQpFhVYuShU6qs51zXBVywHw7pKLg+/ucnna2Lxil3RdEngD0CdNVunE955x8juP/sMEhPhuuuMzpzRo6tq98Fcwq+1ZuuRrWTnZ/Ovje+z5ehXoFxYrC1IrhhMe3UFT9tv5fYrvxe29xyI+mrOvj4YvMVyMPRXcpHOGxELJPDXw1/NtuXO7Yz/9AMjuy8uBpvNyO5nzoQOHWr9nPoCwrnyc+ddRFV4shCAVNWbVhXXkuK+nGbufiiMi6iyVh3l9iuDfy+hPjFYV8258vFH3srD5eN6kaYYDKXzRsSCuAv8wQY/75ptcnkJjm8/4+a8xWQWbYOkJLj2WuOq2tGjoY4TqL4CgjXxMBfb9jPltfms3L2SUlcpzRObM7H3xKrbDo74ne8pcYPNls3qsKn82fESDKXzRsSCuAr8DQl+B4qd9D2yh5vzlnDtNytoXXqWnW3T+e3YO3n0rd9D+/YBrXv6oHTKXaU8sfRd9js/ozxxA069j13fQN92fflx5o+x2+yM6j6KZgnNql7XJW17SEoHZp50DCYYNoW+f+m8EdEuLgJ/ZTDxFUBrBr/KZY8fOcGthV+ycO1/Gbh/G6XWBBb1G87rAyfzZbdLSG+TyqMBBP2iU0UsKlhEdn42H+/6mDPlZ2jWrBljeozBbvs5dpudPm37+H19qEoHZk8THEgwbKp9/0JEmyYf+H3NIFnTgWInCzcW8cSHW7igMJ9ZmxYxY8sqWpWeZWfbrvx2/N283X8cxSmtgLoDr8vtYu3+tVUXUeUdMnrWu7Xqxq0DbsVuszOu5ziaJzUPaPyhKh3EwklHaYUUIjKafOD3FUxqusDqYt2v/siCDTkMPrCdUmsiOf2G88bASXzZ7RLaNE+ieVICJ/0E3qPnjrKkYAnZ+dks2bmE487jWJWV4RnDmTd+Ho6+Dvp36N/gi6hCUTqIhZOOZh+VCBEvmnzgrytoXHR4F7d9vZSp36ygZek5Ctp25Tfj7ubdS6qze4Dic+Vs/NVVVd+7tZsNBzdUdeB8uf9LNBqLTqOddQg/z7yWR8ffSFpyWljfWzBi4aRjLByVCNEUmBb4lVJWIBco0lpfHa711AwmKWUlXP3tam7ZtJhLD+7AldSMj/oM5bVLJ/NV1/4+O3O6pKVwqvQUy3YuIyc/h0UFizh45qDx8+hLq/KbSXFlkqT7oLDw37VWhnc5y/RB0RP4IfpPOsbCUYkQTYGZGf+DwDagVX0LNkZlMOm5P5+b8xYzfctKWpY5OdXTBs8+i/W223ji/3Ipdpaf9zqNpkLtpzwxl1Mtt9HumXVUuCto3aw1V/W+ipLT32PTjh5YaFNrnVKXbphYOCoRoikwJfArpboCDuC3wM/CtqIzZ5i+fhGj3/8rbbbkUZKQxMoBo2n+wH2MmjmtKruvTPLdlFBq2cw5ay5OSy4uyyEAOlkG8MjQR3DYHAztNpT/5h3i4Tc3UdcECVKXbphoPyoRoikwK+N/FvgF0DKsa7Hb4dNPsfbux5+vvp9Xeo2geeeOzL60X1W031O8h8LS93Em5VJi+RqtylC6GcnuS0ktu56Nv/gfurXudt6Pnb9ku8+LqrxJXVoIEa0iHviVUlcDh7XW65VSY+pYbhYwCyAjI6NhK3viCVYXnubegiScFW4AThSf4qfvv8rLW3az4+Rqth3dBkmQ4O5MC9ckUlyZJLsHoEgiPS2lVtCH+rN5qUsLIaKZGRn/cOAapZQdSAZaKaVe1Vrf6r2Q1joLyALjnrsNWtO4ccydt4LTFUWUWHNxWnNxWjailZP9OxOZ0GsMsy6bRWL5Zfx16dmATyrWNetkm9REHp/aX8oVQoioFfHAr7WeC8wF8GT8P68Z9EPlT1/8iVzn3ylL2QmAVbejuWs0Ka5MUtwDWXrb9VXLprcIfKoAX90nCrjlygyemj4gHG9FCCFCpkn38W85vIVkayqpZbeT4sokUfdAYdT202vU4IM5qSjdJ8HxNf8OyPYTwixK+5guN9pkZmbq3NzcoF+nteaDTQd89oY/fe0ACTQR4GvKjESLAgXlrup9T34nQoSeUmq91jqz5uORvWVThCmlmD4onaevHUB6WgoKI9OXABM5vqbMKHfr84I+VF/7IIQIvyZd6qkkveHmCeZ6Brn2QYjIiIvA39RF8xz2dXVA+VpWCBF+TbrUEw8qa+hFxU401XPYL9xYZPbQAKMDKiXRet5jiRZFovX8OZHk2gchIkcCf4yraw77aODrHMv87w9k/vUD5byLECaRUk+Mi4U57P2dY5FAL4Q5JPD7EM0185pkDnshRLCk1FNDtNfMa/JVQ5d6uRCiLpLx1+CvZv7kR1ui8ihAriIWQgRLAn8N/mrjJ86Vc+KccbOWyqMAiI46tVynIIQIhpR6agi0Nh5NnTNCCBEMCfw1+KqZ+xNNnTNCCBEoKfXU4Ktmfra0otY9eUE6Z4QQsSnuA7+/1k3vmrmvGSajuXMmltpRhRCRF9eBv2ZA93fSNpY6ZwJ9T0KI+BXXgb+u6Q5qBslY6ZwJ5j0JIeJTXJ/cjYXpDoLVFN+TECK04jrw+zs5G8snbZviexJChFZcB/6mON1BU3xPQojQiusafyydtA1UU3xPQojQatI3WxdCiHjm72brcZ3xi4aR6wSEiG0S+EVQ5DoBIWJfXJ/cFcGL9ls9CiHqJ4FfBEWuExAi9kngF0GR6wSEiH0S+EVQ5DoBIWKfnNwVQZHrBISIfRL4RdBiZcI6IYRvUuoRQog4I4FfCCHijAR+IYSIMxEP/EqpbkqplUqpbUqpLUqpByM9BiGEiGdmnNytAB7RWm9QSrUE1iullmmtt5owFiGEiDsRz/i11ge11hs8X58GtgHSIiKEEBFiao1fKdUDGAR86eO5WUqpXKVU7pEjRyI9NCGEaLJMC/xKqRbAu8BDWutTNZ/XWmdprTO11pkdOnSI/ACFEKKJMiXwK6USMYL+a1rr98wYgxBCxKuIn9xVSingRWCb1vpPkV5/pMlNS4QQ0caMjH84cBswTim1yfPPbsI4wq7ypiVFxU401TctWbixyOyhCSHiWMQzfq31GkBFer1mqOumJZL1CyHMIlfuhpHctEQIEY0k8IeR3LRECBGNJPCHkdy0RAgRjWQ+/jCSm5YIIaKRBP4wk5uWCCGijZR6hBAizkjgF0KIOCOBXwgh4owEfiGEiDMS+IUQIs4orbXZY6iXUuoIsLeBL28PHA3hcEJFxhUcGVdwZFzBaarj6q61rjWvfUwE/sZQSuVqrTPNHkdNMq7gyLiCI+MKTryNS0o9QggRZyTwCyFEnImHwJ9l9gD8kHEFR8YVHBlXcOJqXE2+xi+EEOJ88ZDxCyGE8CKBXwgh4kxMB36l1GSl1HalVIFSao6P55sppd70PP+lUqqH13NzPY9vV0pNivC4fqaU2qqU+loptVwp1d3rOZfXvYg/jPC4Ziqljnit/26v525XSuV7/t0e4XH92WtMO5RSxV7PhWV7KaUWKKUOK6W+8fO8Ukr9r2fMXyulBns9F85tVd+4bvGM52ul1OdKqYFez+1RSm32bKvcCI9rjFLqpNfv6ldez9X5+w/zuGZ7jekbz/7U1vNcOLdXN6XUSqXUNqXUFqXUgz6WCd8+prWOyX+AFdgJ9AKSgDzg4hrL3Af80/P1TcCbnq8v9izfDOjp+TnWCI5rLJDq+frHlePyfH/GxO01E/irj9e2BXZ5/m/j+bpNpMZVY/kHgAUR2F6jgMHAN36etwOLMO4ffSXwZbi3VYDjGla5PmBK5bg83+8B2pu0vcYA/23s7z/U46qx7FRgRYS2V2dgsOfrlsAOH3+PYdvHYjnjHwIUaK13aa3LgDeAaTWWmQa84vn6HWC8Ukp5Hn9Da12qtd4NFHh+XkTGpbVeqbU+5/l2LdA1ROtu1LjqMAlYprU+rrU+ASwDJps0rpuB10O0br+01quB43UsMg34lzasBdKUUp0J77aqd1xa688964XI7VuBbC9/GrNfhnpcEdm3ALTWB7XWGzxfnwa2ATVv3BG2fSyWA386sM/r+/3U3nBVy2itK4CTQLsAXxvOcXm7C+NTvVKyUipXKbVWKTU9RGMKZlzXeQ4r31FKdQvyteEcF56SWE9ghdfD4dpe9fE37nBuq2DV3Lc0sFQptV4pNcuE8QxVSuUppRYppfp7HouK7aWUSsUInu96PRyR7aWMEvQg4MsaT4VtH4vlO3ApH4/V7E31t0wgr22ogH+2UupWIBMY7fVwhtb6gFKqF7BCKbVZa70zQuP6CHhda12qlPoRxtHSuABfG85xVboJeEdr7fJ6LFzbqz5m7FsBU0qNxQj8I7weHu7ZVh2BZUqpbz0ZcSRswJg35oxSyg4sBGxEyfbCKPN8prX2PjoI+/ZSSrXA+LB5SGt9qubTPl4Skn0sljP+/UA3r++7Agf8LaOUSgBaYxz2BfLacI4LpdQE4FHgGq11aeXjWusDnv93AaswMoGIjEtrfcxrLM8DlwX62nCOy8tN1DgUD+P2qo+/cYdzWwVEKfU94AVgmtb6WOXjXtvqMPA+oStv1ktrfUprfcbzdQ6QqJRqTxRsL4+69q2wbC+lVCJG0H9Na/2ej0XCt4+F48RFJP5hHK3swjj0rzwp1L/GMvdz/sndtzxf9+f8k7u7CN3J3UDGNQjjhJatxuNtgGaer9sD+YToRFeA4+rs9fUMYK2uPpm02zO+Np6v20ZqXJ7l+mGcbFOR2F6en9kD/ycrHZx/4m1duLdVgOPKwDhnNazG482Bll5ffw5MjuC4OlX+7jACaKFn2wX0+w/XuDzPVyaEzSO1vTzv/V/As3UsE7Z9LGQb14x/GGe9d2AE0Uc9j/0aI4sGSAbe9vwhrAN6eb32Uc/rtgNTIjyuj4FDwCbPvw89jw8DNnt2/s3AXREe19PAFs/6VwIXer32Ts92LADuiOS4PN8/Acyr8bqwbS+M7O8gUI6RYd0F/Aj4ked5BfzNM+bNQGaEtlV943oBOOG1b+V6Hu/l2U55nt/xoxEe10+89q21eH0w+fr9R2pcnmVmYjR7eL8u3NtrBEZ55muv35U9UvuYTNkghBBxJpZr/EIIIRpAAr8QQsQZCfxCCBFnJPALIUSckcAvhBBxRgK/EEHyzKy422sWxzae77vX91ohooEEfiGCpLXeB/wDmOd5aB6QpbXea96ohAic9PEL0QCey+3XAwuAe4BB2phdUoioF8uTtAlhGq11uVJqNrAYuEqCvoglUuoRouGmYEwHcInZAxEiGBL4hWgApdSlwESMybMe9twgQ4iYIIFfiCB57uL2D4w51AuB+cAfzB2VEIGTwC9E8O4BCrXWyzzf/x24UCk1uo7XCBE1pKtHCCHijGT8QggRZyTwCyFEnJHAL4QQcUYCvxBCxBkJ/EIIEWck8AshRJyRwC+EEHHm/wOSKwIHeHTyjgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(1,1)\n",
    "ax.scatter(X, y)\n",
    "_ = ax.set_xlabel('X')\n",
    "_ = ax.set_ylabel('y')\n",
    "x = np.linspace(0,2,100)\n",
    "ax.plot(x, x*3 + 4, color='r')\n",
    "ax.plot(x,x*theta_best[1][0] + theta_best[0][0], color='g')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.82652634, 8.19594868]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prediction\n",
    "X_new = np.array([0, 2]).reshape(-1,1)\n",
    "X_new_b = np.c_[np.ones((2,1)), X_new]\n",
    "y_predict = theta_best.T @ X_new_b.T\n",
    "y_predict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Let's do the same with scikit-learn**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([3.82652634]), array([[2.18471117]]))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "lin_reg = LinearRegression()\n",
    "lin_reg.fit(X, y)\n",
    "lin_reg.intercept_, lin_reg.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.82652634],\n",
       "       [8.19594868]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_reg.predict(X_new)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**or directly with lstsq:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[3.82652634],\n",
       "        [2.18471117]]),\n",
       " array([111.32548813]),\n",
       " 2,\n",
       " array([14.5512568 ,  4.03441059]))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.linalg.lstsq(X_b, y, rcond=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient Descent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Batch Gradient Descent\n",
    "\n",
    "$\n",
    "\\frac{\\partial}{\\partial \\theta_j}\\text{MSE}(\\mathbf{\\theta})=\\frac{2}{m}\\sum\\limits_{i=1}^{m}(\\mathbf{\\theta}^\\top\\mathbf{x}^{(i)}-y^{(i)})x_j^{(i)} \n",
    "$\n",
    "\n",
    "vectorized version: \n",
    "\n",
    "$\n",
    "\\nabla_{\\theta}\\text{MSE}(\\mathbf{\\theta}) = \\begin{pmatrix}\n",
    "\\frac{\\partial}{\\partial \\theta_0}\\text{MSE}(\\mathbf{\\theta}) \\\\\n",
    "\\frac{\\partial}{\\partial \\theta_1}\\text{MSE}(\\mathbf{\\theta}) \\\\\n",
    "\\vdots \\\\\n",
    "\\frac{\\partial}{\\partial \\theta_n}\\text{MSE}(\\mathbf{\\theta})\n",
    "\\end{pmatrix} = \\frac{2}{m}\\mathbf{X}^\\top(\\mathbf{X}\\mathbf{\\theta}-\\mathbf{y})\n",
    "$\n",
    "\n",
    "Gradient descent step with learning rate $\\eta$:\n",
    "\n",
    "$\n",
    "\\mathbf{\\theta}^{(\\text{next step})}=\\mathbf{\\theta}-\\eta\\nabla_{\\mathbf{\\theta}}\\text{MSE}(\\mathbf{\\theta})\n",
    "$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.82652634],\n",
       "       [2.18471117]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# example \n",
    "eta = 0.1 # learning rate\n",
    "n_iterations = 1000\n",
    "m = 100\n",
    "\n",
    "theta = np.random.rand(2, 1)\n",
    "for iteration in range(n_iterations):\n",
    "    gradients = 2/m * X_b.T @ (X_b @ theta - y)\n",
    "    theta = theta - eta * gradients\n",
    "theta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x1ba442ae748>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdd1xc15nw8d+dQoeh9y5EESAJhAqqqAIuklziOHY2cRzHSd4U21l7N+XNZjf7buJN2ayTbHY3Hydx3HsEsg3qtiwZ9YoECETvfSgzMO28fyDJKvQ2g3W+/9gaDXOfGUbPPfe5zzlHEUIgSZIkzT0qewcgSZIkTY5M4JIkSXOUTOCSJElzlEzgkiRJc5RM4JIkSXOUZjYP5u/vL6Kjo2fzkJIkSXPeyZMn24UQATc/PqsJPDo6mhMnTszmISVJkuY8RVFqhntcllAkSZLmKJnAJUmS5iiZwCVJkuaoWa2BD8dsNlNfX8/AwIC9QxmRi4sL4eHhaLVae4ciSZJ0jd0TeH19PZ6enkRHR6Moir3DuYUQgo6ODurr64mJibF3OJIkSdfYvYQyMDCAn5+fQyZvAEVR8PPzc+grBEmSbk92H4EDDpu8r3L0+CRJmj47Tjfwy11lNHYbCfV25ZnsBLanhdk7rGE5RAKXJElyBDtON/CDd89jNFsBaOg28oN3zwM4ZBK3ewnFERQWFpKQkEBcXBzPPvusvcORJMlOfrmr7FryvspotvLLXWV2imh0t30Ct1qtfOtb36KgoICLFy/y2muvcfHiRXuHJUmSHTR2Gyf0uL3d9gn82LFjxMXFERsbi5OTEw8++CB5eXn2DkuSJDsI9Xad0OP25lA18H/ZeYGLjT3T+poLQr34yd3JI/59Q0MDERER1/4cHh7O0aNHpzUGSZLmhmeyE26ogQO4atU8k51gx6hG5lAJ3B6G2xNUdp1I0u3p6o1K2YUyCaONlGdKeHg4dXV11/5cX19PaGjorMchSZJj2J4W5rAJ+2a3fQ186dKllJeXU1VVhclk4vXXX2fr1q32DkuSJGlMDjUCtweNRsPvf/97srOzsVqtPProoyQnz/6VgCRJ0kTd9gkc4I477uCOO+6wdxiSJEkTctuXUCRJkuYqmcAlSZLmKJnAJUmS5iiZwCVJkuYomcAlSZLmKJnAJUmS5iiZwIFHH32UwMBAUlJS7B2KJEnSuMkEDjzyyCMUFhbaOwxJkqQJkQkcWLt2Lb6+vvYOQ5IkaUIcayZmwfeh+fz0vmZwKuTKXXYkSfrskSNwSZKkOcqxRuBypCxJkjRucgQuSZI0R8kEDnzhC18gMzOTsrIywsPD+dOf/mTvkCRJksbkWCUUO3nttdfsHYIkSdKEyRG4JEnSHCUTuCRJ0hw1ZgJXFOXPiqK0KopSfN1jvoqi7FEUpfzKf31mNkxJkiTpZuMZgb8A5Nz02PeBfUKI+cC+K3+WJEmSZtGYCVwIcRDovOnhbcBfr/z/X4Ht0xyXJEmSNIbJ1sCDhBBNAFf+GzjSExVFeVxRlBOKopxoa2ub5OEkSZKkm834TUwhxB+FEBlCiIyAgICZPtyk1NXVsX79epKSkkhOTua5556zd0iSJEljmmwfeIuiKCFCiCZFUUKA1ukMarZpNBp+/etfk56eTm9vL0uWLGHz5s0sWLDA3qFJkiSNaLIj8Hzgy1f+/8tA3vSEYx8hISGkp6cD4OnpSVJSEg0NDXaOSpIkaXRjjsAVRXkNyAL8FUWpB34CPAu8qSjKV4Fa4HPTEcy/H/t3SjtLp+Olrkn0TeQfl/3juJ9fXV3N6dOnWb58+bTGIUmSNN3GTOBCiC+M8FcbpzkWu+vr6+O+++7jP//zP/Hy8rJ3OJIkSaNyqLVQJjJSnm5ms5n77ruPhx9+mHvvvdducUiSJI2XnEoPCCH46le/SlJSEt/73vfsHY4kSdK4ONQI3F4OHz7MSy+9RGpqKosXLwbgZz/7GXfccYedI5OkuW/H6QZ+uauMxm4jod6uPJOdwPa0MHuH9ZkgEziwevVqhBD2DkOSPnN2nG7gB++ex2i2AtDQbeQH7w7teyuT+NTJBC5J0oz55a6ya8n7KqPZyi93lc16Av8sXgnIBC5J0oxp7DZO6PGZ8lm9EpA3MSVJmjGh3q4TenymjHYlMJfJBC5J0ox5JjsBV636hsdctWqeyU6Y1Tgc5UpguskELknSjNmeFsbP700lzNsVBQjzduXn96bOetnCUa4EppusgUuSNKO2p4XZvc78THbCDTVwsM+VwHSTCRwYGBhg7dq1DA4OYrFYuP/++/mXf/kXe4clSdI0uXoCkV0on0HOzs7s378fDw8PzGYzq1evJjc3lxUrVtg7NEmSpokjXAlMN1kDBxRFwcPDAxhaE8VsNqMoip2jkiRJGp1DjcCbf/YzBkumdzlZ56REgn/4wzGfZ7VaWbJkCRUVFXzrW9+Sy8lKkuTw5Aj8CrVazZkzZ6ivr+fYsWMUFxfbOyRJmpN2nG5g1bP7ifn++6x6dj87TsvNUWaKQ43AxzNSnmne3t5kZWVRWFhISkqKvcORpDnlszrj0VHJETjQ1tZGd3c3AEajkb1795KYmGjnqCRp7vmsznh0VA41AreXpqYmvvzlL2O1WrHZbDzwwAPcdddd9g5Lkuacz+qMR0clEziwcOFCTp8+be8wJGnOC/V2pWGYZD3XZzw6KllCkSRp2jjK2ie3CzkClyRp2nxWZzwClOvreb7sKMu9/bk3fp29wwFkApckaZp9lmY89pmNvFp+mDdbeikmBphHd/dJmcAlSZIckRCCj5qK+XNlKQdN4Qwo/vjbbOT0fMSGCmc2xq62d4jXyAQuSZIE1Pa18XxZETv1zjQpQTiJCBYNFLOyvodNNfMJjVxJvWs5g9EGwDG2aJMJXJJuA46QbByR0WLmraojvNbQzhlbFEKJJM5azoPt59l8KZBU92QM3n2cte7j4P5iVGoNK8Mf5pwqzCEmLMkEfoXVaiUjI4OwsDDee+89e4cjSdNmuNmRT71xhiffOEPYbZjMhRAcbavgTxXn2T8QRL/iia8wsbH3EOsqFVZ3J+EeEUGF+ynePfMLrGYzAZHRrH/kcRJXrcPNS8eqZ/c7xGbNMoFf8dxzz5GUlERPT4+9Q5GkaTXc7Ehx5b+301T3ZoOeP5cXsaNTUEsIWhFOquk8yxs62VgZQ0xIJm0u9Rxufx19aTPO7u6kbthCStZmAmPmAdBW24vFZHSYCUsygQP19fW8//77/OhHP+I//uM/7B2OJE2rsZKKPUaOs8VktbGj9iSv1NVzwhKBVQkm2lrJfZ172FjuzWISEUE2Ljgd5OjHL4CiEJW6mNUPf4m4jBVonJww9Jg4u6+O0qImOhr6Wbg+3GEmLDlUAv/4zUu01/VN62v6R3iw5oH4UZ/z5JNP8otf/ILe3t5pPbYkOYKRks31PmtT3c901PJ8xWl29/vSo3iiEz6s7f+EVVVmVrUkEhi5jhqPEj44818MnutDFxjEygceJnndRrz8A7FabNQUd1DySRO1xR3YbILAaC/WPZRA3JJAnrnk7hBbtE0pgSuK8hTwGENXZOeBrwghBqYjsNny3nvvERgYyJIlS/jwww/tHY4kTbvh9oO82XhHjo58M7R9wMCLl4/wdusAlYSiFqGkmotZ2tzC+opI5nsvo9ezi9MDu9i/vwKNkzPxy1eSnLWZiAUpKCoV7fW9HHqznLJjzQz0mTGq4bzWTKuflq9nh5By5b06yoSlSSdwRVHCgO8CC4QQRkVR3gQeBF6Y7GuONVKeCYcPHyY/P58PPviAgYEBenp6+OIXv8jLL78867FI0ky4Ptk0dBtR+LQGDuMfOdpzqdiRThxmq43CxmL+WlPFEVMoFsWXCFs127t3se6SB+mDSbiEx1Pmfpy3T76BzWolOC6ezV/7Ngkr1+Ds5o6xz8S5DxsoLWqiva4PlVrBKcqdnSoDZYoFoQAGyy3vdawJS7NxslOEEGM/a7gfHErgR4BFQA+wA/itEGL3SD+TkZEhTpw4ccNjJSUlJCUlTSqG6fbhhx/yq1/9atguFEeKU5KmYrKJZdWz+4ctxYR5u3L4+xtmIlTg1hMHgLefmcVLBUeFD1144yF6SDeeZUVtP6vqEgkPC6WZGk6ee5/e7nbcdN4sWLuBlKxN+IVHYrPaqL3QSUlRE9Xn2rFZBQGRniRmhhC/NIiNv/94Su91uJhdtWp+fm/qpJK4oignhRAZNz8+6RG4EKJBUZRfAbWAEdg9XPJWFOVx4HGAyMjIyR5OkhyWI5cVhjPZqe726ry42kWjaG0EzevBHOJJg1M0u21Wki3FbGs7TFZ5GInOaZj8Bjmv+pBDB59HUamITV/KhqxvEJOWgVqjoaOxj8PvVFB2tBljjwlXTy2pWeEkZobgH+4x5nvqbW3H0tWFxsdnXDFfbyZuFk+lhOIDbANigG7gLUVRviiEuKH2IIT4I/BHGBqBTyHWGZeVlUVWVpa9w5DmkNtpBxp7dF7YhKDPrYOwBVZqPWKoViIItdVzZ/cu1lzWsqQ7GZ+IRKq8zpN35jdYBgfxDYvAb/19vNjmT1WHiqjdBh4rK8W5zkhrTS8qlUJUqh+JmSFEpfqhVt+6KOv171VrtbC0pYSNdSdZ1lJCZ1QTgU88MWrcs3Wym8pNzE1AlRCiDUBRlHeBlYAsHku3jdkaaTmC4W6GzlTnxeXeTv5ccYK8bi3tqfPpFf0sGzhORn03q2viGbCkU26p55Q5j/b9tTi5urJg9XqSszZxtM+DH71bTKBRcJdJTVy3QF/XisbXiVX3xxG/LBg3L6fR3+uWeP7y/HusrjrGuvozeJqNdDl70pNzD/Nzc8eMf7ZOdlNJ4LXACkVR3BgqoWwEToz+I5L02eIoEzpmw0x3XvRbLLxRfYpXG9optoWiCH+SrBfJ7jjEmvJAwvviOWvu4UDPCVx7L6BC4L8gldz7v8f85SvROrvQ1dzPnj8d5Us9WjyFglERnHWyUuxkQesJX980ehnXVFeHPj+flPx8nq2pZVCtpSgkhdOJmdzxyDa2Z4yvDDxbJ7up1MCPKoryNnAKsACnuVIqkaTbhaNM6Jgt071UrBCCj9tqeKHyAvsNfgwoLgQJhdzeXWRWQXrrAoLDtnBaucDeuufRWg0oWk/8VuZy74P34h0UzKDRwqVjLZQWNdFc2UMSgkqNjf1OFi5rbViVoWMp+uFPqtaeHnoKC9Hn5WM8eRIAt+XL8X/863hmb2Gxh8ewPzea2WoznFIfuBDiJ8BPpikWSZpzZrOsMJtm+sZsbX8fL1Qe5912G8344SJ8STOdIr2xjczKecz3WI3Rp58z1r18dPAiaq2WlGUrSFm/mcjUReSdaeLRP1wkpPMc8RYNGgE+Ie6svDeOp49VcLnPdMsxda7aa/8vzGb6Dh1Cn5dP3/79CJMJp9hYAp56Ct3dd6ENDZ3ye5yNddEdaiamJM01MzHSGm/ynKkkO5Ubs6PFZLBY2VFfzMt1DZw2ByMUHxKsF1nXeYiVFd6kGpPwCFtMhddZ/nb2l1gtFgJj5rHh0W+QuGodrh6edLcaePn5YhrPtLPRpjCgqDmvtVDuJnjy7jDS0sP5ToCaZ946i9l2Y89E/6CZgrf3k1Z2hJ7338fa2YnaxwfvBx5At20bLinJKIoy5c9vNk26D3wyHL0PfDRzJU5pbhtv//B09xlfb7L93sPF5KJV8427wyhVGtnd54UBV/xFK+n9Z1laY2RJYzKRgRG0axs4eeEDujuacPH0YsHqLJKzNhEYHYtpwELFyVZKi5poqtBjA2o0Vs47WanQWq+VSK6PL+2nu+kymAHwN3Szvv4Um+pOENnbiuLkhMeGDei2bsVjzWoUrfbmt+Jwpr0P/LMmOjoaT09P1Go1Go2Gm080kjQbxtvVMh3dLyONlid7Y/b6mBRngX9cH8ZgH57tt+IkvFlsPk1aSyPLL0eRqF6GCBJc0B6iqOgvKIqK6MXprPnSl4ldshy1WkNjeTd7X7jI5VOtWEw2vIPcWLE9lkf2X6B3mO3Yr49vUN/LpsbzbKw7ycL2y6gQFPvF8Nzi+/mvPz6D2strXJ/RaJ+TI5AJ/DoHDhzA39/f3mFIt7HxJs+RnjfWolVXjVYmmeyN2Qa9Ea/IAZyjVDS4RlKnqJlnu8S6jqNkVrqSol+AX+hi6nRlfHDuDwxe6Mc7OITVD36JBes24OnrT0+7kVO76ik70kRP+wBaFzXxy4JJWhlCUIwXiqLgdaaS3mHiU2xWHv/O7/mqoYRXj3yMs9VMo7sfryRuZn/EEprd/Qjzdp1w8nbkPn+ZwCXJgYw3eY70PIWhpDNWchltBD/RG7Nnu9v5c+UZVJt0tKrC8BEdbDDuJ622hyX1yTjbVqGLNHG6aRfNByvQOrsQn7malKxNhCUmYzHZqDzdSknRKRrKukGB8AQflt0dS2xaAFon9Q3Huzm+GH0jG+tOsr7uFL6DvfRq3ajJyOLP7gs46xkOV+rak7m57Oh9/g6VwA+88Edaayqn9TUDo2JZ/8jjYz5PURS2bNmCoih8/etf5/HHx/4ZSZqM0S7Jx5s8n8lO4Kk3znDzHSwB40ouo430x3Njtn3QxCvVZ3ijuYdKmz8aoWOR9QwLm6pZdjmUaONCLpiNHOk9g7bnVai0EZqwgC3f+C4JK1ajdXGl6bKeAy+XUnGyFfOAFS9/F5bdHUPCimC8/EYe7W9PC0Pd2c6pv7xBxqUjxPY0YVbUHA9KZF9kBseDkgj08+SZ7IQplz5u+ZzU/ShAY/eEXmbGOFQCt6fDhw8TGhpKa2srmzdvJjExkbVr19o7LOkzZqxL8vF2tWxPC+PJN84Me4zxTCIaa6Q/XAucxSYobK7ixZoKDg/4YsWJaJuee3oOs7RSTXJHCu6eiXzUfYHSlhdwsvbi4qEjfeu9pGRtwjc0nN7OAc4daKb0SBP6ViMaZzVxSwJJygwmJM571C4Qm8FA77596PPyifvkE+JsNkp9IvmvhfdwMGwxPc7uN3wG09HGF+rtSkNPFxrPC2i9zqF2L8fUkUWAeduUXne6OFQCH89IeaaEXun7DAwM5J577uHYsWMygUtTdvNou3/QMuYl+XgTT9gUJhFNpExS0qPnz5Wn2dmloRsPvIQzawc/Iq2hg7SaROKcN2D2M1Hc8SFVZ06jUqtJXrKclPWbiV6Ujs0KlWfbOPTWaepKu0BA6HxvluREMy89ACeXkdOQsNkwHDuGfkcevbt3YzMY0IaG4vf419Bt3cZjb1bNyEQqg9nAwfqDhCX8DX33MRSVBZvJG1PHGtSGNJ65yzH6/B0qgdtLf38/NpsNT09P+vv72b17N//0T/9k77CkOW640fZIJjP1fiqTiMYa6XebLLxeV8xrDe2UWf1RCw9SbWdZ1F5JxuUAEgzJ+ITqqPa6wM6z/4nZPIh/RBRZX3qMpDXrcfX0oqWqh4Ovl1N+ohWT0YKnrwsZd0STuCIEXcDoCXawogJ9Xj76nTuxNDejcnfH845cdFu34paRgaJSXfkMnKZtIpXJauJQwyEKqwr5sP5DjBYjAa4BrAy8i7OlMbS2BRLq7cYzd8kuFIfS0tLCPffcA4DFYuGhhx4iJyfHzlFJc91wN8BGMtqIcaSa+VQnEd080rcKwZ6Wev5aXcpHBh1mtESIfrb1fkJarYXkplSife6i27udU035tFfU4uzmzoKsjaSs30xQbBwGvYnSomZKi0roajag0aqYlx5IYmYwYfE+KKqRSySWjg563v8AfV4eAxcugFqN++pVBP3DM3hs2IDKxWXY9zCVz8BsM3Ok8QiF1YXsr91Pn7kPb2dv7o69m5yYHNID01Gr1HDnuF5u1smJPOM0V+KUHEfM99+/5SbjcEabgDOTE3auutzfxwuVZ3m3w0aH8MRd9JJhOsHC5mYWVc4nQZmHOlhNaesRSko+BiGITFlEStYm4pavRFE0VJ1tp7SombqLHQgBIfN0JK4MIS49ECfXkceJtsFB+g4cQL8jj75Dh8BiwXlBEt7btuF1551oZqCt12qzcrLlJAXVBeyt2Uv3YDeeWk82RG4gNyaXZSHL0Koca3KPnMgjSbNspBuFPm5a3Jw04xoxjreNbbTOluH+blNqMG/XlfFKQyPnzf4owoUU23lyO8tYXKkjSZ9MUHAmjT6V7D3/PIZSPV4BgWTe94WhjX8DAmmr7eWTd6opP97CoMGCh48z6dlR1Hur+PWRKhp3NhP68a3vTwiB8eRJ9Hn59BQWYuvtRRMYiN8jX8Zr61Zc4qd/a0WbsHGu7RwFVQXsrtlNu7EdV40rWRFZ5EbnsipsFU7q0ZeYdUQygUvSDBmpRv2Tu5PHPXoez8Se0TpbgGt/J4AWjYF/qznGE12hDCrOBItB7urfyeI6A0mNqcS53onRx8DZjn18+MkFNFon5i9fSXLWJiKTF2Lss1B2tJnSomN0Nvaj1qqIXRxAYmYw4Ym+5J9t5McjxHKHr2Worp2fj7m+HsXNDa/Nm9Bt24bb8uUo6hv7vadKCMHFzosUVhWyq3oXTf1NOKmcWBO+hpyYHNaGrcVN6zatx5xtMoFL0gyZjoWuxjOxZ7RROoBBbcMnYRBjkDd6jS8WYWCZ5QipLfWkVkYRb1qOR6g7lR5neffcW9isZoLnzWfTY/+HhJVr0Tq7UXO+gw/+p5ia4g6ETRAU48W6hxKYnxGIs5t2xFg8TAbWVZ1B/d3fcrmtChQF98wVBHzn23hu2oTK3Z3pVtFVQUF1AYVVhdT21qJRNKwMW8l30r7D+oj1eDiNvDysI0+bH45M4JI0g6baizyeTpPhRulCpdCl60MbqcLkGkqzomKB7Tx3dOwhtdqZxM6FRPguo9OniSMlb9Fd0YSrl460nDtJydqEf2Q0bXW9HH+/kUvHWhjoM+Omc2LxpggSM0PwDRk+8TZ2G9HYLCxtLmVD3UmWt1xEa7NS7RlE4NN/j9ddd6ENDp705zGSmp4aCqsKKawupKK7ApWiYlnwMh5NeZRNUZvQOevGfA1HnzY/HJnAJcmBjWcUf3WULgC1j4Ju3iCdPsF0qlwJEC3cYfyApPouUupTCTJt4Jy5m6K+YxRdLkZRVMSkLWHtV75CsSqMn+66jM9vSllsrcTXBCqNQszCAJJWhhCR5INqmP0jYahcMXDuHE9e2MGK6lN4mQ10OXuwM2YV+yOW0BEcxZnHsqf1s2nsa2RX9S4Kqgoo6SwBID0wnR8u/yGbozbj7zqxG6AjXcn8/ZtnAcdM4jKBS9IUzfRl91ij+K9tmsdvS85gCPKgW+uHVQywxHqclJYqki+HMt+wmAaLiov9pZzt2onGNojWN4gVDz3CgrUbcPXy5s38co4fKGe7SUGNE01qGx952HjwvkRyMkfeRsxU30DPznz0efmYqqvJUmv5JDiZfZFLOBUQj001VNf2GaV9cCLaDG3srtlNYVUhZ9qGZqKm+qfydMbTZEdnE+w++dH9SPcbrEI47EhcJvAruru7eeyxxyguLkZRFP785z+TmZlp77AkBzdbl903nySe3BKPOsjGS3WXOWLUIcKjiBclZOv3saBKkNC2mBiPdPp99BxuLKCnswqToqXBN4HVuXfw+bvW0NnYz9l9TZQdu4ixx0SwonDK2UKxk5V29VADZMNHFdx7UwK39vbSu2sX+h15GK60BbstW4bf1x5j5Sdg0N7ar919ZW3uyega6GJv7V4Kqwo53nwcgSDeJ54n0p8gOzqbCM+ISb/29Ua63wCOtYDV9WQCv+KJJ54gJyeHt99+G5PJhMFgsHdI0hwwG6vVXT1JGMxWFC8VfbE9PNNVy4DeDV9hJXtwF8mNrSTUJDHfth7XYBfKXU/xzrlXEMJGeFIKKx98ivjlq7Ba1ZQfb+HtZ0/QWtOLSqUQvdCfX5XXU6WxYbtpoHx1VCrMZvoOH6YnP5/effsRg4M4xcQQ8OQT6O6+G23Y0Hv1Kd+PYRqmtveaetlfu5+C6gKONB7BKqxEe0XzjUXfICc6h1jv2Ml9mKMY7n7D9WZ1o2ohrq2iOBqZwIGenh4OHjzICy+8AICTkxNOTnOrJ3Su3T3/rJiNXel/vu8SqhgTbmFudDn5YxUm0m0nSW4pJ7naj7ieFMJ8s2j1qePgxVfoK2/Hw9ePZds/R3LWRnQBwdSVdLHvxQqqzrVhswj8IzxY/bn5xC8LwtXTif/3bBu2m2MWghW2dpp/9jN63v8Aa0cHam9vvO+/H932bbikpNyy+NRUpvcbzAY+qv+IgqoCDjUcwmwzE+YRxiPJj5Abk0u8T/yMbnl29d/L3795FuswExxnbaPqCUyudKgE3r3zMqbG/ml9TadQd7zvnjfqcyorKwkICOArX/kKZ8+eZcmSJTz33HO4z0CL00xwhLvnt+sJZKZ2pTfbBLtam3iptpy6dC9sig+xopxNPQdIqBkgvjUNf/MdeETAhe6DHDp+CrVGw7yMoY1/oxYuRt86QGlRE2VHPqFfb8LFXUvK2jASM0MIiPC84XjXJ15/Yzfr606xqf4kkT0tdGu19KSt4IVFC9jjEUOg1pNnLL5sHyaZTrR1ctA6yKH6QxRUF3Cw/iBGi5FA10AeTHyQ3OhcUvxvPUnMpKtxztpG1VeT9fXvcQLv16ESuL1YLBZOnTrF7373O5YvX84TTzzBs88+y7/+67/aO7Rxsfei845wArGX6d6V/mJvP3+tuUBeu4Vu4YZO2Nho3kNiUwOJNfHEGldRZ4ULfcVYu95BqTYTEBXD+ke+TtLqdag0blScaOHdX56mpaoHRaUQleLHmsxgolP9UWuG7yK5e743Xn5NdOXlkdh0CRUCQ0Iywd/7Jh+HLeIfd1eP+/c71k1Xs9VMUVMRu6p3sa92H/3mfnxdfNk6bys50TmkB6WjUoaPczbMxEbV1wxXGpnCCcqhEvhYI+WZEh4eTnh4OMuXLwfg/vvv59lnn7VLLJPZkXykC67ZqtnZ+wRiT9Pxj73LbOHN+kpebWimzOyFWigsFudZ2FlKQrUnsV0LcbIt5ayxlorOd9CYWhhQOROYtoqtD9xDQFQsDaVdfPxmHZVn2rCabfiGuuj4vP0AACAASURBVLPyvjjilwXhrnMe9rjCaqW/6Aj6/Dx69+wlxGgkMiIC3bf+D8filvGzs300njGiOlt5S0lhor9fq83K8ZbjFFYVsrd2L/pBPZ5OnmyJ2kJOTA7LgpehUTlOOpqOtcRvMVxpZIpXF47zidlRcHAwERERlJWVERgRwxs7PiAgIpbSph6CdC74uM1OPXy8I9nhFjgazmQu4ydTCpmNOrAjm8w/dqsQ7G9v58WaMg70umBBQ6To4D7DbhIbeomtX0Qcd6EN0lDWd4zjpS8ggAbXCJqj7+TB+3PZEBVIaVEThX/8hL6uQZzdNCStDCFpZQgBkZ4jlh4Gyi6hz8uj5733sLS2ovLyQrd1K7ptW3FNSyPvTOMN36/h6sEw9u/XJmycaT1DYXUhu6t30zHQgZvGjfWR68mNziUzNHNOrj8yLlMsjYyXTOBX/O53v+PBLzxEv3GA8Mhofvrr/8JktdHQNXtJaCo7kt9sMpfxky2FzFQd+LOovN/ISzWlvNNqpEO44SFsrLPsZ0FrLfOqo4jrX06Qvz9N3lXsL/4zhkt6dEHBrPr837Fg7QZcPHyoONlK6UdNvFJRiaJAxAJfVt4XR8wifzTa4dcTsbS1oX/vffR5eQyWloJGg8fatei2bsVjfRYq509H6eNdBne4368QgosdFymoKqCwupAWQwvOamfWhq8lNyaXNWFrcNHc2mY4501zaWS8ZAK/YvHixbz2/gFMVtsNj9uEoEU/MCsxTHVHchja1HayNbvJlkKmuw78WdNrsfJOYy0v19dTPOiJSthYKEq4S3+B+bVaYtrSiHXOZNDHwLmu/Xx07AIaJ2fiV6wiZf1mwuIX0Hi5h2PvNXH51AUsJhveQW5k3jOP+GXBePgMXyKxGY307tuPPi+P/sOHwWbDJTWVoP/7f/G6IxeNr++wPzeeK6frf79CCC51Xbo2K7K+rx6NSsPq0NU8teQpsiKycNfOjYaAEQ3oObb3LTpP5fPewEJOe23gmS3xI/+7mKUbrzKBX+fm5H3947OxOvBUdyQP83bl8Pc3TPr4kymFXC25GM1W1IqCVQjCbqMulJHYhOBQZzcv1pSyW6/BhJZQ0c09A7uJb+wkui6V+eYteAZ7UOV2nrwLb2C1mgmJT2Tz498hIXMNgwaF0qImDrxylN6OAUwKlGgtNAdr+MrWcNLTw2857tAWZMfR5+fTu2sXtv5+NKEh+H3ta+i2bcU5duz+6ZG+X2pFwSbEtQHC4lgz/332vymsKqRSX4laUbM8ZDmPL3ycDZEbxrX+iEPrrIRLu6CsAFv1YZYJC13Cg2NKJA3dBn7wt/OgKHb9nssEfh0ntWrYJO6kVo1rYf6pmsiO5DMx4p1oKeTmkotViGtx3K7Ju8Y4yEu1l3iruZcWmxtuwsZK60ckt1USXRvKvJ50IryC6PBp5mjJO3RfbsJN503aHXeTkrUZr4BQLp9qpeB/Smm41A0KOIe5sdvTwgWVGYsCDJj54d+GZgxf/ZwHL18eWqr1vZ1YGpuGtiDLzh5aqnXpp1uQjcdI36+f35vK0vlQWFXIK9X/zY/PlaKgsCRoCQ8lPsSmqE34ufqN6xgO2XZqs0LdMbhUAGWF0D60miP+CbyqupsdhlROi/lYGSpTGc02u9+olwn8OkE6Fxq6jNiuu2mjUhSCdC40d8/88SeyI/l4njdREz0x3M7dJ9frt1rJb2ri5fpqTho9UISNZMrZ0nOWuHpBZHMasaRTbNLzScdRlMvFoFIRt2QZWesfI2phOq3V/ZzZ38Tlk4cwD1rRBbiyfGssCSuCyfnjJzSob5yKbjRb+e+8E6wtPoA+P5+B8+dBpcJ91SoCv/f3eG7cgMp1cvcgbv5+BfsOsmZxI281vsSPz50DYGHAQv5h6T+wJWoLQe5BE3p9h2o7HdBDxT64VAjlu8HYBSoNRK2CJY9AQg74xvLjEXZXsveN+iklcEVRvIHngRRAAI8KIYqmIzB7uNpt0qIfwGS14aRWXetCaZ6lGMbb0TATbU4TPTHczt0nQgiOdvfyYk0pBV1gxIkg0cs20x7im1qIqF3A/IGN+AV4c051kfzyP6CyDdCp9aHEN5Ma7yR+uCydrhbBnh8dwdpjxoSgzkNheXYED+TGXesiuf7z1FrNLG++yMa6k2S0lNIibDgnJhL4j/+I1513oA0MHBrdPlc0pZP72iRXBl3bKKgu4FTLKQqaBEm+STyZ/iTZ0dmEe95avhkvu5/4ryuNUHMYbBZw9YH5WyA+B+I2gsuN5R9HvVE/1RH4c0ChEOJ+RVGcgLm9vQVDSXy22gYd0URODI76pZ5JjQMmXqmr5I3mDuot7jgLK8tsR0npuERUrS/RXYuIcd1Mv3cPZyv30nj0EmaVE6XucZR4JNLhFMh8i4aNvWqaX62kBajX2jjnZuGS1opZgYKiyziHuF37PYTqXNBVlrCx7iRrGs7iaTbS4eLF3uQNfPPfvoNLwqdbkE1ldKsf1LO/dj+F1YUcbTqKVViJ1cXyzcXfJCc6hxhdzPR8hrN94h+lNELmtyA+F8KXgnrkdOioN+onncAVRfEC1gKPAAghTIBpesKS5oLZ+lLbu146YLXxfksrL9VXcrTfFYFCoqjmi/2niKkbILwpnXjbPbgGuVDhfIp3LryEEDYikheSe//fsy1fT4BwZpFJTWKPGmcUulU2ipzNtPlrudQ/eMPxro5G7/Czos/L5w8F7+LU2sSAWsvhkFT2Ry6hLDSRf7tvES4JN34OEx3d9pv7OVB3gF1VuzjUeAiLzUK4RziPpjxKTkwO873nT/tU9lk58Y+zNDJeMzo7cwqmMgKPBdqAvyiKsgg4CTwhhJjexUxmQVlZGZ///Oev/bmyspKf/vSnPPnkk3aMauZNNTHOxpfaXvVSIQSnewy8WFvGex1m+oQzfqKfO817SGiuJ7QunjjDWkJ1AbR613Oo5FV6Ktrw9Atg+b0PkLxuExonH8qONvEVYyU6C5gQXNJaKXayUqexEebjesuo08NkYE3DWTbVneTyC9WgKPhkrqDivr/j//UEUdU/1AXybyN8zuMZ3Q5YBjhYf5DC6kIO1h9k0DpIkFsQDyc+TG5MLgv8Fszo+iMzduKfRGlkImZkduYUKWICK1/d8IOKkgEcAVYJIY4qivIc0COE+PFNz3sceBwgMjJySU1NzQ2vU1JSQlJS0qRimAlWq5WwsDCOHj1KVFTUtccdLc6pGm4259VOg5n6kk7mhLHq2f0TbpmcyompddDM6w21vNbYQpXZDa0YZKk4RmpXCRH17kS2pRGnCcXiZ+FCw0Eqq0+h1mqJy1hByoYthCYkU3O+i9KiJuoudiIEOAW7UtDXy3m1GfOVvHj1s/7lrjJaOnvJaCllY91JljcPbUHW4B1C2mMPTXgLspE+r1BvLT9/2JmC6gIO1B7AYDHg6+LLlqgt5Mbksjhw8bDrj8zU1c+0vO5opZGEnKHSSMQyUI29WbK9r/LGoijKSSFExs2PT2UEXg/UCyGOXvnz28D3b36SEOKPwB8BMjIyZqMbb0r27dvHvHnzbkje9jDWF2qqX7jZvpE02ZH0ROulkzmOyWZjd1sHL9Ze5lCfEzZUzBe1PGQ4SUyjnpCGNOab7sInUEetewnvXfw95tIBgmLj2PDoN0hYuZbeDij9pIm9Lxxh0GDBw8eZJbnRJKwIxjvQjYCbf19b4slWdxDYuBv1h3vxMvXT5ezBezErORSzjMcfzcFvmD7vsdw4urWidq/ExfscZp8Svr2/D52zjtyYXHJjcskIykA9SnKbyaufSY9mp7k0Ag7WFTNBk07gQohmRVHqFEVJEEKUARuBi1MJpqCggObm6e33CA4OJjc3d9zPf/311/nCF74wrTFM1FhfqOn4ws32jaTJnjAmWi+dyHEu9Bl5saacHW1G9MIZb2Eg27qHxJYaguujie1ZQZRbMD3eHZwqf5+2yhpcPL1I3bSF5KxNuPuEceloC3m/KaGzsR+1VkXs4qH9I8MSfFBdt43Y1YRlbmhAv3Mn+h/+kurqanydndEvyeQ59yT2ukUT5OsxpdHf1sUhVPed58XzOxl0PoNK04+zyo0t0ZvIjs4mMyQTrXp809Ls3i1y1QyXRhzmfU7CVLtQvgO8cqUDpRL4ytRDsh+TyUR+fj4///nP7RrHWF+o6fjCzcaNpOlYMXGseunNVyIjbYl19TgdJgtvNzXwSn0jl0yuaISZdHGKxfpiQus1hLekM59MtAEaLvWf4O3zL4CiEL04nRVffoiohRnUl+g5/n4TNRc+QdgEwbFeZD2cQFxGEM6ut/6Tsvb1fboF2fHjALgtXYrfY1/FMzsbtacny8f4LEcjhOB8+3kKqgrYXb2bVmMrLl4u5EasIzc6l9Xhq3FWDz/dfjR2axOdhq6RiZjL7bBT+gSEEGeAW+oykzWRkfJMKCgoID09naCgiU1MmG5jfaGm4ws30x0k07Vi4mg3SnecbuCZt85itl3Zv3GE9y8U8Iny4O9OHeOAXo0FNTGigc8PHCO2sZ2AxoXEG7MJ8vajWVfD/osvYKjoxicklFUP/h0L1m1goM+ZkqImPn7zGAP9Ztx1TqRtjiQxMxif4FvX+RAWC/2HD6PPy6d3376hLciiowl44rt43b0Vp/CpjeyEEJR1lVFQVcCu6l009DWgVWlZHbaap2OeZl34Oty0U+vqndU20RkojYzXXG6HlTMxr/Paa6/ZvXwCY3+hpuMLN9MdJNO5YuJI9dJ/zr9wLXkPx+auwSUKzCEeNGnc6e/Ws9F2mAVtFfjXhxHdvYRYbQiD3kbOd3/IR6fOo3V2IT5zNSnrN+MbGkf58Vbe/0MlHfV9qDQKsYsCSFwZQkSS7w0lEhhKqoMlJejz8tC//wHW9nbUOh3e992HbttWXBYunHJ3R2V3JYXVhRRUFVDdU41aUbMiZAXfXPRN1keux8vJa0qvf70ZbxOd4dLIeDlqj/d4yAR+hcFgYM+ePfzv//6vvUMZ8ws1XV+4mWyLmqkVE6/Xbbx1p3OhUVCFanGNUdPp4otVWFjMKdJ6zhJabyW4OYN4azqe/u5UuRSTd/F1rDYzYYkL2PKN7xK3bBXNFQbOHWii5vwn2GyCwChP1j4Yz/ylQbi431o/Nre00LNz59BSreUVoNXimZWFbvs2PNasQZni/qp1vXXXVvq71HUJBYWlwUv5UvKX2BS5CR8Xnym9/kim/SQ/y6WR8XLUHu/xkAn8Cjc3Nzo6OuwdBjD2F2q2vnBT6XSZqRUThyMAm58T7jE2en19sSpa/EQNGwd3EdfUgk/TAuL6NhHhHkCndyvHyt6h63IT7j6+LNm6neR1mxBCR0lRE6/98ymMvWZcPbUs3BBOYmYIfmEetxzT1t9Pz5499OTn0190BITAdfFigv/5J3jl5KD29p7Se2rub2ZX9S4Kqwop7igGYHHAYr6/7PtsidpCgFvAlF5/vKZ8krdjaWQiHLHHezwm3Qc+GRkZGeLEiRM3PObI/dVdBtO1dVE66qvQOwfOyV/yZEy1T3w2+sxTf7EPY4AVa5gbRq0HbqKPTNthUjrL8G3wJbJ9MXFKKIo/lLYUUVZZhEqtYV7GMlLWbyZ4XioVJ9spLWqirbYXlVoheqE/iZkhRCb7olbf2BctrFb6jxyhJz+fnt17EEYj2vDwod1stt6NU3T0lN5Pu7GdPTV7KKwq5FTrKQCSfJPIjcklOzqbUI/QKb3+rHGQ0shnyUz0gX+mdRlMN6xMaLGJOdMbOh2m2ukyU1cJ/RYreS3tvFxfTdtSPxRhJZWzZPSdIqTegF9TOonme/H38abB4zK7L/6RgfI+/COjWf/lr5GQuY62egulRU3s+lMRNovAP8KD1Q/MJ35pEK6et5Y7Bi5doic/H33+zqEtyDw90d11F7rt23BNT59SXVs/qGdvzV4Kqgs43nwcm7AR5x3Htxd/m5yYHKK87DsfYVwctDRyO5Cf6Aha9AM3LCsLc6c3dDpMR6fLdF2WCiH4pLuPV+oq+aDTzIDQECLauddymPjmOtzq45jftxY/mz897nrKuvdzoPISzu7uJGZlkbp+MxqXYMqONPPms8UY9CZcPLSkrA0jMTOEgAjPW45paW9H/9576PPzGbxYMrQF2erV6H74AzzWr79hC7KJ6jP1caDuAAVVBRQ1FmERFiI9I3ks9TFyonOY7zN/Kh/XqK6WxRq6jVPbgGOOlEY+62QCH8FIu/PYszd0Nqf7OkJrVf2Aidca6nm9sY0GizMuwsAKUcTC7ov4NrgR0prGfGsWbgEuXB48zb7Sl7EpgqjUxdz5+WeISMmg6mw3B99sorW6FkWlEJXiR1JmCFGpfqg1N5ZIbAMD9O7bd2ULsk/AasUlJYWgH/4QrzvvQOM3vs0KhmO0GPmo/iN2Ve3iYP1BTDYTwe7BfHHBF8mJzpnx9Udg+A04YAITwWRpxOHIBD6CkXbnsVdv6GxP97VXa5XBauOD1k5erqviaL8GgUKyKGPLwHHCGvTomtOZb7ybUPcA2nUNfFL6OvrKVrwCgljxuS+QtGYDPe0aSoua+PC141gtNnxD3Vl1fxzxy4Jx87qxRCJsNgwnTqDPy6N3125sfX1ogoPxe/TRoS3I4uIm/V5MVhOHGg5RWF3Ih3UfYrQY8Xf15/74+8mNyWVhwMJh1x+ZKaO1dg57dTlmaSQHwpfJ0ogdyU9+BMPtzmPP3tDZnu47m61VQghO9Rh4qa6ane0G+oWWANHJNushElqr8GiKJqork1hVKMLXzAX9IQ5fPolG68T8FavYkvUknn6xlB1tIe8/K+jrGsTZTcOCVSEkrgzhcEcP39p9ica956+9j1ydCX1+Hj35OzE3NqJyc8NzyxZ027fhtmzZLVuQjffqx2wzc7TpKIVVheyv3U+vuReds447Y+8kNzqXJUFLRl1/ZCaNdfXY2G2UpZE5RiZw4De/+Q3PP/88iqKQmprKX/7yF3zcXIBPd+fRqJQZXalvLDM93XekBDWT77d50MwbjU281thEtckZZzHAUo6Q1nMen3o1Aa1LiDdn4uPrRZ1rGQUlv8d0aYCQuAQ2f+3bxKRlUlfSz/EPmmiuPIaiQMQCP1bdP5+Yhf6otSp2nG7gh38rxmi24jXYT9qpwyj5/05lZ+3QFmQrVxLw1FNDW5C5DT9zcayrH6vNyqnWUxRUFbCnZg/dg914aD3YELmB3JhclocsR6uajW2xRzdSWSxSaWGT6hS5zmfhFxdlaWQOue0TeENDA7/97W+5ePEirq6uPPDAA7z++us88sgjN+zOU9Lrwvqk2U/eVxPrSM2e01HSmc3yzKDNxu62bl6qr+JQj4INFQniMl8aPEpkYzvuLQuZ15dLlHMwvbpOTl8uoLWyClcvHak5uSSv3cig0YuSoiaK8k5jMdvwCXYj8555JCwPxt37xpuLv/mgmPTq02yoO8mylhI0wkalVwhvZtzDD//jSbSBgWPGPPzVj4Wf7y+k1NzGrupdtBvbcdW4khWeRU5MDqvCVk1q/ZGZdLUsZjKbSFfK2ag+zUbVKearGgDocYuFRd+a0DKskn3d9gkcwGKxYDQa0Wq1GAwGQkMdo992rPVEpqukM9PlGSEE5/uMvFJfx99ae+ixafER3dxpO8iC9gpcm4IJa09nvgjFyV9LueEk71x8EVQQm76UzK88jF9EMpeOt1Hwv430dlbi5KohITOExMxggqK9brgBKITAePoM+vw8fvlOPp5mI53OnuTNW8O+iCVU6UJRgJ+MI3nD9Vc5ApVLIxrPc2i9zmJw6uatMifWhK8hJzqHteFrp7z+yIwZ0LNde5QlUe/iVX8AHX2YhZqjtkTe12SzIOsBtqzOtHeU0gQ5VAK/dOlf6e0rmdbX9PRIIj7+xyP+fVhYGE8//TSRkZG4urqyZcsWtmzZMq0xTNZoN50m1fo1gpkqz7SbLLzV1MqrDfWUDzqhFSYyOMaS3tP4NtjQtS4hfnAJwZ6+NHvW8lHpS/Rd7sQ3NJw1D3+ZuGVraa60cP5gE43lx0GBiCRfMu+ZR8wifzRON44QTXV16PPz0eflY66tRXFx4UJEKjuDFnMmIA7bdSPKiVy5BPnr6eQoWq9zqJzbEUKFtX8+boY72f/Nb+HpdGsb4njMeFdRZ+XQzcdLhde6RiJcfWDhnRCfgzZuI6tddKyeviNKs8yhErg9dHV1kZeXR1VVFd7e3nzuc5/j5Zdf5otf/KK9QxsxgSowrdPRp7Nl0GwT7OvQ83J9NQe6bVhRMU9U80XzJ8Q0t+DUnEhMz2Zi1SGYdQOc7/6Qg5XncXJ1JWHlWpLXbQIliNKiZt78+UUsg1Z0ga4s3xZL4opgPHxcbjieVa+np3AX+vx8jCdPgqLgtnw5/t/8Jp6bN1Na3k3Ju+exTbCbpran9tqiUf0BFTgJBathHqbOtZh7k3FVefGje1OnlLynvWxltUD98dEn1MjSyGeKQyXw0UbKM2Xv3r3ExMQQEDC0tsS9997LJ598MisJfKwR2Gz1Yk9Hy2BJn5FXGxp4p6WLTqsWnegiW3xMSmcpro2+BLalEW/NwcvXnWqnC+wsfQuLbZCIBank3PMUwfPTqTzdzf6Xm+lpa0LroiY+I5DEzBCC5+luLJGYzfR9fAh9Xh59Bw4gTCac5s0j4HvfQ3f3XWhDQq49d3va0FKv4xnpNvU1Da0/Ul3IhY4LAKQHpvODZT/A2pvK/+xvo7HbOC1XP9NWthqtayTjKxCfLbtGPsMcKoHbQ2RkJEeOHMFgMODq6sq+ffvIyJi2Jc5HNJ4R2Gz1Yk+2ZbDLbOFvzR280lDHBaMGtbCQxkk+bzhBQMMgbq1pzDfeT4RLAF26Nk6W76Cjsh4PP38ytm8nfuV6Ohuv9Gy/fhoEhCX4sOyuGGIXB6B1/nSkKIRgoLgYfV4+Pe+/j7WrC7WvL96f/zy6bdtwSR55Isxo3TTtxnZ+degtCqsLsDpVARDmOp+nM54mOzqbYPdP96P8u2UT+lhHNaWy1TClEdk1cnu67RP48uXLuf/++0lPT0ej0ZCWlsbjjz8+48cdzwjMEZe5tArBR529vFxfy55OE2ZURIo6vmD9mLjmBtQt84jsyiJOhKDyg7L+oxy98BdUGjVxSzNZl/U1nNxjuXSkmb/9+jLmASte/i4suyuGhBXBePndeHVhbmxEn78TfX4+pspKFCcnPDZuQLd1Kx6rV6NoJ96e1z3Qzd7avRRWFXKs+TgCG1ZbMJbWLZh7FlFDILr5qTck7+k2oasrWRqRRiBXIxyn6Y4z5vvvD9saqABVz945bccZj/GsHHjZMMC/nqngo/5ejBpXPEQPKznEou5iXBvd8W1bQrwpHH9PHU2iipNlBRgHewmMnkfK+k1EpKygprif0qJmulsMaJzVxKUHkJgZQmicN8p1myMMbUG2G31eHoZjx4biyViCbts2vLKzUXtNfNOCXlPvtfVHjjQewSIsRHtF09iQQFfrAmymG3dhmollb6835mc+WmkkIVeWRm4zcjVCB+MIa41cNdLVwL/vLaMv0IlXGmo53a9GETYWqS+w3HiU4IY+bE2pLBjYTrQ2mAHPXs5WH+BAZSkuHp4kbcgiafUG+rq9KP2kiaK88wgBofO9Sc+OYl56AE4un379hMVCf1ER+h15Q1uQDQygjYrE/7vfQbd1K07hE9+h3WA2cLD+IAVVBRxqOITJZiLUPZQvJX+JnOgcEn0Tif3BB8OeSK+WMmaqU2S4q6ufrHZly0Ae/PX60ogvzM8eStiyNCLdRCZwO3GkbZyur7sKwObrhCZCQ22gJ09faiBUNPGA7SCJrdWYGsOI6l5JgjWUOrOZS73nOKt/FRs2ohelcdfD38fTP4ny4+28/4cmBg11ePg6syQ3msTMYHQBn/ZJCyEYLC1Fn5eP/r33sLa3o9Lp0N2zHe9t23BZtGjCCzwNWgeH1h+pKuSj+o8wWowEuAbwQMID5MTksND/xm3NRjuRzvQEp+0Lg9juWwuXPhwqjey9UhoJSJSlEWlcZAKfhOkYlTlSfTvU25W6QROEOaMKd8Ls7I5W9LOOfaTrz+HWqMWlJY0F5pU4WT0pNlaxt/NVzOYOujVelPtmcM/Wu4hW+XBydzNdTWdRa1XMSxvaPzI83ueGEom5pZWe93aiz8tn8NIl0GrxWLcW3bZteKxbh2qCW5CZbWaONB6hsHpo/ZE+cx8+zj5snbeV7Ohs0gPTR1x/ZLQT6YxMcJJdI9I0kgl8gqY6KpvNJWHH0m+18l5rN+qVHpisTijCRjLnyBwsIrypC2vLAub13sk8gikz6TnVeQRjXzFmRUOF+zxK/VbjpgojxayldUcn7XQSHKsj6+EE4jKCcHb99OtlMxjo3bsX/Y48+o8cAZsN10WLCPqnH+OVm4vGZ2L7OlptVk60nKCgqoC9tXvRD+rx1HqyKWoTudG5LAtZhkY19td7tBPpU2+cGfZnJjzBadiukSulkYQcmLdBlkYcmM0mqGzv51RtF6dru1g7P4Dc1JCxf3AW3FYJ/Pot0pzUKoJ0LtfWOhmvqYzKZntJ2OEIITim7+fVhkZ2tvViEGqCRAf32Q6yoKMcS2MQQR0ZJFrD8NV5UedSRkHpf2OyGGlxCabYL4tulziSLG5sN6lxFQq9iuCYs5nf/mA1PsHunx7LasVw7Bj6HXn07NmDMBjQhoXh/42v43X33TjHxEwodpuwcbbtLAVVBeyu3k3HQAeuGlfWR6wnNyb3/7d35tFRX1ee/7yq0r5Ulfa1tIAWdiSwQJjVYBBewHEnnb2T2LGTdJKTdCZOnE5PTzLTfTqn0zOT7jMz3R1n0pOk03ZWG7xIgA0YjA3YSOwgNu37UqoqLbW/+eNXKgRIQjsq/D7ncCj9lvrdevWrW7e+7757WZexjnD95BsIj5VmOOV5FrDG0gAAIABJREFUCp8Xmk9qDntk1khyMZR/TUv1U9LIvMXh9HCmyUZ1ozXgtPuwBRpox0UayEuKucszzB0fGgd+e4s0t89Pi1X7cE7GiU8nf3emfpJPJYpvdbr5bVs3L7a20eDWEyGdrOUYqwZqiGvxE969ioLBVVjCUug39nHmRhXtN24QYzKz4pFHWLB6E4eODxB9soPkQR1eJFfDfJwP99Fg8JNhjgo6b9fVq9qS9r2v4u3oQBcbi/HRRzDuDrQg0028BraUkos9F6mqr6Kqvor2gXYi9BFszNpIRW4FG7I2EGWYnYnfSc1TKGkkJJFSUtc9QHVjn+awG6zUdjgYTs4rSImlYkkapTkmSi1mFiTHotPNbuONyRCyDnyy0fRoLdL8UtJhc/LLF/6ZF154ASklzzzzDN/85jfHfJ7pZI/MRM2RyUTxTp+fqm4bv25u4pjdhx9BsbzM0953sLR34u0swmLdToEvnfBEA9cGq3n58q9ArxWR2vXUpxH6HK6c6GTvPzbg90syUqI5MNjPWZ0HV8APR4XpeX5tCr2//CW2V/bgvHgR9HqtBdnz39VakEXeugT+bly1XqWyrpKq+iqaHE0YdAYezHiQb5R+gy3ZW4gJm/0o6K7zFEoaCTkGXF7ONAWcdWMfNY1WrIOB6DrCwEqLiR1L0ijNMbMy24Qx6t6XAR6PkHTgU4mmx2qRduHCeV544QVOnjxJeHg4FRUVPProoxQUjN6XcDrZIzOROni3KF5KyWnHEC+2tvNyhxWHX0+i7GIXb7Os9xK0JWLsWUmhO4u0qAQ6Yxo5cuXfcdT1kJhlYcNnP0/qgjIazg1y9LftDDkuEhUfzvKt2RSXp5GYEUtKTQut+2rp7rGz036VT9svEPvHD+jw+YhcvJjU7z3PMcsqfnSik9a3h8g48+6EfiXU2+q1SLuuiuu26+iEjjVpa3hm2TM8ZHkIY8TcO8Nb5JVhaeTAv44ijaiskfmGlJL6nkGqG6xBh13bbscfiOMWpsTy8OJUSi1mSnPMLJxn0fVECCkHPjLqvp3haBpuNmEQCCSScL0OvU7g89+Z8dt04ypr164lOlDMf9OmTbz88st85zvfGdWG6WSPzETq4FjResugi39u7ODXLS1cc+oIk24e4DhlzpOYWtyInpUscHyMfJGGx+jkvPUIR6+fJTwqmkXrN7FwzWb6OuOoPd7Oiddq0ekFecuTKC5Px7IkAZ1eC7el3892bytrBg5hP1QVbEFmfOoLGHftIqKgYFK/Elr6W7T6I3VVXOq9hEBQmlrK94u/z8M5D5MYNfU+lDOCkkZChgGXlzPNfdQ09lHdYKWmqY/eATcAsREGVmab+NqWhZTkmCnNNmOMnt/R9USYVw78P19t5nz/6A7K65O4vH4Ys7XBMOKWYxZEhPOV5ESEEEGHPoxOCNauXsn/+Ye/paenh6ioKN5444271kKZaqeamUgdHBnFSwH+5Ej02WG4EmP54fU2FsqrfN53hPzOFlwd+WRYt1DozSDeGEOD4SKvXfkdHr8Ly9IVVDz5LSJii7j6QS+V/9KJ39dBUnYsGz5eQMEDqUTF3vw146qrw7Z3r9aCrKUFER1N/PbtGHfv0lqQ6W9GnXf7ldA52Mn++v1U1ldytussAMuSlvHc6ufYnrt9VpewT4igNFIJDe8qaWQeIqWkoWcwONFY3dBHbYcjGKTlJ8fwUHFKILo2UZAShz7EouuJMK8c+Hi4p+C8RyKlxKAT6ITuVt08cwXf/e53efjhh4mMjiGnYBF9Qz4ut9mnlKVyN6bbpuy5HUU8t/8SrtQwZGYkHkMEMbKXbexhZd95RGscUT2lFDofwhKeQl9MN9XX99J9o4m4pGRWP/kkmcXraL7i48Tr7QzZa4mKC2PZ5iyKy9NIyrpZHtVrtWKvrMS+Zy9DZ85oLcjKy0n+5jeI27p1zBZko/1KEPp+OjnOF6p+xamOU0gkxQnFfKP0G+zI3UF2XPaUx2TaqKyRec+g28vZZlvQWdc0WukJRNcx4XpWWkz8+eYFlFrMlFhMmGb4cztfmVcO/L8VjL1c+mxz37jn6oS4Y5Lydrx+yfKsOyOnp59+mic/+VlarEP85O9+SGp6xpSzVGaLXo+XP7b38h+uThxlSRikh1LeZ63zPZLaBvF0LSXP/hEW+tPQmwW1jpP84fovEGF6CsrWsa78izgHkrn8Xgc1b9Wh0wlyliVSXJ5OztJE9AZNIvG73fS//bZWqvXtI+DxEFFYSMpzzxH/2GOEpd69i03wV4JuCEPcBcLiz6KPuYYQfnqceXxlxVfYkbeDfOM9lB6UNDJvkVLS1Dt0M7putHKpbUR0nRTD5qKUYGZIYer9GV1PhGk7cCGEHvgAaJFSPjZ9k0YnXK8bcyJyOJpu7RsaVeceedxodHZ20u2NoKW5kbeqXuNXr+wHburq98qBe/2SQ712Xmxt40DPIB505Mob/Jl8m4KuBpwdWST1bqDQnUFyjJG2yDreuvJvDN6wkZpfwOanvkRs4jJunLbz1i+68XltJGbG8OBHF1JYlkZ0vPa6pJQM1tRoEskblfhtNvTJSSR85jMYd+8isrh4wjYPegbZUdbKby69ioiuReh8+N0JSOtmvrbmo3ypfP2kl8fPGCprZF4y5PZxtrkvmMpX02ilu1+LrqPD9azIMvHlTfmB6NpMQsy9D6jmCzMRgX8DuARMvkTcJEg1Rt6SeQJa1J1pjgo62LZx0vF0QpBqvDOVzTroZufjT9Bn7cVgMPCXf/Nj4k2m4P6xvjRmkysDTl5q6+J3bZ10efXESTtbeZtVjtMY2iLR965kQf968vSpOOP6Odd9mMM3LhEVF8/ih7eQvXQ9nQ3hnDnYxoDtGhExBhZvyGBReTpJ2bFBBxpsQbZ3L54GrQVZ3LZtGHfvIqa8HGGY2O3h9Dp5p+UdKusqOdJ8BKfPicmchNO6gb6uJaRGFvCdHcVzv+LU74Omk3eWYVXSyD1DSkmzdSi4QOZUg5VLbXa8gcArNzGajQXJ2kSjxURRahyGMQIvxTQduBAiC3gU+FvgWzNi0RgMO+nxcr+940TfIx39MMPpiP/2hzfGPG+sqH2msXt9vNJh5dctLZwZkOjwsVJW83HvMdJabQx2F5Nte5wCbzoxxkjqdGfZe/UlfHjJK1nFzk8+j8+bzZWTXVx4tw2hE1iWJLChPJ3cZUnow7TX4bPbsVdVYdszogVZWRlJX/oycdsfRh8bOyF7PT4P77W9R2VdJQcbDzLoHSQhMoEnFj5BRV4FJSkl6MQ9+OAFpZF9AWmkF3RhkDssjVRAwuRWgCqmjtPj41yL7ZZUvi6HC9AysFZkG3l2Y35Qu06MjbjHFocW043AfwJ8BxizMaAQ4lngWdC638wmY8ks4XrdqDLIaIt7bme0qH2m8EvJUWs/L7a2U9ntwCV1ZMlGPiUPsajnGs6ONIy9ZRS4MsgMT6Q3qp0TV3+H9UY75ows1n7ik5gzSmk47+Tob7rweq5jToum/MkFFK1JI8aofRikx4Pj0NvY9uyl/+BBrQVZfj7Jf/EXWguyjIwJ2ev1e3m//X2q6qt4s+FN7G478eHx7MzbSUVeBatTV0+o/siM01sXmICsvE0a2a6kkTlESkmrzXnTWTdYudhmx+PTPmM5idGsX5hEqcVEicVMcZqKrqfLlD9tQojHgE4p5SkhxOaxjpNS/hT4KWgNHaZ6vYks3hlLZhnLCd9NHtHrxKzo3/VDLl5q6+Gl1nbaPTqiZT8bOErZ0AdEtBjw9S4n2/EAC2Uq0ujnku093rv+PobISIrKN7C+5Kv0tsdz+UQ7/b0NREQbKC5Pp7g8nZTcOIQQSCkZOnde07Vffx1fby96sxnTn/6p1oJs6ZIJadF+6aems4bKukoONByg19lLTFgMD2U/REVeBeXp5YTp5zifdqQ0cmUfdF3WtitpZE5xenxcaLVR3dAXnGzssGvRdWSYjhVZJr644WZ0naSi6xlnOuHSg8AuIcQjQCQQL4T4dynlpLsBSynv6kzGWwo/7GQnIrOMZLyJUZ0QwRWSM9G1aMDrY29XHy+2tHHS4UXgZ5k8zZO+o2S19+DoXEiabSeFnnTMMXG0hF9jX+0LOH39ZBYvYesXv4YurIBrp/o48G9WEFYsixJY9+RC8lYkYQjTnJWnre1mC7Lr1xFhYcQ+9JBWqnXDxFqQSSk5332eyvpK9tXvo3Owk0h9JJuyN1GRW8H6zPVEGmbvl8mojCeNrPq8kkbmgDbbUNBZn2qwcrHVHvz8ZCdEsTY/Ucu7tpgpTo8jTEXXs86MtFQLRODfvlsWymgt1erq6oiLiyMxMXFcJz5eGuHyLNOY+8bj9qh+GL1Oc97m6HCklPT09OBwOMibZPU8KSXHbQO82NrJq519DEkdabKVjfIgy2yXcbUlEWVdycLBLHL0yQzE2DhTf5A26zVizQks3rSV5NwyWmol12q68Lp8mFKjKS5Po2hNGrFmzYn6+gdw7N+Pbe9eBk+cACmJKi3VWpBV7EBvvLt8IKXkivVKsP5IS38LYbow1meupyK3gs3Zm9l/3nrHIiSYxZrmShq5Z7i8Pi602rUVjYHskLbASucIgxZdlwTS+EosJlLi5vgL/UPGWC3V7rkD93g8NDc343Q6x71Gu8056iSlQSdIm4ZOPej2Yh/y4vNL9DpBfJSB6PBbf5hERkaSlZVF2AQb6DY73fy2rZeX2tpodAmiGGKNfIe17pPEtvgZ6llKrn0BBd50wk0GrvfVcKH+CFIHCx9YS37pRgbsqdSe6MDe7SQsUk/B6lSKy9NJy4/XJBKvl4H3jmPbswfHm29qLcgsFoy7d2ktyLIntjDmhu0G++r2UVlfSZ2tDr3QszZ9LTtyd7A1Zyvx4Vpy0Wg9HMN0AgRBjRPu7KU5KcbLGims0HpBZj2gpJFZoN3mDOrW1Y1WzrfcjK4zTVGU5phZZTFRmmNmUXq8iq7nmFl14BNlNAc+USbSePdeMuTzU9lt4z9a2jhmcyERLJFnWS+PkNPZhqM9l0TbMgrd6aRFJNClb6bm6n5szi6SLbks2riNyNjFXD89QEutFQRkFppZtC6d/JJkwsI1p+WsrcX2yh5sr72Kr0trQRa/s0Ir1bpy5YR07WZHc7BoVK21FoFgddpqKnIr2JazjYTIhDvOefBHB0ctxDUak2oIPOqCmoA0UlihpJFZwO31a9r1cN51g5XWQHQdbtCxPNNIaSCNr9RiJiVeRdf3mpBvajyfWpANI6Wk2j7Ii21dvNLRS79fR7Ls5CMcpNRxDk+rGazLyR7YygLS8cQ7uWh9h3eunyEiJobiTZtJW7CG9oYIat7qwuNsJj4pkrLH8yham0Z8oqbBezo76XntdWx79uCqrdVakG3ciHH3LmI3b55QC7KOgQ6taFR9Fee6tcJSK5JX8HzZ8zyc8zAp0eOvsJxMydu7Htt7Q9Oy75BG1IKa2aDD7rwlje9ciy1QmgIyjJGU5Jj5YqAi3+L0eMINKroOFUImAp9PdLg8/K69lxdb27nulETgoky+S7nnOMY2F46uYrIdRRR60oiPi6HReYnTVw/gli5ylq1k4QObcbssXDnZja1rCEOEnoWrUlhUnsYJxwD/sP8KPd02dtqv8Om+88Scrwa/n8gVyzHu2kX8I49MqAVZz1APBxoOUFlXSU1nDRLJooRFVORVsCN3B5mxE//ym1YErqSROcPj83Ox1R501tUN1uD7Fq7XsTQzPlg+tdRinpb8qJg7Ql5Cude4/H72d9t5sbWDw9ZB/AgK5SU2ysMs7GnA3mYhzracAmcG2YYk7FE9nL7xFp32eowpqSzasJWYxBU0nnfTdNkKEjIKTBSXp7OgNJnwSAOvnGri1z99hfV1J1nfeo5or4vOaDO6HY9Q9synici/u5Rgc9l4q/EtquqqONF+Ar/0k2/M13K1cyvINeZO6fVPWgNfFDt2rZGinarWyAzR6XAGiztVN1o522wLVO2EdGNkcJKxNMfMkox4IgzqSzIUUQ58ipxzDPJSWw+/b+/C5tORIHvYwGFWD9YgW2Nx9S4lvz+XAm8aepPgSvf7XG5+D12YgYI168gsXkdvu5Frp7pxD3mJS4ikqDyN4rVpGJO1an6ua9ew7dnLlf/4PYkDVgYMkRzNXM7B7FWcT8wjwxwzrqY84BngUNMhquqqONZ6DK/fS1ZsVnCBTYGpYEbqj4zWyg1uyloPxPfxV4VNLB94V2WNzAIen59LbfaAHKLp183Wm9H1kuHoOlBCNd04O63mFHOPcuCToNvt5Y8dmkRyadBPGB5WyROs971DUns/1s4C0hyLKXSlkRxppF00UHN1P/1uK+kFRRSUbUYKLWfb2j6IIUxHfmkyi8rTySw0I3QCb08P9tffwLZnD84LF0Cv52RSAW9lr+Z4+hLcIxbHCKDuR4/eYuOQd4ijzUepqq/iSPMRXD4XaTFpVORWUJFbweLExbNfNMrnheb375RGkoo0h6061EyLLofrZmPdhj7OtvTh9GjRdWp8xC3OekmGkcgwNc73KyE/iTkVJtP81+OXHOy182JrJ2/29ONFkC+v8jl5mMV913C0ZRBmK8MymMkmUnDGDnG+620O37hAtNHEom1biE8rpeWKjvcre5CylfQFRrZ8ppiFq1IIjzLgd7lw7NPqkPQfPQojWpDFP/ooX/zZ2XFbrrl9bt5tfZfKukoONx1m0DtIYmQiTxY8yc68naxIXjH79UdUGdZZwevzc7ndEVwkU91opalXuxfC9ILFGUY+WWYJ6tcZxsh7V9VRMW+4byPwsdIO/2RVJocudwWd+qe3LaDTaOD37Z10ewXx2FgvD7PG9QH61gj6uxeR07+AQm86MXER1PWf51zdIbx4yC99gOwl63FYU7l2qhvXoJdYcwRFa9IoLk/HlBqN9PsZqq7GtmcP9qp9+B0ODKmpGHc9HmxBNr7N8PTDPuz6D3iz8U0cbgfGCCPbLNvYmbeT1amr0c92hDtm1oiSRqZKT7/rlk7oZ5ttwfc9JS4iGFmXWswszVTR9YedD52EMlbWhAD8BoEvPRqRFY4nPho9Pkrk+2zwHyW9s4eejoUkOBZT6Ewn05BIb0Qnp68foGeghYTMbIrKt6CPWMSN04P0tg6gN+jIL0mmuDyNrOIEdDqBu74+UKr1VTzNzVoLsocf1lqQrVlzSwuykbxS08Lf77tEh+sSxuQLhMefZ8BnIyYshq2WrVTkVrA2Yy1hulmsP6KkkRllOLquabypXTf0DALaQrQlGfGUBDNDTGSaolR0rbiFD52EcnsusgT8iRHIzCh8qZFInZ5sWc8meZCltssMtqfity4jZSCbcn8axPuptR7neOsHGKLCKSzfyNKsL9BeH8WZQ734/V2k5sWz6VNFFKxOISI6DF9fH7bfvITtlT1aCzIhiClfS/LXv0bctm3oYmLGtFdKybnuc9R6KonK30/0UCcGQxQbsjZRkafVH4nQz2IxoFukkQOBWiNKGpkKvQPuEXnXWnQ96Nai66TYCEotJj5VZqE0x8wyFV0rpsF968CH23r5ow34MqMRmRF4I8KJkQ4epIq1zuNEdujoaivEMPAx1vkyMEfF0Wq4xoFrP2PI6yB78TLWbXqWocEcrldbuVYzSLTRy4pt2RSXp5OQHoN0u+k/8jZde/bgOPy21oKsoICU574daEGWOqaNUkpqrbVU1mlFo4brj2zI3EBFXgWbsjYRHTZ638kZYcwONds1h71wq5JG7oLPL6kNaNfDckh9ILrW6wSL0+P52KqsYN51lllF14qZ476UUBxeHz/8oI7ftnfhNkYhpJ8V1LDef5ic7nZ6OvIQ3YtY6csizZfIdW8n9oGTtFivEJeYTNG6zYTHLqPxgpfupn50BkHeck0isSxOQOgEzjNnAqVa38Bns6FPTMT42GMYn9hNRHHxuB/SG303qKyvpKquinp7PQZhYG3GWipyK3jI8hBx4WOWV58eY0kjycWaw1bSyF2xDripabIGq/KdaepjIBhdh2tSiEWTQpZnmYgKV2OpmD73vQb+x+pm/ub4DTri9ZAeiU+nJ0O2sJG3WNl/DldbEvbuIgoG8yjyp9Pl8VDvOIvV9j4+nZ+CB8pJzltDb7uZhnO9+H2SZEscxeXpFD6QSmRsGO7mZs1p79mLu6EBERFB3NatGJ/YTcy6deO2IGuyN2n1R+qruGK9gkBQllbGjrwdbLNswxx595WVU2K8rBG1oGZcfH7JlQ7HLZ3Qb3QPAFp0vSg97mYqn8VMdoKKrhWzw33rwBuHXPywpoF9tj68keFEykHWcZRy7zHi2310dxaSOlBMgSuN9DAzdd4GLjUeYsDdiTUqhfzSjSxIWsn10/0M2d1ExYVRuCaNReXpJGbG4nM4sFdVYd+zl8GA7dFlZRh37yJux45xW5C1D7Rr9Ufqqjjfcx6AlckrqcirYHvOdpKjk8d9bZNJg7yF8aQRlTUyJn2D7mDpVC26ttHv8gKQEBMe7CSzKsfM8izjHVUrFYrZ4r5y4IM+P6939fFiaxfv2oYQ0s8SzrFBHmJBbxPW9hxc3cWs8GaT70/FF+fhYvsxbnScJiIulsI1G4kxr6Tlqp7OBgc6nSB3eZImkSxNROf30X/sGPa9e3G8dRDpchGel4dx926tBVnm2E60e6ib/fX72Ve/j+rOagAWJy5mZ+5OduTuID02fUKvcVLVF31eaD4ZqJ2tpJGJ4PdLrnb231JC9XqXFl3rBBSnxQfT+EotZnISo1V0rbhnhLwDl1Lyvm2Al9p72dPRw4BfkEoHG+RbrBqqxtdmprOjgLyhBSzxZeDy6rk6cBFr33u4/EPkrCghbWE59t406s9Z8XsliVmxLCpPp7BMk0icFy9q+dqvv4Gvpwe9yUT8I49g/MgTRC5dOuYH2Oay8WbDm1TWV/J++/v4pZ+FpoXszNOcdk58zqRf71hpkMFCUUoamRS2IU8wja+m0crpxj4cgejaHB0WXCBTYjGxIstETISKrhXzh5BOI/xZcxc/b2rnhtNHJC7K5DE2+I9i6hikq72AQdsulnszWe9LoNbZwsm+V7E5G+mPMLFp5+OgW0T92SHa33ITGeNg6YZMitelk5wdh6e9HdtLv6B1zx7c1wItyLZswfjEbmLXr0eMUaq1393PoaZDVNZV8l7re3ill5z4HJ5Z9gwVuRUsNC+c1mserSSrRXSwzVENv/iJKsM6Dn6/5HpXf1C7rm60crWzH9Ci66K0eHatzAg67VwVXStClJBw4O80HSF8yMWzHGSR7QbWtkzcfWUkDGZR5k3HHeHndM8Jqm3VDCKpi1mAJ+VPKDcXcuGYC6Gzk7M0kQ3laeQuSwLnEI4DB2j46z0MHr/ZgiztBz8gfmfFmC3IBj2DHGk5QlVdFUebj+L2u0mPSeezSz5LRW4FixIWzZgjyDBF0d7XT6m4ylZ9DVt11RToWrSd/ap570jsTg+ng9p1H6cbrdidWnRtCkTXuwMOe3m2iVgVXSvuE0LiTn6s4xAOazM9nUXgXEOZM42UCCPthkbebvgVDncPEZkLuBS5BYM3nyX+aAweiNUbWPRkNoVrUomONTDw3nHa//IfcBx4Ezk0RFh2Nklf/SrGXY8TbrGMem23z807Le9QVVfF4ebDDHmHSI5K5mNFH6Mit4IVyStmNnoLSCMvJf2R2KHDmIUDj9Rzwl/M7/zbWL39k2xfXz5z1wsx/H7Jje4BrcBToG7I1c5+pAQhoCg1jkeXZ2jdZHLM5CfFqOhacd8SEg48o+1RnM12NssUXDFDnLcd5UjPeaLNZhZs3IA+cglNFyXLrC4iog0UPKD1j0zJicN15Sq2n/4Tra++hrezE118PMZduzDu3kVUScmoH26P38OJthNU1lVyqPEQDo8DU4SJx/IfY2feTkpTSme2/kgwa6QSGt4Fv5fsqAQas7fw3zsK2esoJs6UyHM7itg+D9rHzSUOp4czTbabVfka+7ANeQAwRoVRYjHx2HItul6RbSQuchZLDCgU84yQmMS88quDtF68xPn6w3iEm7ySBzClr6anLYH26/0IAdmLEykuTyNvRRL09WJ77XVse/fiunQJDIZAC7LdxG7ehC7iziXpPr+PUx2nqKqv4kDDAfpcfcSFxfGQ5SF25u2kLL1s5uqPDGeN1FZqRaJu71Azh9LIlFMVZwEpA9F1w83JxtoORzC6LkiJvaWbTH5SDDqdiq4V9z8hPYl51XqKLk89i7d/BLd7IQ0XBmm57seU6qf8IwsoLEsjOtKP462DtP7rHgaOHdNakC1fTupf/RXxj47egswv/ZztOktVfRX76vfRPdRNlCGKzdmb2Zm7kwczHyRcf/d+kxNiHpZhvT1VsaVviO/9UeuXORdOvN/l5UxTXzCNr6apj75BLbqOizRQYjFTsTSNUouZlRYT8Sq6VihuISQceOrCJ+jp6OTSCRfhkUPBhTYpObEMvf8Bth//lJZ9+/APDGDISCfx2We0Uq35dzpEKSUXey+yr05r8Ns20Ea4LpyNWRupyKtgY9ZGogwz1MlkFGlkPmWN/Hhf7S155gBDHh8/3lc74w5cSkl9z2DQWZ9qsHKlw4E/8ANwYUos2xenUhpYKLMgOVZF1wrFXQgJB+7o9WFKiWbt7gXkr0zG11SP7ZWfc/21V/G2tqGLiSGuYgfGXbuJfmA1QndnU4Nr1mvB+iONjkYMwsC6zHV8veTrbMneQmz42CsqJ8wt0kgVdF/RtifPz6yRsbrHT6YD/VgMuLycae7TVjY2aNF174AbgLgIAystJrYvSdNWN2abMUar6FqhmCwh4cA3f6oIX58V++tv0PT3e3CePw86HTHrHyTlW/+JuK0PoYu6M2pusDdQVafVH7nWdw2d0FGWVsZTS59iW842jBEzEP06bXDtTU3LvkMaeWpeL6jKTkPaAAAMWElEQVQZrtg42vbJIKWkoWdwREW+Pi6324PR9YLkGLYWpwS164UpsehVdK1QTJuQcODt/+W/0Pfyy+D1ErFoESnPfxfjo49iSL6zlkhrfyv76vdRWVfJpd5LAJSmlPK9su+xPXc7SVFJ0zdonksjE+W5HUWjLtcfblY8FoNubzAzpCaQGdITiK5jIwysyDbytS0LKckxU5JtwhQ9Q/MICoXiFkLCgYfnWEj43J9h3LWbyKLCO/Z3DXaxv2E/VXVVnO46DcDSxKV8e/W32ZG7g7SYtOkZEGLSyEQZ1rnHy0KRUtLYO3jLqsbL7Q58gfA6PymGzUUplOaYWJVjpiAlTkXXCsUcERJphKNhdVo50HCAffX7eL/9fSSSQnOhVn8kZwfZ8dnTu8Co0kgY5D54M9UvIW9GXst8Ysjt42xzX7D1V02jle5+LbqOCdezItsU7NdYkm3GHKOia4VitgnpNMJh7G47BxsPUlVfxfHW4/ikj9z4XL684stU5FaQb5qm1nyfSCMTRUpJs3VoREW+Pi612fEGouu8pBg2FiYHK/IVpanoWqGYT4SEAz/QcIBXr7/KOy3v4PF7yIzN5HNLPsfOvJ0UmYumvlT6btJI0U7IeiDkpJGxcHp8nG223eKwu/tdgKZ9r8w28aVN+VredbaJxNhZ7MGpUCimzZQduBAiG/glkAb4gZ9KKf9xpgwbSWVdJRe6L/Dxoo+zM28ny5KWTd1pD0sjtVVw7cCt0sjqpwNZI6EvjYyMroebFFxsvRld5yRGs6EgKdikoDgtDoP+zvRLhUIxf5myBi6ESAfSpZTVQog44BTwhJTy4ljnTFUDt7lsxIXHoRNTdDBjSiPDHWq2QmT81J57nuD0+DjfYrtlsrHTcTO6Xp5lDKbxlVhMJKnoWqEIGWZcA5dStgFtgccOIcQlIBMY04FPlUnna38IpJHWviFOBVY1Vjf2cbHVhsenfRlbEqJZtyAx6LBVdK1Q3J/MiAYuhMgFSoATM/F8U2LUWiPD0shTIZ014vL6ON9iD3SU0SLsdrsTgMgwHcuzTDy9Pj8ohyTHqehaofgwMG0HLoSIBf4AfFNKaR9l/7PAswCWMWpuT5mxpJHCCk3LDlFppM02FJRBqhutXGix4/b5AcgyR1GWl0CpxcSqnASK0+MIU9G1QvGhZFp54EKIMOA1YJ+U8n/c7fhp54GP2bx3keawQ1AacXv9XGi13cy7brDSatOi6wiDTtOuLWZKArnXKXGR99hihUIx18y4Bi60NJD/C1yaiPOeFpffgIuvjCGNzE3WyEzVze6wO4MV+aob+zjXYsPt1aLrTFMUq3ITeMaiLZZZlB5PuEFF1wqFYnSmI6E8CHwWOCeEOB3Y9pdSyjemb9ZtnP61JpEMr4Bc8NCcSiNTrZvt9vq52Ga/We+6sS9YPCrcoGN5ppHPr8vV2n9ZzKTEq+haoVBMnOlkobwDzM2yvMf/CaJM90wamWjd7E67MxhZVzdYOddiwzUiui6xmHh6fR6lOWYWq+haoVBMk5BYiUlM4j29/Fj1sVv6hvh/x+o4FXDYwehar2NpZjyfXZsTTOVLM6roWqFQzCyh4cDvMWPVzQb4wasXSTdGUmox84UHcynNMbMkI54IQ+hMpCoUitBEOfBR8Pj8XG5zBNP4nLfJJwB6neAzayx8adOCSTdAUCgUiplAOXCgu98VLO5U3WjlbHMfTo+mXafERVCWl0C4Xsex691097vJvMfd2xUKhQI+hA7c6/Nzud1xS0W+xt5BAAw6wZKMeD7xgCWgXZvINEVNvXCWQqFQzCL3vQPv6XcFq/FVN1o502QLZpQkx0VQajHx6TWaw16WaSQyTGnXCoUiNLivHLjX56e2w0F1Yx81gdzr+p6b0fXijHg+/kA2JYG86yyziq4VCkXoEtIOvHfAfUuBpzPNfQy6teg6KTacEouZT5RZKLVo0XVUuIquFQrF/UPIOHCfX3KlwxF01jWNVm50DwBaRsii9Dg+uior2P4rO0FF1wqF4v4mJBz4X758jr2nW+l3eQFIjNGi64+u1hz28iwj0eEh8VIUCoVixggJr5dpiuIjJZmU5mjatSUhWkXXCoXiQ09IOPCvbll4r01QKBSKeYeqpqRQKBQhinLgCoVCEaIoB65QKBQhinLgCoVCEaIoB65QKBQhinLgCoVCEaIoB65QKBQhinLgCoVCEaIIKeXcXUyILqBhiqcnAd0zaM5MMV/tgvlrm7JrcsxXu2D+2na/2ZUjpUy+feOcOvDpIIT4QEq5+l7bcTvz1S6Yv7YpuybHfLUL5q9tHxa7lISiUCgUIYpy4AqFQhGihJID/+m9NmAM5qtdMH9tU3ZNjvlqF8xf2z4UdoWMBq5QKBSKWwmlCFyhUCgUI1AOXKFQKEKUeeHAhRAVQohaIcQ1IcTzo+yPEEL8JrD/hBAid8S+7wW21wohdsyxXd8SQlwUQpwVQrwlhMgZsc8nhDgd+Ld3ju36vBCia8T1vzhi3+eEEFcD/z43x3b9zxE2XRFC9I3YN5vj9XMhRKcQ4vwY+4UQ4p8Cdp8VQpSO2Deb43U3uz4dsOesEOJdIcSKEfvqhRDnAuP1wUzaNUHbNgshbCPes78esW/c+2CW7XpuhE3nA/dVQmDfrI2ZECJbCHFICHFJCHFBCPGNUY6Z+ftMSnlP/wF64DqQD4QDZ4DFtx3z58C/BB5/AvhN4PHiwPERQF7gefRzaNcWIDrw+CvDdgX+7r+H4/V54H+Ncm4CcCPwvznw2DxXdt12/NeBn8/2eAWeeyNQCpwfY/8jQCUggLXAidkerwnatW74esDOYbsCf9cDSfdwzDYDr033Pphpu2479nHg4FyMGZAOlAYexwFXRvlczvh9Nh8i8DLgmpTyhpTSDbwE7L7tmN3ALwKPfw9sFUKIwPaXpJQuKWUdcC3wfHNil5TykJRyMPDncSBrhq49LbvGYQdwQErZK6W0AgeAintk1yeBF2fo2uMipTwC9I5zyG7gl1LjOGASQqQzu+N1V7uklO8Grgtzd38NX/tuYzYW07k/Z9quubzH2qSU1YHHDuASkHnbYTN+n80HB54JNI34u5k7X3jwGCmlF7ABiRM8dzbtGsnTaN+uw0QKIT4QQhwXQjwxQzZNxq4/CfxM+70QInuS586mXQSkpjzg4IjNszVeE2Es22dzvCbL7feXBPYLIU4JIZ69RzaVCyHOCCEqhRBLAtvmxZgJIaLRnOAfRmyekzETmsRbApy4bdeM32fzoanxaO3lb89tHOuYiZw7VSb83EKIzwCrgU0jNluklK1CiHzgoBDinJTy+hzZ9SrwopTSJYT4Mtqvl4cmeO5s2jXMJ4DfSyl9I7bN1nhNhHtxf00YIcQWNAe+fsTmBwPjlQIcEEJcDkSnc0U1Wn2OfiHEI8ArQAHzZMzQ5JNjUsqR0fqsj5kQIhbtS+ObUkr77btHOWVa99l8iMCbgewRf2cBrWMdI4QwAEa0n1ETOXc27UIIsQ34PrBLSuka3i6lbA38fwM4jPaNPCd2SSl7RtjyArBqoufOpl0j+AS3/bSdxfGaCGPZPpvjNSGEEMuBnwG7pZQ9w9tHjFcn8DIzJx1OCCmlXUrZH3j8BhAmhEhiHoxZgPHusVkZMyFEGJrz/rWU8o+jHDLz99lsCPqTFP8NaKJ9HjcnPZbcdsxXuXUS87eBx0u4dRLzBjM3iTkRu0rQJmwKbttuBiICj5OAq8zQRM4E7Uof8fgjwHF5c7KkLmCfOfA4Ya7sChxXhDaZJOZivEZcI5exJ+Qe5dbJpZOzPV4TtMuCNq+z7rbtMUDciMfvAhUzadcEbEsbfg/RHGFjYPwmdB/Mll2B/cMBXsxcjVngtf8S+Mk4x8z4fTajb/g0XvwjaLO214HvB7b9V7SoFiAS+F3gZj4J5I849/uB82qBnXNs15tAB3A68G9vYPs64Fzg5j0HPD3Hdv0dcCFw/UNA8YhznwqM4zXgC3NpV+DvHwA/uu282R6vF4E2wIMW7TwNfBn4cmC/AP53wO5zwOo5Gq+72fUzwDri/vogsD0/MFZnAu/z92fSrgna9rUR99hxRnzJjHYfzJVdgWM+j5bcMPK8WR0zNHlLAmdHvF+PzPZ9ppbSKxQKRYgyHzRwhUKhUEwB5cAVCoUiRFEOXKFQKEIU5cAVCoUiRFEOXKFQKEIU5cAVCoUiRFEOXKFQKEKU/w/LddIGiYyO1AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# example plot steps gradient descent\n",
    "eta = 0.1 # learning rate\n",
    "n_iterations = 10\n",
    "m = 100\n",
    "fig, ax = plt.subplots(1,1)\n",
    "ax.scatter(X, y)\n",
    "x = np.arange(0, 2, 0.1)\n",
    "theta = np.random.rand(2, 1)\n",
    "for iteration in range(n_iterations):\n",
    "    gradients = 2/m * X_b.T @ (X_b @ theta - y)\n",
    "    theta = theta - eta * gradients\n",
    "    ax.plot(x, theta[0] + theta[1]* x, label=iteration)\n",
    "ax.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stochastic Gradient Descent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**or with Scikit-Learn:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([3.75825769]), array([2.14694259]))"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import SGDRegressor\n",
    "sgd_reg = SGDRegressor(max_iter = 1000, tol = 1e-3, penalty=None, eta0=0.1)\n",
    "sgd_reg.fit(X, y.reshape(-1))\n",
    "sgd_reg.intercept_, sgd_reg.coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Polynomial regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x1ba4429ac48>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAD8CAYAAACFK0QrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAY1UlEQVR4nO3df5BddXnH8c/DZpGNlm6oAcMNKehgVFyblRXTZtpKEGKFwk6wigNOpnXM1LEW0EYTtSodOklNR2CmnbYZoKVDRoMkLlTUQPlhR2aC3bjBgCHFokRuqImFtTIsslme/nHPTe7enHPu2b337rn3e96vGSZ7757sPtldnvvd5/t8n2PuLgBAdzsh7wAAAM0jmQNAAEjmABAAkjkABIBkDgABIJkDQADmZbnIzH4i6ZeSpiQdcfchMztF0jZJZ0r6iaT3u/vz7QkTAJBmJivz8919mbsPRY/XS7rf3c+WdH/0GACQg2bKLJdJui16+zZJw82HAwCYDctyAtTMfizpeUku6Z/cfYuZjbt7f801z7v7gpi/u1bSWkk64cSTzp13yuLYz3GCmUr9feqf3zu7fwkABGr37t0/d/eFaddkqplLWuHuB83sVEn3mdkTWYNw9y2StkjSqxad7YvW3Jh47Wn9fXp4/cqsHxoACsHMnm50TaYyi7sfjP48JOnrks6T9DMzWxR9okWSDjX6OL096Z/u4PhElnAAAHUaJnMze7WZ/Vr1bUkXSXpM0t2S1kSXrZF0V6OP9bqTT1Jfb0/i+0/v78sQMgCgXpaV+WmSvmtmj0r6nqR73P3bkjZJutDMnpR0YfQ4Vf/8Xm1cPaAFMXXxvt4erVu1dEbBAwAqMm2AtsrQ0JCPjo5KkkbGytq8c78Ojk/o9P4+rVu1VMODpTmLBQC6hZntrmkLj5V1A7TlhgdLJG8AaJHckjkAdItuqCSQzAEgxchYWRt27NXE5JQkqTw+oQ079kpSRyV0Bm0BQIrNO/cfTeRVE5NT2rxzf04RxSOZA0CKpPMvnXYuhmQOACmSzr902rkYkjkApFi3aulxhx078VwMG6AAkKK6yUk3CwB0uW44F0OZBQACQDIHgACQzAEgAB1VM++GI7MA0Ik6Jpl3y5FZAOhEHVNm6ZYjswDQiTommXfLkVkA6EQdk8y75cgsAHSijknm3XJkFgA6UcdsgHbLkVkA6EQdk8yl7jgyCwCdqGPKLACA2SOZA0AASOYAEACSOQAEgGQOAAEgmQNAAEjmABAAkjkABIBkDgABIJkDQABI5gAQAJI5AASAZA4AASCZA0AASOYAEIDMydzMesxszMy+ET0+y8weMbMnzWybmZ3YvjABAGlmsjK/WtK+msd/I+kGdz9b0vOSPtzKwAAA2WVK5ma2WNLFkm6OHpuklZLujC65TdJwOwIEADSWdWV+o6RPSXolevwbksbd/Uj0+BlJsfd7M7O1ZjZqZqOHDx9uKlgAQLyGydzMLpF0yN131z4dc6nH/X133+LuQ+4+tHDhwlmGCQBIk+WGziskXWpm75V0kqSTVVmp95vZvGh1vljSwfaFCQDNGxkra/PO/To4PqHT+/u0btXSYG4i33Bl7u4b3H2xu58p6QpJD7j7lZIelPS+6LI1ku5qW5QA0KSRsbI27Nir8viEXFJ5fEIbduzVyFg579Baopk+809L+oSZ/UiVGvotrQkJAFpv8879mpicmvbcxOSUNu/cn1NErZWlzHKUuz8k6aHo7ackndf6kACg9Q6OT8zo+W7DCVAAhXB6f9+Mnu82JHMAhbBu1VL19fZMe66vt0frVi3NKaLWmlGZpZOFvEsNoHnVfBBqnggimVd3qaubG9VdaknBfKMANG94sBRsTgiizJK0S33dvz2eU0QAMLeCWJkn7UY//+KkRsbKwb4SAzheUUuuQazM03ajQ+khBdBY6AeD0gSRzNN2o8uB9JACaCz0g0Fpgkjmw4MlWdzoL0k9Se8AEJzQDwalCSKZS5LHzmyUppLeASA4oR8MShNMMi8lfLOSngcQntAPBqUJJpnHfRNNlZr5ik0PFGIDBCi64cGSNq4eUKm/T6bKYm7j6oFCdLME0ZooTT/dVR6fkOnY3TI4RAQUR8gHg9IEszKXKt/Eh9evVKm/77jbHhVlRxtAMQWVzKuKvKMNoJiCTOZF3tEGUExBJvMi72gDKKYgk/nwYEmXn1s6emCox0yXn1vMTREAxRBMN0utkbGytu8uHz0wNOWu7bvLGvrNU0joQGCKOlirXpAr8yLPZwCKpMiDteoFmczpZgGKgYXbMUEmc7pZgGJg4XZMkMmcbhagGFi4HRNkMo+bz3D5uSVt3rlfZ62/h1ktQCBYuB0TZDeLNH0+Azd8BsJUO5Op6N0swSbzWmmbJEX8pgMhKepgrXpBllnqsUkCIHSFSOZskgAIXSGSOZskQBhGxspasekBGhliFKJmziYJ0P1oZEhXiGQusUkCdDsaGdIVoswCoPvRyJCuMCvzekxaA7rL6f19KsckbhoZKgq5MmfSGtB9aGRI1zCZm9lJZvY9M3vUzB43s+ui588ys0fM7Ekz22ZmJ7Y/3NZg0hrQfeLGdGxcPcBv1JEsZZZfSVrp7i+YWa+k75rZtyR9QtIN7v5VM/tHSR+W9A9tjLVlqL0B3YlGhmQNV+Ze8UL0sDf6zyWtlHRn9PxtkobbEmEbcIgIQGgy1czNrMfM9kg6JOk+Sf8tadzdj0SXPCMp9uXSzNaa2aiZjR4+fLgVMTeN2huA0GRK5u4+5e7LJC2WdJ6kN8ddlvB3t7j7kLsPLVy4cPaRthC1NwChmVFroruPm9lDkpZL6jezedHqfLGkg22Ir22ovQEISZZuloVm1h+93Sfp3ZL2SXpQ0vuiy9ZIuqtdQQIA0mVZmS+SdJuZ9aiS/O9w92+Y2Q8lfdXMrpc0JumWNsYJAEjRMJm7+w8kDcY8/5Qq9fPgcDoUQLcp7HH+JExmA9CNCnmcPw2nQwF0I5J5HU6HAuhGJPM6nA4F0I1I5nU4HQrkg1vCNYcN0DrcYg6YezQeNI9kHoPTocDc4pZwzaPMAiB3NB40j2QOIHc0HjSPZA4gdzQeNI+aOYDc0XjQPJJ5DWayAPmh8aA5JPMIrVEAuhk18wgzWQB0M5J5hNYoAN2MZB6hNQpANyOZR2iNAtDN2ACN0BoFoJuRzGvQGgWgW1FmAYAAsDJvAw4fAZhrJPMW4/ARgDxQZmkxDh8ByAPJvMU4fAQgDyTzFuPwEYA8kMxbjMNHAPLABmiLcfgIQB5I5m3A4SMAc40yCwAEgGQOAAGgzNICnPgEkDeSeZM48QnEY5EztyizNIkTn8DxPjeyV9du26Py+IRcxxY5I2PlvEMLFsm8SZz4BKYbGStr664D8rrnWeS0F8m8SZz4BKbbvHP/cYm8ikVO+zRM5mZ2hpk9aGb7zOxxM7s6ev4UM7vPzJ6M/lzQ/nA7Dyc+genSEjaLnPbJsjI/IumT7v5mScslfczM3iJpvaT73f1sSfdHjwtneLCkjasHVOrvk0kq9fdp4+oBNnpQWEkJ2yQWOW3UsJvF3Z+V9Gz09i/NbJ+kkqTLJL0ruuw2SQ9J+nRbouxwnPgEjlm3aum0Di+pksivXL6E/0/aaEatiWZ2pqRBSY9IOi1K9HL3Z83s1IS/s1bSWklasmRJM7F2DVqyEKosP9vMJ8qHuSdtVdRdaPYaSd+R9NfuvsPMxt29v+b9z7t7at18aGjIR0dHmwq409X3nUuVGjqlF3S7uJ/t6or7+uGB/AIrADPb7e5Daddk6mYxs15J2yVtdfcd0dM/M7NF0fsXSTrUTLChoO8coYr72XZJW3cdoH+8A2TpZjFJt0ja5+5frnnX3ZLWRG+vkXRX68PrPvSdI1RJP8MusVjpAFlW5iskfUjSSjPbE/33XkmbJF1oZk9KujB6XHj0nSNUaT/DLFby1zCZu/t33d3c/W3uviz675vu/r/ufoG7nx39+dxcBNzp6DtHqNatWipLeB+LlfxxArTF6DtHqIYHS7py+ZLjEjqLlc6QuZulFYrQzQKEjtbbuZelm4URuABmhENynYkyCwAEgGQOAAEgmQNAAEjmABAAkjkABIBkDgABoDWxA9HHC2CmSOY5ikvao08/N+1muNW7mksioQNIRDLPSf1s6PL4hNbd+agmp44/kVsdoUsyB5CEmnlO4mZDxyXyKqbSAUhDMs/JTJMzU+kApCGZ52QmyZm7mgNohGSek7i55709pt4Tpg8Y5a7mALJgAzQnSXcwj3uORA6gEeaZA0CHyzLPnDILAASAZA4AASCZA0AA2AAFAsacn+IgmQOBihsZwZyfcJHMgUDFjYyYmJzShh0/YLUeIJI5EKikkRETk6+oHL2P1Xo42AAFApV1ZER1Kie6G8kcCNRM5vkwlbP7kcyBQA0PlrRgfm+ma5nK2f1I5kDAvvCH5xw30K1eX28PUzkDwAZowOgxRtxAt/PftFAPPnGYn4vAkMwDRY8xqoYHS3zPC4AyS6CSeozpWgDCxMo8AHHllKTuBLoWioMyW7GQzLtcXDnlmm17ZCYpZlQ9XQvFQJmteBqWWczsVjM7ZGaP1Tx3ipndZ2ZPRn8uaG+YSBJXTpGkuHuO9J5gdC0UBGW24slSM/8XSe+pe269pPvd/WxJ90ePkYMZlU2s8SUIA2W24mmYzN39PyQ9V/f0ZZJui96+TdJwi+NCRjMpm0xOOSuzgI2MlbVi0wM6a/09OsHiX7kps4Vrtt0sp7n7s5IU/Xlq0oVmttbMRs1s9PDhw7P8dEiybtXShodCarEyC1O1Rl4en5BLmoqps3E4KGxtb0109y3uPuTuQwsXLmz3pyuc4cGSNq4eUH8fx7aLLGnvpMdMJqnU36eNqwfY/AzYbLtZfmZmi9z9WTNbJOlQK4PCzFQPhdS2ovXP79ULLx3R5CvHVmiszMKV9BvXK+768aaL5zga5GG2yfxuSWskbYr+vKtlEWHW6k/60WdcHKf39x2dUV7/PIqhYTI3s69Iepek15rZM5K+oEoSv8PMPizpgKQ/ameQmB2OcRfHulVLp/WVS/wmVjQNk7m7fzDhXRe0OBa0GSv1cMUN1OL7WyycAC0ITgR2t/oX4qTJh3wvi4tBWwXBicDuVd92WB6f0O27Dkx7vGHHXo2MlfMOFTkimRcEJwK7V1LbYS1emEEyL4ikrga6HTpf1hdcXpiLjWReEHEnRel26A5ZX3B5YS42NkALYibdDnS9zK1GX++4tsN6vDCDZF4gWbod6HqZW1m+3tzHE1mQzDFNWtcLyaL1sn69aTtEI9TMMU3SJlp5fILWtzZI+3oDM0EyxzRpm2j0Mrde//z4aZcm8bXGjJDMMU3afHR6mVtrZKysF146Evs+l/haY0aomWOaal32mm17Yt9fHp/QGzZ8U1PuKrHx1pTNO/dPG1Fcj75xzAQrcxxneLCkUkq5pXoXm/L4hNbd+aiWXXevzlp/j1ZseoDSQEYjY+WGdXH6xjETrMwRa92qpbp22x4lrxsrJqdc4xOTkmhjTBI3JGv77vQXPfrGMVOszBFreLDUMJHHoa4+XdKQrLQDQAvm93KLN8wYK3MkKiXcvaYRar3HZBmSVevGDywjiWNWWJkjUVpnSxpqvcfM5IWt1N9HIsessTJHotpj5FlX6CHWepuZVZN0b856JgX3dcPcMvfZVEZnZ2hoyEdHR+fs86G16ueISFLvCabXnDRP4y9OHpfoumFgV6MY4/7NktTf16svXnpOplk3WTaSJeknmy6ezT8BBWBmu919KO0aVubIpJr0Jian1GPWsM88boDUtdv2aPTp53T98MBchx8ry5CrpJr3+MRkps6d4cGSRp9+Tlt3HUhN6GmtoEAW1MzRUG1HhlTpM6+WU5ISWVwSdElbdx3omF70LLfSS6t5Z+3cuX54QDd8YNnRhG117w+xNIW5x8ocDc1mkmJSEqw9pp53CSbLrfQa1byzbnDWTj3shvITug/JHA3N5v6haUmwPD6hdV979OhR9rwOGyXFWNuN0+jGELXXZk3SjLNFO1BmQUOzuX/oulVLjysn1KqfSZLHYaO41ktT5cWlOppgeLCkjasHtCBmumFteSTucBCjDjCXSOZoaDb3Dx0eLOnK5UtSE3q9uT5sVE3UtbXs6ktM9beFakIf+/xFujGqe5sqG5a1pzTjSlHVUQde9/GAdqA1EZnMts77uZG9un3XgUyfo9Tfp4fXr8ylprxi0wOJZaEs0yHPWn9PpvbD6r8RmAlaE9Eys63zPvjE4UzX9fX26Pw3LdTgX92r51+cPPr8XNXT0zY5s8SQ9XAQow7QLpRZ0FZpyau/r/doyWLxgpN0+64D0xJ5Vbvr6SNj5YbloEYxZB19wKgDtAsrc7RV0oq1v69Xe75wkaRspZi4F4WZlmOSrt+8c3+mEknaC1Nt7fzg+IT65/fqhZeOTNvopZ8c7UQyR1vFtfb19fboi5eec/TxVx75acOPU7+izXJ6s3pddbZM3AanlL300WhVXV+Kop8cc4lkjraqX7HGJbWpBpvwcSvaLAeZ6hN+/WeZmJzSJ+94NNOqfDaravrJMZdI5mi7RkmtOuslTtJAqywHmbLMEk/6vL09plefOE+/mDh+gBjQiUjmyN0H33lGbM18xRtO0daP/Hbs38lyenO2nSPcqBrdiG4W5O764QFdtXyJeqzSU9JjpquWL0lM5FK2g0yz6RwxSQ+vX0kiR9dhZY6OcP3wQMPRuPUbipefW9KDTxxOrMVnvSl1LVoH0a2aSuZm9h5JN0nqkXSzu29qSVRAnbjule27y6k3Pk6aJd7X26PLzy1p++7ycV02tA6iW826zGJmPZL+XtIfSHqLpA+a2VtaFRhQK8vs8Ti1s8RrZ6pcPzxwdC5L3KwVoNs0szI/T9KP3P0pSTKzr0q6TNIPWxEYUGs2Y3irkrppaB1ESJpJ5iVJtac9npH0zvqLzGytpLXRw1+Z2WNNfM658lpJP887iAy6Ic6WxNi78MwB65l3Yv3zPnXkZfubS/Y2+/HVHV9LiThbrVvibFj/ayaZx42zOG6vyd23SNoiSWY22mjyVycgztbphhgl4mw14mwtM2s4braZ1sRnJJ1R83ixpINNfDwAwCw1k8z/U9LZZnaWmZ0o6QpJd7cmLADATMy6zOLuR8zszyTtVKU18VZ3f7zBX9sy2883x4izdbohRok4W404W6thnHN6pyEAQHtwnB8AAkAyB4AAzHkyN7NlZrbLzPaY2aiZnTfXMWRhZh83s/1m9riZfSnveNKY2V+YmZvZa/OOJY6ZbTazJ8zsB2b2dTPrzzumWmb2nuh7/SMzW593PHHM7Awze9DM9kU/k1fnHVMSM+sxszEz+0besSQxs34zuzP6udxnZslT3XJkZtdG3+/HzOwrZnZS0rV5rMy/JOk6d18m6fPR445iZuercpr1be5+jqS/zTmkRGZ2hqQLJaXfdy1f90l6q7u/TdJ/SdqQczxHddFYiiOSPunub5a0XNLHOjROSbpa0r68g2jgJknfdvc3SfotdWC8ZlaS9OeShtz9rao0mlyRdH0eydwlnRy9/evqzN70j0ra5O6/kiR3P5RzPGlukPQpxRzY6hTufq+7H4ke7lLlTEKnODqWwt1fllQdS9FR3P1Zd/9+9PYvVUk+HTeLwMwWS7pY0s15x5LEzE6W9HuSbpEkd3/Z3cfzjSrRPEl9ZjZP0nyl5Ms8kvk1kjab2U9VWfF2zCqtxhsl/a6ZPWJm3zGzd+QdUBwzu1RS2d0fzTuWGfgTSd/KO4gacWMpOi5J1jKzMyUNSnok30hi3ajK4uKVvANJ8XpJhyX9c1QOutnMXp13UPXcvaxKjjwg6VlJv3D3e5Oub8s8czP7d0mvi3nXZyVdIOlad99uZu9X5dXx3e2II02DGOdJWqDKr7PvkHSHmb3ec+jjbBDnZyRdNLcRxUuL093viq75rCrlgq1zGVsDmcZSdAoze42k7ZKucff/yzueWmZ2iaRD7r7bzN6Vdzwp5kl6u6SPu/sjZnaTpPWS/jLfsKYzswWq/JZ4lqRxSV8zs6vc/fa469uSzN09MTmb2b+qUlOTpK8pp1/HGsT4UUk7ouT9PTN7RZWBPIfnKr6qpDjNbECVb/KjVrlDz2JJ3zez89z9f+YwREnpX09JMrM1ki6RdEEeL4opumYshZn1qpLIt7r7jrzjibFC0qVm9l5JJ0k62cxud/erco6r3jOSnnH36m82d6qSzDvNuyX92N0PS5KZ7ZD0O5Jik3keZZaDkn4/enulpCdziKGREVVik5m9UdKJ6rDJau6+191Pdfcz3f1MVX5A355HIm8kuonJpyVd6u4v5h1Pna4YS2GVV+xbJO1z9y/nHU8cd9/g7oujn8crJD3QgYlc0f8jPzWz6iTCC9SZo7sPSFpuZvOj7/8FStmozeO2cR+RdFNU0H9Jx8bjdpJbJd0ajet9WdKaDltNdpu/k/QqSfdFv0Xscvc/zTekilmOpcjDCkkfkrTXzPZEz33G3b+ZY0zd7OOStkYv4E9J+uOc4zlOVAK6U9L3VSlPjinlWD/H+QEgAJwABYAAkMwBIAAkcwAIAMkcAAJAMgeAAJDMASAAJHMACMD/AwzjjLKJ69PZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "m = 100\n",
    "mu = 3\n",
    "sigma = 6\n",
    "\n",
    "a = 0.9\n",
    "b=  2\n",
    "\n",
    "X = sigma * np.random.randn(m, 1) - mu\n",
    "y = a * X**2 + X + b + np.random.randn(m, 1)\n",
    "\n",
    "fig, ax = plt.subplots(1,1)\n",
    "ax.set_xlim((-8,8))\n",
    "ax.set_ylim(0, 50)\n",
    "ax.scatter(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([2.04322802]), array([[1.00533088, 0.90181957]]))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "poly_features = PolynomialFeatures(degree = 2, include_bias=False)\n",
    "X_poly = poly_features.fit_transform(X)\n",
    "\n",
    "lin_reg = LinearRegression()\n",
    "lin_reg.fit(X_poly, y)\n",
    "lin_reg.intercept_, lin_reg.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 50)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAD8CAYAAACFK0QrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXzU5bX48c+ZZLIQIGEJkIUdDCBBAogg1auoYAU1Ba1abe1y688uttUWhdr2trdaqNjWbrfW63KtKxZotKKACqiogEBYhbBDmAQSloQtIcs8vz9mBocwW5bJd5bzfr14JfOdb2ZOFs4883zPcx4xxqCUUiq62awOQCmlVOtpMldKqRigyVwppWKAJnOllIoBmsyVUioGaDJXSqkYkBjKSSKyDzgJNAINxpgxItIVmAf0A/YBXzbGHA9PmEoppQJpzsj8amPMSGPMGPftmcB7xpjBwHvu20oppSzQmmmWm4Hn3Z8/DxS2PhyllFItIaGsABWRvcBxwAB/N8Y8JSJVxpgMr3OOG2O6+Pjae4B7AGxJKaMTu+b6fA6bCDkZqWR0sLfsO1FKqXZyqraBvUdP06drB9JTw5+z1q1bd8QYkxnonJDmzIEJxpgyEekBvCMi20MNwhjzFPAUQHLWYJN19xN+z+2ZkcpHMyeG+tBKKWWJH75azPLtFax5+FpS7Alhfz4R2R/snJCmWYwxZe6PFcC/gLHAYRHJcj9RFlAR7HHsCYGfrqyqJpRwlFLKMidr61my9RA3XpLdLok8VEGTuYikiUgnz+fAJGAL8AZwt/u0u4HXgz1Wr84ppAb45rMzUkMIWSmlrPP25kPU1juZPtr3lLFVQhmZ9wRWishGYA2wyBizGJgDXCciO4Hr3LcDyuhgZ/a0fLr4mBdPtScwY3Jes4JXSqn2Nn/9QQZ0T6Ogd0bwk9tR0DlzY8we4BIfx48C1zT3CQsLcigsyGHmgk28+mkpADkZqcyYnEdhQU5zH04ppdpN6bEzrNl7jBmT8xARq8M5T6gXQNvcw1OGUrTBwZcKcpk9Ld+qMJRSKmQL1h9EhIgceFq2nL9Tip0bhmfx5sYyausbrQpDKaWCKip2cPns93ji3Z0kJdj4dO8xq0O6gKW9WW4ZncvJsw0s2XrIyjCUUsqvomIHsxZupqy6FoCzDU5mLdxMUbHD4sjOZ2kyHzegGzkZqcxfd9DKMJRSyq+5S0qoaTJ7UFPfyNwlJRZF5JulydxmE6aPzmXlriNaY66Uikj+clOk5SzLW+BOH5WDMfCvCHvLopRS4H/9S6Sti7E8mfftlsbY/l2Zv+4gofSJUUqp9jRjch62JlWIkbguxvJkDq4LoXuPnGbdfm2HrpSKLOMHdsMAnZITEVzrYmZPy4+48kTL6sy9TcnP4pdvbGX+uoOM6dfV6nCUUuqcomIHxsDr35/AgMyOVofjV0SMzNOSE/ni8Cze3FROTZ3WnCulIoMxhtfWljK6b5eITuQQIckcXFMtp842sHhrudWhKKUUAOsPHGd35WluG9Pb6lCCiphkfln/rvTuqjXnSqnIMe/TUtKSEpgyIsvqUIKKmGRuswkXZ6Xz0a6j9Ju5iAlzlkXcCiulVPw4dbaBNzeVM3VENmnJEXF5MaCISeZFxQ6Wl3y+v4WjqiYil8wqpeLDok1lnKlr5MuXRv4UC0RQMp+7pISzDc7zjkXiklmlVHyY92kpg3p0ZFSfyOpb7k/EJPNoWTKrlIp9uypOsv5AFV8ekxtxfcv9iZhkHi1LZpVSse+1tQdJtAnTRkXW1nCBREwynzE574L9QVMSbRG3ZFYpFdvqG50sXH+Qa4b2oHvHZKvDCVnEXKL1LI2du6QEh3tq5Yv5WRG3ZFYpFdve21bBkVN13BYlFz49IiaZw+f7gwLc+OeVfFZ2AmNM1MxZKaWi32trS+nZOZkrB2daHUqzRMw0S1O3j+1NyeGTbCitsjoUpVScOFRdy4qSCm4ZnUtiQsSmR58iNtqbLskm1Z7AvE9LrQ5FKRUnFqw/iNPAraOja4oFIjiZd0qxM3VEFm9sLOPU2Qarw1FKxTin09VU67L+XenXPc3qcJotYpM5wO1j+3CmrpE3N5ZZHYpSKsat3nuM/UfPRN2FT4+ITuaj+mQwuEdHXtGpFqVUmL22tpRO7nbc0Siik7mIcPvYPmwsrWJb+Qmrw1FKxaiqM3Us2lzOzQXZpCYlBP+CCBTRyRzgSwU5JCXY9EKoUipsFqx3UNfg5Ctj+1odSotFfDLvmpbE5OG9WLj+ILX1uguRUqptGWN4efV+CvpkMCy7s9XhtFjEJ3OA2y/tzYnaBhZvOWR1KEqpGLN67zF2V57mK2P7WB1Kq0RFMh8/oBt9unbglTUHrA5FKRVjXl59gE4piUwdkW11KK0SFcncZhNuu7Q3q/ceY0/lKavDUUrFiKOnzrJ4yyGmj8qN2gufHlGRzAFuHZ1Lok10dK6UajPz1x2krtHJnZdF9xQLRFEy79E5hUkX9+Sf6/RCqFKq9ZxOwytrDnBpvy4M7tnJ6nBaLWqSOcBdl/Wl6kw9izaVWx2KUirKfbz7KPuOnuHOy6K3HNFbVCXz8QO7MSAzjRdW7bc6FKVUlHt5zX66dLBz/fBeVofSJkJO5iKSICLFIvKm+3Z/EVktIjtFZJ6IJIUvzHMxcOdlfdlQWsUWR3W4n04pFaMqTtaydOthpo/KJcUe3Rc+PZozMv8hsM3r9m+BPxhjBgPHgW+1ZWD+3DIqlxS7jZdW6+hcKdUy/1x7kAan4Y4YuPDpEVIyF5FcYArwtPu2ABOB+e5TngcKwxFgU+kd7Nx0STZFxWWcqK1vj6dUSsUQz4XP8QO6MTCzo9XhtJlQR+ZPAA8CTvftbkCVMcbTaPwg4HOzThG5R0TWisjaysrKVgXrcde4vtTUN/Kv9Y42eTylVPz4YGclB4/XcOe42BmVQwjJXESmAhXGmHXeh32canx9vTHmKWPMGGPMmMzMttlTb0RuBiNy03lh1X6M8fm0Sinl00urD9C9YxKThsXGhU+PUEbmE4CbRGQf8Cqu6ZUngAwR8WwInQu06w4Sd43ry66KU6zee6w9n1YpFcX+76O9vPPZYY6cquPqx1dQVBw77+6DJnNjzCxjTK4xph9wO7DMGHMnsBy4xX3a3cDrYYvShxtHZNM5JZEXtUxRKRWComIHjyz6vIbDUVXDrIWbYyaht6bO/CHgARHZhWsO/Zm2CSk0qUkJ3DK6N0u2HqLy5Nn2fGqlVBR6bPF2GpznT8vW1Dcyd0mJRRG1rWYlc2PMCmPMVPfne4wxY40xg4wxtxpj2j2j3jmuD/WNrk1YlVIqkLLqWt/Hq2raOZLwiKoVoE0NzOzIhEHdeHn1ARqdeiFUKeWfPcFX3QZkZ6S2cyThEdXJHFz9WhxVNSzfXmF1KEqpCLWhtIr6RoPddn5CT7UnMGNynkVRta2oT+bXDutJz87JzF1awoQ5y+g/cxET5iyLmYsaSqnW+8cn+0hLSuDXhcPJyUhFgJyMVGZPy6ewwOcSmaiTGPyUyGZPsDG6Txfe8tpSznOVGoiZX5RSqmWOnjrLmxvLuX1sb24f24fbo3x7OH+ifmQOsP5A1QXHauob+dW/t1oQjVIqkrz6aSl1jU6+Nj42Wt36E/Ujc4DDJ3xfpT5+pp6iYoeOzpWKI0XFDuYuKaGsqoas9BTO1DXyhUHdGdQj+jegCCQmRuaBrkbHSg2pUiq4omIHsxZuxlFVg8FVjlhVU09er9hO5BAjyTzQ1WhHjNSQKqWCm7ukhBof20q+vTn2dyeLiWReWJCD+C4hJcHfHUqpmONvAVC5nwVDsSQmkjmAv+aJjdpVUam44W/KNVYWBgUSM8k8x88vy99xpVTsmTE5j9Qm28DF0sKgQGImmfv6JQquOXNdRKRUfCgsyGH2tHzSU+wAZHZMjqmFQYHERGkifL44aO6SknMXPT0TLLqISKn4MXVEFnOXlJCX1YnX/t94q8NpNzEzMgdXov5o5kR6dEq+4L5YanWplPJv6WeHcVTV8K0v9Lc6lHYVU8ncw19/81hpdamU8u/pD/fQt1sHrh3a0+pQ2lVMJvN4vqKtVDxbt/846w9U8c0J/UmwxVdZckwm8xmT80hJPP9bi5cr2krFs2dX7qVzSiK3jM61OpR2F5PJvLAgh1vG5OJ5XbYB00fn6MVPpWJY6bEzvL2lnDsu60NacszUdoQsJr/jomIHC9Y5zlWzOIEF6xyM6dtVE7pSMcbTWMtTxZbVOcXiiKwRkyNzX/0ZtJpFqdjj3VjL47eLS+JyXUlMJnN/VStazaJUbNGB2+diMpn7q1rpFadvv5SKVTpw+1xMJnNfS/sBxvTrYkE0Sqlw0TLkz8VkMvf0Z/DeuLVX52QWbS6nn274rFTMmDE5j6bl5PFahhyT1SzgSuieypWiYgcPzt+E013eor1alIoNfbp1wGkgPdXOiZp6sjNSmTE5Ly7/X8dsMvc2d0kJdY3O8455LpLE4y9dqVjx9Id76JSSyMczJ8Zlbbm3mJxmaUovkigVe/YeOc3bWw7x1XF94z6RQ5wkc71IolTseeqD3dgTbHxjQnx1R/QnLl7OZkzOY9bCzefVoyYn2uLyIolS0azpas/LB3Yj00fL63gUFyNz7+oWcO1ANCI3XefLlYoivlZ7rtt/XCvT3OIimcPnG1fsmzOFr0/oR/GBKp0zVyqK+FrtebbBGZerPX2Jm2TuzbMDyf9+uMfiSJRSodJChsDiMpnndulAQZ8M/u+jfbqISKkooYUMgcVlMi8qdrDpYPUFGz5rQlcqcs2YnIe9yXLPeF3t6UvQZC4iKSKyRkQ2ishWEfmV+3h/EVktIjtFZJ6IJIU/3LYxd0kJZxt8LyJSSkWmqSOy6Jxqx57gSug5GanMnpavhQxuoZQmngUmGmNOiYgdWCkibwMPAH8wxrwqIk8C3wL+FsZY24zOvSkVfd7ecoijp+t48q7RXD+8l9XhRJygI3Pjcsp90+7+Z4CJwHz38eeBwrBEGAY696ZUdDHG8OT7uxmQmcakYT2tDicihTRnLiIJIrIBqADeAXYDVcaYBvcpBwGf73VE5B4RWSsiaysrK9si5lbz1SLXbhOde1MqQn248whby05w75UDsTVtk6iAEJO5MabRGDMSyAXGAkN9nebna58yxowxxozJzMxseaRtqGmL3KQEG2kpidyQn2V1aEopH/62Yje9Oqdwc0G21aFErGYt5zfGVInICmAckCEiie7ReS5QFob4wsa7Re7ykgq+8dynvL7Bwa1jelscmVLK29p9x/hkz1F+NmUoyYkXbjqjXEKpZskUkQz356nAtcA2YDlwi/u0u4HXwxVkuF11USbDsjrztxW7aXT6fIOhlLLIH9/bSfeOSdx5WV+rQ4looUyzZAHLRWQT8CnwjjHmTeAh4AER2QV0A54JX5jhJSJ87+pB7DlymsVbDlkdjlLKbf2B43y48wjfvmIAqUk6Kg8k6DSLMWYTUODj+B5c8+cx4frhvRjQPY2/Lt9FXUMjjy/dQVlVTVzvXKKU1f703k66piVx1zgdlQcTlytAfUmwCfdeNZDPyk/w0AJXZzaDrg5VyiobS6tYUVLJf17RXzefCIEmcy+FI3NIEPG7xZxSqv386b2dZHSw87Xx/awOJSpoMveSlGij0fi+AKqrQ5VqP1sc1by3vYJvTehPRx2Vh0STeRPZ6Sm+j+vqUKXazZ/e20nnlETuntDP6lCihr7kNfHg9UOYMX8j9Y2fj9C1M5tS4dd0S7jrL+5F5xS7xVFFDx2ZN1FYkMOcaSNIcC8Zzk5P0c5sSoWZry3hVpRUaOFBM2gy92H66FxmT8sH4Fc3D9dErlSY+doSrla3hGsWTeZ+TCvIoW+3Djzx7g6Mn4uiSqm2oW2pW0+TuR+JCTbumziYrWUnWPrZYavDUSqmaVvq1tNkHkDhyGz6d0/jiXd34tSeLUqFzYzJeTTtbKuFB82jyTwA1+h8ENvKT7D0M+3ZolS49O+ehtNAp5REBN0SriW0NNGLpzTKuyfLTZdk85dlu3ji3Z1MGtZLG+MrFQaPLy2hSwc7Hzx4NZ20HLFFdGTu5l0a5d2T5c1N5fzgmsFsP3SSxVt1dK5UW1u15ygf7jzCd68apIm8FTSZu/kqjfL0ZLnxkmwGZqbx+3d20NCkb4tSquWMMTy+pISenZP56njtjNgamszdApVGJdiEn0zKY1fFKRau10UMSrWVFSWVrN1/nPsmDibFrv3KW0OTuVuw0qjrh/fikt4Z/OHdHdQ2GcErpZrP6TTMXVJCn64d+LJu19hqmszdZkzOI7XJyMC7NEpEeOj6PMqra3nhk/1WhKhUTHl7yyE+Kz/Bj64dTFKipqLW0p+gW2FBDrOn5ZOTkeq3NOrygd258qJM/rpiFydq660LVqko19Do5HfvlDC4R0duHqnlh21BSxO9FBbkBK1rfXByHlP/vJK/v7+bGZOHtFNkSsWWhcUO9lSe5sm7Rp9raqdaR0fmzTQ8J52bLsnmmZV7qThRa3U4SkWdsw2N/PHdnYzITWfyxT2tDidmaDJvgQeuu4iGRsOflu30eX9RsYMJc5bRf+YiJsxZpm08lfLy8uoDOKpq+MmkPER0VN5WNJm3QL/uadwxtg+vrill35HT593nb/GRJnSl4ERtPX96byfjB3TjisHdrQ4npmgyb6H7rhmEPcHG40vP77ccaPGRUvHuyRW7OX6mnp/eMFRH5W1Mk3kL9eiUwn9e0Z83N5Wz+WD1uePal1kp38qqanhm5V4KR2aTn5tudTgxR5N5K9xz5QC6dLDz2JLt545pX2alfPv9OzswBn48SdvahoMm81bolGLn+xMH8+HOI6woqQCCLz5SKh59VnaCBesP8vUJ/ejdtYPV4cQkTeat9NVxfenXrQO/eWsbDY3OkBYfKRVv5izeTucUO9+7apDVocQsXTTUSkmJNmZ+cSj3vriOeWtLufOyviEtPlIqXny4s5IPdlTysylDSe+gLW7DRUfmbWDyxT0Z278rv1+6g5O6zF+pcxoanTy6aBu5XVK1xW2YaTJvAyLCz6cM4+jpOv5nxW6rw1EqYsxbW8r2Qyf56Q1DSU7UFrfhpMm8DRQVO7j3xXUA/G3Fbp5ZucfiiJSyXnVNPb9buoOx/bvyxeG9rA4n5mkybyXvFZ8ev1m0XVd8qrj3w1eKOXa6jjV7j/GF3y7X/xNhpsm8lXyt+Gw0hkcWfWZRREpZ7wevrGfFjspzt7WtRfhpMm8lfys7j5yqo9Fp2jkapaxXVOzgjY3lFxzXthbhpcm8lQKt7Jz3aWk7RqJUZPj1m/7flWpbi/AJmsxFpLeILBeRbSKyVUR+6D7eVUTeEZGd7o9dwh9u5PG14jMl0cbAzDTmLtlO9RktVVTxo77RydHTdX7v17YW4RPKyLwB+LExZigwDvieiAwDZgLvGWMGA++5b8cdXys+50wfwZ/vGEV1TT1/eHeH1SEq1W6e/3if3/sEtK1FGAVdAWqMKQfK3Z+fFJFtQA5wM3CV+7TngRXAQ2GJMsL5W/F552V9eWHVfm4f25shvTpbEJlS7afiRC1PvLuTYVmd2VN5itoG57n7BLhzXB9dGR1GzVrOLyL9gAJgNdDTnegxxpSLSA8/X3MPcA9Anz59WhNr1CgqdjB3SQmOqhpE4PsvF/PO/Vdq/2YV9Tx/22VVNWRnpDJjct65BP2bt7ZR1+Dkf+4cxYbSKr/nqfAQY0KruBCRjsD7wKPGmIUiUmWMyfC6/7gxJuC8+ZgxY8zatWtbFXCk89SdNy1X/Nr4vvz3zcMtikqp1vP1t+0ZcU8dkc3tT63iBxMH8YC2uG1zIrLOGDMm0DkhVbOIiB1YALxkjFnoPnxYRLLc92cBFa0JNlb4qjsHeHHVfu3boqKar79tA7y46gD3z9tATkYq39GuiJYJpZpFgGeAbcaY33vd9QZwt/vzu4HX2z686OOv9Mpp4HdL9WKoil6BygrLq2v5xY3DSE3S/itWCWVkPgH4KjBRRDa4/90AzAGuE5GdwHXu23HPX+lVWlIC//hkH1sc1T7vVyrSBSsrnDSsZztFonwJmsyNMSuNMWKMGWGMGen+95Yx5qgx5hpjzGD3x2PtEXCk87fT0MNThtI1LZmH/7VZV4aqqDRjch7+LuH37JSsF/gtpitA25i/nYa+cllffjZlKBsPVvPymgNWh6lUsxUW5HDnuD4XJPREmzDrhqGWxKQ+F3I1S1uIh2qWQIwx3Pn0ajY7qln246vI7JRsdUhKNVtRsYPfLt5OeXUtiTZhzrR8bhnT2+qwYlqbVbOotiEi/LpwOGfrnQH7VygVyQoLcpiSnwXAq/eM00QeITSZt7OBmR357tUDeWNjGcu2H7Y6HKWabYujmmc/2ssdY/swpl9Xq8NRbprMLfDdqwZxUc+OPPyvLVp7rqJKo9Pw039tpmtaMjOvH2J1OMqLJnMLJCXamDN9BIdO1PLYYu3vrKLHcx/tZdPBan4+dSjpHexWh6O8aDK3yKg+Xfj65f14YdV+Pt2nVZ0q8u09cpq5S0q4dmgPbrok2+pwVBOazC30k0l55GSk8tCCTdT6aAGgVKRwOg0Pzd9EUqKNRwrztaY8Amkyt1BaciKzp+Wzp/I0f1620+pwlPLrxdX7WbPvGD+fOoxe6SlWh6N8aFYLXNX2rrwok+mjcnny/T1MGtaLS3pnBGwzqlR7Kz12hjlvb+eKwd25dXSu1eEoPzSZW8i777lN4J4X1jJxSA9eXVOKZymXZ1dzQBO6anfGGGYt3IwAc6aP0OmVCKbTLBbx9IZ2uDvROQ0cPnGWV7wSuYfuaq6s8sqaUlbuOsKsG4aSo/t3RjRN5hbx1/fcH93VXLW3/UdP88iiz5gwqBtfGRsfu4RFM03mFmluctZdzVV7anQaHnhtIwk2Ye4tl2Cz6fRKpNNkbpHmJGfd1Vy1tyff3826/cf59c3DdSARJTSZW8RX33N7gmBvMgLSXc1Ve9taVs0T7+5gSn4WN4/UxUHRQqtZLOJJzk1LEAEeW7ydsupabAL/fdNw7hrf18pQVRyprW/k/nkb6NIhiUcKh2v1ShTRZG6hwoIcnyPuwoIctpWf4Oa/fsSKHRWuDQH0P5VqB3OXlLDj8Cme+8aldElLsjoc1Qw6zRKhhmZ1Zub1Q3h3WwUvrtadiVT4LS+p4JmVe/na+L5cndfD6nBUM2kyj2Bfv7wfV16UySNvfsauipNWh6NiWMWJWn7y2kaG9OrET3ULuKikyTyC2WzC47eOIC05kfte2aDNuFRYOJ2G+1/bwOm6Bv7ylQJSmlyYV9FBk3mE69Ephbm3jGBb+QkeXbTN6nBUlCkqdjBhzjL6z1zEhDnLKCp2XHDO397fzUe7jvLLGy9mUI9OFkSp2oIm8yhwzdCefPuK/rywaj//3lhmdTgqSni3jDB83ufHO6Gv23+c37+zgykjsrjtUt3LM5qJMU07gYTPmDFjzNq1a9vt+WJJfaOT2/7+CTsOn+Lf932B/t3TrA5JRbgJc5ad6/3jLdVuo2taMo6qGhJESO9gZ/lPriI9VXcOilQiss4YMybQOToyjxL2BBt/+cooEhOE7760XufPVVD+WkbU1DvPJflGYzhd28Dy7RXtGZoKA03mUSQ7I5U/fHkk28pP8N9vfmZ1OCrChboM/2yjU7tyxgBN5lHm6iE9uPc/BvLy6gMsXH/Q6nBUBGtOPx/tyhn9NJlHoZ9MuohxA7oya+FmtjiqrQ5HRajCghy6dAhtHlybaUU/TeZRKDHBxl+/MopuaUn8vxfWcfTUWatDUhHqv268+IKGbk2l2hO0K2cM0GQepbp1TObvXx1D5amzfP/lYhoanRecE0qNsYpthQU5zJ6WT7bXJsxXXtSdnIxUBMjJSGX2tHztyhkDtNFWFMvPTWf2l/L58T83Muft7fxs6rBz93lqjD27GeleovGrsCCHypNnefStbdx/7UX88NrBVoekwkBH5lFu+uhcvn55P55eufe8C6K+tqXTvUTj04qSCma/vY0vDu/FfRMHWR2OChMdmceA/Jx0khJtPPDaRn7z1jZ+NmWY3+oErVqIH0XFDn7z1jYqTp7FbhOuysvU7d9imCbzKFdU7OBnRVuoa3DNmR85VceP5m1ABPCxuFerFuJDUbGDmQs2Uev+u6h3Gn75xmckJyboNFuMCjrNIiLPikiFiGzxOtZVRN4RkZ3uj13CG6byx9d0CoCvLg12m2jVQpx4bPH2c4ncQ6fZYlsoc+b/B1zf5NhM4D1jzGDgPfdtZYFmTZvoO+y4YIyhrLrW5306zRa7giZzY8wHwLEmh28Gnnd//jxQ2MZxqRA1Z9qkvtHoyCyGnStFnfWW33N0mi12tbSapacxphzA/dHvHlMico+IrBWRtZWVlS18OuXPjMl5QReFeNORWWzybnfrjy4Oim1hL000xjxljBljjBmTmZkZ7qeLO55FIRkhti/VkVls8nftJEFEFwfFiZZWsxwWkSxjTLmIZAHaP9NChQU5FBbkUFTsYO6SEsqqasjoYOdkTT0NXhdCdWQWu/yNyJ3GsHfOlHaORlmhpcn8DeBuYI774+ttFpFqMU9S9ygqdvDY4u3nLoZ9/fJ+OjKLQZsPVuOnElXficWRUEoTXwE+AfJE5KCIfAtXEr9ORHYC17lvqwhTWJDDx7OuYeN/TWJIr048/8k+1u5rei1bRbNdFae4+7k1ZHSwk5J4/n9nfScWX3TbuDhRcaKWKX9ayZFTZzG45lBnTM7TkXoUO3j8DLc++Qn1jYb5945nQ2nVuWm2bP39xpRQto3TFaBx4uPdRzlZW3/urbg23oou3tdDsjNSubRfF/69qZxGp6FHp2Q2lFZdMM2m4os22ooTc5eU6IrAKOVddmhwvRAXbSij0el6aa44eZZZCzdri+M4p8k8TvirLw9Ul6wig7+yQ2/6wqw0mccJf1UNIlB84Hg7R6OaI9SFXrogLL5pMo8TvlaKJifa6JaWxF1Pr+aT3UctikwFE2p5oZYhxjdN5nHCs1LUe7uw304fwaIfXEFWRipff24NK0pca790u7apmiEAAAveSURBVLn2FeznfeuY3KCPoWWISksTFUdPneVrz65hx+GT3H5pH+avO3jeHG2qPUGXgodJ0+394Pyf9wc7Krn3xXV0SEogQYSKk2fJzkjl6iGZLN9eqWWIcUJLE1VIunVM5uVvj+Oef6zlhVX7L7jfc3FNk0XbC7S9X2KCcP+8DQzq0Ynnv3kpPTql+HkUpXSaRbmlp9r5x7fG+r3fUVWj0y1hEKjK6PsvFzOydwav3jNOE7kKSpO5Oic5MYHsdP9JQ2uZ215GB//dLkf1yeCFb11GeogdMVV802SuzvPg9UP89kfXWua2VVTs4FRtg9/7D1XXktKMXvUqvumcuTqPZ178R/M2+LzfUVXDwFlv0WiM9ndppblLSqh3+i9AKPez9ZtSvujIXF2gsCCHnAA1y43uCihHVQ0z5m9k5K+WahljMxUVO4KuvtW6cdUcOjJXPs2YnMf98zb47JHtrb7RUFVTD2jzLn+aNsm6ekgmC9YFftHTunHVXDoyVz4VFuQETeS+6Lz6+Xw1yXpx1YGAvVa6dLBrXb9qNh2ZK79yMlJb1IhLe4R8LpQmWd6euG2kJnHVIjoyV3756ucSCp3r/VxzXthyMlI1kasW05G58suTWOYuKQl5hB6Lc71N57ybU8GTHeK7G4GY+7mp9qW9WVTIfPURSRQQm1DfaLAnCA9cexHfuXrQufMjfRuzYDH6+p4BMlLt/PKmi4N+P//7wR4efWtbSLHsmzOl+d+Aigvam0W1GU/Sq6lvJEHkvDrzm0dm8+9N5fzyja387p0dVJ6qY1CPNH795rZzSdBRVcP98zawdv8xHinMt/i7cWmaqH1V4/ib866qqQ9YuVPX4OTv7+/mz8t3YU9wvdgFEqgUVKlQaDJXQTVNeo3GnJtO8SSymy7JZsLAbjy+dAfPfbwXAZquhzHAS6sOMKZv14gYoQdqcuWJL9Cct68GZMYYFm0uZ+6SEvYfPcOU/Cx+ceMwPtl99Nx0lcB5lUKxODWl2p8mcxVUKEkPXN0XZ0/L565xfZjyp5U+H8u4H8/zuFZOwfhL1N7Hg815e5+7as9RZr+9nY2lVQzp1YnnvzmW/7goE+C8zZajYfpJRR9N5iqoUJKet4uz08lOT6HMz3J0R1UNM/658dxSdqsWG/lL1N7VODMm5/mcM/c+99N9x/jr8l2sKKkkQQSAEzX1HD9d5/NrvBO7Um1FSxNVUP5KDQOVID54/RAkwGM27UlixWIjX6WXguvFxdOawLNDUxcf3Q2TEmwkJdq49clP+HTfMRJtcq7VQVl1rbY6UO1Kk7kKylfSCzbPW1iQw53j+gRM6E2192Ij7630gPPmsj3vFjwJvfgXk3jitpFkdXa1CE60CXWNTuoanPzqpovpnGKnockLlKfVgWnyeEqFg5YmqpC0dJ73Z0WbeXHVgZCeIzs9hY9nXWPJnPKEOcv8zo1np6cwfXQujqoaFm0q52yDkxG56dw9vh83jczGnmCj/8xFIbU/yMlI5aOZE9s2eBXztDRRtZmWzvMu314Z8rlHTtVx0cNvUedVxtde8+kBL3JW1/LnZbtITrRxy+hc7hjbh+E56eedE+riIG11oMJFp1lUWAVKXhmpdgTXyDc7Pdk1beGjHrumvpHHFm8PW4yhTn10TUvi0S/lX5DIIfTWB9rqQIWLjsxVWPkbsWak2tnwX5OA0KZiyqprueVvH5Ofm86wrM707ZbG1rJq/veDPZRX14Y0HVNU7OCxxdspq64ls2My1wztQVpyIi/62MTal0MBNovwXmRUVlVDRgc7p2obzrvQq/XkKpx0zlyFla/l8Kn2hPNavHp2LgokLTmBYVmd2eI44bdM0CYwLKszg3p0xAD1jU4OHDvDzsOnONvg9Pk1oazO9GjufLfWk6u2onPmynJNR6y+klqwRJ5qT+DRQlfyb3Qa9h89za1PfsLRJnXcTgMlh05yorYBm7imZypOnA14YbIxxETeklG11pOr9qTJXIVdsKTm6fXiS9OGVgk2YUBmR475WZDT4DR88ODVgKtCJViq9j1ed43Y05ISqa6p11G1igqazJXl7rist8858wkDu/LSt8f7/JpQVm+2tHJEN6pW0UirWZTlHil09XPxLIVPEOGucX38JnIIbSFTSypHBPho5kRN5Crq6MhcRYRHCvODtsZtekFx+ugclm+v9DsXH+qm1N60dFBFq1YlcxG5HvgjkAA8bYyZ0yZRKdWEr97jC9Y5Am58XFiQw9r9x3hp1YELWs5OH53DgnWOC6pstHRQRasWT7OISALwV+CLwDDgDhEZ1laBKeUtUBveQB4pzOcPt40kJyMVwTUfPnua612Apy+L93GdXlHRqjUj87HALmPMHgAReRW4GfisLQJTyltz2/B681dNo6WDKpa0JpnnAKVetw8ClzU9SUTuAe5x3zwrIlta8ZztpTtwxOogQhANcbZJjPbMfvmSkJjU9LhpbKiT307d3NrHJzp+lqBxtrVoiTPo/F9rkrmv7qYXXGsyxjwFPAUgImuDrWKKBBpn24mGGEHjbGsaZ9sSkaBL51tTmngQ6O11Oxcoa8XjKaWUaqHWJPNPgcEi0l9EkoDbgTfaJiyllFLN0eJpFmNMg4h8H1iCqzTxWWPM1iBf9lRLn6+daZxtJxpiBI2zrWmcbStonO3aNVEppVR46HJ+pZSKAZrMlVIqBrR7MheRkSKySkQ2iMhaERnb3jGEQkTuE5ESEdkqIo9ZHU8gIvITETEi0t3qWHwRkbkisl1ENonIv0Qkw+qYvInI9e7f9S4RmWl1PL6ISG8RWS4i29x/kz+0OiZ/RCRBRIpF5E2rY/FHRDJEZL7773KbiPjv6mYhEbnf/fveIiKviEiKv3OtGJk/BvzKGDMS+IX7dkQRkatxrWYdYYy5GHjc4pD8EpHewHVA4H3XrPUOMNwYMwLYAcyyOJ5zoqgtRQPwY2PMUGAc8L0IjRPgh8A2q4MI4o/AYmPMEOASIjBeEckBfgCMMcYMx1Vocru/861I5gbo7P48ncisTf8OMMcYcxbAGFNhcTyB/AF4EB8LtiKFMWapMabBfXMVrjUJkeJcWwpjTB3gaUsRUYwx5caY9e7PT+JKPhHXi0BEcoEpwNNWx+KPiHQGrgSeATDG1BljqqyNyq9EIFVEEoEOBMiXViTzHwFzRaQU14g3YkZpXi4CrhCR1SLyvohcanVAvojITYDDGLPR6lia4ZvA21YH4cVXW4qIS5LeRKQfUACstjYSn57ANbjwt4lTJBgAVALPuaeDnhaRNKuDasoY48CVIw8A5UC1MWapv/PD0s9cRN4Fevm462HgGuB+Y8wCEfkyrlfHa8MRRyBBYkwEuuB6O3sp8JqIDDAW1HEGifOnwKT2jci3QHEaY153n/MwrumCl9oztiBCaksRKUSkI7AA+JEx5oTV8XgTkalAhTFmnYhcZXU8ASQCo4D7jDGrReSPwEzg59aGdT4R6YLrXWJ/oAr4p4jcZYx50df5YUnmxhi/yVlE/oFrTg3gn1j0dixIjN8BFrqT9xoRceJqyFPZXvF5+ItTRPJx/ZI3imuHnlxgvYiMNcYcascQgcA/TwARuRuYClxjxYtiAFHTlkJE7LgS+UvGmIVWx+PDBOAmEbkBSAE6i8iLxpi7LI6rqYPAQWOM553NfFzJPNJcC+w1xlQCiMhC4HLAZzK3YpqlDPgP9+cTgZ0WxBBMEa7YEJGLgCQirLOaMWazMaaHMaafMaYfrj/QUVYk8mDcm5g8BNxkjDljdTxNREVbCnG9Yj8DbDPG/N7qeHwxxswyxuS6/x5vB5ZFYCLH/X+kVEQ8nQivITJbdx8AxolIB/fv/xoCXKi1Ytu4bwN/dE/o1/J5e9xI8izwrLtdbx1wd4SNJqPNX4Bk4B33u4hVxph7rQ3JpYVtKawwAfgqsFlENriP/dQY85aFMUWz+4CX3C/ge4BvWBzPBdxTQPOB9bimJ4sJsKxfl/MrpVQM0BWgSikVAzSZK6VUDNBkrpRSMUCTuVJKxQBN5kopFQM0mSulVAzQZK6UUjHg/wPdhjs9dDb1EAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(1,1)\n",
    "ax.scatter(X, y)\n",
    "x = np.arange(-15, 15, 0.1)\n",
    "ax.plot(x, lin_reg.intercept_[0] + lin_reg.coef_[0][1]*x**2 + lin_reg.coef_[0][0]*x)\n",
    "ax.set_xlim((-8,8))\n",
    "ax.set_ylim(0, 50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Learning curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "def plot_learning_curves(model, X, y):\n",
    "    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2)\n",
    "    train_errors, val_errors=[], []\n",
    "    for m in range(1, len(X_train)):\n",
    "        model.fit(X_train[:m], y_train[:m])\n",
    "        y_train_predict = model.predict(X_train[:m])\n",
    "        y_val_predict = model.predict(X_val)\n",
    "        train_errors.append(mean_squared_error(y_train[:m], y_train_predict))\n",
    "        val_errors.append(mean_squared_error(y_val, y_val_predict))\n",
    "    plt.plot(np.sqrt(train_errors), \"r-+\", linewidth=2, label='train')\n",
    "    plt.plot(np.sqrt(val_errors), \"b-\", linewidth = 3, label='val')\n",
    "    plt.legend()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD6CAYAAAC4RRw1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2deXxcZfX/3ydNaLqS7pS2dIFSka20pWyCLBYogiA/xAovRESKgoqKQnFLUBAQFKlftSJb+YJAASvYL4JQqChQSsoihVILtNDQLU33krZpcn5/nLm9k8kkmUxmn/N+ve7reZ67nrkz87nnnmcTVcVxHMfJP0qybYDjOI6THC7gjuM4eYoLuOM4Tp7iAu44jpOnuIA7juPkKS7gjuM4eUpCAi4iFSLyiIi8IyKLReQoEekrIk+LyNJI2ifdxjqO4zghkkg7cBGZCfxLVe8QkT2A7sAPgfWqeqOITAP6qOrVbZ2nf//+OmLEiBSY7TiOUzwsXLhwnaoOiF3froCLSG/gDWCURu0sIkuA41V1lYgMBuap6pi2zjVhwgStrq5O6gM4juMUKyKyUFUnxK5PJIQyCqgF7haR10TkDhHpAQxS1VUAkXRgSi12HMdx2iQRAS8FxgF/UNXDgG3AtEQvICJTRaRaRKpra2uTNNNxHMeJJREBrwFqVPXlSPkRTNDXREInRNK18Q5W1dtVdYKqThgwoEUIx3Ecx0mS0vZ2UNXVIrJCRMao6hLgJODtyHIhcGMkfSytljqOU5Q0NDRQU1PD9u3bs21K2ikvL2fo0KGUlZUltH+7Ah7hW8D9kRYo7wMXYd77LBG5GPgQ+EIS9jqO47RJTU0NvXr1YsSIEYhIts1JG6pKXV0dNTU1jBw5MqFjEhJwVX0daFEDinnjjuM4aWP79u0FL94AIkK/fv3oSF1hQfTE/OAD2LIl21Y4jpMuCl28Azr6OfNewB9+GEaMgGHDwBu5OI5TTOS9gD/0kKWbNsGTT2bXFsdxCo+NGzfy+9//vsPHnXbaaWzcuDENFoXkvYBv2BDm16/Pnh2O4+QYVVUpOU1rAt7Y2NjmcU888QQVFRUpsaE1CkrAo/OO4xQ5116bktNMmzaN9957j7Fjx3L44YdzwgkncN5553HwwQcDcNZZZzF+/HgOPPBAbr/99t3HjRgxgnXr1rF8+XIOOOAALrnkEg488EBOPvlk6uvrU2Jb3gt49BuKC7jjFDgiiS8d2b8NbrzxRvbdd19ef/11br75ZhYsWMD111/P22+/DcBdd93FwoULqa6uZvr06dTV1bU4x9KlS7n88st56623qKio4NFHH03J7Ui0HXjO4h644ziZZOLEic3aaU+fPp3Zs2cDsGLFCpYuXUq/fv2aHTNy5EjGjh0LwPjx41m+fHlKbMlrAW9qssrLABdwxylwEhj+ejciHds/QXr06LE7P2/ePJ555hleeuklunfvzvHHHx+3x2jXrl1357t06eIhFIDNm5t/Py7gjuOkml69erGllY4mmzZtok+fPnTv3p133nmH+fPnZ9S2vPbAY1vouIA7jrObysqUnKZfv34cc8wxHHTQQXTr1o1Bgwbt3nbqqacyY8YMDjnkEMaMGcORRx6ZkmsmSkIz8qSKVE/o8NprMG5cWB48GFauTNnpHcfJARYvXswBBxyQbTMyRrzP25kJHXIW98Adxylm8lrAYwV7+3ZbHMdxioG8FvB4vVTT3HPVcRwnZ8hrAY8XMvEwiuM4xYILuOM4Tp6S1wIeL1ziAu44TrGQ1wLuHrjjOLlIz549M3KdvBZw98Adxylm8ronpnvgjuNkgquvvprhw4dz2WWXAVBVVYWI8Pzzz7NhwwYaGhq47rrrOPPMMzNql3vgjuPkDR0ZTbajS1tMmTKFh4Lpv4BZs2Zx0UUXMXv2bF599VWee+45rrzySjLZsx3cA3ccx2mXww47jLVr17Jy5Upqa2vp06cPgwcP5rvf/S7PP/88JSUlfPTRR6xZs4a99torY3bltYC7B+44TqY455xzeOSRR1i9ejVTpkzh/vvvp7a2loULF1JWVsaIESPiDiWbTvJWwFvrNu8C7jiFS4YjFM2YMmUKl1xyCevWreOf//wns2bNYuDAgZSVlfHcc8/xwQcfZNymvBXw1rrMu4A7jpMODjzwQLZs2cKQIUMYPHgw559/PmeccQYTJkxg7NixfOITn8i4TQkJuIgsB7YAjcAuVZ0gIn2Bh4ARwHLgXFXNmHxGC3V5eeiNu4A7jpMu3nzzzd35/v3789JLL8Xdb+vWrRmxpyOtUE5Q1bFRY9JOA+aq6mhgbqScMaI98Kjp6VzAHccpGjrTjPBMYGYkPxM4q/PmJE60UA8bBl26WP7jj2Hnzkxa4jiOkx0SFXAF/iEiC0VkamTdIFVdBRBJB6bDwNaIFvA+faCiIv42x3Hyn0y3r84WHf2ciQr4Mao6DpgMXC4ixyV6ARGZKiLVIlJdW1vbIePaIjqE0qePLQEu4AVCVVW2LXBygPLycurq6gpexFWVuro6ysvLEz4moUpMVV0ZSdeKyGxgIrBGRAar6ioRGQysbeXY24HbwebETNiydogW6YoKF/CC44034NprXcQdhg4dSk1NDal0AHOV8vJyhg4dmvD+7Qq4iPQASlR1SyR/MvAz4HHgQuDGSPpYUhYniXvgBUxNDRxxhOWnToVf/AL69zcxd0EvOsrKyhgZ3VLB2U0iIZRBwL9F5A1gAfB/qvokJtyTRGQpMClSzhhteeA+rVoeU1VltdI7dlj5T3+CAQPgs581j9xxnN20K+Cq+r6qHhpZDlTV6yPr61T1JFUdHUnXp9/cEPfAC5SqKjj66LD8mc9Y+sQTlj73XPN9HaeIydvRCD0GniLSKYLJnLumBl580XpnQXMxBzjxRBs67tvfdo/cKXpyXsDnzoWrr4azzoK//z1c7x54ikiXCKomd+6//MXSyZOhstLOoQrbttn6Hj0snTHD0ldeCQfIcI/cKTJyXsCfegp++Ut47DFYsCBc7x54CrjvPkvT0Tzrqqss/fGPw3h2Ijz8sKVf+EJzQe7e3dJLL7W0ocHSiROhpAROP909cqfoyHkBHzMmzC9ZEubdA+8EVVUWhrjgAiuXlFg5Fa08gnPfcouVr7/ewiGXXtr+uVeuhBdegK5dTZBjqayEX/3KHjjvvGPr+va19P/+z9Kf/zz8AbhH7hQ6qpqxZfz48dpR/vUvVfvHqo4bZ+saG1VFwvUNDapz54blT3+6w5cpPqqrwxsGqj17qt52m+U7S1VVeN799rO0SxdL168P96usbH7cb39r+5x5ZmLXAdUf/aj55wiWo49OzWdxnBwAqNY4mprzHvj++4f5//7X/p2bN4dv/b16QWmpe+AdJrpTxNlnw9atcMUVVp4zJ/m48o4d8Ic/hOXXX7cKx8ZGKw8bZtdZtqxlyCM6fJIIlZVw3XVma1OTrZs0ydIXX7T0hz+E9evdG3cKkpwX8AEDwnFOtm6FVataxr/BBbzDrFtn6UEHwcEHN992xhkWVrnwwo7HlWfNgjVr7Lw//SncfDNMnx5u37bNyqNGWXnpUktXrYLnn4c99rDrJ0K0KAeTGsa2WrnhBujXzz5H8Jljj3WcPCXnBVykZRw8Nv4dnYILeEIEYnbCCSZmqlBfb+v697f03nst/fe/w+PaEj5VuO02y19xRdgVPghsQFgJGbD//vYln3eelU85BXr3Tu4zVVa2vF7QjhxgyBA4/3x7UHiFp1MA5LyAQ8swSuxIhGChlJLIp9m6NWyk4LRCEEIJxBrCttdf+1rzfY891kT24ovbFr4XX4SFC83jPf/8+PvMmGHiWlNj5cBznjfP0kTDJ/GI93B55pkwv3Mn/PnP8OlPW/m++2DXrtaPdZwcJy8EPNYDjxdCKSlpPqSsd6dvh8ADHzCg+frKSgs7qIY3ulcvS++6y9K337Y0VvQuucTSSy+Fbt1aXrOyMswPGRLuG82Xvxy2iOkslZXNvfEgxh9wwQVQVgaf/3zLB5MLupMPxKvZTNeSTCsUVdWHHw4bF5x2muodd4Tlr3wl3G/ffcP1S5Ykdani4eyz7UbNmtX2fqB61VXxW3oErTy2b1ddtMjKpaWqNTWJ2RDdCuXFF9PbaiT63Dt3WnnUqOaf5dvfVn3iCdVt21raEttixnEyCPnaCgXaDqFEe90eB+8A8UIo8aishJtuMolbvdrWBdMfgbXZLi+3SkuwEEjgXbdHtJd71FGJHZMs0d5/WZmlQdw9YPp0OO20MAY/Y4ZVroLHzJ2cJC8EfPToMFS6bBmsjRp5PFq0XcA7QGshlFiiRXbQIEuDJoHQcv66Bx5IPgQSLbKpJtaeykrr9KMaVpgce6ylwef7xjdg770hGJ95y5bWz+c4WSAvBLxbN9hnH8s3Nlo9WYB74EkSCHh7HngssXHl+nprgx2Ug23JCFwmRTH6WqWRYfGff95sX7PGysGr30cfWdq7tz2crrzSPXInJ8gLAYfmFZnRY6JEi7bPi5kgTU1QV2f5fv06dmysyJaXh69H+Uy09z8wMr3rkiXWpGnWrOb7/va3lv7znz6QlpNV8kbAo+PgW7eGeffAk2DDBhPxioowHpwMsSGPdIZA0k28EAvY6IdB08ageWUQcjn+eGv+dNJJ3orFyQp5I+DRHng0HgNPgmTDJ7HEilQhiVY8Qf/Tn8zjfu89WxdU1j77rKWf/7yNedzY6CEWJyPkvYC7B54EQQuU9iownZBoQQ+GAfjqV5vv89e/WiuWICwVxNJjj49XdpwkyBsBjw6hRNOaB+4dedogVR54MVNZCT/7WfMK3RNPtHTTJkv32svqBy66qKVHHl12MXeSJG8EfNiw+J37EvHAGxvTM2dB3uIC3nniie7cufZje/JJKwdjO9xzj6Xdu9sPedw4K193HcyeHc461Na5HScOeSPgJSXWHjyasrJwohaIL+C/+pXt97nPuYjvxkMoqSeo9CwpsQG5wIbRjaa+3saAee01K//kJzaUL1g72Ysvhoce8gpRJ2HyRsChZRiloqJ5C7ZYAX/hBfjBD0y458wJJ3EpetwDTz3xKj1vvbV5iGXrVli+HObPb3l8TY2NNTNlipVHjIAzz7Qhea+9tvkrpQu6EyGvBDy2IjNasGPL69bZcNbRXveHH6bPtrzCBTz9xBPZHj1g+HA44ggrtzXM7gcfwOOPW29RsO/qiCNsjtFrrw1HUWztWrlKPtmaB+S1gEfHvwH23DP0yLdtC1t7BaxYkT7b8goPoWSe9trMB8PsBoJ+2WXNtzc1WQ+266+38qBBNnLjI4+0H3LJJdH08FBKySsBjw2hxHrgJSUm4q3hAh7BPfDM05aoxusA9bvfNRf0a65pvn39evjf/w07GZ11Ftx/v8032FaLl3i2dIRkj/3447Bu4JRT4PLL4Te/MduCiUQ6a1sxEm+IwnQtyQ4nG7BhQ/PRP7/4xZb7jBzZfJ9evcL8V7/aqcsXDsFNWro025Y48YgdujZ2aFtQvfzy+MP7BpNHn366jbX8/e9becEC1fr6+OfryFC5yQz5+9Oftj4cMajusYfqpz6l+sMfWnnz5tZtK9JhfWllONmExRfoArwGzImURwIvA0uBh4A92jtHZwVcVXXgwPB7//rXW24fNy7c3rdv87HDJ03q9OULg5497YZs3JhtS5xESETQJ09uWySDZdAgS3/5Sxv7/MMPExf0+fNt3zlzVLdsaXvfaH77Wzuue3dLv/Sltm0sKbE/8hVXWPntt1V37Wr52YtI3FMh4N8D/hwl4LOAKZH8DOAb7Z0jFQJ+7LHh93zNNS23R/82HnxQ9Y03wvIBB3T68vlPfb3unnihqSnb1jjJ0Jagr15t5cceU73zTtUbbrBy//5ti+YRR5jHfvPNVg5+G5WVqldeGf+Y4cMtnT9ftaEhvm0XXWS/teAPGe/h09aEIcHSrZvZCKq/+IXqI49YfuvW+Pchni15TKcEHBgKzAVOBOYAAqwDSiPbjwKeau88qRDwiy8Ov9Nf/rLl9hUrVL/1LdW777ZyXV24f8+erllaU2M3Y6+9sm2JkyoS8dBVTexeeMHKhx/etmAOHWqvuKDau7fuDnWA6sSJ5iXHxipPP93yjz6q+vrrFqILtn/3u4nbes017Qt67FvFUUdZ/o9/VP3vf+2P3llBz6EHQGcF/BFgPHB8RMD7A+9GbR8GLGrl2KlANVC9zz77dPqDvPCCalmZiXEiIdympvDNzaMGqvraa3YjDj4425Y46aK90EI80bzwwrZF8rOfDQW5srJjAnvccTaNXSK2tvbwqatTffbZxK8ZhAlnzFBdvDgxQW/PliyGbJIWcOB04PeRfCDgA+II+JvtnSsVHriq6vr1qps2Jb7//vuH3+ubb6bEhPzl6aftRpxwQrYtcbJFIqIZb6msjH9sooKeiOAlKuhBftcui+PPm2flT34y/rV79LD0W9+yV/cgnBPE8oPz7dhh87PecouVX3jB5khtz5Z4tqeQzgj4DUANsBxYDXwM3J+tEEoynHRS+D0+8URWTMgdHnjAbsQXvpBtS5xcoS3RjBWpWNoStcbG9o9vj7a83tau3dQUTrLdmqBHLyNHqp5xhuWDeH3sMmCApWefrfrlL6tedpmVH3zQWvisW9e2oHdS3DtdialRHngk/3BMJeZl7R2fLQGPfju8/fasmJA7TJ9uN+Kyy7JtiZOrtCWSbe0bb//OCnhHrt2aLU1Nqu+8Y+WTT07sbSFozhbd7C2R5fDDVc87L3wrmT3bPHoIm3EmQWsCXppwg/GWXA08KCLXYc0L7+zEudLKsGFhvug783gnHqc92utk1Nq+8fbP5ETVrdkiEnbjfuqpcLuITcq9dCm89Race671Ug7+GyI2pnt9PSxaBBMn2vR627ZZh6krroBPfKL5IEuvvGJLwOc/H+Z//3v43vc684lbEk/V07VkywP/4x/DB+RFF2XFhNwhePWbPj3bljhOZuno20JH9g8qSb/ylfa99CTCKaTBA88bhg4N80Xvgfs4KE6x0tG3hY7sHwzCdPfdtgTrLMTcPJ9C8moslGTJuxBKOseD8BCK4xgdHfQr0ZBNBikKAY/2wGtq0vIgTA1NTfDqq+mdENcF3HHSQ1uCniZxLwoBr6iwoZjB6h9ydr7MRx+F8eMtv3Zteq7hIRTHyQzRgp6mt+qiEHCRll54zlFVZbXgAYMGmeHHH5+6a6iGHngwc7rjOHlLUQg45EEcvKoKpk1rvq5PH/jnP1N3jc2bbSaXnj2hvDx153UcJysUjYDnvAcOsGlTmD/zzHAexGBw/87i4RPHKSiKRsBz3gMH85DBOgc89li4/pvftOmGvv/9zsXSvALTcQqKohHwvGgLHnjgN90UNvuHcO64e+/tXAsVF3DHKSiKRsCjPfCcDaEEHnjsxJ7/+Q+cdFIYAjntNHj5Zct3xCP3EIrjFBRFKeA574H37h2uq6yEO+6AuXPDdX//Oxx5JIwe3TGP3D1wxykoikbA86IzTzwPvKrKluiQyjXXWEuSd9+18te+FlZ4tuWRu4A7TkFRNAK+556meQAffxzqXTqpr4eZM+GJJxJ7YNxbeyqH8jrnXDGEe+8N9bahAaqr4bbb4PvczF1vHcHirUNpIjL+wp13Qt++1o68DY98y8otdoyHUBynIBDNoCs6YcIEra6uztj1YjnggHDkxzfegEMOSd+15s6FSy+F996z8pFHwq9/DUcdFX//h2cp535Rmq0rKYEDD7RzfPxxy2MqKuCIjU8ycegqJtY8ykQWMJBatj76FCs/cSIr15by5k1P8HK/03j5ZXPY+1LH/N+9yujLJqX4EzuOky5EZKGqTmixvpgE/OST4emnLT9nDnz2s6m/Rl2dtfa755742889F268EUaODNe98AKcdJKyY4fEP6gDdGcbH9OjzX0uPXMVM/46uNPXchwnM7Qm4EUxnGxAKpsSNjXBiy/CX/4CixdbA4+1a2389507w/1694bt28N1s2bZMeefD1dfDV26wOc+x27x3r/Lu1x03X7MmWPnD56vw4fDMcfAvvva28NLL4WNSqJpT7wB5szvj2o4AqbjOPlJUQl4Z5sSqpqoPvigjTu1alXb+59zDkyfbgI+bZqJN1hv9pkzrVn3nnuGg2sNYC1/H3Ypo6bNZdo0E+i334b99oMhQ1ra8v77sGBBuLz6ql2ra5cG9m5cwWBWMZJlHM4rHM4rnM4cNtCXj9aU8frrcNhhHb8HjuPkDkUl4PE88KYmizFv3275piaLPQ8caHV9paUWFrn3XvjTn8zbbo/Ro+GWW8yzDnjoIZuB6Yc/DIc3UQ3Fu1vXRubsOJ1R/Zp2HzNgAHz60/GvIWLe+L77wpe+ZOsaGmDbT25kzxumITIKGAXyKfjZz+A3S5i8/u/8mfMB+NvfXMAdJ98pKgGP9sA//BAefhh+8hNYsiT+/iLW4m7TpuZhkYD+/eHss+GUU8xDDkS/R4/44Ymjj4Z588yLv+EGi8ODPTAe/NEiJv70FdjzxKQ/X1kZVNw4reWGxkZYv57TmbNbwOdULuCnTU+kd/IIx3HSSlFVYi5aBAcf3Llz9OwJU6aY13vcceahJ8sbb8Djj9uIscfW/gX+3/+Ds86C2bM7Z2Q0QTtyrOnkgL67aIw8t1euhMFel+k4OY9XYtLcA4+mVy+rJCwpsaWhwSokoysJDz8cpk6FL37R9k8Fhx5qCwD3RDrxRPfCTAVRHnafPnAs/2IeJwDWPv3ii1N7OcdxMkdRCfiee1qYI5jspls3+Pa34aqrrB9MLA0NoYjvvXeajWttHJQUc/qkncyLNKX8299cwB0nnymanpgBt90GY8fC5Zdbx5Ybb4wv3mAx5b33zoB4Q/xxUNLAGf9zyu78009b5a3jOPlJ0Qn4lCnw2mvwP/+TIWFOlAx54PvvbwtY787nnkvr5RzHSSNFFULJaTLkgQOcfrp16wdrCTN5cuqvoWotfXr2tNh7SQdchWXL4JlnbCyZvfe2itbBg63VT8+eHTuX4xQy7Qq4iJQDzwNdI/s/oqqVIjISeBDoC7wKXKCqcRrbOQmRIQ8c4IwzQgH/29/sbSSVvTLnzbPY+vvvW7mkxER80CAbQmDffWHUqHDeZlVrf79woVWsBuPVxEPEbtGee5qoDx9uyz77WBPOPn1sqaiwOo6uXW36z/LyzrUYcpxcJJGf9A7gRFXdKiJlwL9F5O/A94BbVfVBEZkBXAz8IY22FjYZ9MCPOcYEbuNG69A0fLh1Pho92iarLy834evaFfbYw4SvtNTyQ4aYCA8dasMARFNfDz/6Edx6a/P1TU3WGaquznqWdoag89PGjfDBBzB/fuLHDhxoLZGGDrV8QwPs2GH1ACL22QcMME+/Rw/bFiyNjXYOEVu6dbOHSEVF+LAoK7N7VFZm96ZLl7Bl05Yt9hVv2gRbt9oxAweGfQe6du3cfcl1Vq2ycN369XZvg6W0NBwpOXpparK0rCz8HXbt2vy+duli64Nte+wROiKxaZCPPr60NP/f5toVcLWG4lsjxbLIosCJwHmR9TOBKlzAk2dzmpoRxqGszCb1+fOfrbxihS3PPpv4OUpLzesdMgT22su84WeeaS7Q3bvbnyrobZoo3brBiSea2K5aFS4bNpj4JcvatbYsXJj8OdLFHntY89TevcM0WHr2bP420bVr+FCN97D4+GMTyro6S8HOEYhmeXlz0SsrsyU4X7ToNTXZg7m+3s67fbsNBREsTU2hLcHxwbnLyqyT3D/+YX0wcpGSkvCz9+plD9PoDnmx+wafs0uX8O0x6EoTuz24r8E96dkTvvrV1Nqf0EuliHQBFgL7Ab8D3gM2ququyC41wJBWjp0KTAXYZ599Omtv4RJ44BkIoYBN9LNkiVXoNjW1v38su3ZZiCQIk8QyebINUz54sHm669fDRx+Fx7z3nglytKe0117Wq/X4402wWrvu5s12bE2NeeEffGAPoPXrbdmwwR4agXe9Y4cJUE5O4hFh587wLcXJHE1N4VvWtm2wenX6rtW/f5YEXFUbgbEiUgHMBg6It1srx94O3A7WEzNJOwufDHrgYC1RqqtNOJYtg6VLTVS3bAlFL/C2Ghosra+3islly2zUxXh0727x9alTQ3EuK7N496BBMG5c5+wuLbVmn337Wiw9UXbtsj/nihUm/HV15hUFHm1Tk02gUVtraX19GEYKPN7oV/z6+jCUs3Gj3auGBlt27rSQS1NTmPbsGYZcune3Y4LOYrW1Zl8hU1YGn/oUjBljnvy2bbY0NoZhqWApKQnzQZgrWILxipqa7J7t3GnLjh3hcBfBgzr2gR39fTQ2Zv6el5Wl/pwdqtZR1Y0iMg84EqgQkdKIFz4UWJl684qIDFZiRrPHHvanGjOmY8dt22ae7+rVFt4IPJezz24+1nmuUFpqse/oAc1yBVUToM2bw1j5li1WDtYFAhY8XKPDGEEoo7HRlvJyi+f362cVuiKhYG7bFopdkAYP6OABFE0Q7+/WzR485eVhyKG01LY3NobnCM4XnLuiwubjPu64liGJbKMa2t7QYPc6eKiuXWufIZrgoREs0DLctGtX8/sRPNAbGlLXgzuaRFqhDAAaIuLdDfgMcBPwHHAO1hLlQuCx1JtXJDQ1hQKejm85DfToAZ/8pC1O5xAJW8oMHJhta4oHkfBB1K2bvfzm4gO+LRLxwAcDMyNx8BJglqrOEZG3gQdF5DrgNeDONNpZ2GzbZu5Ajx4tm3Y4juO0QiKtUP4DtBg5WlXfByamw6iiI8MVmI7jFAZ53gqyQMhwBabjOIWBC3gu4B644zhJ4AKeC7gH7jhOEriA5wLugTuOkwQu4LmAe+CO4ySBC3gu4ALuOE4SuIDnAh5CcRwnCVzAcwH3wB3HSQIX8FzAPXDHcZLABTwXcA/ccZwkcAHPBdwDdxwnCVzAcwH3wB3HSQIX8FzABdxxnCRwAc8FPITiOE4SuIDnAu6BO46TBC7g2WbXLpvQQcQmTnQcx0mQ4hXwqqpsW2Bs2WJp797NJ9hzHMdph+IV8GuvzbYFRhD/9vCJ4zgdpPgEfOlSmDw521aEZEuzu98AAA1XSURBVGk2esdx8p/iEvCqKth/f3jySSuL2JLNcIp74I7jJEkis9IXDlVVsHgxzJpl5REjYMECGDAgeza5B+44TpIUlwcOsGRJmF++HM4+G37846yZ400IHcdJluIS8KYm+O9/LX/ZZTBkCPz733D99aCaHZu8E4/jOElSXAJeUwP19TBwIPzud/D449Ctm2275Zbs2OQeuOM4SVJcAh6ET8aMsXj4+PEm6ABXXZWdCk2vxHQcJ0naFXARGSYiz4nIYhF5S0SuiKzvKyJPi8jSSNon/eZ2klgBV20eOunVC849N7M2eSWm4zhJkogHvgu4UlUPAI4ELheRTwLTgLmqOhqYGynnNkH8e8yYltvOPdd6RZ5xhnnjmcI9cMdxkqRdAVfVVar6aiS/BVgMDAHOBGZGdpsJnJUuI1NGtAceTWUl3H03jBsH778PN98MO3dmxib3wB3HSZIOxcBFZARwGPAyMEhVV4GJPDAw1calnNYEvKoKuneHxx6DvfaydZMnw8aN6bfJPXDHcZIkYQEXkZ7Ao8B3VHVzB46bKiLVIlJdW1ubjI2pob4ePvwQSkth5MiW26uqYNgwWL3ays8+C336wBVXtKzYTGVFp3vgjuMkiWgC7Z9FpAyYAzylqr+OrFsCHK+qq0RkMDBPVeMEl0MmTJig1dXVKTA7Cf7zHzj0UPO+33mn7X1F4MAD4a23rMnh2rXQ2AglJeH2VLUbHz0a3n3X3g723z8153Qcp6AQkYWqOiF2fSKtUAS4E1gciHeEx4ELI/kLgcdSYWjaaC180hovvACTJpl4g3nuFRUwfLiVZ84MmyB2xiP3EIrjOEmSSAjlGOAC4EQReT2ynAbcCEwSkaXApEg5d+mIgFdWwq23wtNPh+tUTWw//NDKX/mKxc2POsqGpo32yDsi6B5CcRwnSRJphfJvVRVVPURVx0aWJ1S1TlVPUtXRkXR9JgxOmo4IeFVVy3biDQ2wfr21UgE4/HBL58+3dMgQuOACuOeelmONtyboO3bYUloK5eWJfxbHcRyKqSdmIODJxplLS61SM6gAPe205ttXrYL77oOLLrLy6NHw9a/byIetCXq09+2z8TiO00GKQ8BVOx4Dj6aysmU51kP/xjea7/Puu/DHP8IXvxhuf/ttyweC7vFvx3E6QUKtUFJF1lqhrF4NgwdbJeT69an3dmNbpYjAxRfDnXe23HfkSFi2DLZvt1Yu48fDoEFh80XHcZwYkm6FUhBEe9/pCFXEeugAd9zR3EOfELn3y5ZZWl5u4g2wZk3qbXIcp+ApPgFPB7GVlPEE/ZVXYMMGa93iOI6TAlzA00Frgl5RAd/5juUvvbT5PrkwP6fjOHlF4Qp4tBBmWsBjiSfoM2Y0D7EEeRdwx3ESpDAFfOvW5k332hpGNhu4SDuOkwIKT8CXLQtnmb/vPmvtEXS+2W+/7NnVFvFi5o7jOO1QWAJeVQWjRplog/WM7NbNBqKC3O3t6B654zhJUHgC/pvfhOV4w8Y6juMUCIUl4AAffRTmzz+/+TZv6eE4TgFRmm0DUk4g4GedBT//uS2Q2jG8HcdxcoDC88Braiz95jeza4fjOE6aKTwBDzzwIUOar/eWHo7jFBiFJeCqrQu4x70dxykwCkvAN2ywJoS9e0OvXtm2xnEcJ60UloC35n07juMUIIUl4EEFpgu44zhFQGEJuHvgjuMUEYUp4EOHZtcOx3GcDFCYAu4euOM4RUBhCbjHwB3HKSIKS8DdA3ccp4hwAXccx8lT2hVwEblLRNaKyKKodX1F5GkRWRpJ+6TXzATYvh3q6qC0FAYOzLY1juM4aScRD/we4NSYddOAuao6GpgbKWeXlSst3XtvKCmsFwvHcZx4tKt0qvo8sD5m9ZnAzEh+JnBWiu3qOF6B6ThOkZGsqzpIVVcBRNLsxyw8/u04TpGR9liDiEwVkWoRqa6trU3fhbwTj+M4RUayAr5GRAYDRNK1re2oqrer6gRVnTAgmC0+HbgH7jhOkZGsgD8OXBjJXwg8lhpzOoELuOM4RUYizQgfAF4CxohIjYhcDNwITBKRpcCkSDm7eCWm4zhFRruTGqvql1rZdFKKbekcHgN3HKfIKIwG001NzduBO47jFAGFIeC1tbBrF/TrB+Xl2bbGcRwnIxSGgHv823GcIqQwBNxboDiOU4QUloB7BabjOEVEYQm4e+CO4xQRLuCO4zh5SmEIuFdiOo5ThBSGgHsM3HGcIqSwBNw9cMdxioj8F/CtW2HzZsv3yf7Mbo7jOJki/wV82bIwL5I9OxzHcTJMfgt4VRUcckhYFrGlqipbFjmO42SMdkcjzGmqqmDFCrjrLiurZtUcx3GcTJLfHjjAiy9m2wLHcZyskN8e+Pr18M470LUr/OAH2bbGcRwno+S3Bz5/vqWHHw4//3l2bXEcx8kw+S3gQfjkqKOya4fjOE4WyG8Bf+klS48+Ort2OI7jZIH8FfBdu+Dlly3vHrjjOEVI/gr4okWwbRuMGgWDBmXbGsdxnIyTvwIexL89fOI4TpGS/wLu4RPHcYqU/BVwr8B0HKfIyU8BX7MG3n8fevSAgw7KtjWO4zhZoVMCLiKnisgSEXlXRKalyqh2CbzvI46A0vzuTOo4jpMsSQu4iHQBfgdMBj4JfElEPpkqw1oQPcKgx78dx3E65YFPBN5V1fdVdSfwIHBmasyKobYWrr3W0tpa+Ne/bL3Hvx3HKWI6E38YAqyIKtcAR3TOnFYYMcLSgQObrz/yyLRcznEcJx/ojAceb/qbFgNyi8hUEakWkera2tqOXaGqyiZo+Pjj+Nv79fMJHBzHKVo6I+A1wLCo8lBgZexOqnq7qk5Q1QkDBgzo2BWqqmyShmCihiAfW3YBdxynCOmMgL8CjBaRkSKyBzAFeDw1ZjmO4zjtkXQMXFV3icg3gaeALsBdqvpWyiyLpbKy7bLjOE6RIZrBeSQnTJig1dXVGbue4zhOISAiC1V1Quz6/OyJ6TiO47iAO47j5Csu4I7jOHmKC7jjOE6e4gLuOI6Tp2S0FYqI1AIfJHl4f2BdCs1JJW5bcrhtyeG2JUc+2zZcVVv0hMyogHcGEamO14wmF3DbksNtSw63LTkK0TYPoTiO4+QpLuCO4zh5Sj4J+O3ZNqAN3LbkcNuSw21LjoKzLW9i4I7jOE5z8skDdxzHcaLICwHP2uTJ8W25S0TWisiiqHV9ReRpEVkaSftkybZhIvKciCwWkbdE5IpcsU9EykVkgYi8EbHt2sj6kSLycsS2hyJDE2ccEekiIq+JyJxcsitiy3IReVNEXheR6si6rH+nETsqROQREXkn8rs7KhdsE5ExkfsVLJtF5Du5YFvEvu9G/geLROSByP+jw7+5nBfwjE+e3D73AKfGrJsGzFXV0cDcSDkb7AKuVNUDgCOByyP3Khfs2wGcqKqHAmOBU0XkSOAm4NaIbRuAi7NgG8AVwOKocq7YFXCCqo6NamqWC98pwG3Ak6r6CeBQ7B5m3TZVXRK5X2OB8cDHwOxcsE1EhgDfBiao6kHYcNxTSOY3p6o5vQBHAU9Fla8BrsmyTSOARVHlJcDgSH4wsCTb9y1iy2PApFyzD+gOvIrNoboOKI33XWfQnqHYn/lEYA42XWDW7YqybznQP2Zd1r9ToDewjEhdWi7ZFmPPycALuWIb4XzCfbE5GeYApyTzm8t5D5z4kycPyZItrTFIVVcBRNKB7eyfdkRkBHAY8DI5Yl8kTPE6sBZ4GngP2KiquyK7ZOu7/Q1wFdAUKffLEbsCFPiHiCwUkamRdbnwnY4CaoG7I+GnO0SkR47YFs0U4IFIPuu2qepHwC3Ah8AqYBOwkCR+c/kg4AlNnuyEiEhP4FHgO6q6Odv2BKhqo9or7VBgInBAvN0yaZOInA6sVdWF0avj7JrN39wxqjoOCyNeLiLHZdGWaEqBccAfVPUwYBvZC+XEJRJH/hzwcLZtCYjE3c8ERgJ7Az2w7zaWdn9z+SDgCU2enGXWiMhggEi6NluGiEgZJt73q+pfcs0+AFXdCMzD4vQVIhJM7ZeN7/YY4HMishx4EAuj/CYH7NqNqq6MpGuxOO5EcuM7rQFqVPXlSPkRTNBzwbaAycCrqromUs4F2z4DLFPVWlVtAP4CHE0Sv7l8EPB8mDz5ceDCSP5CLPaccUREgDuBxar666hNWbdPRAaISEUk3w37ES8GngPOyZZtqnqNqg5V1RHYb+tZVT0/23YFiEgPEekV5LF47iJy4DtV1dXAChEZE1l1EvB2LtgWxZcIwyeQG7Z9CBwpIt0j/9ngvnX8N5fNyoUOBP1PA/6LxUx/lGVbHsDiVg2YB3IxFjOdCyyNpH2zZNunsNeu/wCvR5bTcsE+4BDgtYhti4CfRtaPAhYA72KvuV2z+N0eD8zJJbsidrwRWd4Kfv+58J1G7BgLVEe+178CfXLItu5AHbBn1Lpcse1a4J3If+F/ga7J/Oa8J6bjOE6ekg8hFMdxHCcOLuCO4zh5igu44zhOnuIC7jiOk6e4gDuO4+QpLuCO4zh5igu44zhOnuIC7jiOk6f8f3En7S9W7266AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lin_reg = LinearRegression()\n",
    "plot_learning_curves(lin_reg, X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAeo0lEQVR4nO3de5RU5Znv8e/TF66iICAQGu3WsLwQFaTFJpxMTEwU0QRXghONSVguj6xJPJncnMTkrIyQxMQkM15YE80w0RlzljFy1ESPIeYYgjFHobExRrmooHJpQWgQEEWhu3nOH3sXVd1d3XRXVVcV+/191qrV+917V9XTt1+//dSuvc3dERGRMFSUugARESkehb6ISEAU+iIiAVHoi4gERKEvIhKQqlIX0JNRo0Z5bW1tqcsQETmqrFq1aqe7j862raxDv7a2lqamplKXISJyVDGzTd1tU3tHRCQgCn0RkYAo9EVEAlLWPX0RkVy0trbS3NzMe++9V+pS+tWgQYOoqamhurq61/dR6ItI4jQ3NzNs2DBqa2sxs1KX0y/cnV27dtHc3ExdXV2v76f2jogkznvvvcfIkSMTG/gAZsbIkSP7/N9MkKHf2govv1zqKkSkPyU58FNy+RyDC/1Dh6C+Hk49Fb773VJXIyJSXMGF/ssvw/PPR8uLF5e2FhFJpj179nDHHXf0+X6zZs1iz549/VBRWnCh/+676eWEv7AvIn01f35BHqa70G9vb+/xfkuWLGH48OEFqaE7wYX+gQPp5YMHS1eHiJShBQsK8jA33HADr7zyCpMnT+bcc8/lIx/5CJ/97Gc588wzAbjsssuYOnUqkyZNYtGiRYfvV1tby86dO9m4cSOnn3461157LZMmTeLCCy/k3cwZax6OGPpmdreZ7TCz1Rnrjjezx81sffxxRLzezGyhmW0ws+fN7JyM+8yN919vZnMLUn0OMkM/c1lEEsqs97e+7N+Dm2++mVNOOYXnnnuOn/70p6xcuZKbbrqJtWvXAnD33XezatUqmpqaWLhwIbt27eryGOvXr+e6665jzZo1DB8+nAcffLAgX47ezPT/C5jZad0NwFJ3nwgsjccAFwMT49s84E6I/kgANwLnAdOAG1N/KIpNM30RKbZp06Z1OJZ+4cKFnH322TQ0NLBlyxbWr1/f5T51dXVMnjwZgKlTp7Jx48aC1HLE0Hf3J4E3O62eDdwTL98DXJax/pceWQEMN7NxwEXA4+7+prvvBh6n6x+SolDoiwTGvfe3vuzfB0OHDj28/MQTT/DHP/6R5cuX87e//Y0pU6ZkPdZ+4MCBh5crKytpa2vL7fPvJNee/hh33wYQfzwhXj8e2JKxX3O8rrv1XZjZPDNrMrOmlpaWHMvrXmbot7ZGh3CKiBTSsGHD2LdvX9Zte/fuZcSIEQwZMoQXX3yRFStWFLW2Qp+GIVujy3tY33Wl+yJgEUB9fX3f/pz2Quc+fmsrZPxBFZGQ3XhjQR5m5MiRzJgxgw984AMMHjyYMWPGHN42c+ZMfv7zn3PWWWdx6qmn0tDQUJDn7K1cQ3+7mY1z921x+2ZHvL4ZmJCxXw2wNV5/fqf1T+T43HnpHPoHDyr0RSRWoEM2AX71q19lXT9w4EB+//vfZ92W6tuPGjWK1asPHzvD9ddfX7C6cm3vPAKkjsCZCzycsf4L8VE8DcDeuP3zB+BCMxsRv4B7Ybyu6DqHvo7gEZGQHHGmb2b3Ec3SR5lZM9FRODcDi83sGmAzcHm8+xJgFrAB2A9cDeDub5rZ94Fn4v2+5+6dXxwuimwzfRGRUBwx9N39ym42XZBlXweu6+Zx7gbu7lN1/UChLyIhC/odudnGIiJJFnzoa6YvIiFR6Cv0RSQgwYe+2jsiUmrHHHNM0Z4r+NDXTF9EQhLchdEV+iLS3771rW9x0kkn8aUvfQmA+fPnY2Y8+eST7N69m9bWVn7wgx8we/bsotcW/Exf7R2RZOvLmZX7euvOFVdcwf333394vHjxYq6++mp+85vf8Oyzz7Js2TK+8Y1v4H08cVshaKavmb6IFNiUKVPYsWMHW7dupaWlhREjRjBu3Di+9rWv8eSTT1JRUcHrr7/O9u3bGTt2bFFrU+gr9EWkH8yZM4cHHniAN954gyuuuIJ7772XlpYWVq1aRXV1NbW1tVlPqdzfgg99tXdEkq0EHRQgavFce+217Ny5kz//+c8sXryYE044gerqapYtW8amTZtKUlfwoa+Zvoj0h0mTJrFv3z7Gjx/PuHHjuOqqq/jEJz5BfX09kydP5rTTTitJXQp9hb6I9JMXXnjh8PKoUaNYvnx51v3efvvtYpWko3fU3hGRkAQf+prpi0hIFPoKfZFEKsUx8MWWy+cYfOirvSOSPIMGDWLXrl2JDn53Z9euXQwaNKhP99MLuZrpiyROTU0Nzc3NtLS0lLqUfjVo0CBqamr6dB+FvkJfJHGqq6upq6srdRllSe0dtXdEJCBBhX57e3TLpJm+iIQkqNDPNqtX6ItISIIPfbV3RCQkwYe+ZvoiEhKFvkJfRAISfOirvSMiIQk+9DXTF5GQKPQV+iISkOBDX+0dEQlJ8KGvmb6IhCSv0Dezr5nZGjNbbWb3mdkgM6szs0YzW29m95vZgHjfgfF4Q7y9thCfQF9opi8iocs59M1sPPCPQL27fwCoBK4Afgzc6u4Tgd3ANfFdrgF2u/v7gVvj/YpKM30RCV2+7Z0qYLCZVQFDgG3AR4EH4u33AJfFy7PjMfH2C8zM8nz+PlHoi0jocg59d38d+BdgM1HY7wVWAXvcvS3erRkYHy+PB7bE922L9x+Z6/PnQu0dEQldPu2dEUSz9zrgfcBQ4OIsu6YuXZNtVt/lsjZmNs/MmsysqdAXQNBMX0RCl09752PAa+7e4u6twEPAB4HhcbsHoAbYGi83AxMA4u3HAW92flB3X+Tu9e5eP3r06DzK66q70E/wFdVERDrIJ/Q3Aw1mNiTuzV8ArAWWAXPifeYCD8fLj8Rj4u1/8iJfwDJb6LtDW1vX9SIiSZRPT7+R6AXZZ4EX4sdaBHwL+LqZbSDq2d8V3+UuYGS8/uvADXnUnZPu+vdq8YhIKPK6Rq673wjc2Gn1q8C0LPu+B1yez/Plq6fQHzq0uLWIiJRC8O/I7Wm9iEjSBBX63bVx1N4RkVAEFfrq6YtI6BT6PawXEUkahT6a6YtIOBT6KPRFJBwK/R7Wi4gkjUIfzfRFJBzBhv7Agellhb6IhCLY0B82LPt6EZEkU+ijmb6IhCPY0D/22PSyQl9EQhFs6Ku9IyIhUuijmb6IhCPY0Fd7R0RCFGzoq70jIiFS6KOZvoiEI5jQd1foi4gEE/ptbVHwA1RWwpAh6W1q74hIKIIJ/c6nYBgwID3WTF9EQqHQR6EvIuEINvQzT7im9o6IhCLY0NdMX0RCpNBHoS8i4Qg29NXeEZEQBRv6mumLSIgU+ij0RSQcwYa+2jsiEqJgQ18zfREJkUIfhb6IhCOv0Dez4Wb2gJm9aGbrzGy6mR1vZo+b2fr444h4XzOzhWa2wcyeN7NzCvMp9I7aOyIi+c/0bwcec/fTgLOBdcANwFJ3nwgsjccAFwMT49s84M48n7tPNNMXEckj9M3sWODvgLsA3P2gu+8BZgP3xLvdA1wWL88GfumRFcBwMxuXc+V9pNAXEclvpn8y0AL8p5n91cx+YWZDgTHuvg0g/nhCvP94YEvG/ZvjdR2Y2TwzazKzppaWljzK60jtHRGR/EK/CjgHuNPdpwDvkG7lZGNZ1nmXFe6L3L3e3etHjx6dR3kdaaYvIpJf6DcDze7eGI8fIPojsD3Vtok/7sjYf0LG/WuArXk8f5/0FPqa6YtIKHIOfXd/A9hiZqfGqy4A1gKPAHPjdXOBh+PlR4AvxEfxNAB7U22gYuipvaOZvoiEoirP+38ZuNfMBgCvAlcT/SFZbGbXAJuBy+N9lwCzgA3A/njfoukc+lUZn3l7e3SrrCxmRSIixZdX6Lv7c0B9lk0XZNnXgevyeb58dA59s6jFk5rlHzwIgweXpjYRkWIJ9h25mR9BLR4RCUPQoa8jeEQkNEGGfirsday+iIQmyNDXTF9EQqXQjyn0RSQEQYe+2jsiEpqgQ18zfREJjUI/ptAXkRAEHfpq74hIaIIOfc30RSQ0Cv2YQl9EQhB06Ku9IyKhCTr0NdMXkdAo9GMKfREJQdChr/aOiIQmiNA/dAhaW9Pj1AxfM30RCU0QoZ8Z6NXVUBF/1gp9EQlNEKGfrbXTeVntHREJQdChr5m+iIRGoR9T6ItICIIOfbV3RCQ0QYe+ZvoiEhqFfkyhLyIhCDr01d4RkdAEHfqa6YtIaBT6MYW+iIQg6NBXe0dEQhN06GumLyKhUejHFPoiEoK8Q9/MKs3sr2b2aDyuM7NGM1tvZveb2YB4/cB4vCHeXpvvc/eW2jsiIpFCzPS/AqzLGP8YuNXdJwK7gWvi9dcAu939/cCt8X5FoZm+iEgkr9A3sxrgEuAX8diAjwIPxLvcA1wWL8+Ox8TbL4j373cKfRGRSL4z/duAbwKH4vFIYI+7t8XjZmB8vDwe2AIQb98b79+Bmc0zsyYza2ppacmzvIjaOyIikZxD38wuBXa4+6rM1Vl29V5sS69wX+Tu9e5eP3r06FzL60AzfRGRSFUe950BfNLMZgGDgGOJZv7Dzawqns3XAFvj/ZuBCUCzmVUBxwFv5vH8vdabmb5CX0RCkPNM392/7e417l4LXAH8yd2vApYBc+Ld5gIPx8uPxGPi7X9y9y4z/f7Qm5m+2jsiEoL+OE7/W8DXzWwDUc/+rnj9XcDIeP3XgRv64bmzUntHRCSST3vnMHd/AngiXn4VmJZln/eAywvxfH2l9o6ISCTod+RWV6eXDx6E4jSbRERKJ+jQr6iAqoz/dVpbi1eTiEgpBB36ncd6MVdEki740NeLuSISEoW+Ql9EAhJ86Ku9IyIhCT70NdMXkZAEF/qZId95rNAXkaQLLvTV3hGRkAUf+prpi0hIFPoKfREJSPChr/aOiIQkiNDPnMFrpi8iIQsi9NXeERGJJD7029ujW0rmmTVB7R0RCUviQ7/zLN86XalXM30RCUniQ3/fvvTykCFdtyv0RSQkiQ/9lpb08gkndN2erb3z4IPwr/8Kb7/dv7WJiBRbQS6XWM4yQ3/06K7bO8/0n3gC5sSXdd+zB77//X4tT0SkqBI/09+xI72cbabfOfTvuSc9fvrp/qtLRKQUEh/6R5rpZ7Z33n4bfvvb9Pi11/qvLhGRUkh86GfO9I/U3nnssailk7J5M7S19V9tIiLFlvjQP9ILuZmhv2pVx23t7dDc3D91iYiUQlChf6T2TjZq8YhIkiQ+9PvyQm42Cn0RSZLEh35fDtlMqco4kFWhLyJJkvjQP9JMP1t756qr0ssKfRFJkkSHflsbvPlmejxyZNd9Os/0x46Fz30uPVboi0iSJDr0d+5ML48c2bFtk9J5pv+pT8Epp6THCn0RSZKcQ9/MJpjZMjNbZ2ZrzOwr8frjzexxM1sffxwRrzczW2hmG8zseTM7p1CfRHeO1M+HrjP9OXNgwgSorIzG27bBu+/2T30iIsWWz0y/DfiGu58ONADXmdkZwA3AUnefCCyNxwAXAxPj2zzgzjyeu1eO1M+HjjP90aPhQx+K/iOYMCG9ftOm/qlPRKTYcg59d9/m7s/Gy/uAdcB4YDaQOoPNPcBl8fJs4JceWQEMN7NxOVfeC72Z6U+bBqNGRctf/Wq6BVRXl95HLR4RSYqCnGXTzGqBKUAjMMbdt0H0h8HMUnPs8cCWjLs1x+u2FaKGbI50CgaAY46Bl1+OZvNnnZVeX1cHy5ZFywp9EUmKvEPfzI4BHgS+6u5vWedLU2XsmmWdZ3m8eUTtH0488cS8ajvSKRhSRoyIbpk00xeRJMrr6B0zqyYK/Hvd/aF49fZU2yb+mJpvNwMZnXJqgK2dH9PdF7l7vbvXj+5uet5LvWnvdEehLyJJlM/ROwbcBaxz91syNj0CzI2X5wIPZ6z/QnwUTwOwN9UG6i+9eSG3Owp9EUmifNo7M4DPAy+Y2XPxuu8ANwOLzewaYDNwebxtCTAL2ADsB67O47l7pVAz/Y0b08ttbfCzn8HgwXDttV0vtC4iUs5yDn13/39k79MDXJBlfweuy/X5cpHPTH/sWBg0CN57L3pX71tvwbHHwm23wT/9U7TP8OHw939fuHpFRPpbot+Rm89M3wxqa9Pj114Dd7jrrvS6JUvyKk9EpOgSe2H01lbYvTtaNst+3p0jqauDF1+MllN9/dQY4Kmn8qtRRKTYEhv6nc+7kzqtQl90nuk3NnbcvmFD1ELqa+tIRKRUEtveyaefn5L5Yu6rr8Kvf911n6efzu2xRURKIbGhn08/PyUz9B96qONRPCkKfRE5mij0e5AZ+lsz3kZWU5NeVuiLyNEksaFf6PZOpptvTi83NcGBA7k9vohIsSU29Asx0x8xIjo2P9MJJ8BnPgPvf380PnAAnn02t8cXESm2xIZ+IWb6Zl1n+5dfHp1++YMfTK9Ti0dEjhaJDf1CzPSha+hfeWX0MTP0dby+iBwtEhv6hZjpQ8fQnzABpk+PlmfMSK9/+uno3boiIuUusaFfqJn+ORlX8v3c56Ai/oqdcUa63799u87EKSJHh8S+I7dQM/3PfCY69cL+/fDd76bXV1REs/4//CEaP/UUnHxy7s8jIlIMiZzpHzwIe/dGyxUVcPzxuT9WdTX84Adwyy3R6ZQzdW7xiIiUu0SGfmZrZ9SodEum0PRirogcbRIf+nlecbFH06al/6CsXp3+70JEpFwp9PMwbBicfXa07A4XXBD1+HUkj4iUq0SGfqFexO2NCy9ML69aBTNnwvnnw9KlcOhQ/z63iEhfJTL0izXTB7jxRvjmN6NLK6Y8+SR87GPR+fi/8x1Ytw7efTe67OLWrbBpU3Q0kIhIsSUy9Is50x88GH78Y3jlFfjiF6NTNKRs2QI/+lF0TP+QIdHFXMaPj/4YDB0a/UGaOhUuvTT6j2HGjKhdNG0aPPZY/9YtImFK5HH6xZzpp7zvfXDHHXD99dHhnffdF83se7JzZ8crfGW66qrowi3HHVf4WkUkXJrpF9jJJ8O//Rts2wa//S18+tPRLH/AgCjAx4yJZvtVR/hz++abcOutxalZRMKhmX4/GTAAZs+Obtm0t8Mbb8DmzdFpHAYNgmOOia7De/310T633AJf/nJuF3UXEckmkaFfypl+b1VWRjP+8eM7rp8+He6+G9auhX37otcLfvKT0tQoIsmTyPZOOcz0c1VZCd/7XnqcahWJiBRC4kL/wAF4661oubIyuvrV0eZTn0qf3fPdd+Gmm0pbj4gkR+LaO5WVsGJF1OJ5663+O+9OfzKLTvI2a1Y0XrQILrsMTjsNxo498ovAIiLdSVx8VFXBeeeVuor8zZwZndDt6aehtRU+/vFofUVF9N9LVVW0XFkJkybBT38KZ55Z2ppFpPwdhfPgMJjBD3/Ydf2hQ7BrV3TEz7Zt0Nwcne/nQx+K3gksItIThX4Z+/CH4Ze/hE9+Mnrn7pgx3e+7d2/0rt6HHipefSJy9Cl66JvZTDN7ycw2mNkN/fpk8+dnXy63cQ/bPv95eHjKfJqaouP6DxyAHdf/5PAs/y9/gbHH7AOibXPmwD9/+M/8x39E7xC+/Xa4febvWbgwOhLojjvgV59+kGXLoiuC7d0LB7/7fdraMs4OerR83TqPy6kW1abaCj0uEPMingfYzCqBl4GPA83AM8CV7r422/719fXe1NTU9yfavx/eeSc6SD910H7mcq7j11+P3lXV3h5dMf3VV6Nt7nDKKbBmTdSAb2uD+vroFeXU13f6dFi+POrbADQ0RO/EguhFiJUr0881bRo880x6fO65XbcvXx49tjuvzbiKiyasZf2WTpf2ytFg9jP0uGqGDGpnyPbXqDr5RCoMKiucivUv0j7xNNrbjfZD4Bs3wUm1QPR52qaNWF0tBpg5Fa9uoGLiKVRUgOHYSy9ip592eLutXYNNmhQtG/gLq/FJZ0afGuBr1+GnnXH4y1j10mqqzzqD6qpDVFc5trIx/SJOY2PHF3QaG7GG9NhXdNzujSthWsb+Kxujr22sw2N38/ic15AxXtH9uKdtWcbe2JiltvTYVvbt8QpZW6/GDfHYj7S/Yz3tn+W+1pAxXrEcGqYfHvqKjMeC6Hew17X0bdz5e+QrV8K500j9LvDMM9Hvbsozz2DTMsYrV3b4ecs2/h+/m3X4gI6+MLNV7l6fdaO7F+0GTAf+kDH+NvDt7vafOnWq5+QnP4njMJzbDkb5uTSWugzddNOtgLc778wtAoGm7nK12EfvjAe2ZIybgQ7H2pjZPGAewIknntj3Z5g/HxYsyLnAo9VodvInPsrP+Qee5ywGcJBqWqmmFcM5RAWO0UYVuxjJNsaxjXG8wVhaqeYQFfE+eplHpGx88R/gi/8encO9QO2eYrd3Lgcucvf/Ho8/D0xz9y9n2z/n9k76CaM/mJ2Xy21cRrUcOgTvVg7lne3v8M47sP/kSRx6fs3hrtah+nOpXPUMlZXR4aIVZ54Ba9LdOZ80CX9hzeG5yqGzJ+PPPsehQ9H9/bzz8OWN6bnMjBn4X546PLYPfwj7y18wi8qyGdOx5csPd8XaGmbQuuwpWlujThqXzILfLYk2Zi4Dfskl8Ojv0p/npZdgv8sYXzILW5Len1kXw5LfR/f1ro+X1ziH+5aqNr+k69epYM/VaewOXNr1+3R43GmbX3op/J9H0/t+ouvYHs0Y9/TY+Y778PME4LP6/nWatHEJJ51En/XU3il26E8H5rv7RfH42wDu/qNs+yv0y2BcTrUczbWWUy2q7eisrQ96Cv3K+f30CnE2CxYseAOYv2DBgkcWLFiwH1gI/HD+/Pkt2fZftGjR/Hnz5uX3pOefn3253MblVMvRVFvncTnVotpUW6HHvbRgwYJt8+fPX5RtW1Fn+gBmNgu4DagE7nb3bs8sk/dMX0QkQD3N9It+GgZ3XwIsOeKOIiJScDpUQ0QkIAp9EZGAKPRFRAKi0BcRCUjRj97pCzNrATbl8RCjgJ0FKqfQVFtuVFtuVFtujtbaTnL3rBeLLevQz5eZNXV32FKpqbbcqLbcqLbcJLE2tXdERAKi0BcRCUjSQz/r25DLhGrLjWrLjWrLTeJqS3RPX0REOkr6TF9ERDIo9EVEApLI0C/qxdePXMvdZrbDzFZnrDvezB43s/XxxxElqm2CmS0zs3VmtsbMvlIu9ZnZIDNbaWZ/i2tbEK+vM7PGuLb7zWxAsWvLqLHSzP5qZo+WU21mttHMXjCz58ysKV5X8u9pXMdwM3vAzF6Mf+6ml0NtZnZq/PVK3d4ys6+WQ21xfV+Lfw9Wm9l98e9HTj9viQv9+OLrPwMuBs4ArjSzM0pY0n8BMzutuwFY6u4TgaXxuBTagG+4++lAA3Bd/LUqh/oOAB9197OBycBMM2sAfgzcGte2G7imBLWlfAVYlzEup9o+4u6TM47jLofvKcDtwGPufhpwNtHXr+S1uftL8ddrMjAV2A/8phxqM7PxwD8C9e7+AaLT0l9Brj9v3V0892i90ceLrxepplpgdcb4JWBcvDwOeKnUX7e4loeBj5dbfcAQ4Fmi6ynvBKqyfa+LXFMNUQh8FHgUsDKqbSMwqtO6kn9PgWOB14gPICmn2jrVcyHwVLnURvra4scTnQ7/UeCiXH/eEjfTJ/vF18eXqJbujHH3bQDxxxNKXA9mVgtMARopk/ri9slzwA7gceAVYI+7t8W7lPJ7exvwTeBQPB5J+dTmwP81s1Vmlrr0XDl8T08GWoD/jNtivzCzoWVSW6YrgPvi5ZLX5u6vA/8CbAa2AXuBVeT485bE0Lcs63Rcag/M7BjgQeCr7v5WqetJcfd2j/7drgGmAadn2624VYGZXQrscPdVmauz7Fqqn7sZ7n4OUYvzOjP7uxLV0VkVcA5wp7tPAd6hdG2mrOK++CeB/13qWlLi1xFmA3XA+4ChRN/bznr185bE0G8GJmSMa4CtJaqlO9vNbBxA/HFHqQoxs2qiwL/X3R8qt/oA3H0P8ATR6w7DzSx1xbdSfW9nAJ80s43Ar4laPLeVSW24+9b44w6ivvQ0yuN72gw0u3tjPH6A6I9AOdSWcjHwrLtvj8flUNvHgNfcvcXdW4GHgA+S489bEkP/GWBi/Mr2AKJ/1R4pcU2dPQLMjZfnEvXSi87MDLgLWOfut2RsKnl9ZjbazIbHy4OJfvDXAcuAOaWszd2/7e417l5L9PP1J3e/qhxqM7OhZjYstUzUn15NGXxP3f0NYIuZnRqvugBYWw61ZbiSdGsHyqO2zUCDmQ2Jf2dTX7fcft5K+YJJP77wMQt4magH/D9LXMt9RH24VqKZzjVE/d+lwPr44/Elqu2/Ef1L+DzwXHybVQ71AWcBf41rWw38c7z+ZGAlsIHoX/CBJf7+ng88Wi61xTX8Lb6tSf38l8P3NK5jMtAUf19/C4woo9qGALuA4zLWlUttC4AX49+F/wUMzPXnTadhEBEJSBLbOyIi0g2FvohIQBT6IiIBUeiLiAREoS8iEhCFvohIQBT6IiIB+f8J106jhijUIAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "polynomial_regression = Pipeline([\n",
    "    (\"poly_features\", PolynomialFeatures(degree=5, include_bias=False)),\n",
    "    ('lin_reg', LinearRegression())\n",
    "])\n",
    "plot_learning_curves(polynomial_regression, X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regularized Linear Models\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ridge Regression\n",
    "\n",
    "$\n",
    "J(\\mathbf{\\theta})=\\text{MSE}(\\mathbf{\\theta}) + \\alpha \\frac{1}{2}\\sum_{i=1}^n\\theta_i^2\n",
    "$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[16.11686663]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#example\n",
    "from sklearn.linear_model import Ridge\n",
    "ridge_reg = Ridge(alpha=1, solver = 'cholesky')\n",
    "ridge_reg.fit(X, y)\n",
    "ridge_reg.predict([[1.5]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([11.16256524])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# using Stochastic Gradient Descent\n",
    "sgd_reg = SGDRegressor(penalty = 'l2')\n",
    "sgd_reg.fit(X, y.ravel())\n",
    "sgd_reg.predict([[1.5]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Lasso Regression\n",
    "\n",
    "$\n",
    "J(\\mathbf{\\theta})=\\text{MSE}(\\mathbf{\\theta})+\\alpha\\sum_{i=1}^n |\\theta_i|\n",
    "$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([16.12401786])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import Lasso\n",
    "lasso_reg = Lasso(alpha=0.1)\n",
    "lasso_reg.fit(X,y)\n",
    "lasso_reg.predict([[1.5]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Elastic Net\n",
    "\n",
    "$\n",
    "J(\\mathbf{\\theta})=\\text{MSE}(\\mathbf{\\theta})+r\\alpha\\sum_{i=1}^n |\\theta_i|+\\frac{1-r}{2}\\alpha\\sum_{i=1}^n\\theta_i^2\n",
    "$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([16.14527928])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import ElasticNet\n",
    "elastic_net = ElasticNet(alpha=0.1, l1_ratio=0.5)\n",
    "elastic_net.fit(X,y)\n",
    "elastic_net.predict([[1.5]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression\n",
    "\n",
    "estimated probability: \n",
    "\n",
    "$\n",
    "\\hat{p}=h_\\mathbf{\\theta}(\\mathbf{x})=\\sigma(\\mathbf{x}^\\top \\mathbf{\\theta})\n",
    "$\n",
    "\n",
    "Logistic function:\n",
    "\n",
    "$\n",
    "\\sigma(t)=\\frac{1}{1+e^{-t}}\n",
    "$\n",
    "\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeMAAAD4CAYAAADfEY7UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU9b3/8dcnO4FAICwCCYRVQBbFiNt1QbEFrILWBVuXVh9y9Sf93dbe26vXX7W19fbWa9urt9SWe6XWBdFqRWypqEC1KiLIvhOWQFhDgATIPvP9/TFDHMOETGCSM5N5Px+PyZzlO5PPmTMn75xlvmPOOURERMQ7SV4XICIikugUxiIiIh5TGIuIiHhMYSwiIuIxhbGIiIjHUrz6xV27dnX5+fle/XoREZFW9fnnnx90znULN8+zMM7Pz2fZsmVe/XoREZFWZWZFjc3TYWoRERGPKYxFREQ8pjAWERHxmGfnjMOpra2luLiYqqoqr0tp8zIyMsjNzSU1NdXrUkREEl5MhXFxcTFZWVnk5+djZl6X02Y55ygtLaW4uJh+/fp5XY6ISMJr8jC1mc00swNmtraR+WZmz5hZoZmtNrPRp1tMVVUVOTk5CuIWZmbk5OToCISISIyI5Jzx88D4U8yfAAwK3qYCz55JQQri1qHXWUQkdjR5mNo596GZ5Z+iySTgBRf4LsZPzSzbzHo65/ZGqUYRETkNfr+jxuenus5PTZ2fWp8fn99R53f4/A6/c9T5Avc+v8N34t7v8AfH604Mn2gfHHYOHA6/HxyB018OIDg9MJ/6dl+MB4eD7UOnE7Y9+OvnBaaHavglwCfPd03Mb/wJ0lOTeWDswAhe6TMXjXPGvYFdIePFwWknhbGZTSWw90yfPn2i8KtFROKbc47jNT6OVNRwpKKWIxW1lFfVcry6jooaH8dr6qio/uK+otZHRXUdx2vqqKzx1QdtdZ2fGl8gcGuC0+r8+r7603HiwGHHjNS4CuNwxzvDvgOcczOAGQAFBQV6l4hIm1VeVcveI1XsLw/cDhyt5kB5FfvLqyk9Xs3hYPCWVdZQ6zv1n8Mkg/ZpKWSmJ9ffZ6amkJ2ZRnpKEmknbskN7sNMT0lOIjkJksxITjJSkqx+OCk4nmxfDCcFx5OTvrglmZFkgecwA8PqA8wscBrMTgxzok1weuhwSBuMsPOSgk/c8LlCNTztdvL8U7ePBdEI42IgL2Q8F9gTheeVMLZt28YTTzxBWVkZr7/+utfliCS08qpaNu07yvaS4xQdOs7OQ5XsLD3OzkMVHK6oPal9x4wUenTMoGuHdAb36ECndml0zkwlOzOV7HZpgfvMNDq2S6F9Wgrt01PITEsmPSUpJgNEoicaYTwXmGZms4ELgTKdL245/fv357nnnuOmm27yuhSRhOGcY0dpBev3lLNhbzkb95WzYe9Rdh+prG+TnGT0zm5H35xMJozoSd8umfTu3I4eHTPokZVB947pZKQme7gUEsuaDGMzewW4EuhqZsXAY0AqgHPut8A8YCJQCFQA326pYr1WWVnJ+PHjWbhwIcnJyRQXF/Pxxx9z6623UlNTw7hx41i4cCEpKTH18W0RaaY6n59VxWUs3XGIZTsOs3znYQ4drwECodu/a3tG9+3MNy7sw9CeWQzo1oFe2e1ITVanhnJ6Irma+rYm5jvggahVFMNmzpzJjTfeSHJy4L/bBQsWsH79em699VbS0tK4+uqrefXVV/nmN7/pcaUi0lzFhyv4YHMJH24u4ZPCUo5W1wHQv2t7rhrSnfP7dmZE704M7N5Be7gSddqFC2PVqlV85zvf4eDBg2zcuBHnHI8++ijvvfces2bNAuCjjz7iwQcfJDs7m/nz5/Pmm28yefJkHn744RYN49LSUh555BFWrFjBz372Mx5++OEW+10ibd3O0gr+smYv89bsZc3uMgB6Z7fja6N6ctmgblzYrws5HdI9rlISgbmGH7pqJQUFBa7h9xlv2LCBoUOHAvDjt9exfk95VH/nsF4deey6c07ZpqqqinPPPZcXXniBMWPG8MMf/pCqqip++tOf0rdvX/bt21ffdvz48Tz11FMMHz4cAJ/Px1lnnUVJScmXnvOyyy7j6NGjJ/2up556inHjxkVhyU5P6OstkijKq2p5a+UeXlu6qz6AR+Vlc+2Is7hqSA8GdGuvi6WkRZjZ5865gnDztGfcwPvvv8/o0aMZM2YMACNHjuSdd96htLSU7OzsL7XdtGkTZ599dv14cnIyaWlpHD16lKysrPrpf//731uneBFp1MpdR5i1pIi3V+2lstbHOb068sjEoUwYcRa5nTO9Lk8SXMyGcVN7sC1l7dq1jBgxon58+fLljB49mnbt2n2pL+fS0lI6dep00rceVVdXk5GR8aVpkewZt9Z/4l4dCRHxgnOODzaXMH1RIUt3HCYzLZnJ5/XitjF9GJmb3fQTiLSSmA1jr+Tk5LBw4UIANm/ezJ/+9Cc++eQTOnfujM/no6qqioyMDLZv306vXr2+9NjS0lK6det2UkBHsmfcUiG5efNmjhw5Ur+nL5II/H7Hu+v3MX3RVtbsLqNnpwweu24YN52fS1aGvjZUYo+uw2/gtttu49ixYwwfPpypU6fyyiuvkJOTA8BXvvIVPvroIwCGDBnCwYMHGT58OJ988gkAixYtYuLEiZ7VHs4bb7zB0qVL8fv9Xpci0iqW7zzMjc9+wn0vLedoVS0///oIPviXsXz70n4KYolZ2jNuoEOHDrz99tth502bNo1f/vKXjBs3jg4dOvDZZ599af6sWbP42c9+dlq/t6metdasWUOnTp2a7NN7zpw5zJkzBwj889CuXTseeCAhPnkmCW7PkUp+/s5G3lq5h+5Z6Tx18ygmn9uLFH32V+KAwrgZzjvvPMaOHYvP56v/rPEJNTU1TJ48+UsXdDVHUz1rff755+Tn538pjGtra3nssceoqKigpqaG3/zmN0yePJnJkyfXt3n22Wf54IMPuOKKK06rLpFY5/M7nvtoG798bzN+B9PGDuT+KwfQPl1/3iR+6N3aTHfffXfY6Wlpadx5551R/33r16/nmWeeYdOmTWRlZTF79mx+/OMf06NHD2bMmEFlZSXZ2dls27Yt7OPvv//+qNckEit2HDzOP/9xFcuKDjNuaA8eu24YeV10ZbTEH4VxjBs2bBi//e1vef7558nPz+fKK6+sn7dixQqmT59Oero6JZDE4pzjpSU7+fe/bCAl2fjlLaO44bze+nywxC2dTIkRpaWl3HffffU9a0Vi0qRJfOtb3+IHP/gB77zzTgtXKBIbjlbVMvXFz/nhnLUU5Hfm3e9dzo2jcxXEEtditgcuaXl6vSXebCs5xr0vLGNHaQWPTBzKty/NVwhL3FAPXCIS9xZu3M8/vbKS1JQkXrrnQi4ekON1SSJRozAWkZj3Px9u49//uoFhPTvyuzvOV/eV0uYojEUkZjnn+MW7m/n1okKuHdGTp24eRbs0fX2htD0xF8bOOZ0DagXqo1pind/vePzP63n+kx1MuSCPJ24YQXKS/jZI2xRTV1NnZGRQWlqqoGhhzjlKS0tP+kILkVhR5/PzL6+v5vlPdnDvZf342Y0KYmnbYmrPODc3l+Li4pO+D1iiLyMjg9zcXK/LEDmJ3+948LVVzF21hwevGcx3rhqoo2XS5sVUGKemptKvXz+vyxARjzjn+PHb65i7ag//On4I9185wOuSRFpFTB2mFpHE9uuFhfxhcRH3XtZPQSwJRWEsIjHh5SVF/OK9zdw4ujcPT1BnNJJYFMYi4rl31u7lh3PWctWQ7vz86yNJ0sVakmAUxiLiqQ17y/nuqysZlZfN9G+MJlXfPywJSO96EfFMWUUt9730OR0zUvndHeerQw9JWDF1NbWIJA6/3/HdV1ew50gls6deTPcsfe5dEpf2jEXEE08v2MKiTSU8et05nN+3s9fliHhKYSwirW7Bhv08vWALN52fy+0X9vG6HBHPKYxFpFXtK6viwddWMbx3R346ebh61xJBYSwircg5xw/eWE1NnZ//vm00Gam6YEsEFMYi0ope+rSIDzeX8G/XDqVf1/ZelyMSMxTGItIqtpYc44l5G7hicDedJxZpIKIwNrPxZrbJzArN7KEw8/uY2SIzW2Fmq81sYvRLFZF4Vefz8+Brq8hITebJm0bqPLFIA02GsZklA9OBCcAw4DYzG9ag2f8DXnPOnQdMAX4T7UJFJH5NX7SVVbuO8NPJw+nRUZ8nFmkokj3jMUChc26bc64GmA1MatDGAR2Dw52APdErUUTi2Zb9R/nvhVu4flQvvjayl9fliMSkSMK4N7ArZLw4OC3Uj4DbzawYmAd8J9wTmdlUM1tmZstKSkpOo1wRiSfOOX741lrap6fw2HUND6iJyAmRhHG4kzuuwfhtwPPOuVxgIvCimZ303M65Gc65AudcQbdu3ZpfrYjElTdX7ObTbYf41/FDyOmQ7nU5IjErkjAuBvJCxnM5+TD0PcBrAM65xUAG0DUaBYpIfCqrqOWJv2zg3LxsplyQ1/QDRBJYJGG8FBhkZv3MLI3ABVpzG7TZCVwNYGZDCYSxjkOLJLAn52/kcEUNP508XN9PLNKEJsPYOVcHTAPmAxsIXDW9zsweN7Prg82+D9xrZquAV4BvOecaHsoWkQSxctcRZn22k7suyWd4705elyMS8yL6CkXn3DwCF2aFTns0ZHg9cGl0SxOReOTzO/7fnDV0z0rnwWsGe12OSFxQD1wiElVvrtjN2t3l/NvEoWRlpHpdjkhcUBiLSNRU1fr4xbubGJXbietH6TPFIpFSGItI1Dz30Xb2llXxbxOHqstLkWZQGItIVJQeq+bZv21l3NAeXNg/x+tyROKKwlhEouKZBVuorPXx0IQhXpciEncUxiJyxraVHOPlJTuZckEeA7t38LockbijMBaRM/bkO5tIT0niu+P0USaR06EwFpEzsnLXEd5Zt4+plw+gW5b6nxY5HQpjETkjT7+/mezMVO65rJ/XpYjELYWxiJy2FTsPs2hTCfde1p8O6RF16CciYSiMReS0Pb1gC50zU7nrknyvSxGJawpjETkty3ce5m+bSrj3cu0Vi5wphbGInJan3w/uFV+c73UpInFPYSwizfZ50WE+2FzC1MsH0F57xSJnTGEsIs329IItdGmfxp0X9/W6FJE2QWEsIs2yctcRPtwcuIJae8Ui0aEwFpFm+e3fttIxI4U7tFcsEjUKYxGJ2NaSY8xfv487L87XFdQiUaQwFpGIzfhgG2nJSXzr0nyvSxFpUxTGIhKR/eVVvLliNzcX5NK1g/qgFokmhbGIRGTmR9up8/uZetkAr0sRaXMUxiLSpLLKWl5espOJI3rSJyfT63JE2hyFsYg06eUlRRyrruO+K7RXLNISFMYickpVtT5mfrSDywZ1ZXjvTl6XI9ImKYxF5JTeWrmbg8eqtVcs0oIUxiLSKOccMz/awZCzsrhkQI7X5Yi0WQpjEWnUx4WlbNp/lHv+oR9m5nU5Im2WwlhEGjXz4+107ZDGdaN6eV2KSJumMBaRsLaWHGPhxgPcflFfMlKTvS5HpE1TGItIWM9/vIO05CS+eaG+EEKkpUUUxmY23sw2mVmhmT3USJtbzGy9ma0zs1nRLVNEWtORihpe/7yYSef2oluWur4UaWlNfu2KmSUD04FrgGJgqZnNdc6tD2kzCHgYuNQ5d9jMurdUwSLS8mYv3UVlrY9vX9rP61JEEkIke8ZjgELn3DbnXA0wG5jUoM29wHTn3GEA59yB6JYpIq2l1ufnD5/s4JIBOQzr1dHrckQSQiRh3BvYFTJeHJwWajAw2Mw+NrNPzWx8uCcys6lmtszMlpWUlJxexSLSouav28fesiru1l6xSKuJJIzDfbjQNRhPAQYBVwK3Af9rZtknPci5Gc65AudcQbdu3Zpbq4i0ghcWF5HXpR1jh+hsk0hriSSMi4G8kPFcYE+YNm8552qdc9uBTQTCWUTiyMZ95Xy2/RC3X9iX5CR18iHSWiIJ46XAIDPrZ2ZpwBRgboM2c4CxAGbWlcBh623RLFREWt5LnxaRlpLELQV5TTcWkahpMoydc3XANGA+sAF4zTm3zsweN7Prg83mA6Vmth5YBPyLc660pYoWkeg7WlXLm8t3c93IXnRun+Z1OSIJpcmPNgE45+YB8xpMezRk2AEPBm8iEof+tHw3x2t83HmxOvkQaW3qgUtEcM7x4qdFjMrtxKi8k669FJEWpjAWERZvLaXwwDHuuDjf61JEEpLCWER48dMisjNT+drInl6XIpKQFMYiCW5vWSXvrt/PrQV5+nYmEY8ojEUS3CtLduJ3jtsv0oVbIl5RGIsksJo6P7M+28XYs7uT1yXT63JEEpbCWCSBzV+3j4PHqrlDH2cS8ZTCWCSBvbi4iD5dMrlikPqKF/GSwlgkQW3cV85nOw5x+0V9SFI/1CKeUhiLJKgXFheRrn6oRWKCwlgkAZVX1TJnxW6uH9WL7Ez1Qy3iNYWxSAL60+fFVNT4dOGWSIxQGIskmPp+qPOyGZmrfqhFYoHCWCTBfLK1lK0lx7lTnXyIxAyFsUiCeWHxDjpnpnKt+qEWiRkKY5EEsreskvfW7+eWC9QPtUgsURiLJJBZS3bigNsv1CFqkViiMBZJEDV1fl75bBdXqR9qkZijMBZJEH9du1f9UIvEKIWxSIJ4cXERfXMyuVz9UIvEHIWxSAJYt6eMZUWHueOivuqHWiQGKYxFEsCLi4vISE3i5vPVD7VILFIYi7RxZRW1zFm5m8nn9qZTZqrX5YhIGApjkTbuj5/voqrWrwu3RGKYwlikDfP7A/1QF/TtzDm9Onldjog0QmEs0oZ9sKWEotIK7RWLxDiFsUgb9uLiIrp2SGfCcPVDLRLLFMYibdTO0goWbTrAN8bkkZaiTV0klmkLFWmjXlpSRJIZ31A/1CIxT2Es0gZV1vh4dekuvnpOD87qlOF1OSLSBIWxSBv09qo9lFXWcsdF+V6XIiIRiCiMzWy8mW0ys0Ize+gU7W4yM2dmBdErUUSawznHHxbvYHCPDlzUv4vX5YhIBJoMYzNLBqYDE4BhwG1mNixMuyzg/wJLol2kiERu+c4jrNtTzh0X52OmfqhF4kEke8ZjgELn3DbnXA0wG5gUpt1PgCeBqijWJyLN9MLiHXRIT+GG83p7XYqIRCiSMO4N7AoZLw5Oq2dm5wF5zrk/n+qJzGyqmS0zs2UlJSXNLlZETm1fWRV/Wb2XWwry6JCe4nU5IhKhSMI43HEuVz/TLAn4FfD9pp7IOTfDOVfgnCvo1k3fqSoSbS8s3oHfOb59ab7XpYhIM0QSxsVA6Peu5QJ7QsazgOHA38xsB3ARMFcXcYm0rsoaH7M+28k1w3qQ1yXT63JEpBkiCeOlwCAz62dmacAUYO6Jmc65MudcV+dcvnMuH/gUuN45t6xFKhaRsP60opgjFbXc8w/9vS5FRJqpyTB2ztUB04D5wAbgNefcOjN73Myub+kCRaRpfr9j5kfbGd67Ixfkd/a6HBFppoiu8HDOzQPmNZj2aCNtrzzzskSkOT7cUsLWkuP86tZR+jiTSBxSD1wibcDMj3fQPSuda0f08roUETkNCmOROLdl/1E+3FzCnRf31bczicQpbbkicW7mx9tJT0nStzOJxDGFsUgcO3C0ijeW7+br5+fSpX2a1+WIyGlSGIvEsd9/vIM6n5+pl+njTCLxTGEsEqfKq2p5aXERE4b3JL9re6/LEZEzoDAWiVOzluzkaHUd910xwOtSROQMKYxF4lB1nY+ZH23n0oE5jMjt5HU5InKGFMYicejN5bs5cLSa+68Y6HUpIhIFCmOROOPzO2Z8uI3hvTty6cAcr8sRkShQGIvEmXfX7WPbwePcd8UAdX0p0kYojEXiiHOOZz/YSt+cTCYM7+l1OSISJQpjkTiycOMBVheX8cCVA0lO0l6xSFuhMBaJE845/uv9LfTpkskNo3t7XY6IRJHCWCROLNhwgDW7y5h21UBSk7XpirQl2qJF4oBzjv9asDmwV3ye9opF2hqFsUgceH/DAdbuLuc72isWaZO0VYvEuMC54s30zdFesUhbpTAWiXHvrd/Puj3lfOeqQaRor1ikTdKWLRLD/H7Hr97fQn5OJpPP7eV1OSLSQhTGIjHszRW72bC3nO9dM1h7xSJtmLZukRhVVevjF+9uYmRuJ64bqb1ikbZMYSwSo37/8Q72lFXx8IShJKm3LZE2TWEsEoMOHa/hN4sKuXpIdy4eoG9mEmnrFMYiMeiZBVs4XlPHQxOGeF2KiLQChbFIjNlx8DgvfVrErRf0YVCPLK/LEZFWoDAWiTFPzt9IWkoS37tmkNeliEgrURiLxJC/bylh3pp9/OPlA+ieleF1OSLSShTGIjGius7Ho2+tIz8nk3+8or/X5YhIK0rxugARCfjdB9vYfvA4L9w9hozUZK/LEZFWFNGesZmNN7NNZlZoZg+Fmf+gma03s9VmtsDM+ka/VJG2q6j0OL9eVMi1I3ty+eBuXpcjIq2syTA2s2RgOjABGAbcZmbDGjRbARQ450YCrwNPRrtQkbbKOcejb60jLTmJR7/WcNMSkUQQyZ7xGKDQObfNOVcDzAYmhTZwzi1yzlUERz8FcqNbpkjb9de1+/hgcwkPXjOYHh110ZZIIookjHsDu0LGi4PTGnMP8NdwM8xsqpktM7NlJSUlkVcp0kaVVdby47fXMaxnR+68WGd3RBJVJGEcrlNcF7ah2e1AAfCf4eY752Y45wqccwXduum8mMhjb63l4LEa/uPrI/StTCIJLJKrqYuBvJDxXGBPw0ZmNg54BLjCOVcdnfJE2q6/rN7LnJV7+N64wYzMzfa6HBHxUCT/ii8FBplZPzNLA6YAc0MbmNl5wO+A651zB6JfpkjbcqC8ikfmrGFUXjYPjB3gdTki4rEmw9g5VwdMA+YDG4DXnHPrzOxxM7s+2Ow/gQ7AH81spZnNbeTpRBKec44fvLGaqlofv7pllA5Pi0hknX445+YB8xpMezRkeFyU6xJps15espO/bSrh8Unn0L9bB6/LEZEYoH/JRVrRlv1HeeIvG7hsUFfuuEhXT4tIgMJYpJUcr67j/peX0z49maduHoVZuA8qiEgiUt/UIq3AOccjb65ha8kxXrrnQnXuISJfoj1jkVYw8+Md9R9junRgV6/LEZEYozAWaWF/31LCE39Zz1fP6cG0sQO9LkdEYpDCWKQFbS05xrRZKxjcI4tf3nIuSUk6TywiJ1MYi7SQkqPVfOv3n5GabPzPnQW0T9clGiISnv46iLSA49V13POHpRw8WsPsqReR1yXT65JEJIYpjEWirKrWx70vLGPdnnJ+d/v5jMpTv9Micmo6TC0SRbU+P9NmLeeTraU8dfNIxg3r4XVJIhIHFMYiUVJT5+eBl5fz/oYD/GTycG44L9frkkQkTugwtUgUVNX6eODl5SzYeIDHJ52jri5FpFkUxiJnqLyqlnv/sIwl2w/xxA3D+eaFCmIRaR6FscgZ2FdWxbefX0rhgaM8PeVcJp3b2+uSRCQOKYxFTtPSHYe4/6XlVNbU8dxdF3D54G5elyQicUphLHIaXvq0iB/NXUdu53a8cu+FDOqR5XVJIhLHFMYizVBd5+NHc9fxyme7uPLsbjw95Tw6tUv1uiwRiXMKY5EIrd1dxvdfW8Wm/Ud5YOwAHrzmbJLV17SIRIHCWKQJtT4/z/5tK88s2EKX9mn8/lsXMHZId6/LEpE2RGEscgob9pbzr2+sZnVxGdeP6sXjk84hOzPN67JEpI1RGIuEcfBYNb94dzOvLt1Jp3apTP/GaK4d2dPrskSkjVIYi4SoqfPzh0928MyCLVTW+rjrknz+6epB2hsWkRalMBYBKmt8zF66kxkfbmNvWRVjz+7GI9cOY2D3Dl6XJiIJQGEsCa28qpYXFxcx86PtlB6vYUx+F568aSSXDVIHHiLSehTGknCcc6zcdYRXPtvJ26v2Ulnr4/LB3Zg2diBj+nXxujwRSUAKY0kY+8qqmLdmL68t28XGfUfJTEtm0rm9uP2ivgzv3cnr8kQkgSmMpU3bc6SSd9buY96avSwrOgzAOb068sQNw7l+VC+yMtR7loh4T2EsbUpljY9Pt5fy4eYS/r7lIIUHjgEw5Kwsvn/NYCaO7MmAbrooS0Rii8JY4pZzjj1lVazceYRlRYdYXnSYdXvKqfM70lOSuLB/DlMuyGPskO4KYBGJaQpjiQsVNXVs3n+MjXvL2bjvKBuC92WVtQBkpCYxKjebqZf356L+OYzp14WM1GSPqxYRiYzCWGJCdZ2PkqPV7CurYtfhCopKK9h5qIKdwfsDR6vr22amJXP2WVlMHNGToT2zGJmbzbCeHUlLSfJwCURETl9EYWxm44GngWTgf51z/9FgfjrwAnA+UArc6pzbEd1SJV4456is9XGkojZ4q+FIZS2HK2o4UlFLWWUth47XcOBoNQfKq9hfXsXhitqTnqdnpwzyumRy+eBu9O2SyaAeWQztmUVe50yS9G1JItKGNBnGZpYMTAeuAYqBpWY21zm3PqTZPcBh59xAM5sC/By4tSUKFvD7HT7n8PkdzlE/fGJ66Hy/H/zu5Ol1PkeNz09tnZ/q4H2Nz0+tz09NnZ8anwvc130xrdbnp6rWx/EaH5U1Po7X1FFRHbhvOO53jdefkZpEdrs0enRMJ69LJuf37UyPjhn06JhO944Z5HXOJLdzOx1mFpGEEcme8Rig0Dm3DcDMZgOTgNAwngT8KDj8OvBrMzPn3Cn+JEfPX9fs5RfvbebEr3P1P+rvvjTP1c8LTjsxHlLtl54rOO+k9oQ+7uR54Z7jxHPX/6owdZ7UnkConghSLyQnGanJRnpKMh3SU2iXlkz7tGQy01Lo2SmDdmkp9ePt0wP3nTNTyc5MpVO7NDq3TyW7XRrZmakKWRGRBiIJ497ArpDxYuDCxto45+rMrAzIAQ6GNjKzqcBUgD59+pxmySfr1C6Vs3tkBX/JF3dmFjoJO8W8Lx5nX2r35ceFzGvwQLMm2tfP++Lwami7k+v7cvskM5KTINmMpCSrvz8xPZxYve4AAAcxSURBVHAfuNUPn2gbnH9iepIZaSlGanISaclJpKYE7tNC7lMbjCfrsLCISIuJJIzD/RVuuHsWSRucczOAGQAFBQVR28W7ZGBXLhnYNVpPJyIi0qoiufy0GMgLGc8F9jTWxsxSgE7AoWgUKCIi0tZFEsZLgUFm1s/M0oApwNwGbeYCdwWHbwIWttb5YhERkXjX5GHq4DngacB8Ah9tmumcW2dmjwPLnHNzgeeAF82skMAe8ZSWLFpERKQtiehzxs65ecC8BtMeDRmuAm6ObmkiIiKJQV0WiYiIeExhLCIi4jGFsYiIiMcUxiIiIh4zrz6BZGYlQFEUn7IrDXr8imNaltikZYlNWpbYpGU5WV/nXLdwMzwL42gzs2XOuQKv64gGLUts0rLEJi1LbNKyNI8OU4uIiHhMYSwiIuKxthTGM7wuIIq0LLFJyxKbtCyxScvSDG3mnLGIiEi8akt7xiIiInFJYSwiIuKxuApjM7vZzNaZmd/MChrMe9jMCs1sk5l9tZHH9zOzJWa2xcxeDX4lpOeCtawM3naY2cpG2u0wszXBdstau85ImNmPzGx3yPJMbKTd+OC6KjSzh1q7zkiY2X+a2UYzW21mb5pZdiPtYna9NPU6m1l68P1XGNw28lu/yqaZWZ6ZLTKzDcG/Af8Ups2VZlYW8t57NNxzxYKm3jMW8Exwvaw2s9Fe1NkUMzs75PVeaWblZvbdBm1idr2Y2UwzO2Bma0OmdTGz94I58Z6ZdW7ksXcF22wxs7vCtWkW51zc3IChwNnA34CCkOnDgFVAOtAP2Aokh3n8a8CU4PBvgfu9XqYwNf4CeLSReTuArl7X2ET9PwL+uYk2ycF11B9IC667YV7XHqbOrwApweGfAz+Pp/USyesM/B/gt8HhKcCrXtfdyLL0BEYHh7OAzWGW5Urgz17XGuHynPI9A0wE/goYcBGwxOuaI1imZGAfgY4t4mK9AJcDo4G1IdOeBB4KDj8UbrsHugDbgvedg8Odz6SWuNozds5tcM5tCjNrEjDbOVftnNsOFAJjQhuYmQFXAa8HJ/0BmNyS9TZXsMZbgFe8rqWFjQEKnXPbnHM1wGwC6zCmOOfedc7VBUc/BXK9rOc0RPI6TyKwLUBg27g6+D6MKc65vc655cHho8AGoLe3VbWoScALLuBTINvMenpdVBOuBrY656LZs2KLcs59CBxqMDl0m2gsJ74KvOecO+ScOwy8B4w/k1riKoxPoTewK2S8mJM31BzgSMgf13BtvHYZsN85t6WR+Q5418w+N7OprVhXc00LHlqb2cghnkjWV6y5m8CeSjixul4ieZ3r2wS3jTIC20rMCh5KPw9YEmb2xWa2ysz+ambntGphzdPUeyYet5EpNL4jES/rBaCHc24vBP4JBLqHaRP19ZNyJg9uCWb2PnBWmFmPOOfeauxhYaY1/MxWJG1aTITLdRun3iu+1Dm3x8y6A++Z2cbgf3at6lTLAjwL/ITAa/sTAofd7274FGEe68ln7CJZL2b2CFAHvNzI08TEegkj5reL5jKzDsAbwHedc+UNZi8ncIj0WPBahTnAoNauMUJNvWfibb2kAdcDD4eZHU/rJVJRXz8xF8bOuXGn8bBiIC9kPBfY06DNQQKHelKCewDh2rSYppbLzFKAG4HzT/Ece4L3B8zsTQKHIVv9j36k68jM/gf4c5hZkayvVhHBerkL+BpwtQueLArzHDGxXsKI5HU+0aY4+B7sxMmH7WKCmaUSCOKXnXN/ajg/NJydc/PM7Ddm1tU5F3NfVhDBeyZmtpEITQCWO+f2N5wRT+slaL+Z9XTO7Q2eGjgQpk0xgXPhJ+QSuJbptLWVw9RzgSnBK0P7Efiv67PQBsE/pIuAm4KT7gIa29P2wjhgo3OuONxMM2tvZlknhglcXLQ2XFsvNTivdQPha1wKDLLA1e1pBA5vzW2N+prDzMYD/wpc75yraKRNLK+XSF7nuQS2BQhsGwsb+6fDS8Hz2M8BG5xzv2ykzVknzneb2RgCf99KW6/KyET4npkL3Bm8qvoioOzEodMY1ehRvXhZLyFCt4nGcmI+8BUz6xw8FfeV4LTT5/XVbM25EfjjXgxUA/uB+SHzHiFw5egmYELI9HlAr+BwfwIhXQj8EUj3eplC6nweuK/BtF7AvJDaVwVv6wgcRvW87jDL8SKwBlgdfFP3bLgswfGJBK6I3RrDy1JI4LzQyuDtxFXHcbNewr3OwOME/sEAyAhuC4XBbaO/1zU3shz/QOAw4OqQ9TERuO/EdgNMC66DVQQuuLvE67obWZaw75kGy2LA9OB6W0PIp0di7QZkEgjXTiHT4mK9EPgHYi9QG8yWewhcM7EA2BK87xJsWwD8b8hj7w5uN4XAt8+0FnWHKSIi4rG2cphaREQkbimMRUREPKYwFhER8ZjCWERExGMKYxEREY8pjEVERDymMBYREfHY/we6MnZtKur8JQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def sigma(t):\n",
    "    return 1/(1+ np.exp(-t))\n",
    "fig, ax = plt.subplots(1,1, figsize=(8, 4))\n",
    "x = np.arange(-10, 10, 0.1)\n",
    "ax.plot(x,sigma(x), label = r\"$\\sigma(t) = \\frac{1}{1+ e^{-t}}$\" )\n",
    "_ = ax.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prediction:\n",
    "\n",
    "$\n",
    "\\hat{y}=\\begin{cases}\n",
    "0  & \\text{if} & \\hat{p}\\lt 0.5\\\\\n",
    "1 & \\text{if} & \\hat{p} \\ge 0.5\n",
    "\\end{cases}\n",
    "$\n",
    "\n",
    "Cost Function of a single training instance:\n",
    "\n",
    "$\n",
    "c(\\mathbf{\\theta})=\\begin{cases}\n",
    "-\\log(\\hat{p}) &\\text{if} & y = 1\\\\\n",
    "-\\log(1-\\hat{p}) &\\text{if} & y=0\n",
    "\\end{cases}\n",
    "$\n",
    "\n",
    "Logistic Regression Cost Funtion (log loss):\n",
    "\n",
    "$\n",
    "J(\\mathbf{\\theta})=-\\frac{1}{m}\\sum\\limits_{i=1}^m\\left[y^{(i)}\\log\\left(\\hat{p}^{(i)}\\right)+(1-y^{(i)})\\log\\left(1-\\hat{p}^{(i)}\\right)    \\right]\n",
    "$\n",
    "\n",
    "Logistic cost function partial derivatives:  \n",
    "$\n",
    "\\frac{\\partial}{\\partial\\theta_j}J(\\mathbf{\\theta})=\\frac{1}{m}\\sum\\limits_{i=1}^m\\left(\\sigma\\left(\\mathbf{\\theta}^\\top\\mathbf{x}^{(i)}\\right)-y^{(i)}\\right)x_j^{(i)}\n",
    "$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['data', 'target', 'target_names', 'DESCR', 'feature_names', 'filename']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#example\n",
    "from sklearn import datasets\n",
    "iris = datasets.load_iris()\n",
    "list(iris.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = iris[\"data\"][:, 3].reshape(-1,1 ) # petal width\n",
    "y = (iris['target'] == 2).astype(np.int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='auto', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "log_reg = LogisticRegression()\n",
    "log_reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_new = np.linspace(0, 3, 1000).reshape(-1, 1)\n",
    "y_proba = log_reg.predict_proba(X_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1ba4475c888>]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deZzN9f7A8dd7dsswGLKOmUKI4jaWLmWJom7Dz61LibrVVYq0R26yhJBchYoWtJAk0S2i3Eoio5ClNJYY69gGY/b5/P74DDPGGAdn5nuW9/Px+D7O95zP95zv+9vJez7n8/0sYoxBKaWU9wtwOgCllFLuoQldKaV8hCZ0pZTyEZrQlVLKR2hCV0opHxHk1IkjIyNNdHS0U6dXSimvtGbNmoPGmMqFlTmW0KOjo4mPj3fq9Eop5ZVE5M9zlWmTi1JK+QhN6Eop5SM0oSullI/QhK6UUj5CE7pSSvmI8yZ0EXlHRA6IyIZzlIuIvCoiCSKyXkT+4v4wlVJKnY8rNfTpQKciyjsDdXO3PsDrlx6WUkqpC3XefujGmO9EJLqIQ7oAM42dh3eliESISDVjzF43xXiWDz6AhAQICIDAQLtVrgz33WfL58+HffvOLr/1Vlu+dCkcO5ZXFhBgy5s1s+Vr1kB2NoSE2C00FMqVs8cApKZCcLB9r0hxXaVSypfkmBwysjNIz0onNCiUsKAwt5/DHQOLagC78j1PzH3trIQuIn2wtXiioqIu+oQffABffnnma/Xr5yX08eNh+fIzy2Nj8xL6U0/BunVnlrdrB998Y/e7d4etW88sj4uDzz6z+zExsH+/TeahoTbp33knvPGGLW/Rwj6WKWO3smWhUye45x4wBkaPzisLD4cKFaBePYiOtuU5OfaPhVKq5BljSE5P5mjaUY6lHytyO55+nJNZJ0nNTCU1K7XIx/Ts9NPneOPWN3gw9kG3x+6OhF5YHbXQVTOMMVOBqQCxsbEXvbLGF1/YxJednbflX6djwQJIT88ry8mBoHxXOncupKScWV62bF759Om2Bp+eDhkZdqtePa/8uecgOTmvLD0drr02rzwqyr4/JQWOHLGP9erZsrQ0GDz47GsaNAhGjYLDhyEyEsqXt4m+QgWoVAkeegj+/nc4ccJeX9WqdqtWDSIi9JeCUkXJzslmf8p+Eo8lsvvYbhKPJXIg5QAHTx4k6WQSB08ePL0dSj1EVk5WkZ8nCOVCy1E2pCylgktRKqjU6cfI0pFnPM+/HxYURmhQKNfVuq5YrtMdCT0RqJXveU1gjxs+t0giNkkHFXIFFSoU/d46dYoub9266PJHHy26/OOPz10WFmaTekqK3Y4ft0m/alVbHhQEL7xgXztyxCb4Q4fsewC2b4eePc/8zDJlYNo0+yth92746CP7KyImBurWteVK+TJjDPtO7CPhcELediSBnck7STyWyN7je8k22We8J0ACqFSqEpGlI4ksHUm9SvVoVasVkaUjqVS6EhVLVaRcaLlCt9LBpQkQz+sk6I6EvgDoJyKzgRZAcnG2n3u7U800oaFQseLZ5eXLw9Ch537/lVfC5s32HsHevXbbtSvvF8DatfDkk2eeLzoa3n8f/vpXe/yePdCokY1BKW+z/8R+1u9fb7cD9vGPQ3+Qkply+phACSSmQgzREdHcGHMjNcvVpEZ4DftYrgY1wmsQWTqSwADfats8b0IXkVlAWyBSRBKBF4BgAGPMG8AXwC1AAnAS+GdxBatse339+nYrzC232Br9jh32PsBvv8HGjXm/AObNg3797E3da66xN4KbNYM77jiz2UkpT3As/Rg/7f6JlYkr+THxR+L3xHMg5cDp8mplq9H4ssa0qd2GOhXrULdiXepUrENU+SiCA4MdjNwZ4tQi0bGxsUZnWyx5e/bAihUQHw+rV9vH48ftPYHwcHt/Ye9ee5P4qqu0bV6VrGPpx/h2x7cs3baUZTuWseHABkzuLbmGlRvSvEZzmlzWhMaXNaZxlcZULlPoLLI+TUTWGGNiCytzbPpc5Yzq1eH22+0G9obwjh02mYOtwc+aZferVLE1/m7d4LbbHAlX+ThjDBsObGD+b/NZvHUxKxNXkm2yKRVUiutrX8/tDW+nZc2WNK/RnIiwCKfD9XhaQ1dn2bEDli2z/fW/+MJ2+VyyxJYtXAg33GDb+pW6GMYYVu1exbzN8/j0t09JOJyAIMRWj6Xj5R3peEVHrqt5HaFBepOnMEXV0DWhqyJlZkJSkq3ZHzwIl11m29/j4qBXL9u/Ptj/mirVRdiZvJOZ62YyY90MEg4nEBwQTPuY9nRr0I24K+OoWraq0yF6BW1yURctODivD36lSrByJbz3nm2W+fhjO3p21iy48UZn41SeKTsnm4VbFjJ59WS+3vY1BkPb6LYMvn4wXet31WYUN9OErlwmktcrZvx4WLwY3n4bGjSw5atW2Rr9+frxK9+XnJbM27+8zaSfJrH96HaiykcxtO1Qel3di5gKMU6H57M0oauLEhwMf/ub3U4ZNcqOYr3hBhgyBNq3114y/uZw6mFe+fEVXl31KsczjtM6qjXjOo6jS/0uBAVouilu+l9Yuc2sWfDWWzBmDHToYAcyjRoFbdo4HZkqbkdSj/DKj68wcdVETmSc4PaGtzOw9UD+Uk1n0y5Jnjd2VXmt0qXttAhbt8LkyXYE6y+/OB2VKk6Z2Zm8tuo1rnj1Cl78/kU61enE+r7rmXPHHE3mDtAaunK7sDB4+GE7+2VAbpXh/fftDdXhwwuf8kB5n0UJi3hi8RNsPriZDpd3YPxN47n6squdDsuvaQ1dFZuwMDtVAdj56994w95AnTPnzNkxlXfZd2Ifd3x8B50/6ExmTiaf9fiMr+7+SpO5B9CErkrE0KF2qoFatex883FxkJjodFTqQhhjmLluJg0nN2Th7wsZ2X4kG/puIO7KOETvfnsETeiqxDRtaptdXn7ZLiayapXTESlXJaUkETc7jnvm30ODyg1Y+9Banrv+OR3N6WE0oasSFRRkp/fdscMu2AF2WoETJxwNSxVh2fZlXPPGNXy19Ssm3DyB7+79jvqR55juUzlKE7pyxKn1WQ8cgC5d7IpPmzc7G5M6U3ZONs9/8zw3zryRcqHlWPXAKh5r+ZjPzSHuSzShK0dVqWInAEtOtmuxnlq3VTkrOS2ZuNlxvPj9i9zT5B7W9FlDk6pNnA5LnYcmdOW4tm3tvOxXXgldu9qujco5fxz6g5Zvt+SrrV/x+q2v826XdykTousYegPth648Qs2a8P33djHszEyno/Ffy7Yvo9ucbgRKIEt7LaVNtA7z9Saa0JXHCAuDd9/N66P+668QFaVzr5eUTzZ9wl3z7qJuxbosvHOhTqLlhbTJRXkUETu6NDXVzrV+ww12STxVvKaumcodH99BbPVYvvvnd5rMvZQmdOWRSpWytfWtW20b++7dTkfku0Z/P5oHP3+QW+rewpJeS6hYSudm8Faa0JXHuukmO+f63r12xsZdu5yOyPeM/n40z33zHD0b9+TT7p9SOri00yGpS6AJXXm0Vq1sUk9KghdfdDoa3zLuh3Gnk/mMrjMIDtS1BL2d3hRVHu+66+CHH6BOHacj8R0TfpzAM0ufoUejHkzvOl0HC/kIraErr9Coke0Fc+gQ/OtfcOyY0xF5r3d/eZcnvnqC2xveznv/956uJORDNKErr7JuHUyfbmdrTE11Ohrv8+UfX/Kvhf+i4+Ud+aDbB5rMfYwmdOVV2reHGTPg22+hd2/IyXE6Iu8RvyeeOz6+g6svu5pP/vEJIYEhToek3EwTuvI6d90F48bB3Ll2MWp1ftuObOPWD28lsnQk/73rv4SHhjsdkioGmtCVV3rySXjgAfjwQzh+3OloPNvx9OPcNus2MrMzWXT3IqqFV3M6JFVMtAFNeSURmDLFztIYrpXNc8oxOfSe35vfDv7G4rsX6zzmPk5r6MprBQdDZCSkp8PAgTpFQGFe/O5F5v82n5c7vkyHyzs4HY4qZprQldfbvh1eew3+8Q+dqTG/z377jBf+9wK9ru7FYy0fczocVQJcSugi0klEfheRBBEZWEh5lIgsE5FfRGS9iNzi/lCVKlz9+jBtGixfDoMGOR2NZ9h2ZBu95/cmtnosb/7tTV3E2U+cN6GLSCAwGegMNATuFJGGBQ77NzDHGNMU6AFMcXegShXlrrugXz8YP972fvFnGdkZdJ/bnQAJYO4dcykVXMrpkFQJcaWG3hxIMMZsM8ZkALOBLgWOMUC53P3ywB73haiUa8aPh5YtoX9/SEtzOhrnDFw6kPg98bwd9za1I2o7HY4qQa70cqkB5J/nLhFoUeCYocBXItIfKAMUevdFRPoAfQCioqIuNFalihQSAh99ZEeQhoU5HY0zPt/yORNWTqBfs350a9DN6XBUCXOlhl5Y45sp8PxOYLoxpiZwC/CeiJz12caYqcaYWGNMbOVTy74r5UZRUXZtUmPsikf+ZPex3dwz/x6aVG3CuJvGOR2OcoArCT0RqJXveU3OblK5H5gDYIz5EQgDIt0RoFIX4/XXoWlTWLnS6UhKhjGGBxY+QFpWGh/d/hFhQX76E8XPuZLQVwN1RSRGREKwNz0XFDhmJ3AjgIg0wCb0JHcGqtSF6NnTLjzds6d/jCR96+e3WJSwiLEdxlKvUj2nw1EOOW9CN8ZkAf2AxcBmbG+WjSIyXETicg97EviXiKwDZgH3GmMKNssoVWLKl4f337d91J991uloitf2I9t54qsnuDHmRvo26+t0OMpB4lTejY2NNfHx8Y6cW/mPJ5+EV16BZcvs2qS+Jsfk0G5GO37Z+wsbHt5AVHntbODrRGSNMSa2sDKdy0X5tBEjbDv6yZNOR1I8Xlv1Gt/9+R3vxL2jyVxpQle+rXRpO4LUFwdK7kzeyeBvBtO5TmfubXKv0+EoD6BzuSifJwJZWXYO9RUrnI7GPYwxPPLFIxgMU26dokP7FaAJXfmJ1FSYNAn69PGNCbw+2fwJn2/5nOFthxMdEe10OMpDaEJXfiE83M7IuHEjTJzodDSXJjktmUe/fJSmVZsyoOUAp8NRHkQTuvIbcXFw220wdCjs2nXewz3WoK8HsT9lP1Nvm6qLPKszaEJXfmXiRLuw9OOPOx3Jxflp90+8Ef8GjzZ/lNjqhfZcU35M/7wrvxITY5euq1vX6UguXI7Jof+X/alatirD2w13OhzlgTShK79z771OR3BxZq6byU+7f2Jm15mEh+pCqups2uSi/FJ2NjzyCIwZ43QkrklOS2bg0oFcV/M6el7d0+lwlIfShK78UmCgXVR6xAjvWFx6xHcjOJBygFc7v0rA2TNTKwVoQld+bOxYyMiAwYOdjqRovx38jYmrJnJ/0/v1RqgqkiZ05bfq1IEBA2D6dPj5Z6ejObfHFz9OmeAyjLxxpNOhKA+nCV35tX//GypVgoEDnY6kcEu2LmFRwiKGtBlClTJVnA5HeTjt5aL8Wvny8MEHntmNMcfk8MzSZ4iOiOaRZo84HY7yAprQld+76aa8fWM8Z2bGD3/9kLX71vJhtw8JDQp1OhzlBbTJRSngxAno3NlO4OUJ0rLSGPzNYK6tdi3dG3V3OhzlJTShKwWUKQNpabYboyesQTrpp0nsTN7J2I5jtZuicpn+n6IUtpnlpZcgKckuWeekw6mHGfn9SDrX6Uz7mPbOBqO8iiZ0pXK1aAHdusHLL9vE7pSR340kOS2ZMR28ZBir8hia0JXKZ+RIu/7oyy87c/4/j/7JpNWTuLfJvTS+rLEzQSivpb1clMqnfn345BNo186Z84/4bgQAw9oOcyYA5dU0oStVQNeu9rGkuzAmHE5g+trpPNLsEWqVr1VyJ1Y+Q5tclCrE+vXQtCls3lxy5xz27TBCAkMYdP2gkjup8ima0JUqRPXqsHUrDCuhlo9NSZv4YP0H9Gvej6plq5bMSZXP0YSuVCEiI+HRR2HOHPj11+I/39D/DaVMSBmeafVM8Z9M+SxN6Eqdw5NPQtmyxV9LX7tvLR9v+pjHWjxGZOnI4j2Z8mma0JU6h4oV7WLSn3xi29SLy5BlQ4gIi+DJvz5ZfCdRfkF7uShVhMcfh9q1oUGD4vn8n3b/xMItC3mx3YtEhEUUz0mU39CErlQRIiLgvvuK7/Nf+N8LVCpViUdbPFp8J1F+w6UmFxHpJCK/i0iCiBS6FICI/ENENonIRhH50L1hKuWst9+G++9372eu2bOGRQmLePK6JwkPDXfvhyu/dN6ELiKBwGSgM9AQuFNEGhY4pi4wCGhljLkKeKwYYlXKMfv3wzvvwE8/ue8zRy0fRfnQ8jzc7GH3fajya67U0JsDCcaYbcaYDGA20KXAMf8CJhtjjgAYYw64N0ylnNW/v71JOmKEez5vU9Im5m2eR//m/SkfVt49H6r8nisJvQawK9/zxNzX8qsH1BORH0RkpYh0KuyDRKSPiMSLSHySk9PZKXWBwsPtDdLPP4dffrn0zxu9fDSlg0szoOWAS/8wpXK5ktALm83CFHgeBNQF2gJ3Am+JyFm37I0xU40xscaY2MqVK19orEo5ql8/KFcORo26tM/ZdmQbs36dxUPXPqT9zpVbudLLJRHIP1NQTWBPIcesNMZkAttF5Hdsgl/tliiV8gAREfCf/0CNgr9PL9CY5WMIDAjUfufK7Vypoa8G6opIjIiEAD2ABQWOmQ+0AxCRSGwTzDZ3BqqUJ/jnP89cVPpC7T62m+nrpnNfk/uoHl7dfYEphQsJ3RiTBfQDFgObgTnGmI0iMlxE4nIPWwwcEpFNwDLgaWPMoeIKWiknHTwITz0FCQkX/t6XV7xMdk62ztmiioVLA4uMMV8AXxR4bUi+fQM8kbsp5dMyM2HSJDh6FN56y/X3JaUk8eaaN+l5dU9iKsQUX4DKb+lcLkpdoGrV4IEHYMYM2LnT9ff9Z+V/SMtKY2CrQsfmKXXJNKErdRGeecauZjR2rGvHJ6clM2n1JLo16EaDysU0MYzye5rQlboIUVFwzz22yWXv3vMfP3n1ZI6lH2Pw9YOLPzjlt3RyLqUu0sCBcOKEbVMvSkpGChNWTqBznc40rda0ZIJTfkkTulIX6YorYNas8x837edpHDx5UGvnqthpk4tSl2jDBpg7t/Cy9Kx0xq0YR5vabWgV1apkA1N+RxO6Updo6FDb6yU5+eyyGetmsOf4Hq2dqxKhCV2pSzRokE3mU6ac+XpWThZjfhhDs+rN6HB5B2eCU35FE7pSl+jaa6FTJ5gwAU6ezHt99obZbDuyjcHXD0aksDnulHIvTehKucHgwZCUlDdyNMfkMHr5aBpVacRtV97mbHDKb2gvF6XcoHVruOWWvBr6/N/msylpEx92+5AA0XqTKhma0JVyk88/t6NHjTGM+n4UV1S4gjuuusPpsJQf0YSulJvYZA7jPohnTeJapnV9g6AA/SemSo7+FlTKjb7+Gp7t1YyK2/rS+5reToej/IxWH5Ryo+ArvofKFQj5cShBEuJ0OMrPaA1dKTca/cNIwm+czL5tlVhQcF0vpYqZJnSl3CR+TzyLty7m2QejufxyGDnStqkrVVI0oSvlJqO+H0VEWAT9r+vLwIGQmGg3pUqKJnSl3GDjgY18+tun9G/en3Kh5bjnHti+HWrVcjoy5U80oSvlBqOXj6ZMcBkGtBgAQEgIhIXZudL37XM4OOU3NKErdYm2Ht7KrA2zeCj2ISqVrnT6dWPsCNJ//tPB4JRf0YSu1CUa88MYggKCeOK6J854XQS6doVFi2DNGoeCU35FE7pSl2D3sd1MXzud+5rcR/Xw6meVP/wwlC8Po0Y5EJzyO5rQlboE41aMI8fk8EyrZwotL18e+veHefNg06YSDk75HU3oSl2k/Sf2M3XNVHpd04uYCjHnPG7AAChdGmbOLMHglF/Sof9KXaQJKyeQlpXGoNaDijwuMhJWr4b69UsoMOW3tIau1EU4nHqYyasn071Rd+pVqnfe4xs2hIAA241RqeKiCV2pi/Dqqlc5kXGC51o/5/J7Fi60A4127y7GwJRf04Su1AU6ln6Miasm0rV+Vxpf1tjl9zVqBAcPwvjxxRic8mua0JW6QFNWT+Fo2lEGXz/4gt4XEwN33QVvvmkTu1LupgldqQuQkpHC+B/H06lOJ2Krx17w+wcNgtRU+M9/iiE45fdcSugi0klEfheRBBEZWMRxt4uIEZEL/z9dKS8w7edpHDx5kH9f/++Len+DBtCtG0yaBCkpbg5O+b3zJnQRCQQmA52BhsCdItKwkOPCgUeBVe4OUilPkJaVxrgV42gb3ZZWUa0u+nNefBG++grKlHFjcErhWj/05kCCMWYbgIjMBroABce9jQDGAk+5NUKlPMT0tdPZc3wPM7te2ggh7Y+uiosrTS41gF35nifmvnaaiDQFahljPi/qg0Skj4jEi0h8UlLSBQerlFMysjN4aflLtKzZkvYx7S/589LT4f77YcoUNwSnVC5XEroU8trphbVEJACYADx5vg8yxkw1xsQaY2IrV67sepRKOWz62un8mfwnz9/wPCKF/ZO4MKGhkJBgJ+1KT3dDgErhWkJPBPKvu1IT2JPveTjQCPifiOwAWgIL9Mao8hXpWemM/H4kLWq0oHOdzm773MGD7SAjneNFuYsrCX01UFdEYkQkBOgBnF7P3BiTbIyJNMZEG2OigZVAnDEmvlgiVqqEvfPLO+xM3snwdsPdUjs/pWNHaNYMRo/WKQGUe5w3oRtjsoB+wGJgMzDHGLNRRIaLSFxxB6iUk9Ky0hj5/Uha1WpFx8s7uvWzRWDoULv26LvvuvWjlZ9yabZFY8wXwBcFXhtyjmPbXnpYSnmGaWumsfv4bmZ0neHW2vkpnTvbdvROndz+0coP6fS5Sp1DamYqo5aP4obaN7ilZ0thROzoUaXcQYf+K3UOb655k30n9jG8rXvbzguzfj306AEnTxbraZSP04SuVCFSMlIYvXw07WPa0ya6TbGf7/hx+OgjmDy52E+lfJgmdKUK8Xr86xxIOcCwtsNK5HytWtl29DFjbHJX6mJoQleqgOS0ZEYvH81NV9xE66jWJXbe4cPh0CGYOLHETql8jCZ0pQoY+8NYDqce5qUbXyrR8zZrBnFx8PLLcORIiZ5a+Qjt5aJUPnuP72XCygn0aNSDptWalvj5hw+HBQvs1ABKXShN6ErlM/zb4WTmZDKi3QhHzn/NNXZT6mJok4tSuf449AfTfp5Gn7/0oU7FOo7GMn8+DCl06J5S56YJXalczy97ntCgUJ5v87zTobBihV0IY/16pyNR3kQTulLAmj1r+GjjRzzR8gmqlq3qdDgMGgQREfDss05HoryJJnTl94wxPL3kaSqVqsRTf/WMBbcqVLDT6y5aBEuXOh2N8haa0JXf++z3z1i2YxnD2w2nfFh5p8M57ZFHoHZteOYZyMlxOhrlDbSXi/Jr6VnpPPXVUzSs3JA+1/ZxOpwzhIXZQUbHjjkdifIWmtCVX5v00yS2HtnKop6LCArwvH8OXbo4HYHyJtrkovxWUkoSw78bzi11b+HmOjc7Hc45GQOvvAIvlezAVeWFNKErvzVk2RBSMlJ4uePLTodSJBH4+We7utG2bU5HozyZJnTll9bvX8/Un6fycLOHaVC5gdPhnNeYMRAUBE8+6XQkypNpQld+J8fk0Pe/falYqiJD2w51OhyX1KgB//63HUG6ZInT0ShPpQld+Z0Za2ewYtcKxnYYS8VSFZ0Ox2WPPw5XXAGPPabdGFXhPO+2vlLF6NDJQzy95Gla1WrFPU3ucTqcCxIaCm+9ZZteArQqpgqhCV35lee+fo6jaUeZcusUAsT7smLbtnn72dkQGOhYKMoDed//0UpdpJWJK5n28zQGtBjA1Zdd7XQ4l2TwYLjtNtulUalTNKErv5CZnUnf//alWng1r7kRWpTLLoMvv4RZs5yORHkSTejKL4xbMY61+9byWufXCA8NdzqcS/bII3bJusceg8OHnY5GeQpN6MrnbUraxLBvh3FHwzvo1qCb0+G4RWAgTJ1qk/nTTzsdjfIUmtCVT8vOyea+z+4jPCScSbdMcjoct2rSxA40mjULdu92OhrlCTShK582cdVEVu1exWudX6NKmSpOh+N2w4bB2rV24JFSmtCVz/r94O8M/mYwcVfG0aNRD6fDKRZhYVCvnu3tsmKF09Eop2lCVz4pIzuDnvN6Ujq4NK/f+joi4nRIxeqjj6BVK/j0U6cjUU5yKaGLSCcR+V1EEkRkYCHlT4jIJhFZLyJfi0ht94eqlOuG/m8oa/au4a3b3qJ6eHWnwyl23bpB06bw4INw4IDT0SinnDehi0ggMBnoDDQE7hSRhgUO+wWINcZcDcwFxro7UKVc9e2Ob3lp+Us80PQB/q/B/zkdTokICYGZM+3qRvfeq3O9+CtXaujNgQRjzDZjTAYwGzhjHRVjzDJjzMncpyuBmu4NUynXHEk9Qq9Pe1GnYh0mdJrgdDglqlEjmDDBDjgaP97paJQTXJnLpQawK9/zRKBFEcffD3xZWIGI9AH6AERFRbkYolKuMcbwwMIH2HtiLyvuW0HZkLJOh1TiHnoIfvnF3ihV/seVhF7Y3aRCZ5AQkbuBWKBNYeXGmKnAVIDY2FidhUK51fgfxzNv8zxe7vgyzWo0czocR4jYAUenGGNfU/7BlSaXRKBWvuc1gT0FDxKRDsBgIM4Yk+6e8JRyzbc7vmXg0oH8vcHfeeK6J5wOxyNMnAj/+Ie2p/sTVxL6aqCuiMSISAjQA1iQ/wARaQq8iU3meo9dlai9x/fSfW536lSswztd3vH5LoquCgiAuXPt4CPlH87b5GKMyRKRfsBiIBB4xxizUUSGA/HGmAXAOKAs8HHuP6adxpi4YoxbKQDSs9L5x9x/cDzjOF/3/ppyoeWcDslj9OtnF5cePhyuucZ2bVS+zaUFLowxXwBfFHhtSL79Dm6OS6nzMsbw4OcPsnzncmb/fTZXVbnK6ZA8igi8/jps2gS9e9sbpY0aOR2VKk46UlR5rTE/jGHGuhkMbTOU7o26Ox2ORwoLs6NHIyJ0agB/oEvQKa/0yaZPGPT1IO5sdCdD2gw5/xv8WPXqsP/AwTYAAA28SURBVHkzhHv/NPDqPLSGrrzO8p3LufvTu2lZs6XeBHXRqWT+9ddw112QleVsPKp4aEJXXmXdvnX87cO/EVU+igU9FhAWFOZ0SF4lIcHOn963r65H6ou0yUV5jYTDCdz8/s2Eh4azpNcSKpep7HRIXufBB2HXLhg5EipVgtGjdeCRL9GErrzCn0f/pON7HcnKyWLZPcuIKq9TR1ysESPg0CEYMwaCguxzTeq+QRO68njbj2yn3Yx2JKcns6TXEhpUbuB0SF5NBCZPtu3oiYk6PYAv0YSuPNq2I9toN6Mdx9OPs7TXUq6tfq3TIfmEgAB48828/cOHoUIFTezeTm+KKo+1KWkTbaa34UTGCb7u/bUmczcLCMhL5s2awYABOu+Lt9OErjzS8p3Laf1O69Nt5k2rNXU6JJ8VEQFxcfDaa9CrF2RkOB2Rulja5KI8zqebP+WueXdRu3xtFt29iOiIaKdD8mkBAfDKK3DZZTBokL1hOmcOlNNpcbyO1tCVxzDG8NLyl/j7nL/TpGoTlt+3XJN5CRGBgQPhrbdg6VJ4Qmcg9kpaQ1ceISUjhfsW3MecjXPo0agHb8e9Teng0k6H5Xfuvx/q1IGGuasG5+TYGrzyDvpVKcdtPbyVv77zV+ZumsvYDmP5sNuHmswd1KYNVK4MmZlw880wdqzeLPUWmtCVY4wxvLfuPZq82YSdyTv54q4veLrV0zo3i4fIyLA3TJ99Fjp3hv37nY5InY8mdOWI5LRkes7rSe/5vWlatSnrHlrHzXVudjoslU+ZMvbm6BtvwHffwdVXw6JFTkeliqIJXZW4/275L41fb8ycjXMY0W6EDuX3YCJ2/pfVq20zTP/+tilGeSa9KapKzP4T+3ls8WPM3jCbhpUbsvyO5bSs2dLpsJQLGjWC+Hg7sVdwMJw8Cd98A7feqqNLPYnW0FWxy8zO5NVVr9JgcgPmbZ7HsLbD+OXBXzSZe5mwMKhb1+5PmQK33QadOtkl7pRn0Bq6KjbGGBZuWcjTS55my6EttI9pz6TOk3RyLR8wYICdqXHYMNu23rcvDB1qp+RVztEaunI7YwzLti+j3Yx2dJndBUH4/M7PWdprqSZzHxEcDI89Bn/8YdvYp0yBPn2cjkppDV25jTGGJduWMPzb4fyw6weqla3GpM6T6HNtH4IDg50OTxWDyEg7Fe/DD+e1pW/fDtOnw6OPao29pGkNXV2y1MxU3v75bZq+2ZSb37+ZP5P/ZFLnSWwbsI1Hmj+iydwPXHVV3ujSRYtg+HCoWRP+9S/YsMHZ2PyJJnR10f449AfPLnmWmhNq8sDCB8g22Uz921QS+ifwSPNHdL1PP9W3r03ivXrB++9D48bQpYuuYVoStMlFXZBDJw/x0caPeG/9e6xMXEmgBNK1flf6N+/PDbVv0FGeCrA19qlT7Zql06bBkSN5TTIvvggdOkCLFtrl0d3EOPRnMzY21sTHxztybnVh9hzfw8LfF7JgywKWbF1CZk4mjas0ptfVvbir8V3UKFfD6RCVl0hMhHr1IDUVoqLg//7Pbq1bQ2Cg09F5BxFZY4yJLbRME7oqKDM7k/g98SzdtpQFWxYQv8d+T5dXuJxu9btx99V3c03VaxyOUnmrY8dg3jy7ffUVpKfDrFnQo4ediz0jA6pVczpKz6UJXRUpPSuddfvX8b8d/2PZjmUs37mcExknEIQWNVsQVy+OuCvjaFi5oTapKLc6cQK+/NLO6liuHIwfD089ZZtsbrwRWrWCli2hVi1tnjlFE7o6LTM7ky2HtrB6z2pW717N6j2rWbd/HRnZdt2xBpENaBfdjnYx7WhTuw2Vy1R2OGLlT7Zsgc8+s4tsfP+9bZoJCrK1+lKl7HQDmZl2MFPVqv6Z5DWh+6GUjBS2H93O5qTNbEraxMakjWxK2sSWQ1vIzLGzK4WHhBNbPZZm1ZvRrEYzWke1pmrZqg5HrpSVmQnr19vBSz162Nfat4dly+x+ZKTtQdO2LQwZYl9LSoKKFX27Pb6ohK69XLxQdk42B1IOsO/EPvad2MfO5J1sP7qd7Ue3s+PoDrYf2U7SyaTTxwvC5RUu56oqV3Fbvdu4qspVxFaPpV6legSI9lxVnik4GK691m6nzJ0Lv/5qE/2pbe3avPLWrWHHDrj8crvyUnS0fa17d1u+YwdUqQKlfXT9FJdq6CLSCZgIBAJvGWNeKlAeCswErgUOAd2NMTuK+kytoVvZOdkcTTvK4dTDHEk7wpHUI6cfT712KPXQ6eS99/hekk4mkWPOXEImOCCY2hG1iY6IJiYihpiIGKIjoqkfWZ/6kfUpFVzKoStUquRMnw6bN0NCgq3Z79pl+8BPn277wZcuDWlpUKGCXRS7cmW4807bdz4nByZOtDX/yEi7uEe5clC9uj3eU1xSDV1EAoHJQEcgEVgtIguMMfnnWLsfOGKMqSMiPYAxQPdLD939jDFkm2xyTA7ZOdlkm+wzHnNMDpk5mWRkZ5ze0rPS8/az04t8PT0rnZOZJ0nJTOFExglSMlNIyUg54/FExonT+yczTxYZb6mgUlQsVZFq4dWoWa4msdViqRZejaplq1KtrH2sWa4m1cOrExjgw78zlXLBvfee/VpWln3MyYE334Tdu233yQMHbBPNqfndjx4tfHHsYcNsk86ePVC/vk3yp7ayZaFfP+jaFfbutSNkS5WyM1OWKmW3zp3tTd7Dh+Hbb213zfy/OtzJlSaX5kCCMWYbgIjMBroA+RN6F2Bo7v5cYJKIiCmGBvp3fnmHcSvGFZmQi3rNUPz3DAIlkDIhZSgTXIayIWVP75cPK0/18Or2teAylAkpQ3hIOBVKVaBCWAUqlKpAxVIVT+9XCKtAaFBoscerlC8Lys1ygYHQu/e5j6tQwQ6AOnjQJvrkZHsz9tSUBiEhdhHtY8fytuPH8/4gHD5su2Kmptrt1B+Syy6zCX3jRujWzTb/zJ5dTNfqwjE1gF35nicCLc51jDEmS0SSgUrAwfwHiUgfoA9AVNTFrVATWTqSxlUaExgQSIAEECiBBAYE2kfJfe3U83yPZx1bxGtBAUGEBoYSGhRKSGAIIYEhhAbm7YcEhpyzLDQolOCAYO3ep5SXEbHNLBERtv29oMhImDDh3O+/6qoz113NyrKJPSTEPm/aFH7+2dbsi4srCb2wzFSwmuvKMRhjpgJTwbahu3Dus8RdaftEK6WUJwsKgvDwvOdly9qkXpxc6eKQCNTK97wmsOdcx4hIEFAeOOyOAJVSSrnGlYS+GqgrIjEiEgL0ABYUOGYBcE/u/u3AN8XRfq6UUurcztvkktsm3g9YjO22+I4xZqOIDAfijTELgLeB90QkAVsz71GcQSullDqbSwOLjDFfAF8UeG1Ivv004A73hqaUUupC6DBBpZTyEZrQlVLKR2hCV0opH6EJXSmlfIRj0+eKSBLw50W+PZICo1C9mF6L5/GV6wC9Fk91KddS2xhT6EIFjiX0SyEi8eeabczb6LV4Hl+5DtBr8VTFdS3a5KKUUj5CE7pSSvkIb03oU50OwI30WjyPr1wH6LV4qmK5Fq9sQ1dKKXU2b62hK6WUKkATulJK+QiPTugi0klEfheRBBEZWEh5qIh8lFu+SkSiSz5K17hwLfeKSJKIrM3dHnAizvMRkXdE5ICIbDhHuYjIq7nXuV5E/lLSMbrKhWtpKyLJ+b6TIYUd5zQRqSUiy0Rks4hsFJEBhRzjFd+Li9fiLd9LmIj8JCLrcq9lWCHHuDeHGWM8csNO1bsVuBwIAdYBDQsc8zDwRu5+D+Ajp+O+hGu5F5jkdKwuXMsNwF+ADecovwX4EruKVUtgldMxX8K1tAU+dzpOF66jGvCX3P1wYEsh/395xffi4rV4y/ciQNnc/WBgFdCywDFuzWGeXEM/vTi1MSYDOLU4dX5dgBm5+3OBG8UzF/N05Vq8gjHmO4pejaoLMNNYK4EIEalWMtFdGBeuxSsYY/YaY37O3T8ObMau85ufV3wvLl6LV8j9b30i92lw7lawF4pbc5gnJ/TCFqcu+MWesTg1cGpxak/jyrUA/D335/BcEalVSLk3cPVavcV1uT+ZvxSRq5wO5nxyf7I3xdYG8/O676WIawEv+V5EJFBE1gIHgCXGmHN+L+7IYZ6c0N22OLUHcCXOhUC0MeZqYCl5f7W9jbd8J674GTtvxjXAa8B8h+MpkoiUBT4BHjPGHCtYXMhbPPZ7Oc+1eM33YozJNsY0wa7F3FxEGhU4xK3fiycndF9anPq812KMOWSMSc99Og24toRiczdXvjevYIw5duons7GrdgWLSKTDYRVKRIKxCfADY8y8Qg7xmu/lfNfiTd/LKcaYo8D/gE4Fityawzw5ofvS4tTnvZYC7Zlx2LZDb7QA6J3bq6IlkGyM2et0UBdDRKqeas8UkebYfy+HnI3qbLkxvg1sNsa8co7DvOJ7ceVavOh7qSwiEbn7pYAOwG8FDnNrDnNpTVEnGB9anNrFa3lUROKALOy13OtYwEUQkVnYXgaRIpIIvIC92YMx5g3s2rO3AAnASeCfzkR6fi5cy+1AXxHJAlKBHh5aYWgF9AJ+zW2vBXgOiAKv+15cuRZv+V6qATNEJBD7R2eOMebz4sxhOvRfKaV8hCc3uSillLoAmtCVUspHaEJXSikfoQldKaV8hCZ0pZTyEZrQlVLKR2hCV0opH/H/wAE3qupUGgUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(X_new, y_proba[:,1], \"g-\", label=\"Iris virginica\")\n",
    "plt.plot(X_new, y_proba[:, 0], \"b--\", label = \"Not Iris virginica\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_openml\n",
    "mnist =fetch_openml('mnist_784', version=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['data', 'target', 'frame', 'feature_names', 'target_names', 'DESCR', 'details', 'categories', 'url'])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((70000, 784), (70000,))"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = mnist['data'], mnist['target']\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1ba44848bc8>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAM/UlEQVR4nO3df4ichZ3H8c/nvFTUBoyXNReSaGoJifHg0jrGX0fJUSzGf5KAPRokRNSLfyi0UEHxhPqXyHFt6R9nYXuGpmfOEmjF/BG8yFINRSlZJcZ4wVtP99LUNTsxSCwIUfd7f+yTY407z2xmnplnNt/3C5aZeb7z7PNhyCfPzDwz+zgiBODC9xd1BwDQH5QdSIKyA0lQdiAJyg4k8Zf93NjixYtj5cqV/dwkkMr4+LhOnjzp2WZdld327ZJ+JukiSf8WEU+W3X/lypUaHR3tZpMASjQajZazjp/G275I0r9K2ihpraStttd2+vsA9FY3r9nXS3onIt6NiDOSfi1pUzWxAFStm7Ivk/THGbePF8u+wPYO26O2R5vNZhebA9CNbso+25sAX/rsbUQMR0QjIhpDQ0NdbA5AN7op+3FJK2bcXi7p/e7iAOiVbsp+UNIq21+z/RVJ35O0t5pYAKrW8aG3iPjM9oOS/lPTh952RsRblSUDUKmujrNHxD5J+yrKAqCH+LgskARlB5Kg7EASlB1IgrIDSVB2IAnKDiRB2YEkKDuQBGUHkqDsQBKUHUiCsgNJUHYgCcoOJEHZgSQoO5AEZQeSoOxAEpQdSIKyA0lQdiAJyg4kQdmBJCg7kARlB5Kg7EASlB1IgrIDSXR1FldgkI2MjLSc3XXXXaXrvvzyy6Xz1atXd5SpTl2V3fa4pI8lfS7ps4hoVBEKQPWq2LP/fUScrOD3AOghXrMDSXRb9pC03/ZrtnfMdgfbO2yP2h5tNptdbg5Ap7ot+60R8U1JGyU9YPtb594hIoYjohERjaGhoS43B6BTXZU9It4vLiclPSdpfRWhAFSv47Lbvsz2wrPXJX1H0pGqggGoVjfvxi+R9Jzts7/nPyLihUpS9cCBAwdK5x9++GHpfMuWLVXGQR8cPHiw5azRyHeUuOOyR8S7kv62wiwAeohDb0ASlB1IgrIDSVB2IAnKDiSR5iuuL730Uul8bGysdM6ht8EzNTVVOn/vvfdazo4dO1a6bkR0lGmQsWcHkqDsQBKUHUiCsgNJUHYgCcoOJEHZgSTSHGfftWtX6fyWW27pUxJUZWJionQ+PDzccrZt27bSddesWdNRpkHGnh1IgrIDSVB2IAnKDiRB2YEkKDuQBGUHkkhznL3dd58x/9x3330dr7tq1aoKk8wP7NmBJCg7kARlB5Kg7EASlB1IgrIDSVB2IIkL5jj74cOHS+cnTpzoUxL0y0cffdTxurfddluFSeaHtnt22zttT9o+MmPZFbZftD1WXC7qbUwA3ZrL0/hfSrr9nGWPSBqJiFWSRorbAAZY27JHxAFJp85ZvEnS2b/ztEvS5opzAahYp2/QLYmICUkqLq9sdUfbO2yP2h5tNpsdbg5At3r+bnxEDEdEIyIaQ0NDvd4cgBY6LfsJ20slqbicrC4SgF7otOx7JW0vrm+X9Hw1cQD0Stvj7LaflbRB0mLbxyX9SNKTkvbYvlfSMUnf7WXIudi3b1/p/JNPPulTElSl3WcjxsfHO/7dy5Yt63jd+apt2SNia4vRtyvOAqCH+LgskARlB5Kg7EASlB1IgrIDSVwwX3F9++23u1r/uuuuqygJqvLQQw+Vzj/44IPS+erVq1vOFi5c2FGm+Yw9O5AEZQeSoOxAEpQdSIKyA0lQdiAJyg4kccEcZ+/WDTfcUHeEeen06dOl8xdeeKHl7Jlnnildd//+/R1lOuuxxx5rObv88su7+t3zEXt2IAnKDiRB2YEkKDuQBGUHkqDsQBKUHUiC4+yFU6fOPZ1d/7zxxhul86mpqdL5yMhIy9nx48dL1z1z5kzpfPfu3aXzdtkuueSSlrMbb7yxdN2LL764dP7pp5+WzhuNRuk8G/bsQBKUHUiCsgNJUHYgCcoOJEHZgSQoO5DEBXOcvex4riTZLp3ff//9pfMnnnjivDPNVbvj7BFROl+wYEHL2aWXXlq67rXXXls6v+eee0rn119/fel8w4YNLWdLliwpXXf58uWl83an4V6zZk3pPJu2e3bbO21P2j4yY9njtv9k+1Dxc0dvYwLo1lyexv9S0u2zLP9pRKwrfvZVGwtA1dqWPSIOSKrvs6QAKtHNG3QP2j5cPM1f1OpOtnfYHrU92mw2u9gcgG50WvafS/q6pHWSJiT9uNUdI2I4IhoR0RgaGupwcwC61VHZI+JERHweEVOSfiFpfbWxAFSto7LbXjrj5hZJR1rdF8BgaHuc3fazkjZIWmz7uKQfSdpge52kkDQuqfwgdR889dRTpfOrr766dP7KK69UGee8XHXVVaXzTZs2lc7Xrl3bcnbTTTd1lKkfhoeHS+eTk5Ol82uuuabKOBe8tmWPiK2zLH66B1kA9BAflwWSoOxAEpQdSIKyA0lQdiCJC+Yrru08/PDDdUfAOcr+BPZc3HnnnRUlyYE9O5AEZQeSoOxAEpQdSIKyA0lQdiAJyg4kkeY4Oy48mzdvrjvCvMKeHUiCsgNJUHYgCcoOJEHZgSQoO5AEZQeSoOxAEpQdSIKyA0lQdiAJyg4kQdmBJCg7kARlB5Lg++yYt8bGxkrnN998c5+SzA9t9+y2V9j+ne2jtt+y/f1i+RW2X7Q9Vlwu6n1cAJ2ay9P4zyT9MCKulXSTpAdsr5X0iKSRiFglaaS4DWBAtS17RExExOvF9Y8lHZW0TNImSbuKu+2SxN8IAgbYeb1BZ3ulpG9I+oOkJRExIU3/hyDpyhbr7LA9anu02Wx2lxZAx+ZcdttflfQbST+IiNNzXS8ihiOiERGNoaGhTjICqMCcym57gaaLvjsiflssPmF7aTFfKmmyNxEBVGEu78Zb0tOSjkbET2aM9kraXlzfLun56uMBrU1NTZX+4Ivmcpz9VknbJL1p+1Cx7FFJT0raY/teScckfbc3EQFUoW3ZI+L3ktxi/O1q4wDoFT4uCyRB2YEkKDuQBGUHkqDsQBJ8xRXz1quvvlo6v/vuu/sTZJ5gzw4kQdmBJCg7kARlB5Kg7EASlB1IgrIDSVB2IAnKDiRB2YEkKDuQBGUHkqDsQBKUHUiCsgNJ8H121Gbjxo2l8z179vQpSQ7s2YEkKDuQBGUHkqDsQBKUHUiCsgNJUHYgibbH2W2vkPQrSX8taUrScET8zPbjkv5RUrO466MRsa9XQXHhafd33fm779Way4dqPpP0w4h43fZCSa/ZfrGY/TQi/qV38QBUZS7nZ5+QNFFc/9j2UUnLeh0MQLXO6zW77ZWSviHpD8WiB20ftr3T9qIW6+ywPWp7tNlsznYXAH0w57Lb/qqk30j6QUSclvRzSV+XtE7Te/4fz7ZeRAxHRCMiGkNDQxVEBtCJOZXd9gJNF313RPxWkiLiRER8HhFTkn4haX3vYgLoVtuy27akpyUdjYifzFi+dMbdtkg6Un08AFWZy7vxt0raJulN24eKZY9K2mp7naSQNC7p/p4kBFCJubwb/3tJnmXEMXVgHuETdEASlB1IgrIDSVB2IAnKDiRB2YEkKDuQBGUHkqDsQBKUHUiCsgNJUHYgCcoOJEHZgSQcEf3bmN2U9L8zFi2WdLJvAc7PoGYb1FwS2TpVZbarI2LWv//W17J/aeP2aEQ0agtQYlCzDWouiWyd6lc2nsYDSVB2IIm6yz5c8/bLDGq2Qc0lka1TfclW62t2AP1T954dQJ9QdiCJWspu+3bbb9t+x/YjdWRoxfa47TdtH7I9WnOWnbYnbR+ZsewK2y/aHisuZz3HXk3ZHrf9p+KxO2T7jpqyrbD9O9tHbb9l+/vF8lofu5JcfXnc+v6a3fZFkv5b0m2Sjks6KGlrRPxXX4O0YHtcUiMiav8Ahu1vSfqzpF9FxN8Uy/5Z0qmIeLL4j3JRRDw8INkel/Tnuk/jXZytaOnM04xL2izpbtX42JXk+gf14XGrY8++XtI7EfFuRJyR9GtJm2rIMfAi4oCkU+cs3iRpV3F9l6b/sfRdi2wDISImIuL14vrHks6eZrzWx64kV1/UUfZlkv444/ZxDdb53kPSftuv2d5Rd5hZLImICWn6H4+kK2vOc662p/Hup3NOMz4wj10npz/vVh1ln+1UUoN0/O/WiPimpI2SHiiermJu5nQa736Z5TTjA6HT0593q46yH5e0Ysbt5ZLeryHHrCLi/eJyUtJzGrxTUZ84ewbd4nKy5jz/b5BO4z3bacY1AI9dnac/r6PsByWtsv0121+R9D1Je2vI8SW2LyveOJHtyyR9R4N3Kuq9krYX17dLer7GLF8wKKfxbnWacdX82NV++vOI6PuPpDs0/Y78/0j6pzoytMh1jaQ3ip+36s4m6VlNP637VNPPiO6V9FeSRiSNFZdXDFC2f5f0pqTDmi7W0pqy/Z2mXxoelnSo+Lmj7seuJFdfHjc+LgskwSfogCQoO5AEZQeSoOxAEpQdSIKyA0lQdiCJ/wMY2PpMk6vt1QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# show some digit\n",
    "some_digit = X[2]\n",
    "some_digit_image=some_digit.reshape(28,28)\n",
    "plt.imshow(some_digit_image, cmap='binary')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split training and test data\n",
    "X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training a binary classifier\n",
    "y_train_5 = (y_train == '5')\n",
    "y_test_5 = (y_test == '5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDClassifier(alpha=0.0001, average=False, class_weight=None,\n",
       "              early_stopping=False, epsilon=0.1, eta0=0.0, fit_intercept=True,\n",
       "              l1_ratio=0.15, learning_rate='optimal', loss='hinge',\n",
       "              max_iter=1000, n_iter_no_change=5, n_jobs=None, penalty='l2',\n",
       "              power_t=0.5, random_state=42, shuffle=True, tol=0.001,\n",
       "              validation_fraction=0.1, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import SGDClassifier\n",
    "sgd_clf = SGDClassifier(random_state = 42)\n",
    "sgd_clf.fit(X_train, y_train_5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implementing cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.95035\n",
      "0.96035\n",
      "0.9604\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.base import clone\n",
    "\n",
    "skfolds = StratifiedKFold(n_splits = 3, random_state=None)\n",
    "\n",
    "for train_index, test_index in skfolds.split(X_train, y_train_5):\n",
    "    clone_clf = clone(sgd_clf)\n",
    "    X_train_folds = X_train[train_index]\n",
    "    y_train_folds = y_train_5[train_index]\n",
    "    X_test_fold = X_train[test_index]\n",
    "    y_test_fold = y_train_5[test_index]\n",
    "    \n",
    "    clone_clf.fit(X_train_folds, y_train_folds)\n",
    "    y_pred = clone_clf.predict(X_test_fold)\n",
    "    n_correct = sum(y_pred == y_test_fold)\n",
    "    print(n_correct / len(y_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000\n",
      "[    0     1     2 ... 20331 20342 20359]\n",
      "40000\n",
      "[19964 19965 19966 ... 40088 40125 40127]\n",
      "40000\n",
      "[39988 39989 39990 ... 59997 59998 59999]\n"
     ]
    }
   ],
   "source": [
    "for i, a in skfolds.split(X_train,y_train_5):\n",
    "    print(len(i))\n",
    "    print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.95035, 0.96035, 0.9604 ])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "cross_val_score(sgd_clf, X_train, y_train_5, cv = 3, scoring='accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.91125, 0.90855, 0.90915])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.base import BaseEstimator\n",
    "\n",
    "class Never5Classifier(BaseEstimator):\n",
    "    def fit(self, X, y=None):\n",
    "        pass\n",
    "    def predict(self, X):\n",
    "        return np.zeros((len(X), 1), dtype=bool)\n",
    "\n",
    "never_5_clf = Never5Classifier()\n",
    "cross_val_score(never_5_clf, X_train, y_train_5, cv=3, scoring='accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_predict\n",
    "y_train_pred = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[53892,   687],\n",
       "       [ 1891,  3530]], dtype=int64)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "confusion_matrix(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Precision and recall\n",
    "\n",
    "$\n",
    "\\text{precision} = \\frac{TP}{TP+FP} \\\\\n",
    "\\text{recall} = \\frac{TP}{TP+ FN}\n",
    "$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.8370879772350012, 0.6511713705958311)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score\n",
    "precision_score(y_train_5, y_train_pred), recall_score(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$\n",
    "F_1=\\frac{2}{\\frac{1}{\\text{precision}}+\\frac{1}{\\text{recall}}}=\\frac{TP}{TP+\\frac{FN+FP}{2}}\n",
    "$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7325171197343846"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "f1_score(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The ROC Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Found input variables with inconsistent numbers of samples: [60000, 1]",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-55-e0f7d3a09128>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmetrics\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mroc_curve\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0my_scores\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msgd_clf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecision_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0msome_digit\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mfpr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtpr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mthresholds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mroc_curve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_train_5\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_scores\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mplot_roc_curve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfpr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtpr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_ranking.py\u001b[0m in \u001b[0;36mroc_curve\u001b[1;34m(y_true, y_score, pos_label, sample_weight, drop_intermediate)\u001b[0m\n\u001b[0;32m    769\u001b[0m     \"\"\"\n\u001b[0;32m    770\u001b[0m     fps, tps, thresholds = _binary_clf_curve(\n\u001b[1;32m--> 771\u001b[1;33m         y_true, y_score, pos_label=pos_label, sample_weight=sample_weight)\n\u001b[0m\u001b[0;32m    772\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    773\u001b[0m     \u001b[1;31m# Attempt to drop thresholds corresponding to points in between and\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_ranking.py\u001b[0m in \u001b[0;36m_binary_clf_curve\u001b[1;34m(y_true, y_score, pos_label, sample_weight)\u001b[0m\n\u001b[0;32m    536\u001b[0m         \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"{0} format is not supported\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_type\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    537\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 538\u001b[1;33m     \u001b[0mcheck_consistent_length\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_score\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    539\u001b[0m     \u001b[0my_true\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcolumn_or_1d\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m     \u001b[0my_score\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcolumn_or_1d\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_score\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36mcheck_consistent_length\u001b[1;34m(*arrays)\u001b[0m\n\u001b[0;32m    210\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0muniques\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    211\u001b[0m         raise ValueError(\"Found input variables with inconsistent numbers of\"\n\u001b[1;32m--> 212\u001b[1;33m                          \" samples: %r\" % [int(l) for l in lengths])\n\u001b[0m\u001b[0;32m    213\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    214\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Found input variables with inconsistent numbers of samples: [60000, 1]"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "y_scores = sgd_clf.decision_function([some_digit])\n",
    "fpr, tpr, thresholds = roc_curve(y_train_5, y_scores)\n",
    "\n",
    "def plot_roc_curve(fpr, tpr, label=None):\n",
    "    plt.plot(fpr, tpr, linewidth=2, label=label)\n",
    "    plt.plot([0,1], [0, 1], 'k--')\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}