KNN_InDepth.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fbf9cfb3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:95% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline \n",
    "%config Completer.use_jedi = False\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:95% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22b32c75",
   "metadata": {},
   "source": [
    "# Tasks \n",
    "\n",
    "- [x] Explain Simple and Optimized Implementations of the Algorithm Step by Step"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9afd2a40",
   "metadata": {},
   "source": [
    "## Create Dummy Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b9765d43",
   "metadata": {},
   "outputs": [],
   "source": [
    "from drawdata import draw_scatter\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import plotly\n",
    "import plotly.express as px\n",
    "import math\n",
    "plotly.io.renderers.default = 'iframe_connected'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7b96af77",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e2af41fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# draw_scatter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "457ced1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df = pd.read_clipboard(sep=',')\n",
    "df = pd.read_csv('dummy_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9fc5a0aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.to_csv('dummy_data.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7ca851c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b    644\n",
       "a    635\n",
       "d    613\n",
       "c    550\n",
       "Name: z, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.z.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1625534b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.sample(frac=1).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "951ffae3",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dummy = df.sample(frac=.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e02cfc99",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dummy = df[~df.index.isin(train_dummy.index.tolist())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "80c6095a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((2442, 3), (2198, 3), (244, 3))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape, train_dummy.shape, test_dummy.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e80d0271",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dummy.reset_index(inplace=True, drop=True)\n",
    "test_dummy.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a2fcec0",
   "metadata": {},
   "source": [
    "## KNN From Scratch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd3a8f7b",
   "metadata": {},
   "source": [
    "### Step1: Calculate the Distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1d433bb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import sqrt, pow\n",
    "\n",
    "# calculate minkowski distance\n",
    "def minkowski_distance(r1, r2, p):\n",
    "    distance = 0.0 \n",
    "    for i in range(len(r1)):\n",
    "        distance += abs((r1[i] - r2[i])**p)\n",
    "    return distance**(1/float(p))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0326349b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate the Euclidean distance between two vectors\n",
    "def euclidean_distance(row1, row2):\n",
    "    distance = 0.0\n",
    "    for i in range(len(row1)):\n",
    "        distance += (row1[i] - row2[i])**2\n",
    "    return sqrt(distance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9124cc62",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[635.37532424, 427.78460026],\n",
       "       [109.44707085, 203.86658622],\n",
       "       [411.23549566,   2.82374635],\n",
       "       [424.12776856, 250.00197592],\n",
       "       [434.08204763,  42.97226279]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['x', 'y']].head().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4cf9bfc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "row0 = df[['x', 'y']].head().values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4631a150",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n",
      "749.8462674253082\n",
      "649.1006824858641\n",
      "389.03018002663794\n",
      "586.1056140846025\n",
      "Wall time: 997 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for r in df[['x', 'y']].head().values:\n",
    "    distance = minkowski_distance(row0, r, 1)\n",
    "    print(distance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d9d6e2c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n",
      "749.8462674253082\n",
      "649.1006824858641\n",
      "389.03018002663794\n",
      "586.1056140846025\n",
      "Wall time: 998 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for r in df[['x', 'y']].head().values:\n",
    "    distance = np.linalg.norm(row0- r, ord=1)\n",
    "    print(distance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "df14802d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEGCAYAAACUzrmNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAfe0lEQVR4nO3dfZDd1X3f8fd3l9UKJDB6ggg9IDxSPBUuYHcLOOvxEBwbbBPwxHFHtWnVDq2mHWjsNjVIdYnroRkjkvGkU+xMFOxUqR+IDHakyG1sIsF4TMPDypawJCDIBcNKMsIq2BZBq3349o/7W3S1e3fv0+/8ns7nNaPZu7+9D1/dvfv7nnO+55yfuTsiIiKz6ck7ABERKT4lCxERaUrJQkREmlKyEBGRppQsRESkqbPyDqAbixcv9lWrVuUdhohIqezZs+dn7r6knceUOlmsWrWKoaGhvMMQESkVM/tJu4/RMJSIiDSlZCEiIk0pWYiISFNBk4WZvWBmPzKzvWY2lBxbaGYPmdlzydcFdfffZGaHzOxZM7suZGwiItK6LHoWv+7uV7j7QPL9RmCXu68BdiXfY2ZrgXXApcD1wBfNrDeD+EREpIk8hqFuArYmt7cCH647fr+7j7j788Ah4MrswxPp3vETI+x76TWOnxjJOxSRVISeOuvAd83MgT9x9y3Ahe5+FMDdj5rZBcl9lwGP1T12ODl2BjPbAGwAWLlyZcjYRTqyfe9h7njwKfp6ehidmOCej1zGjVdM+yiLlEronsWgu78T+ABwq5m9Z5b7WoNj0/ZPd/ct7j7g7gNLlrS1pkQkuOMnRrjjwac4OTrBL0fGODk6we0PPqUehpRe0GTh7keSr8eAb1EbVnrZzJYCJF+PJXcfBlbUPXw5cCRkfCJpG371Dfp6zvyz6uvpYfjVN3KKSCQdwZKFmc0zs3MnbwPvB/YDO4D1yd3WA9uT2zuAdWbWb2aXAGuAJ0LFJxLC8gVnMzoxccax0YkJli84O6eIRNIRsmdxIfB9M9tH7aT/bXf/a+Bu4H1m9hzwvuR73P0AsA04CPw1cKu7jweMTyR1i+b3c89HLmNuXw/n9p/F3L4e7vnIZSya3593aCJdsTJfVnVgYMC1N5QU0fETIwy/+gbLF5ytRCGFY2Z76pYztKTUGwmKFNWi+f1KElIp2u5DRESaij5ZaPFUuej3JZKPqIehtHiqXPT7kiKItR4VbbKoXzx1ktpUx9sffIrB1Yuj+gCUhX5fUgQxN1iiHYbS4qnTyjC0o9+X5C321fnR9iyKsHiqCN3ZsrSUivD7krhNNlgme7ZwusESQ+822p5F3ountu89zODm3dx83+MMbt7Njr2HM3ndeiFaSqF6KXn/viQdZejFziT2Bku0PQuAG69YxuDqxZm37osy/p52Syl0LyWv35eko9PPRxF64HC6wXL7lP9DLJ/DqJMF5LN4qijd2TRbSlklQC12K6dOPx+dJJiQySXmBku0w1B5Kkp3Ns2hnaoWoMs8bFIknXw+OhkmzWJ4d9H8fi5fcX5UiQLUs8hFkbqzabWUipIA01SW4n8ZdPL5aLcHXpTh3apSsshJkbqzaQztFCkBpkEnnnR18vloN8EUZXi3qpQsclS18fciJcBu6cQzs05rAu1+PtpNMFXs3RaJkoWkqioJUCeexrodmmv389FOgqla77ZodD0LkRns2Ht42okn5prF8RMjDG7ezcnR00l0bl8Pj95xbaFOyEWZaltkup6FSIqqNKyWhrIMzVWld1s0ShZSCnm1FnXiOU1Dc3HTOgspvCJsjVJGaa8R0ZYrcVPPQgpNU1g7E2qNiIbm4qWehRRaEVaGl20Vd+ittEOtYC7b+xwb9Syk0PIeJy/jKu6yFKLrlfF9jo16FlJoeY6Tl/ViN3kn2HaV9X2OjXoWUnh5jZOXsYUO5VucVtb3OTZKFlIKeUxhLVsLvV7WCbabqc1lfp9jomEokRmUfapoVltpdzu1uezv86SqF+i13YdIE9o+YmZpbgFS5ve5bAX6Trb7CN6zMLNeM/uhme1Mvl9oZg+Z2XPJ1wV1991kZofM7Fkzuy50bCKtiPViN60YfvUNfOLMBqdPeEdTmzt9n+tb9Hm07mMp0GdRs/gE8DRwXvL9RmCXu99tZhuT7+8ws7XAOuBS4CLgb8zsV919PIMYRaQD8+b0MjJ+ZrIYGXfmzenN5PXrW/Qnx8Zxd87uOyvT1n0sBfqgPQszWw58CLiv7vBNwNbk9lbgw3XH73f3EXd/HjgEXBkyPhHpzuunxpnbd+ZpZG5fD6+fmrmNl1brf2qLfnTcGZsg89Z9LAX60MNQfwTcDtS/kxe6+1GA5OsFyfFlwEt19xtOjp3BzDaY2ZCZDb3yyitBghaR1sx0QpzpeJr7fDVa3V8vq5X+VSnQNxNsGMrMbgCOufseM7umlYc0ODat+u7uW4AtUCtwdxpfmYtpIkXRzpqOtPf5atSir5dl6z6GPbNC1iwGgRvN7IPAXOA8M/sK8LKZLXX3o2a2FDiW3H8YWFH3+OXAkRCBlW3mgkiRtXqiTHtsf2qialSz0Hb26QmWLNx9E7AJIOlZ/Ed3v9nM/gBYD9ydfN2ePGQH8DUz+zy1Avca4Im049IupiLpa+VEGWJsf2qiAirdus9THiu47wa2mdktwIvARwHc/YCZbQMOAmPArSFmQsUyc0GkaEJtQzI1UenvOIxMkoW7PwI8ktw+Drx3hvv9PvD7IWOJZeaCSBHFMLZfVdFt9xHLzAWRom4/oUWO5RTlRoJq3UjVaRKHpC3KZAHVn7kg8dIkDgkhumEokaorwqVopXqULEQqptEkjpGx8cz2awqlqDWYWChZiFRM/SSO/t7axgg9PcYN936/q+018pTmNiHNKCk1pmQhUkE3XrGMnbe9G7dasjg5OlHarbOz3AI8y6RUNkoWIhX1+qlx+nvLX7vIqgbTSVKKqRcS7WwokaqrygLUrP4f7e7uENv0ZPUsRCqqKgtQs/p/tJOUYrk6Xj31LCQYbQOfv6osQM3i/9HO3lUx7jGnZCFBxNZFL7IqLEDNquHRalKqyhBfO5QsJHVaQSxpyrrh0UpyDbWDbpEpWUjqYuyiSxhFbnhUZYivVUoWkroYu+gSxmwNj8mf53mirsIQX6s0G0pSV5VZOLErwhqCmRoe+w//XIvnMmbunncMHRsYGPChoaG8w5AZaDZUeRVpgsKOvYfPqA3cecNa7tp5kJOjp5PI3L4eHr3jWn3OWmRme9x9oJ3HaBhKgompi14lRasTTK0NqCaWDyULETlDEU/GUxseqollTzULETlD0ScoqCaWD/UsROQMZVhDENu01SJQshCRaTo5GWc9oWHyNSan0SphhKVkISINtTNBIY/ZU0WasRUD1SxEpCt57MAa466veVOyEJG2TF2sl9XFierl8Zqx0zCUiLSs0dDP4OrFmc+eKvqMrSpSz0JEWjLT0A+Q+VTWPKbPFmH7kzwF61mY2Vzge0B/8joPuPtnzGwh8BfAKuAF4J+4+6vJYzYBtwDjwO+4+3dCxSci7ZltsV4eU1mzfE0V08P2LEaAa939cuAK4HozuxrYCOxy9zXAruR7zGwtsA64FLge+KKZ9QaMT0Ta0GzoZ9H8fi5fcX6mU1izeE0V02uCJQuvOZF825f8c+AmYGtyfCvw4eT2TcD97j7i7s8Dh4ArQ8UnIu2JdeW0iuk1QQvcSc9gD7Aa+IK7P25mF7r7UQB3P2pmFyR3XwY8Vvfw4eTY1OfcAGwAWLlyZcjwgwm1eEm7vEpoMa6cVjG9JmiycPdx4AozOx/4lpm9fZa7W6OnaPCcW4AtUNuiPI04sxRq7FNjqpKV2HYTLsP2J1nIZOqsu79mZo9Qq0W8bGZLk17FUuBYcrdhYEXdw5YDR7KILyuhtn4u2pbSIlUTY49qqmA1CzNbkvQoMLOzgd8AngF2AOuTu60Htie3dwDrzKzfzC4B1gBPhIovD6HGPjWmKhJeHgX8IgnZs1gKbE3qFj3ANnffaWZ/C2wzs1uAF4GPArj7ATPbBhwExoBbk2Gsygg19qkxVREJTZdVzdjUS0SmVVsI9bwiVRT7ZJBOLquqZJEDzYYSyY8mg+ga3KURajZJbLNUJA5pNoI0GaRzShYiKVLvLl1p9wKKeH3xslCyEEmJhjfSFaIX0O5kECX/07TrrEgKtH9Q+kJMCW9ny5Ltew8zuHk3N9/3OIObd7Nj7+GOX7cK1LMQSUHswxshWuChpoS3ssBOtY3plCxEUhDzWpdQw28ht9loNhkk9uTfiJKFSApi3T8odAs8r202Yk7+M1GyCEBFsTjFuH9QFi3w2XoBIf7WJp/zzg+t5a5vH4wq+c9GySJlmhETt9jWuuTZAg/xtzb1Oe+8YS1vv+gtDZNRbI1CzYZqQ7Nr8GpGjMQmrwsihfhba/Scd+082DAZxDhTSj2LFrXSilFRTMoizVZxHsNvIf7WWn3Omeo0a5eex+unxivb01CyaEGrRTwVxaQMQgzfZD38FuJvrdXnbJRUfML54H//Pv291R1+1jBUC1pdHBTrNYqlPKoyVNrJ31qzYeRWn7NRUhkZd06Nlfs9bUY9ixa004qJcUaMlEeVhkrb+VtrtTfVynNOnSY9MjZOT49xcrT87+lslCxa0O4c+thmxEh5VG2otJW/tXbXgrTynPVJZd6cXm649/tn/LzM7+lMlCxapB6DVEGMiwdD9abqk0oM76mSRRvUY5AqiK3hk0VvKob3VAVukQgtmt/P5SvOr+RJbaqsJp5U/T1Vz0JEKi+Gln9oShYiEgUNI3dHw1AiItJU02RhZreZ2YIsghERkWJqpWfxK8CTZrbNzK43MwsdlIhIGTRbFV4lTWsW7v6fzexO4P3AvwTuNbNtwJfc/cehAxQRKaLYLkfQUs3C3R34afJvDFgAPGBm9wSMTUSkkKqyx1Y7WqlZ/I6Z7QHuAR4F/qG7/1vgHwEfCRyfiEjhtLq5aJW00rNYDPyWu1/n7t9w91EAd58AbpjpQWa2wsweNrOnzeyAmX0iOb7QzB4ys+eSrwvqHrPJzA6Z2bNmdl2X/zcRkSCqtsdWK5omC3f/PXf/yQw/e3qWh44Bv+vu/wC4GrjVzNYCG4Fd7r4G2JV8T/KzdcClwPXAF82st53/jEhRxFT4jFGMlyMItijP3Y8CR5PbvzSzp4FlwE3ANcndtgKPAHckx+939xHgeTM7BFwJ/G2oGEVCiK3wGavYVoVnsijPzFYB7wAeBy5MEslkQrkgudsy4KW6hw0nx6Y+1wYzGzKzoVdeeSVo3CLtirHwGbOq7wdVL3iyMLP5wIPAJ939F7PdtcExn3bAfYu7D7j7wJIlS9IKUyQVMRY+JQ5Bk4WZ9VFLFF91928mh182s6XJz5cCx5Ljw8CKuocvB46EjE8kbTEWPiUOwZJFstL7S8DT7v75uh/tANYnt9cD2+uOrzOzfjO7BFgDPBEqPpEQYix8FokmFoQTctfZQeCfAT8ys73Jsf8E3A1sM7NbgBeBjwK4+4FkZfhBajOpbnX38YDxiQQRW+GzKDSxICyrLc4up4GBAR8aGso7jCgcPzGik58U1vETIwxu3s3J0dNDgHP7enj0jmv1eW3AzPa4+0A7j9H1LKQptdik6EJdZ1tO0/UsZFaaCiqhpVFn0MSC8JQsZFaaCiohbd97mMHNu7n5vscZ3LybHXsPd/Q8mlgQnoahZFZqsUko9b3WyeGj2x98isHVizs6yWtiQVjqWUSq1a6/WmwSSohea0wrqrOmnkWE2i1Yt9Ji02wpaZd6reWiZBGZTrv+i+b3z/hzzZaSTkz2Wm+f8tmpSmOjag0oJYvIpD3FMO1xZ4lLVesMVWxAqWYRmbS7/potVX2ht9CoWp2hqtPNlSwik3bBWuPO1ZbW1NaYVLUBpWGoCKXZ9a/6uHPMNMTYmao2oJQsIjVbwbpdVR13jp220OhMVRtQShaSijSTjxRDVVvIWahiA0o1CxFpSAsyu9Nq4b4s1+BQz0JEZlTFFnKRlGmKrZKFiMxKQ4xhlG0CgYahRERyULYptkoWIiI5KNsEAiULEWlbWYqyRVa2CQSqWYhIW8pUlC26Mk0gUM9CSk0t3GxVdd+jPJVlbyz1LKS01MLNnlZ1x0s9i4yoBZyutFq4+r20p2xFWUmPehYZUAs4fWm0cPV7aV9V9z2S5pQsAivbwpuy6LaFq99L5/IuylbtCnRloWQRmMZ4w+i2havfS3fyWtWt3mB+lCwCa9QCPjU+rjHeFHTTwtXYe/moN5ivYAVuM/uymR0zs/11xxaa2UNm9lzydUHdzzaZ2SEze9bMrgsVV9YmW8Bn1b3TEw6PHvpZfkFVSKfTDsu2IErKtz1G1YTsWfwP4F7gz+uObQR2ufvdZrYx+f4OM1sLrAMuBS4C/sbMftXdxwPGl5nB1Yvp7elhLGnJjo67WkQFkPfYu7RHvcF8BetZuPv3gP835fBNwNbk9lbgw3XH73f3EXd/HjgEXBkqtqwNv/oGc3rVIiqisiyIEvUG85Z1zeJCdz8K4O5HzeyC5Pgy4LG6+w0nxypBLSJJg2YBqTeYp6IUuK3BMW94R7MNwAaAlStXhowpNZqbLt3SLKDTdH2NfGSdLF42s6VJr2IpcCw5PgysqLvfcuBIoydw9y3AFoCBgYGGCaWI1CKSTmkWkBRB1tt97ADWJ7fXA9vrjq8zs34zuwRYAzyRcWzBTG4pAQQfH9f2FdWjWUBSBMF6Fmb2deAaYLGZDQOfAe4GtpnZLcCLwEcB3P2AmW0DDgJjwK1VmQmV5fCBhiqqSTUvKQJzL81IzjQDAwM+NDSUdxgzOn5ihMHNuzk5evoPfW5fD4/ecW3qvYuQr6XCav527D08realhoB0ysz2uPtAO48pSoG7krLcUiLUa6m3clqeSVM1L8mbkkVAWQ4fhHgtFVZPK0LS1CwgyZOuZxFQlouIQryWCqs1ujqcTBXjRBL1LALLcvgg7ddSYbWm2RCfajpxKUIvMw9KFhnIcvggzdfSYsKa2ZJmrCeOWMU8NKtkIdPUt5RVWJ05aQLRnjhiFfN1UJQsKqyT4ZGZWspV/0NoplHS3PfSa9GeOGIV89CskkVFdTI8EnMXuxVTh/iKfuJQLSV9MQ/NKllUUKcn/Zi72M3MdOK99ZrV3PvwIeb0FuvEoVpKOLEOzSpZVFCnJ/2it5Tz0ujE6/DmMXA2vOetfOyqlYU4caiHGF6Ma160zqKCOj3p6+Iy0zVaY/GpB57i9gf2vXlsZMz5wiOH8g71TVofIyGoZ1FB3YyrxtrFnkmjXlpvj4EbcHqvyyIN16mHKCEoWVRUNyf9GLvYM2l04h2fcKZem6tIJ+OYi7ASjpJFhemk372pJ95T4xPc9uurWThvDnd9+2BhT8bqIUratEW5SAuOnxjhq4+/yBcefo45vb2MTkxw5w1reftFb9HJWEqnky3KVeCWygm1ydsXHznEyJi/Wei+a+fBjhNFjBvRSXry+PxoGEoqJdT6gjTXoGgNhHQjr8+PehZSGSG3Ek9rhlHjqbj7+N7fHcull6EeTrnkuV2+koVURsj1BZ2uQZl6Mm4U48iY82++8gMGN+9mx97DXcfaqu17DzO4eTcf+9PHeNfndvHVx36S2WtLZ/JcQ6NhKKmM0OsL2p1h1Gi4YHD14mkxAvz9qdqajaxWWte3UCd9+i/3g8HHr7o46GtL5/JcQ6OehVRGiBXoU3sGi+b3c/mK81vqUTQabjpw5Ofc+aG1zO3r4Zy+3mmPa6WVmMbQ0fCrb9BrNu34Z//qoIakCizPXRbUs5BKSXN9QaeFxOMnRnj4mWPTTsaTw00T7tx5w1pWLDiHf/3nQ4yMnW4pNmslplXcXL7gbEbHp/dw+nqtMCvRpbG81tCoZyGF125LutXWf7PX7KSQOFkH+C9/dYDXT41P+/nfnxp/c9rtpRedxx/8duutxEMv/5JPfWNfKsXNRfP7+cxvXjrt+PiEF2Yluswsjc94u9SzkELLa5pgJ1NlG9UBAOae1cPJsTOPTT5Xq63E7XsP86kHnuLUuDd8nk5OGh+/+mKw2tBTX68xPuGFW4kuxaFkIYWV51bbnRQSGyWYef29fOr9b+Nz//uZGYebmm3LMvk+nBqbPmzUbXHz41ddzPWX/krTZKULKYmShRRWnhdj6mQzvkYJZmzc+c3LL2LhvDkdb+zX6H0AmNNrqfQEmiUrLSIUULKQAst7q+12C4mTCeZ3v7GP0WS4aHxigkcP/ayromSj92HOWT38r3/3blZfeG77/7E26EJKMkkFbimsIlyMqd1C4uDqxfTUTYIam+DNInSnRclG78Mf/vZlwRMF6EJKclrhehZmdj3w34Be4D53vzvnkCRHZdtqe/jVN5jT28vI2Nibx7odOjt+YoSLF81j523v5vVT45m+D3n37qQ4CpUszKwX+ALwPmAYeNLMdrj7wXwjkzyV6bocaZ9cG9ULLl9xfgqRtibGCympmN9YoZIFcCVwyN3/L4CZ3Q/cBChZSCnMdnJt9yRUlHpB2Xp33VAxf2ZFSxbLgJfqvh8Grqq/g5ltADYArFy5MrvIRFrU6OTayUkoz9lgU5Wpd9epoiTnoipagXv6ZjVTLnbs7lvcfcDdB5YsWZJRWCLtqS9md7oaXPWCbKmYP7uiJYthYEXd98uBIznFIpKKTk9CRZgNFhMl59kVbRjqSWCNmV0CHAbWAR/LNySR7nRzEoqpXpC3GIv57ShUsnD3MTO7DfgOtamzX3b3AzmHJdKVbk9CMdQLikLJeWbm7s3vVVADAwM+NDSUdxgiLdGUTCkKM9vj7gPtPKZQPQuRMmo1CaiHIGWmZCHSBc3Ll1gUbTaUSGl0OiVWpIyULEQ6pHn5EhMlC5EOaV6+xETJQqRDWjQnMVGBW6QLmpcvsVCyEOmSpsRKDDQMJSIiTSlZiIhIU0oWIpK54ydG2PfSa1qTUiKqWYhIprTqvZzUsxCRzGjVe3kpWYhIZrTqvbyULEQkM1r1Xl5KFiKSGa16Ly8VuEUkU1r1Xk5KFiKSOa16Lx8NQ4mISFNKFiIi0pSShYiINKVkISIiTSlZiIhIU+buecfQMTN7BfhJBi+1GPhZBq/TLsXVvqLGprjaV9TYyhDXxe6+pJ0HlzpZZMXMhtx9IO84plJc7StqbIqrfUWNrapxaRhKRESaUrIQEZGmlCxasyXvAGaguNpX1NgUV/uKGlsl41LNQkREmlLPQkREmlKyEBGRppQsZmFm15vZs2Z2yMw25vD6XzazY2a2v+7YQjN7yMyeS74uqPvZpiTWZ83suoBxrTCzh83saTM7YGafKEJsZjbXzJ4ws31JXJ8tQlx1r9VrZj80s50Fi+sFM/uRme01s6GixGZm55vZA2b2TPJZe1fecZnZ25L3afLfL8zsk3nHlbzOv08+9/vN7OvJ30N6cbm7/jX4B/QCPwbeCswB9gFrM47hPcA7gf11x+4BNia3NwKbk9trkxj7gUuS2HsDxbUUeGdy+1zg75LXzzU2wID5ye0+4HHg6rzjqovvPwBfA3YW5XeZvN4LwOIpx3KPDdgK/Kvk9hzg/CLEVRdfL/BT4OK84wKWAc8DZyffbwP+RZpxBXsjy/4PeBfwnbrvNwGbcohjFWcmi2eBpcntpcCzjeIDvgO8K6MYtwPvK1JswDnAD4CrihAXsBzYBVzL6WSRe1zJ87/A9GSRa2zAecnJz4oU15RY3g88WoS4qCWLl4CF1K5TtDOJL7W4NAw1s8k3f9JwcixvF7r7UYDk6wXJ8VziNbNVwDuoteJzjy0Z6tkLHAMecvdCxAX8EXA7UH8B6iLEBeDAd81sj5ltKEhsbwVeAf4sGbq7z8zmFSCueuuArye3c43L3Q8Dfwi8CBwFfu7u300zLiWLmVmDY0WeZ5x5vGY2H3gQ+KS7/2K2uzY4FiQ2dx939yuoteSvNLO35x2Xmd0AHHP3Pa0+pMGxkL/LQXd/J/AB4FYze88s980qtrOoDcH+sbu/A3id2jBK3nHVXsxsDnAj8I1md21wLMRnbAFwE7UhpYuAeWZ2c5pxKVnMbBhYUff9cuBITrHUe9nMlgIkX48lxzON18z6qCWKr7r7N4sUG4C7vwY8AlxfgLgGgRvN7AXgfuBaM/tKAeICwN2PJF+PAd8CrixAbMPAcNIzBHiAWvLIO65JHwB+4O4vJ9/nHddvAM+7+yvuPgp8E/i1NONSspjZk8AaM7skaUWsA3bkHBPUYlif3F5PrV4weXydmfWb2SXAGuCJEAGYmQFfAp52988XJTYzW2Jm5ye3z6b2B/RM3nG5+yZ3X+7uq6h9jna7+815xwVgZvPM7NzJ29TGuffnHZu7/xR4yczelhx6L3Aw77jq/FNOD0FNvn6ecb0IXG1m5yR/n+8Fnk41rpAFoLL/Az5IbabPj4FP5/D6X6c2/jhKrSVwC7CIWqH0ueTrwrr7fzqJ9VngAwHjeje1LutTwN7k3wfzjg24DPhhEtd+4PeS47m/Z3Wvdw2nC9y5x0WtNrAv+Xdg8nNekNiuAIaS3+dfAgsKEtc5wHHgLXXHihDXZ6k1jvYD/5PaTKfU4tJ2HyIi0pSGoUREpCklCxERaUrJQkREmlKyEBGRppQsRESkKSULERFpSslCRESaUrIQSZGZ/WMzeyq5lsC85PoCs+1PJVIKWpQnkjIz+6/AXOBsavsbfS7nkES6pmQhkrJkL7EngZPAr7n7eM4hiXRNw1Ai6VsIzKd2FcG5Occikgr1LERSZmY7qG1Ffgm1q5TdlnNIIl07K+8ARKrEzP45MObuXzOzXuD/mNm17r4779hEuqGehYiINKWahYiINKVkISIiTSlZiIhIU0oWIiLSlJKFiIg0pWQhIiJNKVmIiEhT/x+qWhFIihzN0AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.sample(100).plot.scatter('x', 'y');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2a001a2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe\n",
       "    scrolling=\"no\"\n",
       "    width=\"100%\"\n",
       "    height=\"545px\"\n",
       "    src=\"iframe_figures/figure_20.html\"\n",
       "    frameborder=\"0\"\n",
       "    allowfullscreen\n",
       "></iframe>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = px.scatter(df, x='x', y='y', color='z')\n",
    "fig"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a977257",
   "metadata": {},
   "source": [
    "### Step2: Get Nearest Neighbours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8086400e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# loacte similar neighbors\n",
    "def get_neighbors(train, test_row, k, p):\n",
    "    distances = []\n",
    "    for train_row in train.values:\n",
    "        dist = minkowski_distance(test_row[:-1], train_row[:-1], p)\n",
    "        distances.append((train_row, dist))\n",
    "    \n",
    "    distances.sort(key=lambda tup: tup[1])\n",
    "    neighbors = []\n",
    "    for i in range(k):\n",
    "        neighbors.append(distances[i][0])\n",
    "    return neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e71d423d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>148.407233</td>\n",
       "      <td>201.718932</td>\n",
       "      <td>c</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              x           y  z\n",
       "187  148.407233  201.718932  c"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dummy.sample(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a8daaf02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([385.2133816200196, 50.72923372522087, 'd'], dtype=object),\n",
       " array([381.6164658612987, 45.18528757843086, 'd'], dtype=object),\n",
       " array([387.5671433108168, 48.48194688235816, 'd'], dtype=object),\n",
       " array([388.0715888983176, 52.31334625473727, 'd'], dtype=object),\n",
       " array([377.8004505103036, 47.529381114710645, 'd'], dtype=object)]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neighbors = get_neighbors(train_dummy, test_dummy.iloc[160].values, 5, 2)\n",
    "neighbors"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5f0570e",
   "metadata": {},
   "source": [
    "### Step3: Make Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "fde8a391",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_classification(train, test_row, k, p):\n",
    "    neighbors = get_neighbors(train, test_row, k, p)\n",
    "    output_values = [row[-1] for row in neighbors]\n",
    "    prediction = max(set(output_values), key=output_values.count)\n",
    "    return prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8ccc6ad4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'d'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction = predict_classification(train_dummy, test_dummy.iloc[160].values, 5, 2)\n",
    "prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03753397",
   "metadata": {},
   "source": [
    "### Test Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "13ba9347",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 854 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "predicted_labels = []\n",
    "labels = []\n",
    "for test_row in test_dummy.values:\n",
    "    predicted_labels.append(predict_classification(train_dummy, test_row, 5, 2))\n",
    "    labels.append(test_row[-1])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "538b414a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(244, 244)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(predicted_labels), len(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d2387798",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9713114754098361"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum([True if x==y else False for x,y in zip(labels, predicted_labels)])/len(test_dummy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "6bb85318",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>548.893697</td>\n",
       "      <td>450.063398</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>309.288837</td>\n",
       "      <td>466.872668</td>\n",
       "      <td>b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>171.742823</td>\n",
       "      <td>165.825046</td>\n",
       "      <td>c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>33.528848</td>\n",
       "      <td>250.071816</td>\n",
       "      <td>c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>71.128670</td>\n",
       "      <td>452.495737</td>\n",
       "      <td>c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2193</th>\n",
       "      <td>21.404658</td>\n",
       "      <td>336.734646</td>\n",
       "      <td>c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2194</th>\n",
       "      <td>750.286876</td>\n",
       "      <td>225.061827</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2195</th>\n",
       "      <td>673.429533</td>\n",
       "      <td>205.252561</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2196</th>\n",
       "      <td>477.136028</td>\n",
       "      <td>24.548314</td>\n",
       "      <td>d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2197</th>\n",
       "      <td>645.415774</td>\n",
       "      <td>481.862782</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2198 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               x           y  z\n",
       "0     548.893697  450.063398  a\n",
       "1     309.288837  466.872668  b\n",
       "2     171.742823  165.825046  c\n",
       "3      33.528848  250.071816  c\n",
       "4      71.128670  452.495737  c\n",
       "...          ...         ... ..\n",
       "2193   21.404658  336.734646  c\n",
       "2194  750.286876  225.061827  a\n",
       "2195  673.429533  205.252561  a\n",
       "2196  477.136028   24.548314  d\n",
       "2197  645.415774  481.862782  a\n",
       "\n",
       "[2198 rows x 3 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dummy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d1d62da",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_labels = test_dummy.z.tolist()\n",
    "# test_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "6526c18c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:4308: SettingWithCopyWarning:\n",
      "\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test_dummy.drop(columns='z', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "b46b9c60",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>508.256193</td>\n",
       "      <td>9.972351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>263.178640</td>\n",
       "      <td>36.146965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-4.152166</td>\n",
       "      <td>175.740393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56.204038</td>\n",
       "      <td>255.740172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>41.424772</td>\n",
       "      <td>360.160187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>427.153861</td>\n",
       "      <td>17.740464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>666.516298</td>\n",
       "      <td>343.719778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>185.022133</td>\n",
       "      <td>481.429239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>447.106788</td>\n",
       "      <td>428.200658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>469.592212</td>\n",
       "      <td>194.226504</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>244 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              x           y\n",
       "0    508.256193    9.972351\n",
       "1    263.178640   36.146965\n",
       "2     -4.152166  175.740393\n",
       "3     56.204038  255.740172\n",
       "4     41.424772  360.160187\n",
       "..          ...         ...\n",
       "239  427.153861   17.740464\n",
       "240  666.516298  343.719778\n",
       "241  185.022133  481.429239\n",
       "242  447.106788  428.200658\n",
       "243  469.592212  194.226504\n",
       "\n",
       "[244 rows x 2 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dummy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7b3ba62",
   "metadata": {},
   "source": [
    "## Combine The Codes to a KNNClassifier Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "5ff172c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "class KNNClassifier():\n",
    "    \n",
    "    def __init__(self, metric_fn='eucledean_distance', norm=2):\n",
    "        self.metric_fn = metric_fn\n",
    "        self.norm = norm\n",
    "            \n",
    "    \n",
    "    def minkowski_distance(self, row1, row2, norm):\n",
    "        distance = 0.0 \n",
    "        for i in range(len(row1)):\n",
    "            distance += abs((row1[i] - row2[i])**norm)\n",
    "        return distance**(1/float(norm))\n",
    "    \n",
    "    def get_distance_metric(self, metric_fn):\n",
    "        self.distance_metrics = {'minkowski_distance': (self.minkowski_distance, self.norm),\n",
    "                                 'euclidean_distance': (self.minkowski_distance, 2),\n",
    "                                 'manhattan_distance': (self.minkowski_distance, 1),\n",
    "                                }\n",
    "        if self.metric_fn != 'custom_metric':                        \n",
    "            return self.distance_metrics[metric_fn]\n",
    "        else:\n",
    "            return (metric_fn, None)\n",
    "    \n",
    "    \n",
    "    def get_neighbors(self, k, train, test_row, distance_metric, norm):\n",
    "        \n",
    "        distances = []\n",
    "        for train_row in train.values:\n",
    "            dist = distance_metric(test_row, train_row[:-1], norm)\n",
    "            distances.append((train_row, dist))\n",
    "            \n",
    "        distances.sort(key=lambda tup: tup[1])\n",
    "        \n",
    "        neighbors = []\n",
    "        for i in range(k):\n",
    "            neighbors.append(distances[i][0])\n",
    "        \n",
    "        return neighbors\n",
    "    \n",
    "    def predict_instance(self, k, train, test_row):\n",
    "        distance_metric, norm = self.get_distance_metric(self.metric_fn)\n",
    "        self.neighbors = self.get_neighbors(k, train, test_row, distance_metric, norm)\n",
    "        output_values = [row[-1] for row in self.neighbors]\n",
    "        self.prediction = max(set(output_values), key=output_values.count)\n",
    "        return self.prediction\n",
    "    \n",
    "    def predict_batch(self, k, train, test):\n",
    "        self.predicted_labels = []\n",
    "        for test_row in test.values:\n",
    "            self.predicted_labels.append(self.predict_instance(k, train, test_row))\n",
    "        return self.predicted_labels\n",
    "    \n",
    "    def learning_metrics(self, test_labels):\n",
    "        self.misclassified_instances = [True if x==y else False for x,y in zip(test_labels, self.predicted_labels)]\n",
    "        self.accuracy = sum(self.misclassified_instances)/len(test_labels)\n",
    "        self.error_rate = 1 - self.accuracy\n",
    "        self.misclassified_instances_indices =  [i for i, x in enumerate(self.misclassified_instances) if not x]\n",
    "        return {'accuracy': self.accuracy, 'error_rate':self.error_rate}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "13238cd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "knnclassifier = KNNClassifier(metric_fn='euclidean_distance')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "d02e28d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([577.09984428, 317.56038476])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dummy.iloc[100].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "4b5b8f42",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knnclassifier.predict_instance(5, train_dummy, test_dummy.iloc[100].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "08700550",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "predicted_labels = knnclassifier.predict_batch(5, train_dummy, test_dummy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "9d4eb22d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.9713114754098361, 'error_rate': 0.02868852459016391}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knnclassifier.learning_metrics(test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "130ac77f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[33, 38, 40, 43, 139, 193, 230]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knnclassifier.misclassified_instances_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "5733e866",
   "metadata": {},
   "outputs": [],
   "source": [
    "knnclassifier = KNNClassifier(metric_fn='minkowski_distance', norm=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "8cf16bdd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([577.09984428, 317.56038476]), 'a')"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dummy.iloc[100].values, test_labels[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "144c6c29",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a'"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knnclassifier.predict_instance(10, train_dummy, test_dummy.iloc[100].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "f9872116",
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_labels = knnclassifier.predict_batch(10, train_dummy, test_dummy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "e697e214",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.9836065573770492, 'error_rate': 0.016393442622950838}"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knnclassifier.learning_metrics(test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "2dab5b36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.016393442622950838"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knnclassifier.error_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "e1162ad3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9836065573770492"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knnclassifier.accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "f6ae6f9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[40, 43, 139, 193]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knnclassifier.misclassified_instances_indices"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6734267",
   "metadata": {},
   "source": [
    "## Apply to Real Word Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "9769ea2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "2767c47a",
   "metadata": {},
   "outputs": [],
   "source": [
    "iris = datasets.load_iris()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "e271d1af",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data=iris.data, columns=iris.feature_names)\n",
    "df['target'] = iris.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "71b9d836",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal length (cm)</th>\n",
       "      <th>sepal width (cm)</th>\n",
       "      <th>petal length (cm)</th>\n",
       "      <th>petal width (cm)</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>6.7</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.2</td>\n",
       "      <td>2.3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>6.3</td>\n",
       "      <td>2.5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>6.2</td>\n",
       "      <td>3.4</td>\n",
       "      <td>5.4</td>\n",
       "      <td>2.3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>5.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.1</td>\n",
       "      <td>1.8</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>150 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)  \\\n",
       "0                  5.1               3.5                1.4               0.2   \n",
       "1                  4.9               3.0                1.4               0.2   \n",
       "2                  4.7               3.2                1.3               0.2   \n",
       "3                  4.6               3.1                1.5               0.2   \n",
       "4                  5.0               3.6                1.4               0.2   \n",
       "..                 ...               ...                ...               ...   \n",
       "145                6.7               3.0                5.2               2.3   \n",
       "146                6.3               2.5                5.0               1.9   \n",
       "147                6.5               3.0                5.2               2.0   \n",
       "148                6.2               3.4                5.4               2.3   \n",
       "149                5.9               3.0                5.1               1.8   \n",
       "\n",
       "     target  \n",
       "0         0  \n",
       "1         0  \n",
       "2         0  \n",
       "3         0  \n",
       "4         0  \n",
       "..      ...  \n",
       "145       2  \n",
       "146       2  \n",
       "147       2  \n",
       "148       2  \n",
       "149       2  \n",
       "\n",
       "[150 rows x 5 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "f6d49594",
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffle data\n",
    "df = df.sample(frac=1).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "ad7bd7d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "25b5cf21",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = df.sample(frac=0.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "63166b17",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = df[~df.index.isin(train.index)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "16c7b129",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((45, 5), (105, 5), (150, 5))"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.shape, train.shape, df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "8104f12d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train.reset_index(drop=True, inplace=True)\n",
    "test.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "1c55466c",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_targets = test.target.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "d9f46b15",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:4308: SettingWithCopyWarning:\n",
      "\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test.drop(columns='target', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "0afc5b67",
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "1f2976a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_standardized = pd.DataFrame(scaler.fit_transform(train.iloc[:,:-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "3b16c456",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_standardized['target'] = train.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "9f080f48",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_standardized = pd.DataFrame(scaler.transform(test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "f2de637e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.647808</td>\n",
       "      <td>0.398829</td>\n",
       "      <td>0.394246</td>\n",
       "      <td>0.390260</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.018825</td>\n",
       "      <td>0.156765</td>\n",
       "      <td>0.509554</td>\n",
       "      <td>0.390260</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.330951</td>\n",
       "      <td>0.882957</td>\n",
       "      <td>-1.277730</td>\n",
       "      <td>-1.356313</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.771480</td>\n",
       "      <td>-0.085299</td>\n",
       "      <td>0.797826</td>\n",
       "      <td>1.062019</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.018825</td>\n",
       "      <td>0.640893</td>\n",
       "      <td>1.086098</td>\n",
       "      <td>1.196370</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3  target\n",
       "0  0.647808  0.398829  0.394246  0.390260       1\n",
       "1  1.018825  0.156765  0.509554  0.390260       1\n",
       "2 -1.330951  0.882957 -1.277730 -1.356313       0\n",
       "3  0.771480 -0.085299  0.797826  1.062019       2\n",
       "4  1.018825  0.640893  1.086098  1.196370       2"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_standardized.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "d761cce4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.884532</td>\n",
       "      <td>-0.569427</td>\n",
       "      <td>1.316715</td>\n",
       "      <td>0.927667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.276791</td>\n",
       "      <td>-0.085299</td>\n",
       "      <td>0.624863</td>\n",
       "      <td>0.793315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.341571</td>\n",
       "      <td>-0.085299</td>\n",
       "      <td>0.163628</td>\n",
       "      <td>0.121556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.959934</td>\n",
       "      <td>1.125021</td>\n",
       "      <td>-1.393038</td>\n",
       "      <td>-1.221961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.217899</td>\n",
       "      <td>-1.295619</td>\n",
       "      <td>0.682517</td>\n",
       "      <td>1.062019</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3\n",
       "0  1.884532 -0.569427  1.316715  0.927667\n",
       "1  0.276791 -0.085299  0.624863  0.793315\n",
       "2 -0.341571 -0.085299  0.163628  0.121556\n",
       "3 -0.959934  1.125021 -1.393038 -1.221961\n",
       "4 -0.217899 -1.295619  0.682517  1.062019"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_standardized.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "fbb5329b",
   "metadata": {},
   "outputs": [],
   "source": [
    "knnclassifier = KNNClassifier(metric_fn='minkowski_distance', norm=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "fc874341",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0   -0.959934\n",
       " 1    1.609149\n",
       " 2   -1.335384\n",
       " 3   -1.087609\n",
       " Name: 10, dtype: float64,\n",
       " 0)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_standardized.iloc[10], test_targets[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "511c2915",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knnclassifier.predict_instance(10, train_standardized, test_standardized.iloc[10].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "16dc4c9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_classes = knnclassifier.predict_batch(10, train_standardized, test_standardized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "8da48bdc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 1.0, 'error_rate': 0.0}"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knnclassifier.learning_metrics(test_labels=test_targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "9a421b97",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knnclassifier.misclassified_instances_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "43801ce5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-72-fc4806014102>:1: SettingWithCopyWarning:\n",
      "\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test['target'] = test_targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "573a685c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)',\n",
       "       'petal width (cm)', 'target'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "f788f96c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe\n",
       "    scrolling=\"no\"\n",
       "    width=\"100%\"\n",
       "    height=\"545px\"\n",
       "    src=\"iframe_figures/figure_74.html\"\n",
       "    frameborder=\"0\"\n",
       "    allowfullscreen\n",
       "></iframe>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = px.scatter_3d(test, x='sepal length (cm)', y='sepal width (cm)', z='petal length (cm)', color='target')\n",
    "fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "86c3d3ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-75-19e34d8cdd89>:1: SettingWithCopyWarning:\n",
      "\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test['predicted_targets'] = predicted_classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "55f6669a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe\n",
       "    scrolling=\"no\"\n",
       "    width=\"100%\"\n",
       "    height=\"545px\"\n",
       "    src=\"iframe_figures/figure_76.html\"\n",
       "    frameborder=\"0\"\n",
       "    allowfullscreen\n",
       "></iframe>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = px.scatter_3d(test, x='sepal length (cm)', y='sepal width (cm)', z='petal length (cm)', color='predicted_targets')\n",
    "fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "297f2977",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e4c5fbb9",
   "metadata": {},
   "source": [
    "## Compare results to Sklearn KNN Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "fa52579f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "9e4bb597",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = KNeighborsClassifier(n_neighbors=10, p=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "4f34b267",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(n_neighbors=10)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(train_standardized.iloc[:,:-1], train_standardized.iloc[:,-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "d4346ab8",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_test = clf.predict(test_standardized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "d97dee16",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 2, 1, 0, 2, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 2, 1, 0, 2, 2, 1, 1,\n",
       "       0, 2, 2, 1, 1, 0, 0, 1, 2, 1, 0, 1, 1, 2, 0, 1, 1, 2, 0, 0, 2, 2,\n",
       "       0])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "56f10e9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "14426d08",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(test_targets, y_pred_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cfe30df8",
   "metadata": {},
   "source": [
    "## Effect of Neigbors on Prediction Accuracy on Iris Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "9d6969d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracies = []\n",
    "clf = KNNClassifier(metric_fn='euclidean_distance')\n",
    "\n",
    "for k in range(1,100):\n",
    "    _ = clf.predict_batch(k=k, train=train_standardized, test=test_standardized)\n",
    "    accuracies.append((k, clf.learning_metrics(test_targets)['accuracy']))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "3be40720",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k</th>\n",
       "      <th>accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.933333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.977778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.977778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   k  accuracy\n",
       "0  1  0.933333\n",
       "1  2  0.977778\n",
       "2  3  0.977778\n",
       "3  4  1.000000\n",
       "4  5  1.000000"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc = pd.DataFrame(accuracies, columns=['k', 'accuracy'])\n",
    "acc.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "884a368d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe\n",
       "    scrolling=\"no\"\n",
       "    width=\"100%\"\n",
       "    height=\"545px\"\n",
       "    src=\"iframe_figures/figure_86.html\"\n",
       "    frameborder=\"0\"\n",
       "    allowfullscreen\n",
       "></iframe>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = px.line(data_frame=acc, x='k', y='accuracy')\n",
    "fig"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed8a67a6",
   "metadata": {},
   "source": [
    "## Custom KNN Classifier Implementation Using Numpy Optimizations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "56bdad0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_excel('../datasets/diabetes.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "0f49f067",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>148</td>\n",
       "      <td>72</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>33.6</td>\n",
       "      <td>0.627</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "      <td>66</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.351</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>183</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>23.3</td>\n",
       "      <td>0.672</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>89</td>\n",
       "      <td>66</td>\n",
       "      <td>23</td>\n",
       "      <td>94</td>\n",
       "      <td>28.1</td>\n",
       "      <td>0.167</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>137</td>\n",
       "      <td>40</td>\n",
       "      <td>35</td>\n",
       "      <td>168</td>\n",
       "      <td>43.1</td>\n",
       "      <td>2.288</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \\\n",
       "0            6      148             72             35        0  33.6   \n",
       "1            1       85             66             29        0  26.6   \n",
       "2            8      183             64              0        0  23.3   \n",
       "3            1       89             66             23       94  28.1   \n",
       "4            0      137             40             35      168  43.1   \n",
       "\n",
       "   DiabetesPedigreeFunction  Age  Outcome  \n",
       "0                     0.627   50        1  \n",
       "1                     0.351   31        0  \n",
       "2                     0.672   32        1  \n",
       "3                     0.167   21        0  \n",
       "4                     2.288   33        1  "
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "d0d37b4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from scipy.stats import mode\n",
    "from sklearn.neighbors import KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "c0ae795a",
   "metadata": {},
   "outputs": [],
   "source": [
    "class KNNClassifier2():\n",
    "    def __init__(self, k, norm=2):\n",
    "        self.k = k\n",
    "        self.norm = norm\n",
    "        \n",
    "    def fit(self, X_train, y_train):\n",
    "        '''just store the training data'''\n",
    "        self.X_train = X_train\n",
    "        self.y_train = y_train\n",
    "        # no. of examples, no. of features\n",
    "        self.m, self.n = self.X_train.shape\n",
    "        \n",
    "    def predict(self, X_test):\n",
    "        self.X_test = X_test\n",
    "        self.m_test, self.n_test = self.X_test.shape\n",
    "        \n",
    "        y_predict = np.zeros(self.m_test)\n",
    "        \n",
    "        for i in range(self.m_test):\n",
    "            x = self.X_test[i]\n",
    "            \n",
    "            neighbors = np.zeros(self.k)\n",
    "            \n",
    "            neighbors = self.find_neighbors(x)\n",
    "            \n",
    "            y_predict[i] = mode(neighbors)[0][0]\n",
    "        \n",
    "        return y_predict\n",
    "    \n",
    "    def find_neighbors(self, x):\n",
    "        distances = np.zeros(self.m)\n",
    "        \n",
    "        for i in range(self.m):\n",
    "            d = self.distance(x, self.X_train[i])\n",
    "            distances[i] = d\n",
    "        \n",
    "        # sort Y_train according to euclidean_distance_array and \n",
    "        # store into Y_train_sorted\n",
    "        \n",
    "        inds = distances.argsort()\n",
    "        y_train_sorted = self.y_train[inds]\n",
    "        \n",
    "        return y_train_sorted[:self.k]\n",
    "    \n",
    "    def distance(self, x, x_train):\n",
    "        return np.linalg.norm(x-x_train, ord=self.norm)\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "8d963090",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.iloc[:, :-1].values\n",
    "y = df.iloc[:, -1].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "99facd5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=1/3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "1b38d2bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_1 = np.concatenate((X_train, y_train.reshape(-1,1)), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "a6954efe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13.0</td>\n",
       "      <td>145.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>22.2</td>\n",
       "      <td>0.245</td>\n",
       "      <td>57.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.0</td>\n",
       "      <td>180.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.271</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.0</td>\n",
       "      <td>116.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>25.6</td>\n",
       "      <td>0.201</td>\n",
       "      <td>30.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>300.0</td>\n",
       "      <td>33.7</td>\n",
       "      <td>0.467</td>\n",
       "      <td>58.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>29.5</td>\n",
       "      <td>0.287</td>\n",
       "      <td>37.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0      1     2     3      4     5      6     7    8\n",
       "0  13.0  145.0  82.0  19.0  110.0  22.2  0.245  57.0  0.0\n",
       "1   3.0  180.0  64.0  25.0   70.0  34.0  0.271  26.0  0.0\n",
       "2   5.0  116.0  74.0   0.0    0.0  25.6  0.201  30.0  0.0\n",
       "3   8.0  176.0  90.0  34.0  300.0  33.7  0.467  58.0  1.0\n",
       "4   4.0  144.0  58.0  28.0  140.0  29.5  0.287  37.0  0.0"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_df = pd.DataFrame(X_train_1)\n",
    "X_train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "db6c215d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>37.7</td>\n",
       "      <td>0.370</td>\n",
       "      <td>27.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>37.6</td>\n",
       "      <td>0.191</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>137.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>168.0</td>\n",
       "      <td>43.1</td>\n",
       "      <td>2.288</td>\n",
       "      <td>33.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24.6</td>\n",
       "      <td>0.237</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.587</td>\n",
       "      <td>36.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0      1     2     3      4     5      6     7\n",
       "0  5.0   95.0  72.0  33.0    0.0  37.7  0.370  27.0\n",
       "1  4.0  110.0  92.0   0.0    0.0  37.6  0.191  30.0\n",
       "2  0.0  137.0  40.0  35.0  168.0  43.1  2.288  33.0\n",
       "3  0.0  101.0  65.0  28.0    0.0  24.6  0.237  22.0\n",
       "4  5.0   44.0  62.0   0.0    0.0  25.0  0.587  36.0"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_df = pd.DataFrame(X_test)\n",
    "X_test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "b77a39bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "0b30a07a",
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "c78d8e93",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_df_standardized = pd.DataFrame(scaler.fit_transform(X_train_df.iloc[:,:-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "1aca6e5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_df_standardized['targets'] = X_train_df.iloc[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "a0377352",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>targets</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.479882</td>\n",
       "      <td>0.881615</td>\n",
       "      <td>0.644672</td>\n",
       "      <td>-0.149411</td>\n",
       "      <td>0.146128</td>\n",
       "      <td>-0.987948</td>\n",
       "      <td>-0.730291</td>\n",
       "      <td>2.147907</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.331953</td>\n",
       "      <td>1.982541</td>\n",
       "      <td>-0.207005</td>\n",
       "      <td>0.219421</td>\n",
       "      <td>-0.147966</td>\n",
       "      <td>0.313386</td>\n",
       "      <td>-0.642131</td>\n",
       "      <td>-0.650767</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.230414</td>\n",
       "      <td>-0.030581</td>\n",
       "      <td>0.266149</td>\n",
       "      <td>-1.317378</td>\n",
       "      <td>-0.662631</td>\n",
       "      <td>-0.612988</td>\n",
       "      <td>-0.879486</td>\n",
       "      <td>-0.289648</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.073965</td>\n",
       "      <td>1.856721</td>\n",
       "      <td>1.023195</td>\n",
       "      <td>0.772669</td>\n",
       "      <td>1.543076</td>\n",
       "      <td>0.280302</td>\n",
       "      <td>0.022464</td>\n",
       "      <td>2.238187</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.050769</td>\n",
       "      <td>0.850160</td>\n",
       "      <td>-0.490897</td>\n",
       "      <td>0.403837</td>\n",
       "      <td>0.366699</td>\n",
       "      <td>-0.182885</td>\n",
       "      <td>-0.587878</td>\n",
       "      <td>0.342311</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6  \\\n",
       "0  2.479882  0.881615  0.644672 -0.149411  0.146128 -0.987948 -0.730291   \n",
       "1 -0.331953  1.982541 -0.207005  0.219421 -0.147966  0.313386 -0.642131   \n",
       "2  0.230414 -0.030581  0.266149 -1.317378 -0.662631 -0.612988 -0.879486   \n",
       "3  1.073965  1.856721  1.023195  0.772669  1.543076  0.280302  0.022464   \n",
       "4 -0.050769  0.850160 -0.490897  0.403837  0.366699 -0.182885 -0.587878   \n",
       "\n",
       "          7  targets  \n",
       "0  2.147907      0.0  \n",
       "1 -0.650767      0.0  \n",
       "2 -0.289648      0.0  \n",
       "3  2.238187      1.0  \n",
       "4  0.342311      0.0  "
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_df_standardized.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "c92e3a2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_df_standardized = pd.DataFrame(scaler.transform(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "c73a9b41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((72, 9), (36, 8), (72,), (36,))"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_df_standardized.shape, X_test_df_standardized.shape, y_train.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "622394bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "model1 = KNNClassifier(metric_fn='euclidean_distance')\n",
    "model2 = KNNClassifier2(k=5, norm=2)\n",
    "model3 = KNeighborsClassifier(n_neighbors=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "d3497ed0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier()"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model2.fit(X_train, y_train)\n",
    "model3.fit(X_train_df_standardized.iloc[:, :-1], y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "23ab99f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred1 = model1.predict_batch(5, X_train_df_standardized, X_test_df_standardized)\n",
    "y_pred2 = model2.predict(X_test)\n",
    "y_pred3 = model3.predict(X_test_df_standardized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "41c1abc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "acc1 = accuracy_score(y_test, y_pred1)\n",
    "acc2 = accuracy_score(y_test, y_pred2)\n",
    "acc3 = accuracy_score(y_test, y_pred3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "9a290f10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.6388888888888888, 0.6388888888888888, 0.6388888888888888)"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc1, acc2, acc3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "abe60ff9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "f2a5c7c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.68      0.83      0.75        23\n",
      "           1       0.50      0.31      0.38        13\n",
      "\n",
      "    accuracy                           0.64        36\n",
      "   macro avg       0.59      0.57      0.56        36\n",
      "weighted avg       0.61      0.64      0.61        36\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, y_pred1))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "246a9e60",
   "metadata": {},
   "source": [
    "## Custom KNN Regressor Implementation Using Numpy Optimizations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "af0b3964",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = datasets.load_diabetes()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "3ff0d563",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data.data, columns=data.feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "6f9818fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['target'] = data.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "a7d7d210",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>bmi</th>\n",
       "      <th>bp</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>s3</th>\n",
       "      <th>s4</th>\n",
       "      <th>s5</th>\n",
       "      <th>s6</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.038076</td>\n",
       "      <td>0.050680</td>\n",
       "      <td>0.061696</td>\n",
       "      <td>0.021872</td>\n",
       "      <td>-0.044223</td>\n",
       "      <td>-0.034821</td>\n",
       "      <td>-0.043401</td>\n",
       "      <td>-0.002592</td>\n",
       "      <td>0.019908</td>\n",
       "      <td>-0.017646</td>\n",
       "      <td>151.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.001882</td>\n",
       "      <td>-0.044642</td>\n",
       "      <td>-0.051474</td>\n",
       "      <td>-0.026328</td>\n",
       "      <td>-0.008449</td>\n",
       "      <td>-0.019163</td>\n",
       "      <td>0.074412</td>\n",
       "      <td>-0.039493</td>\n",
       "      <td>-0.068330</td>\n",
       "      <td>-0.092204</td>\n",
       "      <td>75.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.085299</td>\n",
       "      <td>0.050680</td>\n",
       "      <td>0.044451</td>\n",
       "      <td>-0.005671</td>\n",
       "      <td>-0.045599</td>\n",
       "      <td>-0.034194</td>\n",
       "      <td>-0.032356</td>\n",
       "      <td>-0.002592</td>\n",
       "      <td>0.002864</td>\n",
       "      <td>-0.025930</td>\n",
       "      <td>141.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.089063</td>\n",
       "      <td>-0.044642</td>\n",
       "      <td>-0.011595</td>\n",
       "      <td>-0.036656</td>\n",
       "      <td>0.012191</td>\n",
       "      <td>0.024991</td>\n",
       "      <td>-0.036038</td>\n",
       "      <td>0.034309</td>\n",
       "      <td>0.022692</td>\n",
       "      <td>-0.009362</td>\n",
       "      <td>206.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.005383</td>\n",
       "      <td>-0.044642</td>\n",
       "      <td>-0.036385</td>\n",
       "      <td>0.021872</td>\n",
       "      <td>0.003935</td>\n",
       "      <td>0.015596</td>\n",
       "      <td>0.008142</td>\n",
       "      <td>-0.002592</td>\n",
       "      <td>-0.031991</td>\n",
       "      <td>-0.046641</td>\n",
       "      <td>135.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>437</th>\n",
       "      <td>0.041708</td>\n",
       "      <td>0.050680</td>\n",
       "      <td>0.019662</td>\n",
       "      <td>0.059744</td>\n",
       "      <td>-0.005697</td>\n",
       "      <td>-0.002566</td>\n",
       "      <td>-0.028674</td>\n",
       "      <td>-0.002592</td>\n",
       "      <td>0.031193</td>\n",
       "      <td>0.007207</td>\n",
       "      <td>178.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>438</th>\n",
       "      <td>-0.005515</td>\n",
       "      <td>0.050680</td>\n",
       "      <td>-0.015906</td>\n",
       "      <td>-0.067642</td>\n",
       "      <td>0.049341</td>\n",
       "      <td>0.079165</td>\n",
       "      <td>-0.028674</td>\n",
       "      <td>0.034309</td>\n",
       "      <td>-0.018118</td>\n",
       "      <td>0.044485</td>\n",
       "      <td>104.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>439</th>\n",
       "      <td>0.041708</td>\n",
       "      <td>0.050680</td>\n",
       "      <td>-0.015906</td>\n",
       "      <td>0.017282</td>\n",
       "      <td>-0.037344</td>\n",
       "      <td>-0.013840</td>\n",
       "      <td>-0.024993</td>\n",
       "      <td>-0.011080</td>\n",
       "      <td>-0.046879</td>\n",
       "      <td>0.015491</td>\n",
       "      <td>132.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>440</th>\n",
       "      <td>-0.045472</td>\n",
       "      <td>-0.044642</td>\n",
       "      <td>0.039062</td>\n",
       "      <td>0.001215</td>\n",
       "      <td>0.016318</td>\n",
       "      <td>0.015283</td>\n",
       "      <td>-0.028674</td>\n",
       "      <td>0.026560</td>\n",
       "      <td>0.044528</td>\n",
       "      <td>-0.025930</td>\n",
       "      <td>220.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>441</th>\n",
       "      <td>-0.045472</td>\n",
       "      <td>-0.044642</td>\n",
       "      <td>-0.073030</td>\n",
       "      <td>-0.081414</td>\n",
       "      <td>0.083740</td>\n",
       "      <td>0.027809</td>\n",
       "      <td>0.173816</td>\n",
       "      <td>-0.039493</td>\n",
       "      <td>-0.004220</td>\n",
       "      <td>0.003064</td>\n",
       "      <td>57.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>442 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          age       sex       bmi        bp        s1        s2        s3  \\\n",
       "0    0.038076  0.050680  0.061696  0.021872 -0.044223 -0.034821 -0.043401   \n",
       "1   -0.001882 -0.044642 -0.051474 -0.026328 -0.008449 -0.019163  0.074412   \n",
       "2    0.085299  0.050680  0.044451 -0.005671 -0.045599 -0.034194 -0.032356   \n",
       "3   -0.089063 -0.044642 -0.011595 -0.036656  0.012191  0.024991 -0.036038   \n",
       "4    0.005383 -0.044642 -0.036385  0.021872  0.003935  0.015596  0.008142   \n",
       "..        ...       ...       ...       ...       ...       ...       ...   \n",
       "437  0.041708  0.050680  0.019662  0.059744 -0.005697 -0.002566 -0.028674   \n",
       "438 -0.005515  0.050680 -0.015906 -0.067642  0.049341  0.079165 -0.028674   \n",
       "439  0.041708  0.050680 -0.015906  0.017282 -0.037344 -0.013840 -0.024993   \n",
       "440 -0.045472 -0.044642  0.039062  0.001215  0.016318  0.015283 -0.028674   \n",
       "441 -0.045472 -0.044642 -0.073030 -0.081414  0.083740  0.027809  0.173816   \n",
       "\n",
       "           s4        s5        s6  target  \n",
       "0   -0.002592  0.019908 -0.017646   151.0  \n",
       "1   -0.039493 -0.068330 -0.092204    75.0  \n",
       "2   -0.002592  0.002864 -0.025930   141.0  \n",
       "3    0.034309  0.022692 -0.009362   206.0  \n",
       "4   -0.002592 -0.031991 -0.046641   135.0  \n",
       "..        ...       ...       ...     ...  \n",
       "437 -0.002592  0.031193  0.007207   178.0  \n",
       "438  0.034309 -0.018118  0.044485   104.0  \n",
       "439 -0.011080 -0.046879  0.015491   132.0  \n",
       "440  0.026560  0.044528 -0.025930   220.0  \n",
       "441 -0.039493 -0.004220  0.003064    57.0  \n",
       "\n",
       "[442 rows x 11 columns]"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "fc208dc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "class KNNRegressor():\n",
    "    def __init__(self, k, norm=2):\n",
    "        self.k = k\n",
    "        self.norm = norm\n",
    "        \n",
    "    def fit(self, X_train, y_train):\n",
    "        '''just store the training data'''\n",
    "        self.X_train = X_train\n",
    "        self.y_train = y_train\n",
    "        # no. of examples, no. of features\n",
    "        self.m, self.n = self.X_train.shape\n",
    "    \n",
    "    def find_neighbors(self, x):\n",
    "        distances = np.zeros(self.m)\n",
    "        \n",
    "        for i in range(self.m):\n",
    "            d = self.distance(x, self.X_train[i])\n",
    "            distances[i] = d\n",
    "        \n",
    "        # sort Y_train according to euclidean_distance_array and \n",
    "        # store into Y_train_sorted\n",
    "        \n",
    "        inds = distances.argsort()\n",
    "        y_train_sorted = self.y_train[inds]\n",
    "        \n",
    "        return y_train_sorted[:self.k]\n",
    "    \n",
    "    def distance(self, x, x_train):\n",
    "        return np.linalg.norm(x-x_train, ord=self.norm)\n",
    "    \n",
    "    def predict(self, x_test):\n",
    "        self.x_test = x_test\n",
    "        self.m_test, self.n_test = self.x_test.shape\n",
    "        \n",
    "        y_predict = np.zeros(self.m_test)\n",
    "        \n",
    "        for i in range(self.m_test):\n",
    "            x = self.x_test[i]\n",
    "            \n",
    "            neighbors = np.zeros(k)\n",
    "            neighbors = self.find_neighbors(x)\n",
    "            \n",
    "            y_predict[i] = np.mean(neighbors)\n",
    "            \n",
    "        return y_predict\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "d6b84ed6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "a238250c",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.iloc[:, :-1].values\n",
    "y = df.iloc[:, -1].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "c77fcfc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=1/3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "228a6f57",
   "metadata": {},
   "outputs": [],
   "source": [
    "model1 = KNNRegressor(k=5, norm=2)\n",
    "model2 = KNeighborsRegressor(n_neighbors=5, p=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "a0899f69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsRegressor()"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1.fit(X_train, y_train)\n",
    "model2.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "a5a7176c",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred1 = model1.predict(X_test)\n",
    "y_pred2 = model2.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "d19a5a42",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "b383e0a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "err1 = mean_squared_error(y_test, y_pred1)\n",
    "err2 = mean_squared_error(y_test, y_pred2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "e9a5bc5d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3180.2686486486487, 3180.2686486486487)"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "err1, err2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "1744a0a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(219.0, 125.6), (70.0, 160.2), (202.0, 153.0), (230.0, 236.6), (111.0, 153.4)]"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(y_test[:5], y_pred1[:5]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "417c7825",
   "metadata": {},
   "source": [
    "## KNN Classifier for Digit Recognition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "fb7b7a8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from sklearn import datasets\n",
    "from skimage import exposure\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "a8da2e33",
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist = datasets.load_digits()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "4608057b",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(np.array(mnist.data), mnist.target, test_size=0.25, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "664644ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "f75303a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training data points: 1212\n",
      "validation data points: 135\n",
      "testing data points: 450\n"
     ]
    }
   ],
   "source": [
    "print(\"training data points: {}\".format(len(y_train)))\n",
    "print(\"validation data points: {}\".format(len(y_val)))\n",
    "print(\"testing data points: {}\".format(len(y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b35937e3",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "57c65965",
   "metadata": {},
   "outputs": [],
   "source": [
    "k_values = range(1, 30, 2)\n",
    "\n",
    "accuracies = []\n",
    "\n",
    "for k in range(1, 30, 2):\n",
    "    model = KNeighborsClassifier(n_neighbors=k)\n",
    "    model.fit(X_train, y_train)\n",
    "    score = model.score(X_val, y_val)\n",
    "    accuracies.append(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "38f515ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe\n",
       "    scrolling=\"no\"\n",
       "    width=\"100%\"\n",
       "    height=\"545px\"\n",
       "    src=\"iframe_figures/figure_134.html\"\n",
       "    frameborder=\"0\"\n",
       "    allowfullscreen\n",
       "></iframe>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "acc_df = pd.DataFrame(zip(k_values, accuracies), columns=['k_values', 'accuracies'])\n",
    "fig = px.line(data_frame=acc_df, x='k_values', y='accuracies')\n",
    "fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "c1a40466",
   "metadata": {},
   "outputs": [],
   "source": [
    "k = 5 \n",
    "model = KNeighborsClassifier(n_neighbors=k_values[k])\n",
    "model.fit(X_train, y_train)\n",
    "predictions = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "a2f309ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVALUATION ON TESTING DATA\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00        43\n",
      "           1       0.95      1.00      0.97        37\n",
      "           2       1.00      1.00      1.00        38\n",
      "           3       1.00      0.98      0.99        46\n",
      "           4       0.98      1.00      0.99        55\n",
      "           5       0.98      0.93      0.96        59\n",
      "           6       0.98      1.00      0.99        45\n",
      "           7       1.00      1.00      1.00        41\n",
      "           8       0.97      0.95      0.96        38\n",
      "           9       0.94      0.96      0.95        48\n",
      "\n",
      "    accuracy                           0.98       450\n",
      "   macro avg       0.98      0.98      0.98       450\n",
      "weighted avg       0.98      0.98      0.98       450\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"EVALUATION ON TESTING DATA\")\n",
    "print(classification_report(y_test, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "8be7e59d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[43,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0, 37,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0, 38,  0,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0, 45,  0,  0,  0,  0,  1,  0],\n",
       "       [ 0,  0,  0,  0, 55,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0, 55,  1,  0,  0,  3],\n",
       "       [ 0,  0,  0,  0,  0,  0, 45,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0, 41,  0,  0],\n",
       "       [ 0,  2,  0,  0,  0,  0,  0,  0, 36,  0],\n",
       "       [ 0,  0,  0,  0,  1,  1,  0,  0,  0, 46]], dtype=int64)"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "180db8f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted digit :  2\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPUAAAD4CAYAAAA0L6C7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAMQ0lEQVR4nO3df6jV9R3H8ddr12zmykubG9GtroO4NBbTsEiE1axGm6GB+6OokTIQoo1ig8gFrX/qrxjujyGIZUJtsaxkzrJVS1uwfqhJZdZo4vDaNpNh6SQtfe+Pe9ysXXe/55zvj3PfPh9w6Z4fnM/raC8/3/M93+/344gQgDw+13QAAOWi1EAylBpIhlIDyVBqIJkJVbyo7ZS71CdNmlTreNOmTattrIkTJ9Y21htvvFHbWEeOHKltrLpFhEe7v5JSZzU0NFTreA8++GBtYw0ODqYca9++fbWN1SvY/AaSodRAMpQaSIZSA8lQaiAZSg0kQ6mBZCg1kAylBpIpVGrbV9t+x/a7tu+oOhSAzo1Zatt9kn4p6TuSvibpettfqzoYgM4UmakvkfRuROyIiMOSHpE0v9pYADpVpNRnS9p13O3h1n2fYnux7U22N5UVDkD7ipylNdrpXf9zamVELJe0XMp76iUwHhSZqYclnXPc7QFJ71UTB0C3ipT6VUnn255me6Kk6yT9ttpYADo15uZ3RHxi+4eSnpbUJ+mBiNhWeTIAHSl05ZOIeFLSkxVnAVACjigDkqHUQDKUGkiGUgPJUGogGUoNJEOpgWRcxaLzdR77ffnll9c1lNasWVPbWFK9q0v09/fXNladf44LFy6sbay6nWjZHWZqIBlKDSRDqYFkKDWQDKUGkqHUQDKUGkiGUgPJUGogGUoNJFNkhY4HbO+x/WYdgQB0p8hM/aCkqyvOAaAkY5Y6Il6Q9M8asgAoQaGriRZhe7GkxWW9HoDOlFZqlt0BegN7v4FkKDWQTJGvtH4t6U+ShmwP2/5B9bEAdKrIWlrX1xEEQDnY/AaSodRAMpQaSIZSA8lQaiAZSg0kQ6mBZEo79rspS5curW2snTt31jaWJF177bW1jVXnUjh1Lid0MmKmBpKh1EAylBpIhlIDyVBqIBlKDSRDqYFkKDWQDKUGkqHUQDJFrlF2ju3nbW+3vc32rXUEA9CZIsd+fyLpJxGxxfbpkjbbfiYi3qo4G4AOFFl2528RsaX1+35J2yWdXXUwAJ1p6ywt24OSZkh6eZTHWHYH6AGFS237C5Iek3RbRHz42cdZdgfoDYX2fts+RSOFfjgiHq82EoBuFNn7bUn3S9oeET+vPhKAbhSZqWdL+r6kOba3tn6+W3EuAB0qsuzOi5JcQxYAJeCIMiAZSg0kQ6mBZCg1kAylBpIZ9xfzP9nt2bNHBw8e7Pp15s6dq3Xr1pWQCE2j1OPcwYMHFdH9UbkjxxghAza/gWTG/Uzd399f21gbNmyobSyp/rW7BgcHax0P1WCmBpKh1AmtXr1aCxYs0HnnnadJkyZpaGhIS5Ys0f79+5uOhhpQ6oTuu+8+9fX16d5779X69et18803a9myZbrqqqt09OjRpuOhYuP+MzX+19q1azV16tT/3L7ssst05pln6qabbtKGDRs0Z86cBtOhaszUCR1f6GMuvvhiSdLu3bvrjoOaUeqTxMaNGyVJF1xwQcNJUDVKfRLYvXu37rrrLl155ZWaOXNm03FQMUqd3IEDBzR//nxNmDBBK1eubDoOasCOssQ++ugjzZs3Tzt27NDGjRs1MDDQdCTUYMxS2/68pBckndp6/uqI+FnVwdCdjz/+WAsWLNArr7yiZ599VhdeeGHTkVCTIjP1IUlzIuJA61LBL9p+KiJeqjgbOnT06FHdcMMNeu6557Ru3TpdeumlTUdCjYpceDAkHWjdPKX1w8X6e9gtt9yiRx99VHfeeacmT56sl17677+/AwMDbIYnV/Ri/n22t0raI+mZiBh12R3bm2xvKjkj2vTUU09Jku655x7NmjXrUz8rVqxoOB2qVmhHWUQckTTddr+kJ2x/PSLe/MxzWHanR9R9dhd6S1tfaUXEPkkbJF1dRRgA3Suy7M7U1gwt25MkXSnp7YpzAehQkc3vsyStst2nkX8EfhMRv6s2FoBOFdn7/bpG1qQGMA5wmCiQDIeJjnOnnnpqKVcCPffcc0tIg17ATD3OHTp0qJTXef3110t5HTSPUgPJUGogGUoNJEOpgWQoNZDMuP9Ka/r06U1HqEydy+BMmTKltrHWrFlT21gnI2ZqIBlKDSRDqYFkKDWQDKUGkqHUQDKUGkiGUgPJUGogGUoNJFO41K0L+r9mm4sOAj2snZn6VknbqwoCoBxFl90ZkDRXEmu2AD2u6Ey9VNLtko6e6AmspQX0hiIrdFwjaU9EbP5/z4uI5RExMyJmlpYOQNuKzNSzJc2zvVPSI5Lm2H6o0lQAOjZmqSNiSUQMRMSgpOsk/SEibqw8GYCO8D01kExblzOKiA0aWcoWQI9ipgaSodRAMpQaSIZSA8lQaiAZSg0kQ6mBZMb9sjv79u1rOkJl7r777trG+uCDD2oba+vWrbWNdTJipgaSodRAMpQaSIZSA8lQaiAZSg0kQ6mBZCg1kAylBpKh1EAyhQ4TbV1JdL+kI5I+4TLAQO9q59jvb0XE3sqSACgFm99AMkVLHZJ+b3uz7cWjPYFld4DeUHTze3ZEvGf7y5Kesf12RLxw/BMiYrmk5ZJkO0rOCaCgQjN1RLzX+u8eSU9IuqTKUAA6V2SBvMm2Tz/2u6RvS3qz6mAAOlNk8/srkp6wfez5v4qI9ZWmAtCxMUsdETskfaOGLABKwFdaQDKUGkiGUgPJUGogGUoNJEOpgWQoNZCMI8o/TDvrsd+Dg4O1jlfn8jRTpkypbazMFi1aVMs4a9eu1d69ez3aY8zUQDKUGkiGUgPJUGogGUoNJEOpgWQoNZAMpQaSodRAMpQaSKZQqW33215t+23b223PqjoYgM4Uve73LyStj4jv2Z4o6bQKMwHowpiltn2GpG9KWihJEXFY0uFqYwHoVJHN769Kel/SStuv2V7Ruv73p7DsDtAbipR6gqSLJC2LiBmS/iXpjs8+KSKWR8RMlrkFmlWk1MOShiPi5dbt1RopOYAeNGapI+LvknbZHmrddYWktypNBaBjRfd+/0jSw6093zsk1XN5BwBtK1TqiNgqic/KwDjAEWVAMpQaSIZSA8lQaiAZSg0kQ6mBZCg1kAylBpIpekQZUJpVq1bVNlbd65/t3LmzlnEOHTp0wseYqYFkKDWQDKUGkqHUQDKUGkiGUgPJUGogGUoNJEOpgWTGLLXtIdtbj/v50PZtNWQD0IExDxONiHckTZck232Sdkt6otpYADrV7ub3FZL+EhF/rSIMgO61e0LHdZJ+PdoDthdLWtx1IgBdKTxTt675PU/So6M9zrI7QG9oZ/P7O5K2RMQ/qgoDoHvtlPp6nWDTG0DvKFRq26dJukrS49XGAdCtosvuHJT0xYqzACgBR5QByVBqIBlKDSRDqYFkKDWQDKUGkqHUQDKUGkjGEVH+i9rvS2r39MwvSdpbepjekPW98b6ac15ETB3tgUpK3Qnbm7Ke4ZX1vfG+ehOb30AylBpIppdKvbzpABXK+t54Xz2oZz5TAyhHL83UAEpAqYFkeqLUtq+2/Y7td23f0XSeMtg+x/bztrfb3mb71qYzlcl2n+3XbP+u6Sxlst1ve7Xtt1t/d7OaztSuxj9TtxYI+LNGLpc0LOlVSddHxFuNBuuS7bMknRURW2yfLmmzpGvH+/s6xvaPJc2UdEZEXNN0nrLYXiXpjxGxonUF3dMiYl/DsdrSCzP1JZLejYgdEXFY0iOS5jecqWsR8beI2NL6fb+k7ZLObjZVOWwPSJoraUXTWcpk+wxJ35R0vyRFxOHxVmipN0p9tqRdx90eVpL/+Y+xPShphqSXG45SlqWSbpd0tOEcZfuqpPclrWx9tFhhe3LTodrVC6X2KPel+Z7N9hckPSbptoj4sOk83bJ9jaQ9EbG56SwVmCDpIknLImKGpH9JGnf7eHqh1MOSzjnu9oCk9xrKUirbp2ik0A9HRJbLK8+WNM/2To18VJpj+6FmI5VmWNJwRBzbolqtkZKPK71Q6lclnW97WmvHxHWSfttwpq7ZtkY+m22PiJ83nacsEbEkIgYiYlAjf1d/iIgbG45Vioj4u6Rdtodad10hadzt2Gx3gbzSRcQntn8o6WlJfZIeiIhtDccqw2xJ35f0hu2trft+GhFPNhcJBfxI0sOtCWaHpEUN52lb419pAShXL2x+AygRpQaSodRAMpQaSIZSA8lQaiAZSg0k82/4mPg2paHGDwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted digit :  0\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPUAAAD4CAYAAAA0L6C7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAMMUlEQVR4nO3d7Y9cZR3G8euyYJWWQvEphC5sDYWkJbg1hGCaEChqipJdQngBwSYQQ3mDoYHEoO/8B0x9YQylog0gRCm0DSEgSbtBiSB9Utk+CG4WuqIW0y0PpWmD/nyxU1PpYs/MnIeZX7+fZNOds5O9r0l79T5z5pxzOyIEII9PNB0AQLkoNZAMpQaSodRAMpQaSOaMKn6p7ZSH1AcGBpqOUJmpqanaxnr//fdrGyuziPBM2yspdVb33Xdf0xEqs3HjxtrGGh0drW2s0xG730AylBpIhlIDyVBqIBlKDSRDqYFkKDWQDKUGkqHUQDKFSm17he19tl+3fX/VoQB07pSltj1L0o8lXS9psaRbbS+uOhiAzhSZqa+U9HpEjEfEMUmPSxqpNhaAThUp9QWS9p/weLK17X/YXmV7m+1tZYUD0L4iV2nNdHnXSZdWRsRaSWulvJdeAv2gyEw9KenEC4kXSHqrmjgAulWk1K9IWmR7oe1PSrpF0uZqYwHo1Cl3vyPiQ9t3S3pO0ixJD0XEWOXJAHSk0J1PIuIZSc9UnAVACTijDEiGUgPJUGogGUoNJEOpgWQoNZAMpQaScRWLzmc99/vQoUNNR6hMna9tYmKitrGuueaa2saq28ctu8NMDSRDqYFkKDWQDKUGkqHUQDKUGkiGUgPJUGogGUoNJEOpgWSKrNDxkO0Dtl+tIxCA7hSZqX8uaUXFOQCU5JSljogXJB2sIQuAEhS6m2gRtldJWlXW7wPQmdJKzbI7QG/g6DeQDKUGkinykdZjkn4n6VLbk7a/XX0sAJ0qspbWrXUEAVAOdr+BZCg1kAylBpKh1EAylBpIhlIDyVBqIJnSzv1uytDQUG1jbdy4sbaxJGn16tW1jVXnsjujo6O1jTUyMlLbWJK0adOmWsebCTM1kAylBpKh1EAylBpIhlIDyVBqIBlKDSRDqYFkKDWQDKUGkilyj7IB21tt77E9ZvueOoIB6EyRc78/lHRfROywfbak7bafj4jdFWcD0IEiy+78LSJ2tL5/T9IeSRdUHQxAZ9q6Ssv2oKSlkl6e4WcsuwP0gMKltj1X0gZJqyPi3Y/+nGV3gN5Q6Oi37TM1XehHI+LJaiMB6EaRo9+W9FNJeyLih9VHAtCNIjP1MkkrJS23vav19Y2KcwHoUJFld34ryTVkAVACzigDkqHUQDKUGkiGUgPJUGogmb6/mf/p7ujRozpy5EjXv2f27Nk6evRoCYnQNErd544cOaKI7s/KnT7HCBmw+w0k0/cz9UUXXVTbWBMTE7WNJdW7vlWd6lxLa/78+bWN1SuYqYFkKHVS+/fv180336xzzjlH8+bN00033aQ333yz6VioAaVO6IMPPtDy5cu1d+9erV+/Xg8//LBee+01XXvttTp8+HDT8VCxvn9PjZM9+OCDGh8f1759+3TxxRdLki6//HItWrRIDzzwgO69996GE6JKzNQJbd68WVddddV/Cy1JCxcu1LJly3piUXRUi1InNDY2pssuu+yk7UuWLNHu3dwENjtKndDBgwdn/CjnvPPO09TUVAOJUCdKndRMZ4iVceYZeh+lTmj+/Pk6ePDgSdunpqZOy5MxTjdFbjz4Kdu/t/2H1rI7P6gjGDq3ZMkSjY2NnbR99+7dWrx4cQOJUKciM/VRScsj4kuShiStsH1VpanQleHhYb300ksaHx//77aJiQm9+OKLGh4ebjAZ6lBk2Z2IiPdbD89sffHmrIfdeeedGhwc1MjIiDZt2qTNmzdrZGREAwMDuuuuu5qOh4oVvZn/LNu7JB2Q9HxEzLjsju1ttreVnBFtmjNnjrZs2aJLLrlEK1eu1G233aaFCxdqy5Ytmjt3btPxULFCZ5RFxL8kDdk+V9JTti+LiFc/8hyW3ekhF154oTZs2NB0DDSgraPfEXFI0qikFVWEAdC9Ike/P9eaoWX705K+KmlvxbkAdKjI7vf5ktbbnqXp/wR+GRFPVxsLQKeKLLvzR02vSQ2gD3BGGZAM11P3udmzZ5dyJ1BuEZwHM3WfK6uIFDoPSg0kQ6mBZCg1kAylBpKh1EAyff+R1tKlnBeDjzc4ONh0hNoxUwPJUGogGUoNJEOpgWQoNZAMpQaSodRAMpQaSIZSA8lQaiCZwqVu3dB/p21uOgj0sHZm6nsk7akqCIByFF12Z4Gkb0paV20cAN0qOlOvkfRdSf/+uCewlhbQG4qs0HGDpAMRsf3/PS8i1kbEFRFxRWnpALStyEy9TNKw7QlJj0tabvuRSlMB6FiR9am/FxELImJQ0i2StkTEtypPBqAjfE4NJNPW7YwiYlTTS9kC6FHM1EAylBpIhlIDyVBqIBlKDSRDqYFkKDWQTN8vu7Nz587axmKJn3LUuRTO6OhobWP1CmZqIBlKDSRDqYFkKDWQDKUGkqHUQDKUGkiGUgPJUGogGUoNJFPoNNHWnUTfk/QvSR9yG2Cgd7Vz7ve1EfHPypIAKAW730AyRUsdkn5te7vtVTM9gWV3gN5QdPd7WUS8Zfvzkp63vTciXjjxCRGxVtJaSbIdJecEUFChmToi3mr9eUDSU5KurDIUgM4VWSBvju2zj38v6euSXq06GIDOFNn9/oKkp2wff/4vIuLZSlMB6NgpSx0R45K+VEMWACXgIy0gGUoNJEOpgWQoNZAMpQaSodRAMpQaSKbvl9155513ahvrxhtvrG0sqd4lY+pcCmdoaKi2sW6//fbaxuoVzNRAMpQaSIZSA8lQaiAZSg0kQ6mBZCg1kAylBpKh1EAylBpIplCpbZ9r+wnbe23vsf2VqoMB6EzRc79/JOnZiLjZ9iclnVVhJgBdOGWpbc+TdLWk2yUpIo5JOlZtLACdKrL7/UVJb0v6me2dtte17v/9P1h2B+gNRUp9hqQvS/pJRCyVdFjS/R99UkSsjYgrWOYWaFaRUk9KmoyIl1uPn9B0yQH0oFOWOiL+Lmm/7Utbm66TtLvSVAA6VvTo93ckPdo68j0u6Y7qIgHoRqFSR8QuSbxXBvoAZ5QByVBqIBlKDSRDqYFkKDWQDKUGkqHUQDKUGkim79fSqnO9qTVr1tQ2liRt3bq1trHeeOON2saqe02y0w0zNZAMpQaSodRAMpQaSIZSA8lQaiAZSg0kQ6mBZCg1kMwpS237Utu7Tvh61/bqGrIB6MApTxONiH2ShiTJ9ixJf5X0VLWxAHSq3d3v6yT9JSLqO1EYQFvavaDjFkmPzfQD26skreo6EYCuFJ6pW/f8Hpb0q5l+zrI7QG9oZ/f7ekk7IuIfVYUB0L12Sn2rPmbXG0DvKFRq22dJ+pqkJ6uNA6BbRZfd+UDSZyrOAqAEnFEGJEOpgWQoNZAMpQaSodRAMpQaSIZSA8lQaiAZR0T5v9R+W1K7l2d+VtI/Sw/TG7K+Nl5Xcy6KiM/N9INKSt0J29uyXuGV9bXxunoTu99AMpQaSKaXSr226QAVyvraeF09qGfeUwMoRy/N1ABKQKmBZHqi1LZX2N5n+3Xb9zedpwy2B2xvtb3H9pjte5rOVCbbs2zvtP1001nKZPtc20/Y3tv6u/tK05na1fh76tYCAX/W9O2SJiW9IunWiNjdaLAu2T5f0vkRscP22ZK2S7qx31/XcbbvlXSFpHkRcUPTecpie72k30TEutYddM+KiEMNx2pLL8zUV0p6PSLGI+KYpMcljTScqWsR8beI2NH6/j1JeyRd0GyqctheIOmbktY1naVMtudJulrSTyUpIo71W6Gl3ij1BZL2n/B4Ukn+8R9ne1DSUkkvNxylLGskfVfSvxvOUbYvSnpb0s9aby3W2Z7TdKh29UKpPcO2NJ+z2Z4raYOk1RHxbtN5umX7BkkHImJ701kqcIakL0v6SUQslXRYUt8d4+mFUk9KGjjh8QJJbzWUpVS2z9R0oR+NiCy3V14madj2hKbfKi23/UizkUozKWkyIo7vUT2h6ZL3lV4o9SuSFtle2DowcYukzQ1n6ppta/q92Z6I+GHTecoSEd+LiAURMajpv6stEfGthmOVIiL+Lmm/7Utbm66T1HcHNttdIK90EfGh7bslPSdplqSHImKs4VhlWCZppaQ/2d7V2vb9iHimuUgo4DuSHm1NMOOS7mg4T9sa/0gLQLl6YfcbQIkoNZAMpQaSodRAMpQaSIZSA8lQaiCZ/wDUtPrmt40YzgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted digit :  6\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPUAAAD4CAYAAAA0L6C7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAMbUlEQVR4nO3dXYhc9R3G8edJorY2xoB9ibjSvFgM9aKxLtISCBqrpipmlQoqCimFmItolhaL6Y32Ri+Ekl6EYkiTFJoamtTEUoKxkG5aoVjzZqtGiw1b3G6ixhpMKxjS/HqxkxLbrXtm5rzM/vx+YHHnzDD/Z1wf/2fOnDl/R4QA5DGl6QAAykWpgWQoNZAMpQaSodRAMtOqeFLbKQ+pT58+vdbx5syZU9tYo6OjtY31zjvv1DZWZhHh8bZXUuqs+vv7ax1v06ZNtY31yCOP1DZWna/r44jdbyAZSg0kQ6mBZCg1kAylBpKh1EAylBpIhlIDyVBqIJlCpba9xPZrtl+3/VDVoQB0bsJS254qaa2kr0v6oqS7bH+x6mAAOlNkpr5a0usRcTgiTkraImlptbEAdKpIqS+R9MZZt0da2z7E9nLbe23vLSscgPYV+ZbWeF/v+p+vVkbEOknrpLxfvQQmgyIz9YikS8+63Sepvi/fAmhLkVK/IOkLtufYPlfSnZJ+WW0sAJ2acPc7Ik7ZXilpl6SpkjZExMuVJwPQkUJXPomInZJ2VpwFQAk4owxIhlIDyVBqIBlKDSRDqYFkKDWQDKUGkmGFjjYsW7as1vFmzpxZ21hDQ0O1jYVqMVMDyVBqIBlKDSRDqYFkKDWQDKUGkqHUQDKUGkiGUgPJUGogmSIrdGyw/Zbtl+oIBKA7RWbqTZKWVJwDQEkmLHVE/FbS32vIAqAEpX1Ly/ZyScvLej4AnSmt1Cy7A/QGjn4DyVBqIJkiH2k9Ken3ki63PWL7W9XHAtCpImtp3VVHEADlYPcbSIZSA8lQaiAZSg0kQ6mBZCg1kAylBpJxRPmnaWc997uKf1cf5emnn65trIGBgdrGQjkiwuNtZ6YGkqHUQDKUGkiGUgPJUGogGUoNJEOpgWQoNZAMpQaSodRAMkWuUXap7d/YPmT7Zdur6ggGoDNFrvt9StJ3ImK/7Qsk7bP964h4peJsADpQZNmdIxGxv/X7CUmHJF1SdTAAnWlrhQ7bsyVdKen5ce5j2R2gBxQute3pkn4haTAi3vvv+1l2B+gNhY5+2z5HY4XeHBFPVRsJQDeKHP22pB9LOhQRP6g+EoBuFJmpF0q6V9Ji2wdbPzdVnAtAh4osu/OcpHEvmwKg93BGGZAMpQaSodRAMpQaSIZSA8m0dZooes+KFSt09OjRrp/nvPPO0wcffFBCIjSNUk9yR48eLWXlkLFzjJABu99AMszUbXjxxRebjgBMiJkaSIZSJ7Zz504tWrRI06dP14wZM9Tf36/du3c3HQsVo9RJPfHEE1q6dKmuuuoqbd++XVu3btUdd9yh999/v+loqBjvqRMaHh7W4OCgHn/8cQ0ODv5n+4033thcKNSGmTqhDRs2aMqUKVqxYkXTUdAASp3Qc889p/nz52vLli2aN2+epk2bpssuu0xr165tOhpqwO53QqOjoxodHdWDDz6oRx99VPPmzdPWrVu1cuVKnTp1SqtWcen2zCh1QqdPn9aJEye0adMm3X777ZKkxYsXa3h4WI899pgeeOABziBLjN3vhC666CJJ0vXXX/+h7TfccIPefPNNHTlypIlYqEmRCw9+wvYfbL/YWnbn+3UEQ+euuOKKcbefOUd8yhT+X55Zkb/uB5IWR8SXJC2QtMT2VypNha7cdtttkqRdu3Z9aPuuXbvU19enWbNmNRELNSly4cGQ9I/WzXNaP1ysv4fddNNNuvbaa3Xffffp2LFjmjt3rrZt26Znn31WGzdubDoeKlboQJntqZL2SbpM0tqIYNmdHmZbO3bs0OrVq/Xwww/r3Xff1fz587V582bdfffdTcdDxdzOd3Ftz5S0XdL9EfHSRzwu5Ux+8ODBWscbHh6e8DEDAwN8n/pjKiLG/aO1dcQkIo5LGpK0pPtIAKpQ5Oj3Z1oztGx/UtLXJL1acS4AHSrynvpiST9pva+eIunnEfGramMB6FSRo99/1Nia1AAmAc5CAJLh3O9JbtasWaUcueYSwXm09ZFW4SflI61SFPlIqywDAwO1jYVylPKRFoDeR6mBZCg1kAylBpKh1EAyk/7o94IFC+oaSgcOHKhtLEnas2dPbWMNDQ3VNtaOHTtqG6vuTyzqxNFv4GOCUgPJUGogGUoNJEOpgWQoNZAMpQaSodRAMpQaSIZSA8kULrXtqbYP2Oaig0APa2emXiXpUFVBAJSjUKlt90m6WdL6auMA6FbRmXqNpO9KOv3/HmB7ue29tveWEQxAZ4qs0HGLpLciYt9HPS4i1kVEf0T0l5YOQNuKzNQLJd1qe1jSFkmLbf+00lQAOjZhqSNidUT0RcRsSXdK2h0R91SeDEBH+JwaSKatFToiYkhjS9kC6FHM1EAylBpIhlIDyVBqIBlKDSRDqYFkKDWQTFufU/ei48ePNx2hMrNnz246QiXWrFlT21h1L7szODhY63jjYaYGkqHUQDKUGkiGUgPJUGogGUoNJEOpgWQoNZAMpQaSodRAMoVOE21dSfSEpH9JOsVlgIHe1c6539dGxLHKkgAoBbvfQDJFSx2SnrW9z/by8R7AsjtAbyi6+70wIkZtf1bSr22/GhG/PfsBEbFO0jpJsh0l5wRQUKGZOiJGW/98S9J2SVdXGQpA54oskPcp2xec+V3SDZJeqjoYgM4U2f3+nKTtts88/mcR8UylqQB0bMJSR8RhSV+qIQuAEvCRFpAMpQaSodRAMpQaSIZSA8lQaiAZSg0kM+mX3RkeHm46QmXqXFJoaGiotrGuueaa2sZasGBBbWP1CmZqIBlKDSRDqYFkKDWQDKUGkqHUQDKUGkiGUgPJUGogGUoNJFOo1LZn2t5m+1Xbh2x/tepgADpT9NzvH0p6JiK+YftcSedXmAlAFyYste0ZkhZJWiZJEXFS0slqYwHoVJHd77mS3pa00fYB2+tb1//+EJbdAXpDkVJPk/RlST+KiCsl/VPSQ//9oIhYFxH9LHMLNKtIqUckjUTE863b2zRWcgA9aMJSR8RRSW/Yvry16TpJr1SaCkDHih79vl/S5taR78OSvlldJADdKFTqiDgoiffKwCTAGWVAMpQaSIZSA8lQaiAZSg0kQ6mBZCg1kAylBpJxRJT/pHb5T9oDli1bVut4a9asqW2sCy+8sLax9uzZU9tYAwMDtY0l1bv+WUR4vO3M1EAylBpIhlIDyVBqIBlKDSRDqYFkKDWQDKUGkqHUQDITltr25bYPnvXznu3BGrIB6MCE1yiLiNckLZAk21Ml/U3S9mpjAehUu7vf10n6S0T8tYowALpX9BLBZ9wp6cnx7rC9XNLyrhMB6Erhmbp1ze9bJW0d736W3QF6Qzu731+XtD8i3qwqDIDutVPqu/R/dr0B9I5CpbZ9vqTrJT1VbRwA3Sq67M77ki6qOAuAEnBGGZAMpQaSodRAMpQaSIZSA8lQaiAZSg0kQ6mBZKpadudtSe1+PfPTko6VHqY3ZH1tvK7mfD4iPjPeHZWUuhO292b9hlfW18br6k3sfgPJUGogmV4q9bqmA1Qo62vjdfWgnnlPDaAcvTRTAygBpQaS6YlS215i+zXbr9t+qOk8ZbB9qe3f2D5k+2Xbq5rOVCbbU20fsP2rprOUyfZM29tsv9r623216Uztavw9dWuBgD9r7HJJI5JekHRXRLzSaLAu2b5Y0sURsd/2BZL2SRqY7K/rDNvfltQvaUZE3NJ0nrLY/omk30XE+tYVdM+PiOMNx2pLL8zUV0t6PSIOR8RJSVskLW04U9ci4khE7G/9fkLSIUmXNJuqHLb7JN0saX3TWcpke4akRZJ+LEkRcXKyFVrqjVJfIumNs26PKMl//GfYni3pSknPNxylLGskfVfS6YZzlG2upLclbWy9tVhv+1NNh2pXL5Ta42xL8zmb7emSfiFpMCLeazpPt2zfIumtiNjXdJYKTJP0ZUk/iogrJf1T0qQ7xtMLpR6RdOlZt/skjTaUpVS2z9FYoTdHRJbLKy+UdKvtYY29VVps+6fNRirNiKSRiDizR7VNYyWfVHqh1C9I+oLtOa0DE3dK+mXDmbpm2xp7b3YoIn7QdJ6yRMTqiOiLiNka+1vtjoh7Go5Viog4KukN25e3Nl0nadId2Gx3gbzSRcQp2ysl7ZI0VdKGiHi54VhlWCjpXkl/sn2wte17EbGzuUgo4H5Jm1sTzGFJ32w4T9sa/0gLQLl6YfcbQIkoNZAMpQaSodRAMpQaSIZSA8lQaiCZfwOFYRQ+ZvlEiwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted digit :  3\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPUAAAD4CAYAAAA0L6C7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAMc0lEQVR4nO3df6jddR3H8deru7Wydjlgs4l3uNpCjXBzyrYYtOvMWCU2IWQXChbBBCkcCc31l/23f8z1RwRjOcM0sZWXCGmJc1iQ5u6cqfuha0y9WWrIxR+BF+e7P+4ZTL12v+ec749z33s+4OI95x7O53WYLz7f8z3n+/k4IgQgj480HQBAuSg1kAylBpKh1EAylBpIZk4VT2q7tlPqAwMDdQ2lJUuW1DaWJM2fP7+2sSYnJ2sb69ixY7WNVefrqltEeLr7XcVHWnWWutVq1TWURkdHaxtLktauXVvbWM8//3xtYw0PD9c21smTJ2sbq24fVmoOv4FkKDWQDKUGkqHUQDKUGkiGUgPJUGogGUoNJEOpgWQKldr2etvHbB+3fUvVoQB0b8ZS2x6Q9DNJX5X0eUkjtj9fdTAA3SkyU6+UdDwiTkTEpKR7JX2j2lgAulWk1BdIevGM2+Pt+97D9mbbB2wfKCscgM4VufRyuitBPnAVVkTslLRTqvcqLQDvVWSmHpe06IzbQ5JeqiYOgF4VKfXjkj5n+zO2Pyppo6TfVxsLQLdmPPyOiHdsf0/SXkkDku6IiGcqTwagK4WWM4qIByQ9UHEWACXgG2VAMpQaSIZSA8lQaiAZSg0kQ6mBZCg1kEwl2+7UadOmTbWNtXz58trGkqQrr7yytrF27NhR21gbNmyobaw6X1e/YKYGkqHUQDKUGkiGUgPJUGogGUoNJEOpgWQoNZAMpQaSodRAMkV26LjD9iu2n64jEIDeFJmp75S0vuIcAEoyY6kj4hFJr9WQBUAJSrtKy/ZmSZvLej4A3Smt1Gy7A/QHzn4DyVBqIJkiH2n9WtJfJV1ke9z2d6uPBaBbRfbSGqkjCIBycPgNJEOpgWQoNZAMpQaSodRAMpQaSIZSA8nM+m136txWZXR0tLaxJOnkyZO1jTUxMVHbWKgWMzWQDKUGkqHUQDKUGkiGUgPJUGogGUoNJEOpgWQoNZAMpQaSKbJG2SLbD9s+YvsZ2zfVEQxAd4p89/sdSTdHxEHb8yWN2X4wIg5XnA1AF4psu/OviDjY/v0NSUckXVB1MADd6egqLduLJV0m6bFp/sa2O0AfKFxq25+U9FtJWyLi9ff/nW13gP5Q6Oy37bmaKvTdEfG7aiMB6EWRs9+W9AtJRyLiJ9VHAtCLIjP1GknflrTO9qH2z9cqzgWgS0W23fmLJNeQBUAJ+EYZkAylBpKh1EAylBpIhlIDyVDqWW7evHmlPM/IyEgpz4PmzfodOs52b7/9tiJ6/1bu1HeMkAEzNZAMM3UH6tzbSpI2bdo042PuvPPO0sZbu3Ztac/1/+zfv7+Wcc5WzNRAMpQ6ob1792rdunVauHCh5s2bp6GhIV1//fU6fJjFas4GHH4n9Nprr+nyyy/XjTfeqAULFuiFF17Q9u3btXr1aj311FO68MILm46IClHqhEZGRj7wEdXKlSt18cUXa8+ePbr55psbSoY6cPh9ljj33HMlSXPnzm04CapGqRM7deqUJicn9dxzz+mGG27QwoULtXHjxqZjoWIcfie2atUqjY2NSZKWLl2qffv26bzzzms4FarGTJ3YXXfdpUcffVT33HOPBgcHdfXVV9f+WTvqR6kTu+SSS7Rq1SqNjIzooYce0ptvvqnt27c3HQsVK7Lw4Mds/832k+1td35cRzCUq9VqaenSpTp+/HjTUVCxIjP125LWRcQyScslrbe9utJUKN3LL7+so0ePasmSJU1HQcWKLDwYkt5s35zb/mGx/j523XXXacWKFbr00ks1ODioZ599VrfffrvmzJnDZ9RngUJnv20PSBqTtFTSzyKCbXf62OrVq3Xffffptttu0+TkpBYtWqTh4WFt27ZNixcvbjoeKlao1BFxStJy2y1J99v+QkQ8/b7HsO1On9i6dau2bt3adAw0pKOz3xExIWm/pPVVhAHQuyJnvxe0Z2jZ/rikL0s6WnEuAF0qcvh9vqRftt9Xf0TSfRHxh2pjAehWkbPff9fUntQAZgG+UQYkwwUds1yr1SplJVAWTsiDUs9yGzZsKOV5du/eXcrzoHkcfgPJUGogGUoNJEOpgWQoNZCMy9hc7QNPygUdpZiYmKhtrDqXOarzSrEytyUqYsuWLbWNFRHTfpbJTA0kQ6mBZCg1kAylBpKh1EAylBpIhlIDyVBqIBlKDSRDqYFkCpfa9oDtJ2yz6CDQxzqZqW+SdKSqIADKUajUtockfV3SrmrjAOhV0Zl6h6QfSnr3wx5ge7PtA7YPlBEMQHeK7NBxjaRXImLs/z0uInZGxBURcUVp6QB0rMhMvUbStbZPSrpX0jrbv6o0FYCuzVjqiNgWEUMRsVjSRkn7IuJblScD0BU+pwaS6Wgx/4jYr6mtbAH0KWZqIBlKDSRDqYFkKDWQDKUGkqHUQDKUGkiGbXf6WKvVqm2sOrf4ufXWW2sbq85tcKR6/83Ydgc4S1BqIBlKDSRDqYFkKDWQDKUGkqHUQDKUGkiGUgPJUGogmULLGbVXEn1D0ilJ77AMMNC/Olmj7MqI+E9lSQCUgsNvIJmipQ5Jf7I9ZnvzdA9g2x2gPxQ9/F4TES/ZPk/Sg7aPRsQjZz4gInZK2ilx6SXQpEIzdUS81P7vK5Lul7SyylAAuldkg7xP2J5/+ndJX5H0dNXBAHSnyOH3pyXdb/v04++JiD9WmgpA12YsdUSckLSshiwASsBHWkAylBpIhlIDyVBqIBlKDSRDqYFkKDWQTCeXXp716twuRpL2799f21iHDh2qbazh4eHaxhodHa1trH7BTA0kQ6mBZCg1kAylBpKh1EAylBpIhlIDyVBqIBlKDSRDqYFkCpXadsv2HttHbR+x/cWqgwHoTtHvfv9U0h8j4pu2PyrpnAozAejBjKW2PSjpS5I2SVJETEqarDYWgG4VOfz+rKRXJe22/YTtXe31v9+DbXeA/lCk1HMkrZD084i4TNJbkm55/4MiYmdEXME2t0CzipR6XNJ4RDzWvr1HUyUH0IdmLHVE/FvSi7Yvat91laTDlaYC0LWiZ7+/L+nu9pnvE5K+U10kAL0oVOqIOCSJ98rALMA3yoBkKDWQDKUGkqHUQDKUGkiGUgPJUGogGUoNJOOIKP9J7fKf9EO0Wq26hqp1bytJWrZsWa3j1eXJJ5+sbaw69+2SpImJidrGighPdz8zNZAMpQaSodRAMpQaSIZSA8lQaiAZSg0kQ6mBZCg1kMyMpbZ9ke1DZ/y8bntLDdkAdGHGNcoi4pik5ZJke0DSPyXdX20sAN3q9PD7Kkn/iIjnqwgDoHdFlwg+baOkX0/3B9ubJW3uORGAnhSeqdtrfl8r6TfT/Z1td4D+0Mnh91clHYyIl6sKA6B3nZR6RB9y6A2gfxQqte1zJF0t6XfVxgHQq6Lb7vxX0rkVZwFQAr5RBiRDqYFkKDWQDKUGkqHUQDKUGkiGUgPJUGogmaq23XlVUqeXZ35K0n9KD9Mfsr42XldzLoyIBdP9oZJSd8P2gaxXeGV9bbyu/sThN5AMpQaS6adS72w6QIWyvjZeVx/qm/fUAMrRTzM1gBJQaiCZvii17fW2j9k+bvuWpvOUwfYi2w/bPmL7Gds3NZ2pTLYHbD9h+w9NZymT7ZbtPbaPtv/tvth0pk41/p66vUHAs5paLmlc0uOSRiLicKPBemT7fEnnR8RB2/MljUnaMNtf12m2fyDpCkmDEXFN03nKYvuXkv4cEbvaK+ieExETDcfqSD/M1CslHY+IExExKeleSd9oOFPPIuJfEXGw/fsbko5IuqDZVOWwPSTp65J2NZ2lTLYHJX1J0i8kKSImZ1uhpf4o9QWSXjzj9riS/M9/mu3Fki6T9FjDUcqyQ9IPJb3bcI6yfVbSq5J2t99a7LL9iaZDdaofSu1p7kvzOZvtT0r6raQtEfF603l6ZfsaSa9ExFjTWSowR9IKST+PiMskvSVp1p3j6YdSj0tadMbtIUkvNZSlVLbnaqrQd0dEluWV10i61vZJTb1VWmf7V81GKs24pPGIOH1EtUdTJZ9V+qHUj0v6nO3PtE9MbJT0+4Yz9cy2NfXe7EhE/KTpPGWJiG0RMRQRizX1b7UvIr7VcKxSRMS/Jb1o+6L2XVdJmnUnNjvdIK90EfGO7e9J2itpQNIdEfFMw7HKsEbStyU9ZftQ+74fRcQDzUVCAd+XdHd7gjkh6TsN5+lY4x9pAShXPxx+AygRpQaSodRAMpQaSIZSA8lQaiAZSg0k8z+xxwARyVfragAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted digit :  7\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPUAAAD4CAYAAAA0L6C7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAALxUlEQVR4nO3d24td5R3G8edpDtYjYmtCSIKJUIRQiBGRSkBTbcukauJFL5KoWCnkSnGwINo7/wFNL6oQo45D0kij8YCIVtFoC2KdJNPWGC1psDiNNpGaQ400an69mB2IOnbW3nud5pfvB4bMPrDfZ5s8vmuvvdZ6HRECkMe3mg4AoFyUGkiGUgPJUGogGUoNJDO9ihe1zS71EsyaNau2sWbPnl3bWDNnzqxtrD179tQ2liQdOnSotrEiwhPdX0mpUY41a9bUNtbg4GBtY11wwQW1jXX99dfXNpYkPf3007WONxE2v4FkKDWQDKUGkqHUQDKUGkiGUgPJUGogGUoNJEOpgWQKldr2gO13be+xfVfVoQD0btJS254m6TeSlktaJGm17UVVBwPQmyIz9WWS9kTE3og4JukxSSurjQWgV0VKPVfS+yfdHuvc9yW219oesT1SVjgA3StyltZEp3d97dTKiFgvab3EqZdAk4rM1GOS5p90e56kfdXEAdCvIqV+U9L3bC+0PVPSKknPVBsLQK8m3fyOiM9t3yrpBUnTJD0cEbsqTwagJ4WufBIRz0l6ruIsAErAEWVAMpQaSIZSA8lQaiAZSg0kQ6mBZCg1kAwrdHShzlUsJOm+++6rbaxXX321trHqXKFjyZIltY0lsUIHgApQaiAZSg0kQ6mBZCg1kAylBpKh1EAylBpIhlIDyVBqIJkiK3Q8bHu/7bfqCASgP0Vm6iFJAxXnAFCSSUsdEa9J+ncNWQCUoLSztGyvlbS2rNcD0JvSSs2yO0A7sPcbSIZSA8kU+Uprs6TXJV1ke8z2L6qPBaBXRdbSWl1HEADlYPMbSIZSA8lQaiAZSg0kQ6mBZCg1kAylBpJh2Z0ujI6O1jrewoULaxvr4MGDtY318ccf1zbWtm3bahurLZipgWQoNZAMpQaSodRAMpQaSIZSA8lQaiAZSg0kQ6mBZCg1kEyRa5TNt/2K7d22d9m+vY5gAHpT5NjvzyX9MiJ22D5b0nbbL0bE2xVnA9CDIsvufBAROzq/H5G0W9LcqoMB6E1XZ2nZXiBpiaQ3JniMZXeAFihcattnSXpC0mBEHP7q4yy7A7RDob3ftmdovNCbImJrtZEA9KPI3m9LekjS7oi4t/pIAPpRZKZeKukmSVfZHu38/LTiXAB6VGTZnT9Kcg1ZAJSAI8qAZCg1kAylBpKh1EAylBpIhov5T3H79+/X0aNH+36d008/XaeddloJidA0Sj3FHT16VBH9H5Vrm1InweY3kAwzdRfqXG9KkoaGhiZ9zrJly0ob76mnnirttdqizP8+RbRh7S5maiAZSp3QsmXLZHvCn4GBgabjoWJsfid0//336/DhL5/y/vrrr+uOO+7QihUrGkqFulDqhBYtWvS1+x588EHNnDlTq1ataiAR6sTm9yng008/1ZYtW3TdddfpvPPOazoOKkapTwFbt27VkSNHdPPNNzcdBTWg1KeA4eFhzZo1S8uXL286CmpAqZPbt2+fXnrpJd1www2aPp1dKKcCSp3cxo0bdfz4cTa9TyFFLjz4bdt/sv3nzrI799QRDOUYHh7W4sWLtXjx4qajoCZFZur/SroqIhZLuljSgO0fVJoKpRgZGdGuXbuYpU8xRS48GJL+07k5o/PDxfqngOHhYU2fPl1r1qxpOgpqVPRi/tNsj0raL+nFiJhw2R3bI7ZHSs6IHnz22WfavHmzBgYGNHv27KbjoEaFdodGxBeSLrZ9rqQnbX8/It76ynNYdqdFZsyYoQMHDjQdAw3oau93RByUtE0SZwUALVVk7/f5nRlatk+X9CNJ71ScC0CPimx+z5H0qO1pGv+fwO8i4tlqYwHoVZG933/R+JrUAKYAjigDkuFg4Cluzpw5Gl9tuP/XQQ6UeorbvHlz0xHQMmx+A8lQaiAZSg0kQ6mBZCg1kAx7v7uwbt26Wse78sorax2vLocOHaptrJ07d9Y2VlswUwPJUGogGUoNJEOpgWQoNZAMpQaSodRAMpQaSIZSA8lQaiCZwqXuXNB/p20uOgi0WDcz9e2SdlcVBEA5ii67M0/SNZI2VBsHQL+KztTrJN0p6fg3PYG1tIB2KLJCx7WS9kfE9v/3vIhYHxGXRsSlpaUD0LUiM/VSSStsvyfpMUlX2d5YaSoAPZu01BFxd0TMi4gFklZJejkibqw8GYCe8D01kExXlzOKiG0aX8oWQEsxUwPJUGogGUoNJEOpgWQoNZAMpQaSodRAMo6I8l/ULv9FW2DlypVNR6jMPffcU9tYdS5fNDQ0VNtYdYsIT3Q/MzWQDKUGkqHUQDKUGkiGUgPJUGogGUoNJEOpgWQoNZAMpQaSKXQ5o86VRI9I+kLS51wGGGivbq5R9sOI+KiyJABKweY3kEzRUoek39vebnvtRE9g2R2gHYpufi+NiH22Z0l60fY7EfHayU+IiPWS1kt5T70EpoJCM3VE7Ov8uV/Sk5IuqzIUgN4VWSDvTNtnn/hd0k8kvVV1MAC9KbL5PVvSk7ZPPP+3EfF8pakA9GzSUkfEXkmLa8gCoAR8pQUkQ6mBZCg1kAylBpKh1EAylBpIhlIDybDsTostWLCgtrG2bdtW21h1vq/MWHYHOEVQaiAZSg0kQ6mBZCg1kAylBpKh1EAylBpIhlIDyVBqIJlCpbZ9ru3Hbb9je7fty6sOBqA3Ra/7/WtJz0fEz2zPlHRGhZkA9GHSUts+R9IVkn4uSRFxTNKxamMB6FWRze8LJR2Q9IjtnbY3dK7//SUsuwO0Q5FST5d0iaQHImKJpE8k3fXVJ0XE+oi4lGVugWYVKfWYpLGIeKNz+3GNlxxAC01a6oj4UNL7ti/q3HW1pLcrTQWgZ0X3ft8maVNnz/deSbdUFwlAPwqVOiJGJfFZGZgCOKIMSIZSA8lQaiAZSg0kQ6mBZCg1kAylBpKh1EAyRY8oQwMGBwdrG2toaKi2sVAtZmogGUoNJEOpgWQoNZAMpQaSodRAMpQaSIZSA8lQaiCZSUtt+yLboyf9HLY9WEM2AD2Y9DDRiHhX0sWSZHuapH9KerLaWAB61e3m99WS/h4R/6giDID+dXtCxypJmyd6wPZaSWv7TgSgL4Vn6s41v1dI2jLR4yy7A7RDN5vfyyXtiIh/VRUGQP+6KfVqfcOmN4D2KFRq22dI+rGkrdXGAdCvosvuHJX0nYqzACgBR5QByVBqIBlKDSRDqYFkKDWQDKUGkqHUQDKUGkjGEVH+i9oHJHV7euZ3JX1Ueph2yPreeF/NuSAizp/ogUpK3QvbI1nP8Mr63nhf7cTmN5AMpQaSaVOp1zcdoEJZ3xvvq4Va85kaQDnaNFMDKAGlBpJpRaltD9h+1/Ye23c1nacMtufbfsX2btu7bN/edKYy2Z5me6ftZ5vOUibb59p+3PY7nb+7y5vO1K3GP1N3Fgj4m8YvlzQm6U1JqyPi7UaD9cn2HElzImKH7bMlbZd0/VR/XyfYvkPSpZLOiYhrm85TFtuPSvpDRGzoXEH3jIg42HCsrrRhpr5M0p6I2BsRxyQ9Jmllw5n6FhEfRMSOzu9HJO2WNLfZVOWwPU/SNZI2NJ2lTLbPkXSFpIckKSKOTbVCS+0o9VxJ7590e0xJ/vGfYHuBpCWS3mg4SlnWSbpT0vGGc5TtQkkHJD3S+WixwfaZTYfqVhtK7QnuS/M9m+2zJD0haTAiDjedp1+2r5W0PyK2N52lAtMlXSLpgYhYIukTSVNuH08bSj0maf5Jt+dJ2tdQllLZnqHxQm+KiCyXV14qaYXt9zT+Uekq2xubjVSaMUljEXFii+pxjZd8SmlDqd+U9D3bCzs7JlZJeqbhTH2zbY1/NtsdEfc2nacsEXF3RMyLiAUa/7t6OSJubDhWKSLiQ0nv276oc9fVkqbcjs1uF8grXUR8bvtWSS9Imibp4YjY1XCsMiyVdJOkv9oe7dz3q4h4rrlIKOA2SZs6E8xeSbc0nKdrjX+lBaBcbdj8BlAiSg0kQ6mBZCg1kAylBpKh1EAylBpI5n+Dz8v43mbGdwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# displaying results for a few digits\n",
    "for i in np.random.randint(0, high=len(y_test), size=(5,)):\n",
    "  # grab the image and classify it\n",
    "  image = X_test[i]\n",
    "  prediction = model.predict([image])[0]\n",
    "  \n",
    "  # show the prediction\n",
    "  imgdata = np.array(image, dtype='float')\n",
    "  pixels = imgdata.reshape((8,8))\n",
    "  plt.imshow(pixels,cmap='gray')\n",
    "  plt.annotate(prediction,(3,3),bbox={'facecolor':'white'},fontsize=16)\n",
    "  print(\"Predicted digit : \",prediction)\n",
    "  plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "5f386a59",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe\n",
       "    scrolling=\"no\"\n",
       "    width=\"100%\"\n",
       "    height=\"545px\"\n",
       "    src=\"iframe_figures/figure_139.html\"\n",
       "    frameborder=\"0\"\n",
       "    allowfullscreen\n",
       "></iframe>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "k_values = range(1, 30, 2)\n",
    "\n",
    "accuracies = []\n",
    "\n",
    "for k in range(1, 30, 2):\n",
    "    model = KNeighborsClassifier(n_neighbors=k)\n",
    "    model.fit(X_train, y_train)\n",
    "    score = model.score(X_test, y_test)\n",
    "    accuracies.append(score)\n",
    "\n",
    "acc_df = pd.DataFrame(zip(k_values, accuracies), columns=['k_values', 'accuracies'])\n",
    "fig = px.line(data_frame=acc_df, x='k_values', y='accuracies')\n",
    "fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "050efe40",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbd10965",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cb8b80b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3c9ed39",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}