10_precision_recall.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "932d610a-041d-4039-ab62-6d6734d4fa45",
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "import pandas as pd\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1cf19996-007c-4042-977b-2fdbd94cdfec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Source code credit for this function: https://gist.github.com/shaypal5/94c53d765083101efc0240d776a23823\n",
    "def print_confusion_matrix(confusion_matrix, class_names, figsize = (10,7), fontsize=14):\n",
    "    \"\"\"Prints a confusion matrix, as returned by sklearn.metrics.confusion_matrix, as a heatmap.\n",
    "    \n",
    "    Arguments\n",
    "    ---------\n",
    "    confusion_matrix: numpy.ndarray\n",
    "        The numpy.ndarray object returned from a call to sklearn.metrics.confusion_matrix. \n",
    "        Similarly constructed ndarrays can also be used.\n",
    "    class_names: list\n",
    "        An ordered list of class names, in the order they index the given confusion matrix.\n",
    "    figsize: tuple\n",
    "        A 2-long tuple, the first value determining the horizontal size of the ouputted figure,\n",
    "        the second determining the vertical size. Defaults to (10,7).\n",
    "    fontsize: int\n",
    "        Font size for axes labels. Defaults to 14.\n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    matplotlib.figure.Figure\n",
    "        The resulting confusion matrix figure\n",
    "    \"\"\"\n",
    "    df_cm = pd.DataFrame(\n",
    "        confusion_matrix, index=class_names, columns=class_names, \n",
    "    )\n",
    "    fig = plt.figure(figsize=figsize)\n",
    "    try:\n",
    "        heatmap = sns.heatmap(df_cm, annot=True, fmt=\"d\")\n",
    "    except ValueError:\n",
    "        raise ValueError(\"Confusion matrix values must be integers.\")\n",
    "    heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(), rotation=0, ha='right', fontsize=fontsize)\n",
    "    heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(), rotation=45, ha='right', fontsize=fontsize)\n",
    "    plt.ylabel('Truth')\n",
    "    plt.xlabel('Prediction')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3c521a04-bcfe-41e0-8ad5-8de86edfeb5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "truth =      [\"Dog\",\"Not a dog\",\"Dog\",\"Dog\",\"Dog\", \"Not a dog\", \"Not a dog\", \"Dog\",\"Dog\", \"Not a dog\"]\n",
    "prediction = [\"Dog\",\"Dog\",\"Dog\",\"Not a dog\",\"Dog\", \"Not a dog\", \"Dog\",\"Not a dog\", \"Dog\", \"Dog\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b021d82e-bbf2-41cc-bd98-8f4cc623bb91",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA10AAAKiCAYAAAAtyLYnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABSiElEQVR4nO3deZyN9f//8eeZ7RjLjMyHMbKnrNmpyR4RIm0fWmRLTdmHZMqaNJF8kJASSSUZyZakDAkxtixDka0xk61mMjiznPP9w23O18VgRtc1Z8Y87t/bdft1rvO+rut9fL+38zuveV7v12VzuVwuAQAAAAAs4eXpCQAAAADArYyiCwAAAAAsRNEFAAAAABai6AIAAAAAC1F0AQAAAICFKLoAAAAAwEIUXQAAAABgIYouAAAAALAQRRcAAAAAWIiiCwAAAAAsRNEFAAAAIN+JjIyUzWbTwIEDrztu3bp1qlevngoUKKCKFStq5syZ2b4WRRcAAACAfGXr1q2aNWuWatased1xhw8fVrt27dSkSRPt2LFDr776qvr376+oqKhsXY+iCwAAAEC+ce7cOT399NP64IMPdNttt1137MyZM1W2bFlNnjxZVatW1XPPPaeePXtq4sSJ2bomRRcAAACAPMvhcCgpKcmwORyOa47v06eP2rdvr1atWt3w3Js2bVLr1q0N+9q0aaOYmBilpqZmeY4+WR6JfCX19O+engIAmGpCvRGengIAmOq1o596egrXlJO/JSOnzdOYMWMM+0aNGqXRo0dfNXbBggXavn27tm7dmqVzJyQkKDg42LAvODhYaWlpOn36tEJCQrJ0HoouAAAAAHlWRESEwsPDDfvsdvtV444fP64BAwZo9erVKlCgQJbPb7PZDK9dLlem+6+HogsAAACAuZzpOXYpu92eaZF1pW3btunkyZOqV6+ee196errWr1+vadOmyeFwyNvb23BMyZIllZCQYNh38uRJ+fj4KCgoKMtzpOgCAAAAcMtr2bKldu/ebdjXo0cPValSRa+88spVBZckhYaGatmyZYZ9q1evVv369eXr65vla1N0AQAAADCXy+npGVylSJEiqlGjhmFfoUKFFBQU5N4fERGhuLg4zZs3T5IUFhamadOmKTw8XL1799amTZs0e/Zsff7559m6Nt0LAQAAAEBSfHy8jh075n5doUIFrVy5UtHR0apdu7bGjh2rqVOn6rHHHsvWeW2ujJVgwGXoXgjgVkP3QgC3mlzdvTA+Nseu5RtSNceudbNIugAAAADAQhRdAAAAAGAhGmkAAAAAMJUrFzbS8CSSLgAAAACwEEkXAAAAAHM5SbouR9IFAAAAABYi6QIAAABgLtZ0GZB0AQAAAICFSLoAAAAAmMuZ7ukZ5CokXQAAAABgIZIuAAAAAOZiTZcBSRcAAAAAWIikCwAAAIC5eE6XAUkXAAAAAFiIpAsAAACAqVys6TIg6QIAAAAAC5F0AQAAADAXa7oMSLoAAAAAwEIkXQAAAADMxZouA5IuAAAAALAQSRcAAAAAcznTPT2DXIWkCwAAAAAsRNEFAAAAABbi9kIAAAAA5qKRhgFJFwAAAABYiKQLAAAAgLl4OLIBSRcAAAAAWIikCwAAAIC5WNNlQNIFAAAAABYi6QIAAABgLtZ0GZB0AQAAAICFSLoAAAAAmMrlSvf0FHIVki4AAAAAsBBJFwAAAABz0b3QgKQLAAAAACxE0gUAAADAXHQvNCDpAgAAAAALkXQBAAAAMBdrugxIugAAAADAQiRdAAAAAMzl5DldlyPpAgAAAAALUXQBAAAAgIW4vRAAAACAuWikYUDSBQAAAAAWIukCAAAAYC4ejmxA0gUAAAAAFiLpAgAAAGAu1nQZkHQBAAAAgIVIugAAAACYizVdBiRdAAAAAGAhki4AAAAA5iLpMiDpAgAAAAALkXQBAAAAMJXLle7pKeQqJF0AAAAAYCGSLgAAAADmYk2XAUkXAAAAAFiIpAsAAACAuVwkXZcj6QIAAAAAC5F0AQAAADAXa7oMSLoAAAAAwEIkXQAAAADMxZouA5IuAAAAALAQRRcAAAAAWIjbCwEAAACYi0YaBiRdAAAAAGAhki4AAAAA5qKRhgFJFwAAAABYiKQLAAAAgLlY02VA0gUAAAAAFiLpAgAAAGAuki4Dki4AAAAAsBBJFwAAAABz0b3QgKQLAAAAACxE0gUAAADAXKzpMiDpAgAAAAALkXQBAAAAMBdrugxIugAAAADAQiRdAAAAAMzFmi4Dki4AAAAAsBBFFwAAAABzuZw5t2XDjBkzVLNmTQUEBCggIEChoaH65ptvrjk+OjpaNpvtqm3//v3Zui63FwIAAADIF0qXLq233npLlSpVkiR9/PHHevjhh7Vjxw5Vr179mscdOHBAAQEB7tfFixfP1nUpugAAAADkCx06dDC8HjdunGbMmKHNmzdft+gqUaKEihYtetPX5fZCAAAAAOZyOnNsczgcSkpKMmwOh+OGU0xPT9eCBQuUnJys0NDQ646tU6eOQkJC1LJlS61duzbb/xwUXQAAAADyrMjISAUGBhq2yMjIa47fvXu3ChcuLLvdrrCwMH311VeqVq1apmNDQkI0a9YsRUVFafHixapcubJatmyp9evXZ2uONpfL5crWEcgXUk//7ukpAICpJtQb4ekpAICpXjv6qaencE0XFr6eY9fyeviVq5Itu90uu92e6fiUlBQdO3ZMf//9t6KiovThhx9q3bp11yy8rtShQwfZbDYtXbo0y3NkTRcAAACAPOt6BVZm/Pz83I006tevr61bt2rKlCl6//33s3T8vffeq/nz52drjhRdAAAAAMyVh26mc7lcWVoDlmHHjh0KCQnJ1jUougAAAADkC6+++qratm2rMmXK6J9//tGCBQsUHR2tVatWSZIiIiIUFxenefPmSZImT56s8uXLq3r16kpJSdH8+fMVFRWlqKiobF2XogsAAACAuZzZe2hxTvnzzz/VtWtXxcfHKzAwUDVr1tSqVav0wAMPSJLi4+N17Ngx9/iUlBQNGTJEcXFx8vf3V/Xq1bVixQq1a9cuW9elkQYyRSMNALcaGmkAuNXk6kYan4/KsWv5Pzkmx651s0i6AAAAAJgrlyZdnsJzugAAAADAQiRdAAAAAMzlIum6HEkXAAAAAFiIpAsAAACAuVjTZUDSBQAAAAAWIukCAAAAYC6eSmVA0gUAAAAAFiLpAgAAAGAu1nQZkHQBAAAAgIVIugAAAACYi6TLgKQLAAAAACxE0QUAAAAAFuL2QgAAAADmcnF74eVIugAAAADAQiRdAAAAAEzlcvJw5MuRdAEAAACAhUi6AAAAAJiLlvEGJF0AAAAAYCGSLgAAAADmonuhAUkXAAAAAFiIpAsAAACAueheaEDSBQAAAAAWIukCAAAAYC66FxqQdAEAAACAhUi6AAAAAJiLpMuApAsAAAAALETSBQAAAMBcLroXXo6kCwAAAAAsRNIFAAAAwFys6TIg6bLQkSNHZLPZDFvBggVVqlQptWzZUiNHjtShQ4c8PU0AAAAAFiLpygF33HGHnnnmGUmSw+HQyZMntWXLFo0dO1Zvvvmmhg4dqnHjxslms3l4pgAAAADMRtGVAypVqqTRo0dftf/HH3/Us88+q8jISHl7e2vs2LE5PzkgF/hg3hea8v5cPfPEwxo2MMzT0wGAbLvvpY6q/GB9Bd1RSmkXU/THtt/0w1sLdPb3eE9PDfAMJ400LsfthR7UpEkTffvtt7Lb7ZowYYKOHz/ufi8tLU3/+9//VKtWLfn7+yswMFAtWrTQihUrMj3X+fPnNXToUJUpU0YFChRQjRo19MEHHyg6Olo2my3Tog/IDXbHHtCipd/orkoVPD0VALhpZe+pom3z1mhup1H67Jm35OXjrac+GSZff7unpwYgF6Do8rC77rpLnTt3VkpKipYsWSJJcrlc6ty5s8LDw3Xx4kX16dNHTz31lH755Rc99NBDmjp1quEc6enpeuihh/T2228rKChIAwYMUGhoqAYPHqxJkyZ54FMBWXP+/AUNG/O2Rr8yQAFFCnt6OgBw0xZ0m6BfFq3X6d/idDL2mJYPeV+Bpf+jknfzByXkUy5nzm15AEVXLtCsWTNJ0tatWyVJ8+fP1+LFi9WsWTPt3r1bEydO1IwZM7Rr1y6VKFFCQ4YM0eHDh93Hz507V2vXrlXHjh21fft2jR8/Xh988IG2bNmi7777ziOfCciKN955T01DGyi0QR1PTwUATGUvUlCSdPHvcx6eCYDcgKIrFyhVqpQk6fTp05IuFVGSNGHCBPn5+bnHlS5dWoMGDVJqaqo+/fRT9/758+dLksaOHSsvr///X2mVKlXUrVs3q6cP3JSVa6IV++shDQzr4empAIDpWo14Wse27NepX//w9FQAz3C6cm7LAyi6cgHXFU/s3rFjh/z9/dWwYcOrxjZv3lyStHPnTve+Xbt2qVChQqpZs+ZV4++7774bXt/hcCgpKcmwORyO7H0IIBvi/zyltya/r8iRL8tu97vxAQCQh7QZ210lqpTVkn7veXoqAHIJiq5cID7+Umej4sWLS5KSkpIUHByc6diSJUtKkhITE937kpKS3Mde6VrnuVxkZKQCAwMN2/gpM7P1GYDs2HfgN53962917tVPtZq2V62m7RWzY7c+XbRUtZq2V3p6uqenCAA3pfWYZ3VXq7qa/+Q4/ZNw1tPTATzG5XTm2JYX0DI+F4iOjpYkNWjQQJIUEBCgP//8M9OxGfsDAgLc+wICAnTq1Knrjr+eiIgIhYeHG/Z5/RN3w+OAm3Vvvdr66pMZhn3Dx01ShXJl1OuZJ+Tt7e2hmQHAzWvzejdVblNfn3R+Q4nHM///lwHkTyRdHvbrr79q4cKFstvteuSRRyRJderU0YULF7Rly5arxq9bt06SVLt2bfe+WrVqKTk5Wb/88stV4zdu3HjDOdjtdgUEBBg2u50Wt7BOoUIFdWfF8obN37+AigYU0Z0Vy3t6egCQbQ++0V01OjXSkv7vKSX5ogoVD1Sh4oHysft6emqAZ7Cmy4Ciy4M2bNigNm3ayOFwKCIiQrfffrskuZtfREREKDU11T0+Li5OkyZNko+Pj55++mn3/oz/HjFihJyXRaz79+/Xxx9/nBMfBQCAfK1e1wdUILCQui4coYEx091btQ6hnp4agFyA2wtzwMGDB90PJ05JSdHJkyf1888/a8+ePfL29tbw4cM1cuRI9/iuXbtq8eLF+vrrr1WzZk099NBDSk5O1sKFC3XmzBm98847qlixont8jx499Mknn2jp0qWqV6+e2rRpo7Nnz2rBggV64IEHtGzZMkNXQyA3mjttgqenAAA3bVy5p288CMhP8sjzs3IKRVcOOHTokMaMGSNJ8vf3V9GiRVWlShWNGDFC3bp10x133GEYb7PZtGjRIk2ZMkUff/yx3n33Xfn5+alu3boKDw9Xx44dDeO9vb21cuVKjRo1Sp9//rkmT56sO+64Q++8846KFSumZcuWGdaAAQAAAMg5FF0WKl++/FXt4LPKx8dHgwcP1uDBg7M0vlChQpo4caImTpxo2D98+HBJUuXKlW9qHgAAAEC25ZG1VjmFe85uERlt5y+3b98+TZ06VUWLFlWzZs08MCsAAAAAJF23iBdffFFHjhxRw4YNddttt+nQoUNatmyZUlNTNXv2bBUsWNDTUwQAAEB+kUeen5VTKLpuEU888YRmzpypxYsXKzExUYULF1azZs00ePBgtWnTxtPTAwAAAPItiq5bxNNPP21oIw8AAAB4DGu6DFjTBQAAAAAWIukCAAAAYC6e02VA0gUAAAAAFqLoAgAAAAALcXshAAAAAHPRSMOApAsAAAAALETSBQAAAMBULh6ObEDSBQAAAAAWIukCAAAAYC7WdBmQdAEAAACAhUi6AAAAAJiLpMuApAsAAAAALETSBQAAAMBcLroXXo6kCwAAAAAsRNIFAAAAwFys6TIg6QIAAAAAC5F0AQAAADCVi6TLgKQLAAAAACxE0gUAAADAXCRdBiRdAAAAAGAhki4AAAAA5nLynK7LkXQBAAAAgIUougAAAADAQtxeCAAAAMBcNNIwIOkCAAAAAAuRdAEAAAAwF0mXAUkXAAAAAFiIpAsAAACAqVwukq7LkXQBAAAAgIVIugAAAACYizVdBiRdAAAAAPKFGTNmqGbNmgoICFBAQIBCQ0P1zTffXPeYdevWqV69eipQoIAqVqyomTNnZvu6FF0AAAAAzOV05dyWDaVLl9Zbb72lmJgYxcTE6P7779fDDz+svXv3Zjr+8OHDateunZo0aaIdO3bo1VdfVf/+/RUVFZWt63J7IQAAAIB8oUOHDobX48aN04wZM7R582ZVr179qvEzZ85U2bJlNXnyZElS1apVFRMTo4kTJ+qxxx7L8nUpugAAAACYypWDa7ocDoccDodhn91ul91uv+5x6enp+vLLL5WcnKzQ0NBMx2zatEmtW7c27GvTpo1mz56t1NRU+fr6ZmmO3F4IAAAAIM+KjIxUYGCgYYuMjLzm+N27d6tw4cKy2+0KCwvTV199pWrVqmU6NiEhQcHBwYZ9wcHBSktL0+nTp7M8R5IuAAAAAObKwaQrIiJC4eHhhn3XS7kqV66snTt36u+//1ZUVJS6deumdevWXbPwstlshtcZzyC7cv/1UHQBAAAAyLOycivh5fz8/FSpUiVJUv369bV161ZNmTJF77///lVjS5YsqYSEBMO+kydPysfHR0FBQVm+JkUXAAAAAHM5PT2BrHO5XFetCcsQGhqqZcuWGfatXr1a9evXz/J6Lok1XQAAAADyiVdffVU//vijjhw5ot27d+u1115TdHS0nn76aUmXblV89tln3ePDwsJ09OhRhYeHKzY2Vh999JFmz56tIUOGZOu6JF0AAAAATJWT3Quz488//1TXrl0VHx+vwMBA1axZU6tWrdIDDzwgSYqPj9exY8fc4ytUqKCVK1dq0KBBeu+991SqVClNnTo1W+3iJYouAAAAAPnE7Nmzr/v+3Llzr9rXrFkzbd++/V9dl6ILAAAAgLlyadLlKazpAgAAAAALUXQBAAAAgIW4vRAAAACAufJQy/icQNIFAAAAABYi6QIAAABgqtzaMt5TSLoAAAAAwEIkXQAAAADMxZouA5IuAAAAALAQSRcAAAAAU7Gmy4ikCwAAAAAsRNIFAAAAwFys6TIg6QIAAAAAC5F0AQAAADCVi6TLgKQLAAAAACxE0gUAAADAXCRdBiRdAAAAAGAhki4AAAAApmJNlxFJFwAAAABYiKQLAAAAgLlIugxIugAAAADAQhRdAAAAAGAhbi8EAAAAYCoaaRiRdAEAAACAhUi6AAAAAJiKpMuIpAsAAAAALETSBQAAAMBUJF1GJF0AAAAAYCGSLgAAAADmctk8PYNchaQLAAAAACxE0gUAAADAVKzpMiLpAgAAAAALkXQBAAAAMJXLyZquy5F0AQAAAICFSLoAAAAAmIo1XUYkXQAAAABgIZIuAAAAAKZy8ZwuA5IuAAAAALAQSRcAAAAAU7Gmy4ikCwAAAAAsRNEFAAAAABbi9kIAAAAApuLhyEYkXQAAAABgIZIuAAAAAKZyuTw9g9yFpAsAAAAALETSBQAAAMBUrOkyIukCAAAAAAuRdAEAAAAwFUmXEUkXAAAAAFiIpAsAAACAqeheaETSBQAAAAAWIukCAAAAYCrWdBmRdAEAAACAhUi6AAAAAJjK5SLpuhxJFwAAAABYiKQLAAAAgKlcTk/PIHch6QIAAAAAC5F0AQAAADCVkzVdBiRdAAAAAGAhki4AAAAApqJ7oRFJFwAAAABYiKILAAAAACzE7YUAAAAATOVycnvh5Ui6AAAAAMBCJF0AAAAATOVyeXoGuQtJFwAAAABYiKQLAAAAgKlY02V0U0WX0+nUwYMHdfLkSTmdTsN7TZs2NWViAAAAAHAryHbRtXnzZj311FM6evSoXFfcrGmz2ZSenm7a5AAAAADkPU4ejmyQ7aIrLCxM9evX14oVKxQSEiKbjX9QAAAAALiWbBddv/32mxYtWqRKlSpZMR8AAAAAeZyLpMsg290L77nnHh08eNCKuQAAAADALSdLSdcvv/zi/u9+/fpp8ODBSkhI0N133y1fX1/D2Jo1a5o7QwAAAAB5Cs/pMspS0VW7dm3ZbDZD44yePXu6/zvjPRppAAAAAIBRloquw4cPWz0PAAAAALcIuhcaZanoKleunPu/169fr/vuu08+PsZD09LStHHjRsNYAAAAAMjvst1Io0WLFjp79uxV+xMTE9WiRQtTJgUAAAAg73K5bDm25QXZLroy1m5d6cyZMypUqJApkwIAAACAW0WWn9P16KOPSrrUNKN79+6y2+3u99LT0/XLL7/ovvvuM3+GAAAAAPKU3Nq9MDIyUosXL9b+/fvl7++v++67T+PHj1flypWveUx0dHSmd/TFxsaqSpUqWbpulouuwMBASZeSriJFisjf39/9np+fn+6991717t07q6cDAAAAgBy1bt069enTRw0aNFBaWppee+01tW7dWvv27bvhXXsHDhxQQECA+3Xx4sWzfN0sF11z5syRJJUvX15DhgzhVkIAAAAAecqqVasMr+fMmaMSJUpo27Ztatq06XWPLVGihIoWLXpT1832mq5Ro0ZRcAEAAAC4JqfLlmPbv5GYmChJKlas2A3H1qlTRyEhIWrZsqXWrl2bretkOenKUKFChUwbaWT4/fffs3tKAAAAALgpDodDDofDsM9utxt6UGTG5XIpPDxcjRs3Vo0aNa45LiQkRLNmzVK9evXkcDj0ySefqGXLloqOjr5hOpbB5nJlb5nblClTDK9TU1O1Y8cOrVq1Si+//LKGDRuWndMhl0rs0crTUwAAUwV9GuvpKQCAqdJS4jw9hWvaevsjOXatFb1racyYMYZ9o0aN0ujRo697XJ8+fbRixQpt2LBBpUuXztY1O3ToIJvNpqVLl2ZpfLaTrgEDBmS6/7333lNMTEx2TwcAAAAANy0iIkLh4eGGfTdKufr166elS5dq/fr12S64JOnee+/V/Pnzszw+22u6rqVt27aKiooy63QAAAAA8qicXNNlt9sVEBBg2K5VdLlcLvXt21eLFy/WDz/8oAoVKtzU59uxY4dCQkKyPD7bSde1LFq0KEsL0AAAAADAE/r06aPPPvtMX3/9tYoUKaKEhARJlx6PlfFIrIiICMXFxWnevHmSpMmTJ6t8+fKqXr26UlJSNH/+fEVFRWUrcMp20VWnTh1DIw2Xy6WEhASdOnVK06dPz+7pAAAAANxicumzkTVjxgxJUvPmzQ3758yZo+7du0uS4uPjdezYMfd7KSkpGjJkiOLi4uTv76/q1atrxYoVateuXZavm+1GGlcuUvPy8lLx4sXVvHnzLD+RGbkfjTQA3GpopAHgVpObG2lsLvVojl3r3hOLc+xaNytbSVdaWprKly+vNm3aqGTJklbNCQAAAEAe9m+fn3WryVYjDR8fH7344otX9cEHAAAAAGQu290L77nnHu3YscOKuQAAAAC4Bbhcthzb8oJsN9J46aWXNHjwYP3xxx+qV6+eChUqZHi/Zs2apk0OAAAAAPK6LBddPXv21OTJk9W5c2dJUv/+/d3v2Ww2uVwu2Ww2paenmz9LAAAAAHmG09MTyGWyXHR9/PHHeuutt3T48GEr5wMAAAAAt5QsF10ZneXLlStn2WQAAAAA5H0u5Y21VjklW400Ln8oMgAAAADgxrLVSOOuu+66YeF19uzZfzUhAAAAAHmb0+XpGeQu2Sq6xowZo8DAQKvmAgAAAAC3nGwVXV26dFGJEiWsmgsAAACAW4CTNV0GWV7TxXouAAAAAMi+LBddGd0LAQAAAABZl+XbC51OHnEGAAAA4MZoGW+UrZbxAAAAAIDsyVYjDQAAAAC4Ee6RMyLpAgAAAAALkXQBAAAAMBVruoxIugAAAADAQiRdAAAAAEzFmi4jki4AAAAAsBBJFwAAAABTkXQZkXQBAAAAgIVIugAAAACYiu6FRiRdAAAAAGAhki4AAAAApnISdBmQdAEAAACAhUi6AAAAAJjKyZouA5IuAAAAALAQSRcAAAAAU7k8PYFchqQLAAAAACxE0QUAAAAAFuL2QgAAAACmcnp6ArkMSRcAAAAAWIikCwAAAICpnDZaxl+OpAsAAAAALETSBQAAAMBUtIw3IukCAAAAAAuRdAEAAAAwFd0LjUi6AAAAAMBCJF0AAAAATOWkeaEBSRcAAAAAWIikCwAAAICpnCLquhxJFwAAAABYiKQLAAAAgKl4TpcRSRcAAAAAWIikCwAAAICp6F5oRNIFAAAAABYi6QIAAABgKqenJ5DLkHQBAAAAgIVIugAAAACYiu6FRiRdAAAAAGAhii4AAAAAsBC3FwIAAAAwFS3jjUi6AAAAAMBCJF0AAAAATEXLeCOSLgAAAACwEEkXAAAAAFORdBmRdAEAAACAhUi6AAAAAJjKRfdCA5IuAAAAALAQSRcAAAAAU7Gmy4ikCwAAAAAsRNIFAAAAwFQkXUYkXQAAAABgIZIuAAAAAKZyeXoCuQxJFwAAAABYiKQLAAAAgKmcPKfLgKQLAAAAACxE0gUAAADAVHQvNCLpAgAAAAALUXQBAAAAgIW4vRAAAACAqbi90IikCwAAAAAsRNIFAAAAwFQ8HNmIpAsAAAAALETSBQAAAMBUPBzZiKQLAAAAACxE0gUAAADAVHQvNCLpAgAAAJAvREZGqkGDBipSpIhKlCihTp066cCBAzc8bt26dapXr54KFCigihUraubMmdm6LkUXAAAAAFO5cnDLjnXr1qlPnz7avHmzvvvuO6Wlpal169ZKTk6+5jGHDx9Wu3bt1KRJE+3YsUOvvvqq+vfvr6ioqCxfl9sLAQAAAOQLq1atMryeM2eOSpQooW3btqlp06aZHjNz5kyVLVtWkydPliRVrVpVMTExmjhxoh577LEsXZeiCwAAAICpnDn4pC6HwyGHw2HYZ7fbZbfbb3hsYmKiJKlYsWLXHLNp0ya1bt3asK9NmzaaPXu2UlNT5evre8PrcHshAAAAgDwrMjJSgYGBhi0yMvKGx7lcLoWHh6tx48aqUaPGNcclJCQoODjYsC84OFhpaWk6ffp0luZI0gUAAADAVDnZvTAiIkLh4eGGfVlJufr27atffvlFGzZsuOFYm8344DGXy5Xp/muh6AIAAACQZ2X1VsLL9evXT0uXLtX69etVunTp644tWbKkEhISDPtOnjwpHx8fBQUFZel6FF0AAAAATJVzK7qyx+VyqV+/fvrqq68UHR2tChUq3PCY0NBQLVu2zLBv9erVql+/fpbWc0ms6QIAAACQT/Tp00fz58/XZ599piJFiighIUEJCQm6cOGCe0xERISeffZZ9+uwsDAdPXpU4eHhio2N1UcffaTZs2dryJAhWb4uRRcAAAAAUzlzcMuOGTNmKDExUc2bN1dISIh7++KLL9xj4uPjdezYMffrChUqaOXKlYqOjlbt2rU1duxYTZ06Ncvt4iVuLwQAAACQT2Q0wLieuXPnXrWvWbNm2r59+01fl6ILAAAAgKmcWWvql29weyEAAAAAWIiiCwAAAAAsxO2FAAAAAEzlzLVN4z2DpAsAAAAALETSBQAAAMBU5FxGJF0AAAAAYCGSLgAAAACmyu5Di291JF0AAAAAYCGSLgAAAACmonuhEUkXAAAAAFiIpAsAAACAqci5jEi6AAAAAMBCJF0AAAAATEX3QiOSLgAAAACwEEkXAAAAAFPRvdCIpAsAAAAALETSBQAAAMBU5FxGJF0AAAAAYCGSLgAAAACmonuhEUkXAAAAAFiIogsAAAAALMTthQAAAABM5aKVhgFJFwAAAABYiKQLAAAAgKlopGFE0gUAAAAAFiLpAgAAAGAqJ2u6DEi6AAAAAMBCJF0AAAAATEXOZUTSlQuUL19e5cuX9/Q0AAAAAFjAo0XXkSNHZLPZZLPZ9NBDD2U6Jjo6WjabTWFhYf/qWs2bN5fNZvtX5wAAAABwY065cmzLC3JN0rVixQqtX7/e09MAAAAAAFPliqKrfPny8vLy0iuvvOLpqQAAAAD4l5w5uOUFuaLoqly5srp27arNmzdr8eLFWT7u2LFj6tWrl26//Xb5+fmpdOnS6tWrl44fP24YZ7PZtG7dOvd/Z2zdu3e/4TW++uorPfnkk6pUqZIKFiyowMBANWnSRFFRUdn6jJL09ddfq0GDBvL391dwcLB69+6tv/7665rjz5w5o0GDBqlChQqy2+0qUaKEOnfurH379mU6/siRI+rcubOKFSumwoULq1mzZlq/fr1Gjx4tm82m6OjobM8ZsIJfiw4q/PosBUz/WgHTv1ah16bK5+4Gnp4WANy0Jo3v0ZKv5urYkW1KS4lTx45tPD0lALlIrule+Prrr2vBggV69dVX9fDDD8vb2/u643/77Tc1btxYJ0+eVIcOHVS9enXt3btXH330kZYvX66ffvpJlSpVkiSNGjVKc+fO1dGjRzVq1Cj3OWrXrn3DeUVERMjPz0+NGzdWSEiITp06paVLl+rxxx/X1KlT1a9fvyx9vnnz5qlbt24KCAhQ165dVbRoUS1fvlytWrVSSkqK/Pz8DOPPnDmje++9VwcPHlTz5s3VpUsXHTlyRIsWLdKKFSv03XffKTQ01D0+Li5O9913n+Lj49WuXTvVqlVLBw4cUOvWrdWiRYsszRHIKc6zp3Rx0Ydy/hknSfJt1FoF+7+uc6PC5Dxx1MOzA4DsK1SooH75ZZ/mfvyFFi380NPTATzOlUfWWuWUXFN0lS1bVn369NGkSZM0e/ZsPf/889cdHxYWppMnT+r99983jJ01a5ZeeOEFhYWFac2aNZKk0aNHKzo6WkePHtXo0aOzNa+VK1eqYsWKhn3nzp3TfffdpxEjRqhXr14qWLDgdc+RlJSkfv36qVChQtq6davuuusuSdK4cePUqlUrxcfHq1y5coZjhg4dqoMHDyoiIkJvvvmme3/37t314IMPqlu3btq/f7+8vC6FlcOGDVN8fLzefvttDRkyxD1+7ty56tGjR7Y+M2C1tF2bDa8di+fIr0UHed9RlaILQJ606tu1WvXtWk9PA0AulStuL8zw2muvKTAwUGPGjNH58+evOe748eP64YcfVK1aNfXu3dvwXu/evVW1alV9//33V91meDOuLLgkqXDhwurevbsSExO1devWG55jyZIlSkpKUs+ePd0FlyT5+vpq3LhxV41PSUnR559/rqCgIA0fPtzwXps2bdSmTRv99ttv2rhxoyTJ4XDoyy+/VHBwsPr3728Y361bN1WpUiVLnxXwCJuXfBs2l81eQOmHMr91FgAA5C2s6TLKVUVXsWLF9Morr+jEiROaPHnyNcft2LFDktSsWbOr2sDbbDY1bdpUkrRr165/PaeTJ08qPDxcVatWVcGCBd3rwQYPHixJOnHixA3PkTGPJk2aXPVeaGiofHyMgeP+/ft14cIFNWzYMNMUrXnz5pKknTt3SpIOHDggh8Oh+vXrX3Wbos1mM9yGmBmHw6GkpCTD5kjPK/8njLzKq3QFBcxYpoAPvpF/t4E6P220nCeOeXpaAAAApstVRZckDRw4UKVKldKECRN05syZTMckJSVJkoKDgzN9v2TJkpKkxMTEfzWXs2fPqkGDBvrf//6noKAg9erVS8OHD9eoUaP08MMPS7pUsNxIxjxKlChx1Xve3t4KCgoy7Mvu58sYX7x48UzHX+s8GSIjIxUYGGjYJv1y5LrHAP+WM/64zo16Qclv9JNj7TL5PzdUXqXKenpaAADABK4c/J+8INcVXf7+/ho9erQSExMNa5kuFxAQIEn6888/M30/Y3/GuJs1e/ZsHTt2TG+88YY2bNigd999V2PHjtXo0aN17733Zvk8gYGBki6lZldKT0+/qrjM7ufL+H9PnTp13fHXEhERocTERMMWXrP8dY8B/rX0NDlPnlD6kV/lWDRbzmO/y++BRz09KwAAANPluqJLknr27KkqVarovffe07FjV99ulNF1cP369XK5jNWty+XSjz/+aBgnyd0NMT09PcvzOHTokCSpY8eOV72XcY2sqFWr1jWP2bRpk9LS0gz7qlSpogIFCmjr1q2Zrm3LaH+f8fkqV64su92ubdu2KSUlxTDW5XJp8+bNV57CwG63KyAgwLDZvXPl/2ngVmaTbD6+np4FAAAwAWu6jHLlL2tvb2+9+eabcjgcev311696v2zZsmrRooW7RfzlPvroI+3du1f333+/ypQp495frFgxSdIff/yR5XlkdBTcsGGDYf9nn32mlStXZvk8Dz/8sAICAvTRRx/p119/de9PTU29qlGGJPn5+enJJ5/U6dOnFRkZaXhvzZo1+uabb1SpUiU1atRI0qWi6fHHH1dCQoKmTp1qGD9v3jzFxsZmea5ATrA/1lPed9aQLShYXqUryP5oD3lXqaXUTd97emoAcFMKFSqoWrWqq1at6pKkCuXLqlat6ipTppSHZwYgN8g1LeOv9Mgjjyg0NFSbNm3K9P0ZM2aocePG6t27t5YtW6Zq1app3759Wrp0qYoXL64ZM2YYxt9///1atGiRnnjiCbVr104FChTQ3Xffrfbt219zDl27dtX48ePVr18/rV27VuXKldMvv/yiNWvW6NFHH83yg5wDAwM1depUde/eXQ0aNFCXLl0UGBio5cuXy9/fXyEhIVcdM378eK1bt05vvPGGNm7cqHvuucf9nK6CBQtqzpw57nbx0qV1WWvWrNHLL7+stWvXqnbt2jpw4ICWL1+uBx98UKtWrTKMBzzJK+A2FXx+mGyBxeS6kCzn8cM6/06E0vZt9/TUAOCm1K9XS9+vWeR+/c7E0ZKkj+ctVK/nBnloVgByi1xbdEmXCo+MToRXqly5smJiYjRmzBitWrVKK1asUPHixdW9e3eNGjXqqude9e7dW0eOHNGCBQs0btw4paWlqVu3btctukqXLq1169Zp6NChWrNmjdLS0lS3bl2tXr1ax48fz3LRJV1q3R4YGKg33nhDH3/8sQIDA9WxY0dNmDBBderUuWp88eLF9fPPP2vs2LH6+uuv9eOPPyowMFAPP/ywRo0apRo1ahjGlylTRps2bdIrr7yi1atXKzo6WvXq1dPq1av15ZdfSvr3a9wAs1yY846npwAAplq3fpN8/G739DSAXMPpyhsNLnKKzXXloijccho3bqxNmzYpMTFRhQsXztIxiT1aWTwrAMhZQZ9yqzWAW0taSpynp3BNXcvlXHOsT45mPQjxFO43u4XEx8dfte/TTz/VTz/9pFatWmW54AIAAAD+DVcObnlBrr69ENlTo0YN1alTR9WqVZO3t7d27typ6OhoFSlSRBMnTvT09AAAAIB8iaLrFhIWFqZly5YpJiZGycnJKl68uJ566imNGDFCVapU8fT0AAAAkE8480wGlTMoum4h48aN07hx4zw9DQAAAACXoegCAAAAYCoXSZcBjTQAAAAAwEIkXQAAAABM5fT0BHIZki4AAAAAsBBJFwAAAABT0b3QiKQLAAAAACxE0gUAAADAVHQvNCLpAgAAAAALkXQBAAAAMBXdC41IugAAAADAQiRdAAAAAEzlcrGm63IkXQAAAABgIYouAAAAALAQtxcCAAAAMBUPRzYi6QIAAAAAC5F0AQAAADAVLeONSLoAAAAAwEIkXQAAAABM5WJNlwFJFwAAAABYiKQLAAAAgKnoXmhE0gUAAAAAFiLpAgAAAGAql4uk63IkXQAAAABgIZIuAAAAAKbiOV1GJF0AAAAAYCGSLgAAAACm4jldRiRdAAAAAGAhki4AAAAApuI5XUYkXQAAAABgIZIuAAAAAKbiOV1GJF0AAAAA8oX169erQ4cOKlWqlGw2m5YsWXLd8dHR0bLZbFdt+/fvz9Z1SboAAAAAmCq3rulKTk5WrVq11KNHDz322GNZPu7AgQMKCAhwvy5evHi2rkvRBQAAACBfaNu2rdq2bZvt40qUKKGiRYve9HW5vRAAAABAnuVwOJSUlGTYHA6HqdeoU6eOQkJC1LJlS61duzbbx1N0AQAAADCVKwf/JzIyUoGBgYYtMjLSlM8REhKiWbNmKSoqSosXL1blypXVsmVLrV+/PlvnsbloLYJMJPZo5ekpAICpgj6N9fQUAMBUaSlxnp7CNTUvnXO/Jb89tOKqZMtut8tut1/3OJvNpq+++kqdOnXK1vU6dOggm82mpUuXZvkY1nQBAAAAMJUzB3OdrBRYZrr33ns1f/78bB3D7YUAAAAAkEU7duxQSEhIto4h6QIAAABgqty6funcuXM6ePCg+/Xhw4e1c+dOFStWTGXLllVERITi4uI0b948SdLkyZNVvnx5Va9eXSkpKZo/f76ioqIUFRWVretSdAEAAADIF2JiYtSiRQv36/DwcElSt27dNHfuXMXHx+vYsWPu91NSUjRkyBDFxcXJ399f1atX14oVK9SuXbtsXZdGGsgUjTQA3GpopAHgVpObG2k0uv3+HLvWT3E/5Ni1bhZrugAAAADAQtxeCAAAAMBUzly7qsszSLoAAAAAwEIkXQAAAABMRdsII5IuAAAAALAQSRcAAAAAU7Gmy4ikCwAAAAAsRNIFAAAAwFQuki4Dki4AAAAAsBBJFwAAAABT0b3QiKQLAAAAACxE0QUAAAAAFuL2QgAAAACmomW8EUkXAAAAAFiIpAsAAACAqWikYUTSBQAAAAAWIukCAAAAYCrWdBmRdAEAAACAhUi6AAAAAJjKRdJlQNIFAAAAABYi6QIAAABgKifdCw1IugAAAADAQiRdAAAAAEzFmi4jki4AAAAAsBBJFwAAAABTsabLiKQLAAAAACxE0gUAAADAVKzpMiLpAgAAAAALkXQBAAAAMBVruoxIugAAAADAQiRdAAAAAEzFmi4jki4AAAAAsBBFFwAAAABYiNsLAQAAAJiKRhpGJF0AAAAAYCGSLgAAAACmopGGEUkXAAAAAFiIpAsAAACAqVwup6enkKuQdAEAAACAhUi6AAAAAJjKyZouA5IuAAAAALAQSRcAAAAAU7l4TpcBSRcAAAAAWIikCwAAAICpWNNlRNIFAAAAABYi6QIAAABgKtZ0GZF0AQAAAICFSLoAAAAAmMpJ0mVA0gUAAAAAFiLpAgAAAGAqF90LDUi6AAAAAMBCFF0AAAAAYCFuLwQAAABgKlrGG5F0AQAAAICFSLoAAAAAmMpJIw0Dki4AAAAAsBBJFwAAAABTsabLiKQLAAAAACxE0gUAAADAVE6SLgOSLgAAAACwEEkXAAAAAFOxpsuIpAsAAAAALETSBQAAAMBUPKfLiKQLAAAAACxE0gUAAADAVKzpMiLpAgAAAAALkXQBAAAAMBXP6TIi6QIAAAAAC5F0AQAAADCVi+6FBiRdAAAAAGAhki4AAAAApmJNlxFJFwAAAABYiKILAAAAACzE7YUAAAAATMXDkY1IugAAAADAQiRdAAAAAExFy3gjki4AAAAAsBBJFwAAAABTsabLiKQLAAAAACxE0QUAAADAVC6XK8e27Fi/fr06dOigUqVKyWazacmSJTc8Zt26dapXr54KFCigihUraubMmdn+96DoAgAAAJAvJCcnq1atWpo2bVqWxh8+fFjt2rVTkyZNtGPHDr366qvq37+/oqKisnVd1nQBAAAAMFVuXdHVtm1btW3bNsvjZ86cqbJly2ry5MmSpKpVqyomJkYTJ07UY489luXzkHQBAAAAyLMcDoeSkpIMm8PhMOXcmzZtUuvWrQ372rRpo5iYGKWmpmb5PCRdyFTgnDWengLyAYfDocjISEVERMhut3t6OrjFpc3x9AyQH/C9BlySlhKXY9caPXq0xowZY9g3atQojR49+l+fOyEhQcHBwYZ9wcHBSktL0+nTpxUSEpKl85B0AfAYh8OhMWPGmPbXKADwNL7XgJwXERGhxMREwxYREWHa+W02m+F1RvOOK/dfD0kXAAAAgDzLbrdbliyXLFlSCQkJhn0nT56Uj4+PgoKCsnweki4AAAAAyERoaKi+++47w77Vq1erfv368vX1zfJ5KLoAAAAA5Avnzp3Tzp07tXPnTkmXWsLv3LlTx44dk3TpVsVnn33WPT4sLExHjx5VeHi4YmNj9dFHH2n27NkaMmRItq7L7YUAPMZut2vUqFEsNgdwy+B7DcjdYmJi1KJFC/fr8PBwSVK3bt00d+5cxcfHuwswSapQoYJWrlypQYMG6b333lOpUqU0derUbLWLlySbK7uPcQYAAAAAZBm3FwIAAACAhSi6AAAAAMBCFF0AAAAAYCGKLgAAAACwEEUXAAAAAFiIogsAAAAALETRBQAAAAAWougCYAoe+QfgVuJ0OiVJZ8+elcPh8PBsAOR1FF0A/jWXyyWbzaZjx47p4MGDnp4OAPxrXl5e2r9/v6pUqaKPP/5YFy9e9PSUAORhFF0A/jWbzaY//vhDlStX1ujRo7V//35PTwkAbkpGwiVJ+/fvV+HChTVixAgtXrxYFy5c8ODMAORlPp6eAIC8y+l0ysvLSxcuXND58+dVv359rVixQkWLFlXfvn1VpUoVT08RALIs4ztt165dmj9/vtauXauCBQvqyJEjGjhwoNLT0/Xf//5Xdrvd01MFkMdQdAG4KRk/TrZv36633npL27ZtU6FChZSYmKg5c+YoLS1N4eHhuuuuuzw9VQC4IZfL5f5Oa9WqlSpWrKjGjRurUaNGWrRokb7++mv17dtXLpdLnTt3pvACkC02F6vfAdykXbt2qVmzZqpRo4bat2+vLl26aP78+fr222+1adMm9erVS0OGDKHwApAnnDlzRm3btlVCQoI++OADtWnTxv3enDlzNHHiRP3xxx+aNm2aHn/8cfn7+3twtgDyEoouANnmcrl07tw5Pfnkk1q3bp2++OILtWvXzv3+rl27NH78eC1YsEC9e/dWeHi4Kleu7MEZA8CNHT9+XNWrV1fHjh01f/58SZLD4XCnWvPnz1ffvn3l4+OjqVOn6pFHHqHwApAlNNIAkG02m00+Pj46ePCg6tat6y640tLSJEm1atXSiBEj1Lx5c82bN0/vvvuuYmNjPTllALihkydP6ty5cwoKCpIkpaSkyG63ux+J8cwzz6hz5846e/as+vfvr6ioKB6XASBLKLoAZJvL5dKpU6eUkJCg8+fP68KFC0pJSZGPz/8vE61ataqeeOIJORwOzZs3T9OmTdORI0c8N2kAuIHbb79dZcqUUVRUlI4fPy4/Pz/3IzFSU1MlSb169dIdd9yh4OBg9e3bVxs3bpRk7HoIAFei6AKQbTabTWXLllXLli21e/du7dmzR35+fu4fHenp6ZKkZ599VlWrVlW1atX0/vvv67PPPpPT6eTHCYBcqWTJkmrdurVOnDihkSNH6s8//5TNZlNaWpp8fX0lSXv37tXZs2f1/PPPy+VyqUePHjpz5oy8vPhJBeDa+IYAkG0ZRVOLFi2UkpKiLl266Pfff5eXl5dSU1Pl7e0tSVq3bp3i4uI0YsQItW7dWm+//bYOHz7MjxMAHpPx/ZWWliaHw6Hk5GSlpKS433/nnXcUGhqqefPmaeTIkfrjjz/cKf6BAwf07bff6v7779eAAQP03//+VwcPHtTu3bs98lkA5B388gFwTRk/Ts6dO6f4+Hjt2bNHp0+fdhdNffv21ZNPPqnDhw+rffv2+uWXX9x/DY6NjdXixYtVtWpV97qvxMRELVq0yGOfB0D+lvGoiz179uj5559X7dq1Va1aNTVp0kRLly7Vn3/+qYCAAL333ntq2LChPvjgA91///2aOHGi3nzzTb3wwgtauHChGjduLElq3bq1JOn333/35McCkAfwnC4Ambr8IaHDhw9XTEyMTp06pf/85z/q1auX2rZtq8aNG+vTTz9Venq6Fi5cqPvuu0+tWrVSiRIltGHDBu3fv19vv/22QkJC9MADD0iS4S/KAJBTMp7DtXXrVj344IPy8vJSpUqVFBAQoK1bt6pLly7q1auX+vXrp9q1a2vJkiUaNGiQlixZoqFDh0qSgoKC9M4772jAgAGSpLVr18rHx0fVqlXz5EcDkAdQdAG4SsaPk5iYGLVq1UpFihRR06ZN5e3trW3btikyMlKrVq3S6NGj1aFDBy1YsEC1a9fWN998o+XLl8vf31+lSpXSu+++qz59+kiSPv30U0midTwAj7DZbDp8+LC6dOmi8uXL6/XXX1f79u3ldDr1+eefa86cOZo+fbpSU1M1fPhwlS5dWp999pl27typY8eOqUiRIipatKjq1KkjSVq0aJGWLFmiOnXq6I477vDwpwOQ2/GcLgCZOnHihB588EFJ0sSJE9230ezfv18ffvihJk2apBo1amjKlClq0aKFJCkpKUmHDh1SwYIF5e3trUqVKkmSlixZopdffll+fn5as2aNQkJCPPOhAOQb6enp7vWlGWbPnq2+fftq8uTJeuGFFwzvbd++XSNGjND333+vDz74QF27dr3muT/88ENNmTJFcXFx2rBhA0kXgBtiTReATO3atUu//fabOnfu7C64JKlKlSqaOHGihg0bpj179uiLL75w3zIYEBCgOnXqqHLlyu6C66233tLw4cN19uxZffHFFxRcACz10ksv6bfffpO3t/dVnVI3bdokh8Ohe+65R9Klwizjb89169ZV79695e3trTFjxujcuXNXndvhcKh169YaOHCg0tPT9eOPP1JwAcgSii4gH7s86L6y3fuePXvkcDjctwNe+ePlxRdfVKNGjTR//nzFxcVdde6LFy9qwIABmjhxonx9fbV+/XrVqFHDqo8CAJo8ebJmzpypp556yt0p9fLvrnLlysnLy8v9neXl5SWbzeb+LuzUqZNatmypY8eO6a+//rrq/Ha7XQMGDNCwYcO0atUqVa9ePWc+GIA8j6ILyMcyHviZnp6u8+fPS5L7dpwyZcpIkvbt2yfJWKBlvH/vvffq/Pnz2r9//1XnLlCggJ599lm99957WrFiBT9OAFhu4MCBeuGFF7Rt2zY9/vjj7sIrLS1N0qWiy+l0auzYsTp69Kih4MoozkqVKnXd5wm2b99eERERKlu2bM58KAC3BIouIJ86ePCgoqKi1KlTJ91///1q27atXn/9de3atUvp6emqX7++goODNWnSJO3du1fe3t7uW3EyfoyUKFFCklSsWLFMr1GvXj3997//ValSpXLscwHInzIKqxkzZui5557Tjh079Pjjj+vQoUPu52w9++yz6tixo7Zs2aLx48fr+PHjstlski6lXvv27dPGjRvVsGFDFS1a9JrXunKtGADcCEUXkA8tWbJEPXv21BNPPKHVq1dr//79+umnnzR69Gg988wz+t///qdKlSqpb9++SkpK0pNPPqnY2Fh5e3vLZrPJy8tLsbGx+vrrr3XXXXcpODj4mtfK+EEDAFby8fFxry+dNWuW+vTpo127dumJJ57Q4cOH3ePGjx+v0NBQzZw5Uz169NDmzZt15swZbdq0SZGRkdqzZ4969+6twMBAT30UALcgWsYD+cysWbM0ZMgQlS9fXlOnTtVjjz2mwMBAbdmyRV988YW++OILRURE6Pz58xo5cqSOHj2qDz/8UI0aNdL48eN19913KykpSR9++KE2btyo6dOnq3z58p7+WADyOafTKT8/P+3du1fbt2+Xj4+PihYtqp07d+qJJ57Ql19+qQoVKuiOO+7QrFmz9PLLL2vVqlVq0aKFAgIC5HA4lJKSogkTJqhHjx6SLt1WzR+OAJiBlvFAPjJt2jT1799fjz/+uAYNGqTQ0FDD+6dPn9Y333yjoUOH6uTJk5o6dar69OmjIUOG6P3331dycrKkS7fhFC5cWKNGjdKgQYMk8eMEgOdkfP9s3bpVHTt2lN1uV4MGDXTx4kUdOXJEe/fuVd26dbVw4UJVrFjRfdykSZMUGxurnTt3qnnz5mrevLnat28v6f8fEA8AZqDoAvKJWbNmKSwsTM8++6zCw8NVs2ZNSVf/sLhw4YK+/PJLDRo0SP/5z3+0YsUKVapUSd9//737L8h169ZV7dq11bRp00zPAQA57Y8//lDTpk1VuHBhvf3222rTpo2cTqfOnTunXr16KSoqSnXr1nUnXpe78plefKcBMBtFF5APzJs3T927d1ft2rW1YMEC3XXXXe6OXZmlU6dPn9aoUaM0Y8YMvfbaaxo7duw1z82PEwC5wapVq9SuXTuNHj1aI0eOlCSlpqbK19dXkvTkk0/qiy++MBReaWlp8vHxIakHYDl+KQH5wG233SYfHx/9+uuv+vbbbyXJ0Cr5Sv/5z3/UuXNnSdL333+vxMRE9/O7rmyjTMEFIDc4ffq0JKlOnTqSLqVXvr6+7u+uefPmqWHDhtq+fbs6d+6sgwcPursaUnABsBq/loB8oEOHDlq6dKm8vLwUERGhd955R5KuenDo5e655x6FhITIy8tLBQsWdN96Q5EFIDcKCAiQJC1cuFCJiYnu7yxvb2+lpqbKx8dHderUUZEiRRQTE6M2bdooKSnpmn98AgAz8esJyCcefPBBLViwQF5eXho1apQmTZok6erCK+MHSFJSkv766y/5+/tfszADgNyiffv2atiwob7//ntt3LjR/b3ldDrl6+srm82m4OBghYaGqkuXLnrhhRcUEBBAygUgR1B0AflIu3bt3IXXyJEjMy28Mn6ALF26VBcvXlSPHj1kt9v5azCAXCtjTVaPHj107tw5vfLKK1q7dq3OnTvnTudjY2O1cuVKNWzYUJ999pmGDh3qPhYArEbRBeQzWSm8tm3bpunTp6thw4bu9RH8NRhAbpXx0PbOnTurT58++v3339WzZ0+NHDlSW7Zs0dKlSzV69Gjt2bNH1atXdx9HAw0AOYXuhUA+tXLlSnXp0kVOp1NjxozR4MGDJV36a3BkZKS++uorffDBB+rSpYuHZwoAN5ZRQP3111/6+OOPNWfOHO3evdv9vpeXl8aPH+/+rgOAnETRBeRjlxdekZGReuihhzR+/HjNmjVLkyZN0sCBAyXx12AAeUPGd1VKSoqSk5P1xRdf6OTJkypWrJhq1Kih5s2bS+JRFwByHkUXkM9dXnhVr15dW7duVWRkpF555RVJ/DgB4DlW/MGH7zQAnkDRBUCrVq1Sp06dlJKSYki4+HECwFMyvn/++usvxcfH66+//lKpUqVUoUIFT08NALKNoguApEvdCk+cOKGwsDBJFFwAPCfj+2fHjh0aOHCgYmJidOHCBRUqVEijRo3Sk08+qdtvv93T0wSALKPoAnAVCi4AnpJxS2FMTIweeOABBQYGqn379qpcubLWrFmjlStXqn///goLC9Ndd93l6ekCQJb4eHoCAHIfCi4AnmKz2XTo0CF169ZNd955p0aMGKEOHTpIkn799Vc5nU5NmTJFycnJGjJkiO68804PzxgAboyiCwAA5Bqpqan65JNPlJycrKFDh7oLruHDh2v69Ol65plnlJiYqA8//FC+vr7q16+fKleu7OFZA8D1UXQBAIBcIy0tTZs3b9a9996rbt26SZIiIyP15ptvKiwsTMOGDdPJkye1fft2zZgxQz4+PnrppZe41RBArsaaLgAAkKscPXpUdrtdJUuW1LJly9S9e3c1a9ZMb775pqpUqSJJ6ty5s5YuXSqHw6HOnTtr5syZCgwM9PDMASBzJF0AACDXcLlcKleunPv1li1b5HA41L9/f1WpUkXp6eny9vaWJLVo0UJBQUGqX78+BReAXI2iCwAA5BoZD0N2uVxyOp2Kjo6W3W7XHXfcIUny9vbWrl27tHv3br344ot67rnn5O/v7z7G7IcpA4AZaFEGAAA85lqrHGw2m7y9vfXQQw/pr7/+0ty5c3Xq1Clt3rxZb7/9tuLi4lStWjUKLgB5Amu6AABAjkhNTZW3t7e8vLz0119/6bbbbrvhMT///LOGDh2qDRs2KDg4WOfPn1dycrImTJigQYMG5cCsAeDfI+kCAACWioqK0tmzZ+Xr6ysvLy/9/PPPevTRR7V+/fobHnvPPfdowoQJGjNmjIKDg/XQQw9p3rx57oLL6XRaPX0A+NdIugAAgGU2bdqkRo0aqUKFCjp06JB+/fVXVa9eXfXr19f06dNVp06dax575S2DDodDdrvd/drpdPIwdwB5At9UAADAMrVq1dJzzz2nw4cPq2LFiqpVq5ZCQ0M1ZsyY6xZckq5ao3V5wSWJggtAnkHSBQAALHF5e/fHHntMy5cvl4+PjxYsWKAOHTpIogEGgPyBogsAAFgmJSVFTqdTBQsWVEBAgJKSklStWjX9/PPPKlSokNLS0uTjwxNsANzaKLoAAIDlvv/+e128eFGrV6/Wu+++q6pVq+qnn35S0aJFr1l4kYIBuFVQdAEAAFNdXixdWTilp6drwIABmj59uqHwyrgVMT4+XqdOnVLNmjU9NX0AMB0rUAEAgGmcTqdsNptOnz6tgwcPavXq1dq5c6fS0tIkSd7e3nr77bfVp08fxcbGqlGjRjp79qy8vb11+PBhvfnmm3r++ef122+/efiTAIB5uIkaAACYIqOF+/bt29W/f3/t3LlT58+flySFhoZqwoQJuvvuuxUQEKC3335bNptN06ZNU6NGjfT888/rxx9/1JIlS/Tmm2/qzjvv9PCnAQDzcHshAAD41zJuI9y+fbuaN2+ukJAQtWrVSnfeeaeWLFmi9evX6/bbb9frr7+uRx55REWLFtXFixc1ZswYzZ49W6dPn1bRokU1cuRIDRw40HBOAMjrKLoAAIApTp8+rY4dO+rMmTOaOnWq2rRpI+nSQ42nT5+uqVOnKiUlRXPmzFHr1q0lSampqdq6datOnTqloKAgNW7cWBIPPgZwa6HoAgAAN+XKwujAgQOqW7euevbsqXfffVfSpYLLbrfL4XDoo48+0oABA1SjRg399NNP8vf3z9J5ASCv4xsNAABky759+5SWliYvLy85nU73/oMHD+rChQu66667JP1/weVyuWS329WjRw+1bdtWO3fu1OrVq695fgouALcavtUAAECWffrpp2rQoIGmTZum9PR0Q+FVsmRJSdLXX3+t1NRUd8Fls9mUmpqqAgUKqGfPnpKk+Ph4j30GAMhpFF0AACBLXC6XvL295e3trXfeeUczZsxwJ16SdPfdd6tBgwb64Ycf9P777yslJcVdcPn6+kqSEhISJEllypTx2OcAgJxG0QUAALLEZrPp0Ucf1dy5c5WSkqI33nhDM2fOdD+Dy8/PTxMmTFDJkiU1efJk97iMgmv//v1atmyZSpUqpdKlS3vyowBAjqKRBgAAyJbU1FQtXbpUL730kmw2m4YPH66wsDD5+PgoMTFRCxcu1IgRI/T333+rXbt26tmzp+Lj47V06VKtWLFCU6ZMUb9+/Tz9MQAgx1B0AQCALElPT5e3t7ek6xdef//9tzZu3Khhw4Zpz5497uNDQkI0bNgwd8HFc7gA5BcUXQAAIFPLly/XiRMn1KlTJwUFBbkLrgwXL17UihUr9OKLL8rLy0uvvfaaXnzxRfn4+EiSkpOT9e233youLk7ly5dX6dKlVadOHUm0hQeQv1B0AQCAq7z77rsaMGCAJKl8+fIqW7asunbtqho1auiee+5xj0tLS9NXX32ll156SV5eXhoxYoQ78boWEi4A+c21vxEBAEC+lJKSomXLlkmSihUrJqfTqbi4OPXu3VuS1Lx5c9199916+umndfvtt+uJJ56Qr6+v+vXrp7Fjx8rpdOqll16Sj4+P4ZbEDBRcAPIbki4AAHCV06dP67nnntPSpUvVrl07DR48WP/8848WLVqk7777Tn/++ae8vb0VFBSkp556SpUqVZKfn58GDBigihUrqkePHurfv/91Ey8AyC8ougAAQKbOnj2rp556SqtXr9bDDz+s999/XyVKlNCpU6e0bds2fffdd/rhhx8UGxurlJQUFStWTKmpqfrnn39UunRpLV26VLVr1/b0xwAAj6PoAgAA13T27Fl17dpV33zzjR5++GFNmDBBd955p/v9pKQkHTp0SD/88IO+++477d69W/Hx8ZowYYKGDBniwZkDQO5B0QUAAK7rysJr0qRJqlChglwul1wul6EL4cGDB5WYmKh69epJomkGAEgUXQAAIAuuVXhltH7PrGEGbeEB4BKKLgAAkCXXS7xIswDg2vjzEwAAyJJixYrpk08+Udu2bfX111/r5Zdf1u+//07BBQA3QNIFAACy5ezZs+revbuWL1+u+++/X1999ZWKFCni6WkBQK5F0gUAALKlWLFimjNnjho3bqwHH3yQggsAboCkCwAA3BSHwyG73S6JLoUAcD0UXQAA4F+h4AKA6+P2QgAA8K9QcAHA9VF0AQAAAICFKLoAAAAAwEIUXQAAAABgIYouAAAAALAQRRcAAAAAWIiiCwAAAAAsRNEFALhljB49WrVr13a/7t69uzp16vSvzmnGOQAA+RtFFwDAct27d5fNZpPNZpOvr68qVqyoIUOGKDk52dLrTpkyRXPnzs3S2CNHjshms2nnzp03fQ4AADLj4+kJAADyhwcffFBz5sxRamqqfvzxRz333HNKTk7WjBkzDONSU1Pl6+tryjUDAwNzxTkAAPkbSRcAIEfY7XaVLFlSZcqU0VNPPaWnn35aS5Yscd8S+NFHH6lixYqy2+1yuVxKTEzU888/rxIlSiggIED333+/du3aZTjnW2+9peDgYBUpUkS9evXSxYsXDe9feWug0+nU+PHjValSJdntdpUtW1bjxo2TJFWoUEGSVKdOHdlsNjVv3jzTczgcDvXv318lSpRQgQIF1LhxY23dutX9fnR0tGw2m77//nvVr19fBQsW1H333acDBw6Y+K8JAMhLKLoAAB7h7++v1NRUSdLBgwe1cOFCRUVFuW/va9++vRISErRy5Upt27ZNdevWVcuWLXX27FlJ0sKFCzVq1CiNGzdOMTExCgkJ0fTp0697zYiICI0fP14jRozQvn379Nlnnyk4OFiStGXLFknSmjVrFB8fr8WLF2d6jqFDhyoqKkoff/yxtm/frkqVKqlNmzbueWV47bXX9M477ygmJkY+Pj7q2bPnTf9bAQDyNm4vBADkuC1btuizzz5Ty5YtJUkpKSn65JNPVLx4cUnSDz/8oN27d+vkyZOy2+2SpIkTJ2rJkiVatGiRnn/+eU2ePFk9e/bUc889J0l64403tGbNmqvSrgz//POPpkyZomnTpqlbt26SpDvuuEONGzeWJPe1g4KCVLJkyUzPkXE75Ny5c9W2bVtJ0gcffKDvvvtOs2fP1ssvv+weO27cODVr1kySNGzYMLVv314XL15UgQIFbv4fDgCQJ5F0AQByxPLly1W4cGEVKFBAoaGhatq0qd59911JUrly5dxFjyRt27ZN586dU1BQkAoXLuzeDh8+rEOHDkmSYmNjFRoaarjGla8vFxsbK4fD4S70bsahQ4eUmpqqRo0auff5+vqqYcOGio2NNYytWbOm+79DQkIkSSdPnrzpawMA8i6SLgBAjmjRooVmzJghX19flSpVytAso1ChQoaxTqdTISEhio6Ovuo8RYsWvanr+/v739Rxl3O5XJIkm8121f4r913++TLeczqd/3oOAIC8h6QLAJAjChUqpEqVKqlcuXI37E5Yt25dJSQkyMfHR5UqVTJs//nPfyRJVatW1ebNmw3HXfn6cnfeeaf8/f31/fffZ/q+n5+fJCk9Pf2a56hUqZL8/Py0YcMG977U1FTFxMSoatWq1/1MAID8i6QLAJDrtGrVSqGhoerUqZPGjx+vypUr68SJE1q5cqU6deqk+vXra8CAAerWrZvq16+vxo0b69NPP9XevXtVsWLFTM9ZoEABvfLKKxo6dKj8/PzUqFEjnTp1Snv37lWvXr1UokQJ+fv7a9WqVSpdurQKFChwVbv4QoUK6cUXX9TLL7+sYsWKqWzZspowYYLOnz+vXr165cQ/DQAgD6LoAgDkOjabTStXrtRrr72mnj176tSpUypZsqSaNm3q7jbYuXNnHTp0SK+88oouXryoxx57TC+++KK+/fbba553xIgR8vHx0ciRI3XixAmFhIQoLCxMkuTj46OpU6fq9ddf18iRI9WkSZNMb29866235HQ61bVrV/3zzz+qX7++vv32W912222W/FsAAPI+myvjBnUAAAAAgOlY0wUAAAAAFqLoAgAAAAALUXQBAAAAgIUougAAAADAQhRdAAAAAGAhii4AAAAAsBBFFwAAAABYiKILAAAAACxE0QUAAAAAFqLoAgAAAAALUXQBAAAAgIUougAAAADAQv8H+aIQWaAQ4QoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1000x700 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm = confusion_matrix(truth,prediction)\n",
    "print_confusion_matrix(cm,[\"Dog\",\"Not a dog\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9a44e1c6-3a1f-4db7-8fcb-9a0e3a15cbd2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         Dog       0.57      0.67      0.62         6\n",
      "   Not a dog       0.33      0.25      0.29         4\n",
      "\n",
      "    accuracy                           0.50        10\n",
      "   macro avg       0.45      0.46      0.45        10\n",
      "weighted avg       0.48      0.50      0.48        10\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(truth, prediction))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a724a91-d4a2-4b74-ae55-b05746e0d587",
   "metadata": {},
   "source": [
    "### F1 Score for \"Dog\" Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c9fc8506-14a2-4f9b-8641-3b282a4aaa83",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6159677419354839"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "2*(0.57*0.67/(0.57+0.67))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "413fec5f-77b8-4e6d-8594-ad3c62f1373d",
   "metadata": {},
   "source": [
    "### F1 Score for \"Not a Dog\" Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5a790bf6-6ea1-4cda-8c2f-012517f4fb7e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2844827586206896"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "2*(0.33*0.25/(0.33+0.25))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97c9fc89-df6c-4031-9278-d2ea7213d202",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}