TopicRank.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import spacy\n",
    "import pytextrank\n",
    "import json\n",
    "from rake_nltk import Rake\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define const file paths\n",
    "data_dir = \"Data Collection/\"\n",
    "dataset_dir = os.path.join(data_dir, \"dataset\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get all terms from index_terms.json\n",
    "#  note. this gets all leaves in the json i.e. lowest level terms\n",
    "term_tree = json.load(open(os.path.join(data_dir, \"index_terms.json\")))\n",
    "\n",
    "def get_leafs(term_tree):\n",
    "    terms = set()\n",
    "    queue = [term_tree]\n",
    "\n",
    "    while(queue):\n",
    "        curr_dict = queue.pop()\n",
    "        for key in curr_dict:\n",
    "            if(curr_dict[key]):\n",
    "                queue.append(curr_dict[key])\n",
    "            else:\n",
    "                terms.add(key.lower())\n",
    "    return terms\n",
    "\n",
    "terms = get_leafs(term_tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define models (TopicRank and RAKE)\n",
    "# Input: string of entire article text\n",
    "# Output: list of predicted keywords for the article (all lowercase)\n",
    "\n",
    "def topicRank(text):\n",
    "\n",
    "    # load a spaCy model, depending on language, scale, etc.\n",
    "    nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "    # add PyTextRank to the spaCy pipeline\n",
    "    nlp.add_pipe(\"topicrank\")\n",
    "    doc = nlp(text)\n",
    "    \n",
    "    tr = doc._.textrank\n",
    "        \n",
    "    topics = set()\n",
    "\n",
    "    for node in tr.node_list:\n",
    "        for topic in node:\n",
    "            topics.add(str(topic).lower())\n",
    "            \n",
    "    return topics\n",
    "\n",
    "def rake(text):\n",
    "    r = Rake(min_length=1, max_length=8, include_repeated_phrases=False)\n",
    "    r.extract_keywords_from_text(text)\n",
    "    ranked_keywords = r.get_ranked_phrases()\n",
    "    top_keywords = ranked_keywords[:1000000]\n",
    "    \n",
    "    keywords = set()\n",
    "    for keyword in top_keywords:\n",
    "        keywords.add(str(keyword).lower())\n",
    "    return keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define performance metrics\n",
    "def iou(pred, exp):\n",
    "    return len(pred & exp) / len(pred | exp) * 100\n",
    "\n",
    "def accuracy(pred, exp):\n",
    "    return len(pred & exp) / len(exp) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic Rank IOU Score: 7.6923076923076925%\n",
      "Topic Rank Accuracy: 25.0%\n",
      "RAKE IOU Score: 11.11111111111111%\n",
      "RAKE Accuracy: 25.0%\n",
      "Combined IOU Score: 7.6923076923076925%\n",
      "Combined Accuracy: 25.0%\n",
      "\n",
      "Topic Rank IOU Score: 0.0%\n",
      "Topic Rank Accuracy: 0.0%\n",
      "RAKE IOU Score: 0.0%\n",
      "RAKE Accuracy: 0.0%\n",
      "Combined IOU Score: 0.0%\n",
      "Combined Accuracy: 0.0%\n",
      "\n",
      "Topic Rank IOU Score: 25.0%\n",
      "Topic Rank Accuracy: 25.0%\n",
      "RAKE IOU Score: 25.0%\n",
      "RAKE Accuracy: 25.0%\n",
      "Combined IOU Score: 25.0%\n",
      "Combined Accuracy: 25.0%\n",
      "\n",
      "Topic Rank IOU Score: 5.714285714285714%\n",
      "Topic Rank Accuracy: 18.181818181818183%\n",
      "RAKE IOU Score: 2.941176470588235%\n",
      "RAKE Accuracy: 9.090909090909092%\n",
      "Combined IOU Score: 5.405405405405405%\n",
      "Combined Accuracy: 18.181818181818183%\n",
      "\n",
      "Topic Rank IOU Score: 0.0%\n",
      "Topic Rank Accuracy: 0.0%\n",
      "RAKE IOU Score: 0.0%\n",
      "RAKE Accuracy: 0.0%\n",
      "Combined IOU Score: 0.0%\n",
      "Combined Accuracy: 0.0%\n",
      "\n",
      "Topic Rank IOU Score: 0.0%\n",
      "Topic Rank Accuracy: 0.0%\n",
      "RAKE IOU Score: 0.0%\n",
      "RAKE Accuracy: 0.0%\n",
      "Combined IOU Score: 0.0%\n",
      "Combined Accuracy: 0.0%\n",
      "\n",
      "Topic Rank IOU Score: 0.0%\n",
      "Topic Rank Accuracy: 0.0%\n",
      "RAKE IOU Score: 0.0%\n",
      "RAKE Accuracy: 0.0%\n",
      "Combined IOU Score: 0.0%\n",
      "Combined Accuracy: 0.0%\n",
      "\n",
      "Topic Rank IOU Score: 0.0%\n",
      "Topic Rank Accuracy: 0.0%\n",
      "RAKE IOU Score: 0.0%\n",
      "RAKE Accuracy: 0.0%\n",
      "Combined IOU Score: 0.0%\n",
      "Combined Accuracy: 0.0%\n",
      "\n",
      "Topic Rank IOU Score: 11.11111111111111%\n",
      "Topic Rank Accuracy: 100.0%\n",
      "RAKE IOU Score: 10.0%\n",
      "RAKE Accuracy: 100.0%\n",
      "Combined IOU Score: 9.090909090909092%\n",
      "Combined Accuracy: 100.0%\n",
      "\n",
      "Topic Rank IOU Score: 0.0%\n",
      "Topic Rank Accuracy: 0.0%\n",
      "RAKE IOU Score: 0.0%\n",
      "RAKE Accuracy: 0.0%\n",
      "Combined IOU Score: 0.0%\n",
      "Combined Accuracy: 0.0%\n",
      "\n",
      "Topic Rank IOU Score: 0.0%\n",
      "Topic Rank Accuracy: 0.0%\n",
      "RAKE IOU Score: 0.0%\n",
      "RAKE Accuracy: 0.0%\n",
      "Combined IOU Score: 0.0%\n",
      "Combined Accuracy: 0.0%\n",
      "\n",
      "Topic Rank IOU Score: 0.0%\n",
      "Topic Rank Accuracy: 0.0%\n",
      "RAKE IOU Score: 0.0%\n",
      "RAKE Accuracy: 0.0%\n",
      "Combined IOU Score: 0.0%\n",
      "Combined Accuracy: 0.0%\n",
      "\n",
      "Topic Rank IOU Score: 0.0%\n",
      "Topic Rank Accuracy: 0.0%\n",
      "RAKE IOU Score: 0.0%\n",
      "RAKE Accuracy: 0.0%\n",
      "Combined IOU Score: 0.0%\n",
      "Combined Accuracy: 0.0%\n",
      "\n",
      "Topic Rank IOU Score: 0.0%\n",
      "Topic Rank Accuracy: 0.0%\n",
      "RAKE IOU Score: 0.0%\n",
      "RAKE Accuracy: 0.0%\n",
      "Combined IOU Score: 0.0%\n",
      "Combined Accuracy: 0.0%\n",
      "\n",
      "Topic Rank IOU Score: 0.0%\n",
      "Topic Rank Accuracy: 0.0%\n",
      "RAKE IOU Score: 6.666666666666667%\n",
      "RAKE Accuracy: 33.33333333333333%\n",
      "Combined IOU Score: 5.88235294117647%\n",
      "Combined Accuracy: 33.33333333333333%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Test RAKE, TopicRank, and Combined model performance\n",
    "accuracies = {\"rake\":[], \"topicrank\":[], \"combined\":[]}\n",
    "ious = {\"rake\":[], \"topicrank\":[], \"combined\":[]}\n",
    "\n",
    "preds = {\"rake\":[], \"topicrank\":[], \"combined\":[]}\n",
    "ground_truths = []\n",
    "\n",
    "for doi in os.listdir(dataset_dir):\n",
    "    text_path = os.path.join(dataset_dir, doi, f\"{doi}.txt\")\n",
    "    if(not os.path.exists(text_path)):\n",
    "        continue\n",
    "    text = open(text_path, encoding=\"utf-8\", errors=\"ignore\").read()\n",
    "    \n",
    "    terms_path = os.path.join(dataset_dir, doi, f\"{doi}.json\")\n",
    "    if(not os.path.exists(terms_path)):\n",
    "        continue\n",
    "    \n",
    "    gt_term_tree = json.load(open(terms_path))\n",
    "    gt_terms = get_leafs(gt_term_tree)\n",
    "    \n",
    "    topicRank_topics = topicRank(text)\n",
    "    topicRank_terms = topicRank_topics.intersection(terms)\n",
    "    \n",
    "    rake_topics = rake(text)\n",
    "    rake_terms = rake_topics.intersection(terms)\n",
    "    \n",
    "    combined_terms = topicRank_terms | rake_terms\n",
    "    \n",
    "    topicrank_iou = iou(topicRank_terms, gt_terms)\n",
    "    topicrank_accuracy = accuracy(topicRank_terms, gt_terms)\n",
    "    print(f\"Topic Rank IOU Score: {topicrank_iou}%\")\n",
    "    print(f\"Topic Rank Accuracy: {topicrank_accuracy}%\")\n",
    "    \n",
    "    rake_iou = iou(rake_terms, gt_terms)\n",
    "    rake_accuracy = accuracy(rake_terms, gt_terms)\n",
    "    print(f\"RAKE IOU Score: {rake_iou}%\")\n",
    "    print(f\"RAKE Accuracy: {rake_accuracy}%\")\n",
    "    \n",
    "    combined_iou = iou(combined_terms, gt_terms)\n",
    "    combined_accuracy = accuracy(combined_terms, gt_terms)\n",
    "    print(f\"Combined IOU Score: {combined_iou}%\")\n",
    "    print(f\"Combined Accuracy: {combined_accuracy}%\")\n",
    "    print()\n",
    "    \n",
    "    accuracies['topicrank'].append(topicrank_accuracy)\n",
    "    ious['topicrank'].append(topicrank_iou)\n",
    "    \n",
    "    accuracies['rake'].append(rake_accuracy)\n",
    "    ious['rake'].append(rake_iou)\n",
    "    \n",
    "    accuracies['combined'].append(combined_accuracy)\n",
    "    ious['combined'].append(combined_iou)\n",
    "    \n",
    "    preds['topicrank'].append(topicRank_terms)\n",
    "    preds['rake'].append(rake_terms)\n",
    "    preds['combined'].append(combined_terms)\n",
    "    \n",
    "    ground_truths.append(gt_terms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rake accruacy: 13.58%\n",
      "topicrank accruacy: 12.96%\n",
      "combined accruacy: 16.05%\n",
      "\n",
      "rake IOU: 2.73%\n",
      "topicrank IOU: 2.78%\n",
      "combined IOU: 2.80%\n"
     ]
    }
   ],
   "source": [
    "# Print out model metrics\n",
    "for model in accuracies:\n",
    "    print(f\"{model} accruacy: {np.mean(accuracies[model]):.2f}%\")\n",
    "    \n",
    "print()\n",
    "\n",
    "for model in ious:\n",
    "    print(f\"{model} IOU: {np.mean(ious[model]):.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGiCAYAAAABVwdNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAoAUlEQVR4nO3de3xU9Z3/8feEwCSEJJAAk0QCRk0hXEVACKi0EESt9gHSuihb0VJwNdwERLACD0UIsAqWi1KQDSwCsuii4AWXBoVCISA3sSTcWUIhSSkkA0FCSL6/P/pj6iygJJlkJl9fz8djHo/OmTMnn6HT5NUz55xxGGOMAAAALBXk7wEAAACqErEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArObX2Nm0aZMeeeQRxcXFyeFw6MMPP/R63BijiRMnKjY2VqGhoUpJSdGhQ4e81jl79qwGDBigiIgI1a9fX4MGDdKFCxeq8VUAAIBA5tfYKSoqUrt27TRv3rzrPj5jxgzNnj1b8+fPV2ZmpsLCwtS7d29dunTJs86AAQP0l7/8RevXr9fHH3+sTZs2aciQIdX1EgAAQIBzBMoXgTocDq1evVp9+vSR9I+9OnFxcRo9erTGjBkjSSosLJTL5dLixYvVv39/ZWVlqWXLltqxY4c6duwoSVq3bp0eeughnTx5UnFxcf56OQAAIEAE+3uAGzl27Jhyc3OVkpLiWRYZGanOnTtr69at6t+/v7Zu3ar69et7QkeSUlJSFBQUpMzMTPXt2/e62y4uLlZxcbHnfllZmc6ePavo6Gg5HI6qe1EAAMBnjDE6f/684uLiFBR04w+rAjZ2cnNzJUkul8trucvl8jyWm5urxo0bez0eHBysqKgozzrXk5aWpldeecXHEwMAAH/IyclRkyZNbvh4wMZOVRo/frxGjRrluV9YWKimTZsqJydHERERfpwMAADcLLfbrfj4eIWHh3/vegEbOzExMZKkvLw8xcbGepbn5eXpzjvv9KyTn5/v9bwrV67o7Nmznudfj9PplNPpvGZ5REQEsQMAQA3zQ4egBOx1dhISEhQTE6OMjAzPMrfbrczMTCUnJ0uSkpOTVVBQoJ07d3rW2bBhg8rKytS5c+dqnxkAAAQev+7ZuXDhgg4fPuy5f+zYMe3Zs0dRUVFq2rSpRo4cqddee02JiYlKSEjQhAkTFBcX5zljKykpSQ888IAGDx6s+fPnq6SkREOHDlX//v05EwsAAEjyc+x89dVX+tnPfua5f/U4moEDB2rx4sUaO3asioqKNGTIEBUUFOiee+7RunXrFBIS4nnOsmXLNHToUPXs2VNBQUHq16+fZs+eXe2vBQAABKaAuc6OP7ndbkVGRqqwsJBjdgAAqCFu9u93wB6zAwAA4AvEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArBbQsVNaWqoJEyYoISFBoaGhuv322zV58mQZYzzrGGM0ceJExcbGKjQ0VCkpKTp06JAfpwYAAIEkoGNn+vTpevvttzV37lxlZWVp+vTpmjFjhubMmeNZZ8aMGZo9e7bmz5+vzMxMhYWFqXfv3rp06ZIfJwcAAIHCYb67myTAPPzww3K5XFq0aJFnWb9+/RQaGqp3331XxhjFxcVp9OjRGjNmjCSpsLBQLpdLixcvVv/+/W/q57jdbkVGRqqwsFARERFV8loAAIBv3ezf74Des9O1a1dlZGTo4MGDkqS9e/dq8+bNevDBByVJx44dU25urlJSUjzPiYyMVOfOnbV169Ybbre4uFhut9vrBgAA7BTs7wG+z7hx4+R2u9WiRQvVqlVLpaWlmjJligYMGCBJys3NlSS5XC6v57lcLs9j15OWlqZXXnml6gYHAAABI6D37PzXf/2Xli1bpuXLl2vXrl1asmSJXn/9dS1ZsqRS2x0/frwKCws9t5ycHB9NDAAAAk1A79l54YUXNG7cOM+xN23atNH//u//Ki0tTQMHDlRMTIwkKS8vT7GxsZ7n5eXl6c4777zhdp1Op5xOZ5XODgAAAkNA79m5ePGigoK8R6xVq5bKysokSQkJCYqJiVFGRobncbfbrczMTCUnJ1frrAAAIDAF9J6dRx55RFOmTFHTpk3VqlUr7d69WzNnztRvfvMbSZLD4dDIkSP12muvKTExUQkJCZowYYLi4uLUp08f/w4PAAACQkDHzpw5czRhwgQ999xzys/PV1xcnJ555hlNnDjRs87YsWNVVFSkIUOGqKCgQPfcc4/WrVunkJAQP04OAAACRUBfZ6e6cJ0dAABqHiuuswMAAFBZxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKwW8LHz17/+Vf/6r/+q6OhohYaGqk2bNvrqq688jxtjNHHiRMXGxio0NFQpKSk6dOiQHycGAACBJKBj59y5c+rWrZtq166tzz77TPv379cbb7yhBg0aeNaZMWOGZs+erfnz5yszM1NhYWHq3bu3Ll265MfJAQBAoHAYY4y/h7iRcePGacuWLfrTn/503ceNMYqLi9Po0aM1ZswYSVJhYaFcLpcWL16s/v3739TPcbvdioyMVGFhoSIiInw2PwAAqDo3+/c7oPfsrFmzRh07dtSvfvUrNW7cWO3bt9fChQs9jx87dky5ublKSUnxLIuMjFTnzp21devWG263uLhYbrfb6wYAAOwU0LFz9OhRvf3220pMTNTnn3+uZ599VsOHD9eSJUskSbm5uZIkl8vl9TyXy+V57HrS0tIUGRnpucXHx1fdiwAAAH4V0LFTVlamu+66S1OnTlX79u01ZMgQDR48WPPnz6/UdsePH6/CwkLPLScnx0cTAwCAQBPQsRMbG6uWLVt6LUtKStKJEyckSTExMZKkvLw8r3Xy8vI8j12P0+lURESE1w0AANgpoGOnW7duOnDggNeygwcPqlmzZpKkhIQExcTEKCMjw/O42+1WZmamkpOTq3VWAAAQmIL9PcD3ef7559W1a1dNnTpVjz32mLZv364FCxZowYIFkiSHw6GRI0fqtddeU2JiohISEjRhwgTFxcWpT58+/h0eAAAEhICOnU6dOmn16tUaP368Xn31VSUkJOjNN9/UgAEDPOuMHTtWRUVFGjJkiAoKCnTPPfdo3bp1CgkJ8ePkAAAgUAT0dXaqC9fZAQCg5rHiOjsAAACVVe7YufXWW/Xqq696zogCAAAIZOWOnZEjR+q///u/ddttt6lXr1567733VFxcXBWzAQAAVFqFYmfPnj3avn27kpKSNGzYMMXGxmro0KHatWtXVcwIAABQYZU+QLmkpERvvfWWXnzxRZWUlKhNmzYaPny4nn76aTkcDl/NWaU4QBkAgJrnZv9+V/jU85KSEq1evVrp6elav369unTpokGDBunkyZN66aWX9Mc//lHLly+v6OYBAAB8otyxs2vXLqWnp2vFihUKCgrSk08+qVmzZqlFixaedfr27atOnTr5dFAAAICKKHfsdOrUSb169dLbb7+tPn36qHbt2tesk5CQoP79+/tkQAAAgMood+wcPXrU891UNxIWFqb09PQKDwUAAOAr5T4bKz8/X5mZmdcsz8zM1FdffeWToQAAAHyl3LGTmpqqnJyca5b/9a9/VWpqqk+GAgAA8JVyx87+/ft11113XbO8ffv22r9/v0+GAgAA8JVyx47T6VReXt41y0+fPq3g4ID+EnUAAPAjVO7Yuf/++zV+/HgVFhZ6lhUUFOill15Sr169fDocAABAZZV7V8zrr7+u++67T82aNVP79u0lSXv27JHL5dLSpUt9PiAAAEBllDt2brnlFn399ddatmyZ9u7dq9DQUD399NN6/PHHr3vNHQAAAH+q0EE2YWFhGjJkiK9nAQAA8LkKH1G8f/9+nThxQpcvX/Za/otf/KLSQwEAAPhKha6g3LdvX+3bt08Oh0NXvzT96jecl5aW+nZCAACASij32VgjRoxQQkKC8vPzVbduXf3lL3/Rpk2b1LFjR3355ZdVMCIAAEDFlXvPztatW7VhwwY1bNhQQUFBCgoK0j333KO0tDQNHz5cu3fvroo5AQAAKqTce3ZKS0sVHh4uSWrYsKFOnTolSWrWrJkOHDjg2+kAAAAqqdx7dlq3bq29e/cqISFBnTt31owZM1SnTh0tWLBAt912W1XMCAAAUGHljp2XX35ZRUVFkqRXX31VDz/8sO69915FR0dr5cqVPh8QAACgMhzm6ulUlXD27Fk1aNDAc0ZWTeN2uxUZGanCwkJFRET4exwAAHATbvbvd7mO2SkpKVFwcLC++eYbr+VRUVE1NnQAAIDdyhU7tWvXVtOmTbmWDgAAqDHKfTbW7373O7300ks6e/ZsVcwDAADgU+U+QHnu3Lk6fPiw4uLi1KxZM4WFhXk9vmvXLp8NBwAAUFnljp0+ffpUwRgAAABVwydnY9V0nI0FAEDNUyVnYwEAANQ05f4YKygo6HtPM+dMLQAAEEjKHTurV6/2ul9SUqLdu3dryZIleuWVV3w2GAAAgC/47Jid5cuXa+XKlfroo498sblqxTE7AADUPNV+zE6XLl2UkZHhq80BAAD4hE9i59tvv9Xs2bN1yy23+GJzAAAAPlPuY3b+7xd+GmN0/vx51a1bV++++65PhwMAAKiscsfOrFmzvGInKChIjRo1UufOndWgQQOfDgcAAFBZ5Y6dp556qgrGAAAAqBrlPmYnPT1dq1atumb5qlWrtGTJEp8MBQAA4Cvljp20tDQ1bNjwmuWNGzfW1KlTfTIUAACAr5Q7dk6cOKGEhIRrljdr1kwnTpzwyVAAAAC+Uu7Yady4sb7++utrlu/du1fR0dE+GQoAAMBXyh07jz/+uIYPH64vvvhCpaWlKi0t1YYNGzRixAj179+/KmYEAACosHKfjTV58mQdP35cPXv2VHDwP55eVlamJ598kmN2AABAwKnwd2MdOnRIe/bsUWhoqNq0aaNmzZr5erZqw3djAQBQ89zs3+9y79m5KjExUYmJiRV9OgAAQLUo9zE7/fr10/Tp069ZPmPGDP3qV7/yyVAAAAC+Uu7Y2bRpkx566KFrlj/44IPatGmTT4YCAADwlXLHzoULF1SnTp1rlteuXVtut9snQwEAAPhKuWOnTZs2Wrly5TXL33vvPbVs2dInQwEAAPhKuQ9QnjBhgh599FEdOXJEPXr0kCRlZGRo+fLlev/9930+IAAAQGWUO3YeeeQRffjhh5o6daref/99hYaGql27dtqwYYOioqKqYkYAAIAKq/B1dq5yu91asWKFFi1apJ07d6q0tNRXs1UbrrMDAEDNc7N/v8t9zM5VmzZt0sCBAxUXF6c33nhDPXr00LZt2yq6OQAAgCpRro+xcnNztXjxYi1atEhut1uPPfaYiouL9eGHH3JwMgAACEg3vWfnkUceUfPmzfX111/rzTff1KlTpzRnzpyqnA0AAKDSbnrPzmeffabhw4fr2Wef5WsiAABAjXHTe3Y2b96s8+fPq0OHDurcubPmzp2rM2fOVOVsAAAAlXbTsdOlSxctXLhQp0+f1jPPPKP33ntPcXFxKisr0/r163X+/PmqnBMAAKBCKnXq+YEDB7Ro0SItXbpUBQUF6tWrl9asWePL+aoFp54DAFDzVPmp55LUvHlzzZgxQydPntSKFSsqsykAAIAqUanYuapWrVrq06dPle/VmTZtmhwOh0aOHOlZdunSJaWmpio6Olr16tVTv379lJeXV6VzAACAmsMnsVMdduzYoT/84Q9q27at1/Lnn39ea9eu1apVq7Rx40adOnVKjz76qJ+mBAAAgaZGxM6FCxc0YMAALVy4UA0aNPAsLyws1KJFizRz5kz16NFDHTp0UHp6uv785z9/79Wci4uL5Xa7vW4AAMBONSJ2UlNT9fOf/1wpKSley3fu3KmSkhKv5S1atFDTpk21devWG24vLS1NkZGRnlt8fHyVzQ4AAPwr4GPnvffe065du5SWlnbNY7m5uapTp47q16/vtdzlcik3N/eG2xw/frwKCws9t5ycHF+PDQAAAkS5vhuruuXk5GjEiBFav369QkJCfLZdp9Mpp9Pps+0BAIDAFdB7dnbu3Kn8/HzdddddCg4OVnBwsDZu3KjZs2crODhYLpdLly9fVkFBgdfz8vLyFBMT45+hAQBAQAnoPTs9e/bUvn37vJY9/fTTatGihV588UXFx8erdu3aysjIUL9+/ST940KHJ06cUHJysj9GBgAAASagYyc8PFytW7f2WhYWFqbo6GjP8kGDBmnUqFGKiopSRESEhg0bpuTkZHXp0sUfIwMAgAAT0LFzM2bNmqWgoCD169dPxcXF6t27t9566y1/jwUAAAJEpb4byxZ8NxYAADVPtXw3FgAAQKAjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYLWAjp20tDR16tRJ4eHhaty4sfr06aMDBw54rXPp0iWlpqYqOjpa9erVU79+/ZSXl+eniQEAQKAJ6NjZuHGjUlNTtW3bNq1fv14lJSW6//77VVRU5Fnn+eef19q1a7Vq1Spt3LhRp06d0qOPPurHqQEAQCBxGGOMv4e4WX/729/UuHFjbdy4Uffdd58KCwvVqFEjLV++XL/85S8lSdnZ2UpKStLWrVvVpUuX626nuLhYxcXFnvtut1vx8fEqLCxUREREtbwWAABQOW63W5GRkT/49zug9+z8X4WFhZKkqKgoSdLOnTtVUlKilJQUzzotWrRQ06ZNtXXr1htuJy0tTZGRkZ5bfHx81Q4OAAD8psbETllZmUaOHKlu3bqpdevWkqTc3FzVqVNH9evX91rX5XIpNzf3htsaP368CgsLPbecnJyqHB0AAPhRsL8HuFmpqan65ptvtHnz5kpvy+l0yul0+mAqAAAQ6GrEnp2hQ4fq448/1hdffKEmTZp4lsfExOjy5csqKCjwWj8vL08xMTHVPCUAAAhEAR07xhgNHTpUq1ev1oYNG5SQkOD1eIcOHVS7dm1lZGR4lh04cEAnTpxQcnJydY8LAAACUEB/jJWamqrly5fro48+Unh4uOc4nMjISIWGhioyMlKDBg3SqFGjFBUVpYiICA0bNkzJyck3PBMLAAD8uAT0qecOh+O6y9PT0/XUU09J+sdFBUePHq0VK1aouLhYvXv31ltvvVWuj7Fu9tQ1AAAQOG7273dAx051IXYAAKh5rLzODgAAQHkROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwWkBfQRkAUPPdOu4Tf48APzs+7ed+/fns2QEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjYsKVjEupgV/X0wLAH7s2LMDAACsxp4dwHLsXQR7F/Fjx54dAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWsiZ158+bp1ltvVUhIiDp37qzt27f7eyQAABAArIidlStXatSoUZo0aZJ27dqldu3aqXfv3srPz/f3aAAAwM+C/T2AL8ycOVODBw/W008/LUmaP3++PvnkE/3Hf/yHxo0bd836xcXFKi4u9twvLCyUJLndbp/PVlZ80efbRM1SFe+r8uA9CN6D8Leqeg9e3a4x5vtXNDVccXGxqVWrllm9erXX8ieffNL84he/uO5zJk2aZCRx48aNGzdu3Cy45eTkfG8r1Pg9O2fOnFFpaalcLpfXcpfLpezs7Os+Z/z48Ro1apTnfllZmc6ePavo6Gg5HI4qnffHxu12Kz4+Xjk5OYqIiPD3OPgR4j0If+M9WHWMMTp//rzi4uK+d70aHzsV4XQ65XQ6vZbVr1/fP8P8SERERPA/cvgV70H4G+/BqhEZGfmD69T4A5QbNmyoWrVqKS8vz2t5Xl6eYmJi/DQVAAAIFDU+durUqaMOHTooIyPDs6ysrEwZGRlKTk7242QAACAQWPEx1qhRozRw4EB17NhRd999t958800VFRV5zs6C/zidTk2aNOmajw2B6sJ7EP7Ge9D/HMb80PlaNcPcuXP17//+78rNzdWdd96p2bNnq3Pnzv4eCwAA+Jk1sQMAAHA9Nf6YHQAAgO9D7AAAAKsROwAAwGrEDqrU4sWLuWAjqsytt96qN998099j6Msvv5TD4VBBQYG/R0EAOH78uBwOh/bs2XPDdarrPXMzs/wYWHHqOYDA9tOf/lR33nmnz8Nkx44dCgsL8+k2gerQtWtXnT59+qau/ovKI3ZQYZcvX1adOnX8PQZ+xBo1alTh55aWlsrhcCgoiB3cqH516tThKv/ViP+V46b99Kc/1dChQzVy5Eg1bNhQvXv31syZM9WmTRuFhYUpPj5ezz33nC5cuHDDbfztb39Tx44d1bdvXxUXF6usrExpaWlKSEhQaGio2rVrp/fff78aXxWq2lNPPaWNGzfq97//vRwOhxwOh44fP66NGzfq7rvvltPpVGxsrMaNG6crV654nnf1/TZ06FBFRkaqYcOGmjBhgr57tYz/+zFWQUGBnnnmGblcLoWEhKh169b6+OOPJf3zI9U1a9aoZcuWcjqdOnHihHbs2KFevXqpYcOGioyMVPfu3bVr1y6v1+BwOPTOO++ob9++qlu3rhITE7VmzZobvuaLFy/qwQcfVLdu3fhoKwCUlZVpxowZuuOOO+R0OtW0aVNNmTJFkrRv3z716NFDoaGhio6O1pAhQ7x+hz311FPq06ePpk6dKpfLpfr16+vVV1/VlStX9MILLygqKkpNmjRRenr6NT83OztbXbt29bwXN27c6Hns/36MdfX9+fnnnyspKUn16tXTAw88oNOnT3tt85133lFSUpJCQkLUokULvfXWW16Pb9++Xe3bt1dISIg6duyo3bt3++qfsWb73u9EB76je/fupl69euaFF14w2dnZJjs728yaNcts2LDBHDt2zGRkZJjmzZubZ5991vOc9PR0ExkZaYwx5sSJE6Z58+Zm4MCB5sqVK8YYY1577TXTokULs27dOnPkyBGTnp5unE6n+fLLL/3xElEFCgoKTHJyshk8eLA5ffq0OX36tDl58qSpW7euee6550xWVpZZvXq1adiwoZk0aZLneVffbyNGjDDZ2dnm3XffNXXr1jULFizwrNOsWTMza9YsY4wxpaWlpkuXLqZVq1bmf/7nf8yRI0fM2rVrzaeffmqM+cd7sXbt2qZr165my5YtJjs72xQVFZmMjAyzdOlSk5WVZfbv328GDRpkXC6Xcbvdnp8jyTRp0sQsX77cHDp0yAwfPtzUq1fP/P3vfzfGGPPFF18YSebcuXPm3LlzpmvXrub+++83RUVFVf8PjB80duxY06BBA7N48WJz+PBh86c//cksXLjQXLhwwcTGxppHH33U7Nu3z2RkZJiEhAQzcOBAz3MHDhxowsPDTWpqqsnOzjaLFi0ykkzv3r3NlClTzMGDB83kyZNN7dq1TU5OjjHGmGPHjnneM++//77Zv3+/+e1vf2vCw8PNmTNnjDHe7xlj/vn+TElJMTt27DA7d+40SUlJ5oknnvDM8u6775rY2FjzwQcfmKNHj5oPPvjAREVFmcWLFxtjjDl//rxp1KiReeKJJ8w333xj1q5da2677TYjyezevbta/q0DFbGDm9a9e3fTvn37711n1apVJjo62nP/auxkZ2eb+Ph4M3z4cFNWVmaMMebSpUumbt265s9//rPXNgYNGmQef/xx378A+E337t3NiBEjPPdfeukl07x5c897wRhj5s2bZ+rVq2dKS0s9z0lKSvJa58UXXzRJSUme+9+Nnc8//9wEBQWZAwcOXHeG9PR0I8ns2bPne2ctLS014eHhZu3atZ5lkszLL7/suX/hwgUjyXz22WfGmH/+4crKyjJt27Y1/fr1M8XFxT/wr4Lq4Ha7jdPpNAsXLrzmsQULFpgGDRqYCxcueJZ98sknJigoyOTm5hpj/hE7zZo187wvjTGmefPm5t577/Xcv3LligkLCzMrVqwwxvwzdqZNm+ZZp6SkxDRp0sRMnz7dGHP92JFkDh8+7HnOvHnzjMvl8ty//fbbzfLly71ew+TJk01ycrIxxpg//OEPJjo62nz77beex99++21ixxjDMTsolw4dOnjd/+Mf/6i0tDRlZ2fL7XbrypUrunTpki5evKi6detKkr799lvde++9euKJJ7w+cjh8+LAuXryoXr16eW3z8uXLat++fZW/FvhPVlaWkpOT5XA4PMu6deumCxcu6OTJk2ratKkkqUuXLl7rJCcn64033lBpaalq1arltc09e/aoSZMm+slPfnLDn1unTh21bdvWa1leXp5efvllffnll8rPz1dpaakuXryoEydOeK333eeFhYUpIiJC+fn5Xuv06tVLd999t1auXHnNfPCPrKwsFRcXq2fPntd9rF27dl4HuXfr1k1lZWU6cOCAXC6XJKlVq1Zex3a5XC61bt3ac79WrVqKjo6+5v3w3S+jDg4OVseOHZWVlXXDWevWravbb7/dcz82NtazzaKiIh05ckSDBg3S4MGDPetcuXLFc5BzVlaW2rZtq5CQkOvO8GNG7KBcvvtL4fjx43r44Yf17LPPasqUKYqKitLmzZs1aNAgXb582RM7TqdTKSkp+vjjj/XCCy/olltukSTP5+KffPKJZ9lVfGEeyis0NPSm1vluPEnSwIED9fe//12///3v1axZMzmdTiUnJ+vy5cte69WuXdvrvsPhUFlZmdeyn//85/rggw+0f/9+tWnTpoKvBL50M++LH3K9/+5v5v3gi59j/v8xald/Xy5cuPCa730krH8YByijwnbu3KmysjK98cYb6tKli37yk5/o1KlT16wXFBSkpUuXqkOHDvrZz37mWee7B4necccdXrf4+PjqfjmoQnXq1FFpaannflJSkrZu3ep1sPGWLVsUHh6uJk2aeJZlZmZ6bWfbtm1KTEy87i/3tm3b6uTJkzp48GC5ZtuyZYuGDx+uhx56SK1atZLT6dSZM2fKtY2rpk2bpoEDB6pnz57av39/hbYB30pMTFRoaKgyMjKueSwpKUl79+5VUVGRZ9mWLVsUFBSk5s2bV/pnb9u2zfOfr1y5op07dyopKalC23K5XIqLi9PRo0ev+X2ZkJAg6R+v5+uvv9alS5euO8OPGbGDCrvjjjtUUlKiOXPm6OjRo1q6dKnmz59/3XVr1aqlZcuWqV27durRo4dyc3MVHh6uMWPG6Pnnn9eSJUt05MgR7dq1S3PmzNGSJUuq+dWgKt16663KzMzU8ePHdebMGT333HPKycnRsGHDlJ2drY8++kiTJk3SqFGjvD4uOHHihEaNGqUDBw5oxYoVmjNnjkaMGHHdn9G9e3fdd9996tevn9avX69jx47ps88+07p16753tsTERC1dulRZWVnKzMzUgAEDKrU34PXXX9eAAQPUo0cPZWdnV3g78I2QkBC9+OKLGjt2rP7zP/9TR44c0bZt27Ro0SINGDBAISEhGjhwoL755ht98cUXGjZsmH796197PsKqjHnz5mn16tXKzs5Wamqqzp07p9/85jcV3t4rr7yitLQ0zZ49WwcPHtS+ffuUnp6umTNnSpKeeOIJORwODR48WPv379enn36q119/vdKvwwbEDiqsXbt2mjlzpqZPn67WrVtr2bJlSktLu+H6wcHBWrFihVq1aqUePXooPz9fkydP1oQJE5SWlqakpCQ98MAD+uSTTzz/TwV2GDNmjGrVqqWWLVuqUaNGKikp0aeffqrt27erXbt2+rd/+zcNGjRIL7/8stfznnzySX377be6++67lZqaqhEjRmjIkCE3/DkffPCBOnXqpMcff1wtW7bU2LFjvfYoXc+iRYt07tw53XXXXfr1r3+t4cOHq3HjxpV6vbNmzdJjjz2mHj16lHtPE3xvwoQJGj16tCZOnKikpCT9y7/8i/Lz81W3bl19/vnnOnv2rDp16qRf/vKX6tmzp+bOneuTnztt2jRNmzZN7dq10+bNm7VmzRo1bNiwwtv77W9/q3feeUfp6elq06aNunfvrsWLF3t+X9arV09r167Vvn371L59e/3ud7/T9OnTffJaajqH+e5+ZAAIEFV11WUAPz7s2QEAAFYjdgAAgNX4GAsAAFiNPTsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAq/0/B29+4TD+ckYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGiCAYAAAABVwdNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAkJUlEQVR4nO3dfVhUdf7/8dcAOqDCKKADrFhUpnhvWopWllJ0u5dl25psUZm2hampmX4Lvbox1C0tzbLMBbeybcssb9K2pbIyJdNqbQU1wxVLMFNmEnNU+Pz+2F+zzYoFCs746fm4rrmunXPOHN7HPQvPPXMYHMYYIwAAAEuFBXsAAACAhkTsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsFNXbef/99XX311UpKSpLD4dDrr78esN4Yo0mTJikxMVFRUVFKT0/X1q1bA7bZu3evMjMzFRMTo+bNm2vo0KHav3//STwKAAAQyoIaO5WVleratavmzJlT4/rp06dr1qxZmjt3rgoLC9W0aVNlZGTo4MGD/m0yMzP1r3/9S2+//baWLVum999/X8OHDz9ZhwAAAEKcI1T+EKjD4dDixYs1cOBASf+5qpOUlKSxY8dq3LhxkiSPxyO32638/HwNHjxYRUVF6tChg9atW6eePXtKklauXKkrrrhCO3fuVFJSUrAOBwAAhIiIYA9wLCUlJSorK1N6erp/mcvlUq9evbRmzRoNHjxYa9asUfPmzf2hI0np6ekKCwtTYWGhrrnmmhr37fP55PP5/M+rq6u1d+9excXFyeFwNNxBAQCAemOM0ffff6+kpCSFhR37zaqQjZ2ysjJJktvtDljudrv968rKytSqVauA9REREYqNjfVvU5Pc3Fw98MAD9TwxAAAIhtLSUrVu3fqY60M2dhrSxIkTNWbMGP9zj8ejNm3aqLS0VDExMUGcDAAA1JbX61VycrKio6N/druQjZ2EhARJUnl5uRITE/3Ly8vL1a1bN/82u3fvDnjdkSNHtHfvXv/ra+J0OuV0Oo9aHhMTQ+wAAHCK+aVbUEL2c3ZSUlKUkJCggoIC/zKv16vCwkKlpaVJktLS0lRRUaH169f7t3nnnXdUXV2tXr16nfSZAQBA6AnqlZ39+/fryy+/9D8vKSnRZ599ptjYWLVp00ajR4/Www8/rLZt2yolJUU5OTlKSkry/8ZWamqqLrvsMg0bNkxz587V4cOHNWLECA0ePJjfxAIAAJKCHDuffPKJLr74Yv/zH++jycrKUn5+vsaPH6/KykoNHz5cFRUVOv/887Vy5UpFRkb6X/Piiy9qxIgRGjBggMLCwjRo0CDNmjXrpB8LAAAITSHzOTvB5PV65XK55PF4uGcHAIBTRG1/fofsPTsAAAD1gdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDVQjp2qqqqlJOTo5SUFEVFRenMM8/UQw89JGOMfxtjjCZNmqTExERFRUUpPT1dW7duDeLUAAAglIR07EybNk1PP/20nnzySRUVFWnatGmaPn26Zs+e7d9m+vTpmjVrlubOnavCwkI1bdpUGRkZOnjwYBAnBwAAocJhfnqZJMRcddVVcrvdmj9/vn/ZoEGDFBUVpRdeeEHGGCUlJWns2LEaN26cJMnj8cjtdis/P1+DBw+u1dfxer1yuVzyeDyKiYlpkGMBAAD1q7Y/v0P6yk6fPn1UUFCgLVu2SJI+//xzffjhh7r88sslSSUlJSorK1N6err/NS6XS7169dKaNWuOuV+fzyev1xvwAAAAdooI9gA/Z8KECfJ6vWrfvr3Cw8NVVVWlKVOmKDMzU5JUVlYmSXK73QGvc7vd/nU1yc3N1QMPPNBwgwMAgJAR0ld2/va3v+nFF1/UwoULtWHDBi1YsECPPvqoFixYcEL7nThxojwej/9RWlpaTxMDAIBQE9JXdu655x5NmDDBf+9N586d9e9//1u5ubnKyspSQkKCJKm8vFyJiYn+15WXl6tbt27H3K/T6ZTT6WzQ2QEAQGgI6Ss7Bw4cUFhY4Ijh4eGqrq6WJKWkpCghIUEFBQX+9V6vV4WFhUpLSzupswIAgNAU0ld2rr76ak2ZMkVt2rRRx44d9emnn2rGjBm69dZbJUkOh0OjR4/Www8/rLZt2yolJUU5OTlKSkrSwIEDgzs8AAAICSEdO7Nnz1ZOTo7uvPNO7d69W0lJSbr99ts1adIk/zbjx49XZWWlhg8froqKCp1//vlauXKlIiMjgzg5AAAIFSH9OTsnC5+zAwDAqceKz9kBAAA4UcQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsFvKx8/XXX+sPf/iD4uLiFBUVpc6dO+uTTz7xrzfGaNKkSUpMTFRUVJTS09O1devWIE4MAABCSUjHzr59+9S3b181atRIK1as0KZNm/TYY4+pRYsW/m2mT5+uWbNmae7cuSosLFTTpk2VkZGhgwcPBnFyAAAQKhzGGBPsIY5lwoQJWr16tT744IMa1xtjlJSUpLFjx2rcuHGSJI/HI7fbrfz8fA0ePLhWX8fr9crlcsnj8SgmJqbe5gcAAA2ntj+/Q/rKzpIlS9SzZ0/97ne/U6tWrdS9e3fNmzfPv76kpERlZWVKT0/3L3O5XOrVq5fWrFlzzP36fD55vd6ABwAAsFNIx85XX32lp59+Wm3bttVbb72lO+64QyNHjtSCBQskSWVlZZIkt9sd8Dq32+1fV5Pc3Fy5XC7/Izk5ueEOAgAABFVIx051dbXOOeccPfLII+revbuGDx+uYcOGae7cuSe034kTJ8rj8fgfpaWl9TQxAAAINSEdO4mJierQoUPAstTUVO3YsUOSlJCQIEkqLy8P2Ka8vNy/riZOp1MxMTEBDwAAYKeQjp2+fftq8+bNAcu2bNmi0047TZKUkpKihIQEFRQU+Nd7vV4VFhYqLS3tpM4KAABCU0SwB/g5d999t/r06aNHHnlE119/vT7++GM9++yzevbZZyVJDodDo0eP1sMPP6y2bdsqJSVFOTk5SkpK0sCBA4M7PAAACAkhHTvnnnuuFi9erIkTJ+rBBx9USkqKHn/8cWVmZvq3GT9+vCorKzV8+HBVVFTo/PPP18qVKxUZGRnEyQEAQKgI6c/ZOVn4nB0AAE49VnzODgAAwIkidgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDVIuqycffu3eVwOI5a7nK5dPbZZ2v06NFKTU2tt+EAAABOVJ1iZ+DAgTUur6io0IYNG9StWze988476tu3b33MBgAAcMIcxhhTXzu77777tHbtWhUUFNTXLk8Kr9crl8slj8ejmJiYYI8DAABqobY/v+v1np0hQ4Zo48aN9blLAACAE1KvsRMeHq7q6ur63CUAAMAJqdfYee2119ShQ4f63CUAAMAJqdMNyrNmzapxucfj0fr167V8+XKtWLGiXgYDAACoD3WKnZkzZ9a4PCYmRu3atdP777+vtLS0ehkMAACgPtQpdkpKShpqDgAAgAZxQvfs7NmzR3v27KmvWQAAAOpdnWOnoqJC2dnZio+Pl9vtltvtVnx8vEaMGKGKiooGGBEAAOD41eltrL179yotLU1ff/21MjMz/X8aYtOmTcrPz1dBQYE++ugjtWjRokGGBQAAqKs6xc6DDz6oxo0ba9u2bXK73Uetu/TSS/Xggw8e80ZmAACAk61Ob2O9/vrrevTRR48KHUlKSEjQ9OnTtXjx4nobDgAA4ETVKXZ27dqljh07HnN9p06dVFZWdsJDAQAA1Jc6xU58fLy2b99+zPUlJSWKjY090ZkAAADqTZ1iJyMjQ/fdd58OHTp01Dqfz6ecnBxddtll9TYcAADAiXIYY0xtN965c6d69uwpp9Op7OxstW/fXsYYFRUV6amnnpLP59Mnn3yi5OTkhpy53tX2T8QDAIDQUduf33X6bazWrVvro48+UnZ2tiZOnKgfO8nhcOiSSy7Rk08+ecqFDgAAsFudYkeSzjjjDK1YsUL79u3T1q1bJUlnnXUW9+oAAICQVKfYufbaa2u13WuvvXZcwwAAANS3OsWOy+VqqDkAAAAaRJ1iJy8vr6HmAAAAaBAn9FfPAQAAQh2xAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsdkrFztSpU+VwODR69Gj/soMHDyo7O1txcXFq1qyZBg0apPLy8uANCQAAQsopEzvr1q3TM888oy5dugQsv/vuu7V06VK98sorWrVqlb755htde+21QZoSAACEmlMidvbv36/MzEzNmzdPLVq08C/3eDyaP3++ZsyYof79+6tHjx7Ky8vTRx99pLVr1x5zfz6fT16vN+ABAADsdErETnZ2tq688kqlp6cHLF+/fr0OHz4csLx9+/Zq06aN1qxZc8z95ebmyuVy+R/JyckNNjsAAAiukI+dv/71r9qwYYNyc3OPWldWVqbGjRurefPmAcvdbrfKysqOuc+JEyfK4/H4H6WlpfU9NgAACBERwR7g55SWlmrUqFF6++23FRkZWW/7dTqdcjqd9bY/AAAQukL6ys769eu1e/dunXPOOYqIiFBERIRWrVqlWbNmKSIiQm63W4cOHVJFRUXA68rLy5WQkBCcoQEAQEgJ6Ss7AwYM0MaNGwOW3XLLLWrfvr3uvfdeJScnq1GjRiooKNCgQYMkSZs3b9aOHTuUlpYWjJEBAECICenYiY6OVqdOnQKWNW3aVHFxcf7lQ4cO1ZgxYxQbG6uYmBjdddddSktLU+/evYMxMgAACDEhHTu1MXPmTIWFhWnQoEHy+XzKyMjQU089FeyxAABAiHAYY0ywhwg2r9crl8slj8ejmJiYYI8DAABqobY/v0P6BmUAAIATRewAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqxA4AALAasQMAAKxG7AAAAKsROwAAwGrEDgAAsBqxAwAArEbsAAAAqxE7AADAasQOAACwGrEDAACsRuwAAACrETsAAMBqIR07ubm5OvfccxUdHa1WrVpp4MCB2rx5c8A2Bw8eVHZ2tuLi4tSsWTMNGjRI5eXlQZoYAACEmpCOnVWrVik7O1tr167V22+/rcOHD+vSSy9VZWWlf5u7775bS5cu1SuvvKJVq1bpm2++0bXXXhvEqQEAQChxGGNMsIeorW+//VatWrXSqlWrdOGFF8rj8ahly5ZauHChrrvuOklScXGxUlNTtWbNGvXu3bvG/fh8Pvl8Pv9zr9er5ORkeTwexcTEnJRjAQAAJ8br9crlcv3iz++QvrLzvzwejyQpNjZWkrR+/XodPnxY6enp/m3at2+vNm3aaM2aNcfcT25urlwul/+RnJzcsIMDAICgOWVip7q6WqNHj1bfvn3VqVMnSVJZWZkaN26s5s2bB2zrdrtVVlZ2zH1NnDhRHo/H/ygtLW3I0QEAQBBFBHuA2srOztYXX3yhDz/88IT35XQ65XQ662EqAAAQ6k6JKzsjRozQsmXL9O6776p169b+5QkJCTp06JAqKioCti8vL1dCQsJJnhIAAISikI4dY4xGjBihxYsX65133lFKSkrA+h49eqhRo0YqKCjwL9u8ebN27NihtLS0kz0uAAAIQSH9NlZ2drYWLlyoN954Q9HR0f77cFwul6KiouRyuTR06FCNGTNGsbGxiomJ0V133aW0tLRj/iYWAAD4dQnpXz13OBw1Ls/Ly9PNN98s6T8fKjh27Fi99NJL8vl8ysjI0FNPPVWnt7Fq+6trAAAgdNT253dIx87JQuwAAHDqsfJzdgAAAOqK2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgtYhgD2C70ycsD/YICLLtU68M6tfnHATnIIIt2OcgV3YAAIDViB0AAGA1YgcAAFiN2AEAAFYjdgAAgNWIHQAAYDVrYmfOnDk6/fTTFRkZqV69eunjjz8O9kgAACAEWBE7L7/8ssaMGaPJkydrw4YN6tq1qzIyMrR79+5gjwYAAILMig8VnDFjhoYNG6ZbbrlFkjR37lwtX75cf/7znzVhwoSjtvf5fPL5fP7nHo9HkuT1eut9tmrfgXrfJ04tDXFe1QXnIDgHEWwNdQ7+uF9jzM9vaE5xPp/PhIeHm8WLFwcsv+mmm8xvf/vbGl8zefJkI4kHDx48ePDgYcGjtLT0Z1vhlL+ys2fPHlVVVcntdgcsd7vdKi4urvE1EydO1JgxY/zPq6urtXfvXsXFxcnhcDTovL82Xq9XycnJKi0tVUxMTLDHwa8Q5yCCjXOw4Rhj9P333yspKelntzvlY+d4OJ1OOZ3OgGXNmzcPzjC/EjExMfyPHEHFOYhg4xxsGC6X6xe3OeVvUI6Pj1d4eLjKy8sDlpeXlyshISFIUwEAgFBxysdO48aN1aNHDxUUFPiXVVdXq6CgQGlpaUGcDAAAhAIr3sYaM2aMsrKy1LNnT5133nl6/PHHVVlZ6f/tLASP0+nU5MmTj3rbEDhZOAcRbJyDwecw5pd+X+vU8OSTT+pPf/qTysrK1K1bN82aNUu9evUK9lgAACDIrIkdAACAmpzy9+wAAAD8HGIHAABYjdgBAABWI3bQoPLz8/nARjSY008/XY8//niwx9B7770nh8OhioqKYI+CELB9+3Y5HA599tlnx9zmZJ0ztZnl18CKXz0HENouuugidevWrd7DZN26dWratGm97hM4Gfr06aNdu3bV6tN/ceKIHRy3Q4cOqXHjxsEeA79iLVu2PO7XVlVVyeFwKCyMC9w4+Ro3bsyn/J9E/K8ctXbRRRdpxIgRGj16tOLj45WRkaEZM2aoc+fOatq0qZKTk3XnnXdq//79x9zHt99+q549e+qaa66Rz+dTdXW1cnNzlZKSoqioKHXt2lWvvvrqSTwqNLSbb75Zq1at0hNPPCGHwyGHw6Ht27dr1apVOu+88+R0OpWYmKgJEyboyJEj/tf9eL6NGDFCLpdL8fHxysnJ0U8/LeN/38aqqKjQ7bffLrfbrcjISHXq1EnLli2T9N+3VJcsWaIOHTrI6XRqx44dWrdunS655BLFx8fL5XKpX79+2rBhQ8AxOBwOPffcc7rmmmvUpEkTtW3bVkuWLDnmMR84cECXX365+vbty1tbIaC6ulrTp0/XWWedJafTqTZt2mjKlCmSpI0bN6p///6KiopSXFychg8fHvA97Oabb9bAgQP1yCOPyO12q3nz5nrwwQd15MgR3XPPPYqNjVXr1q2Vl5d31NctLi5Wnz59/OfiqlWr/Ov+922sH8/Pt956S6mpqWrWrJkuu+wy7dq1K2Cfzz33nFJTUxUZGan27dvrqaeeClj/8ccfq3v37oqMjFTPnj316aef1tc/46ntZ/8mOvAT/fr1M82aNTP33HOPKS4uNsXFxWbmzJnmnXfeMSUlJaagoMC0a9fO3HHHHf7X5OXlGZfLZYwxZseOHaZdu3YmKyvLHDlyxBhjzMMPP2zat29vVq5cabZt22by8vKM0+k07733XjAOEQ2goqLCpKWlmWHDhpldu3aZXbt2mZ07d5omTZqYO++80xQVFZnFixeb+Ph4M3nyZP/rfjzfRo0aZYqLi80LL7xgmjRpYp599ln/NqeddpqZOXOmMcaYqqoq07t3b9OxY0fz97//3Wzbts0sXbrUvPnmm8aY/5yLjRo1Mn369DGrV682xcXFprKy0hQUFJjnn3/eFBUVmU2bNpmhQ4cat9ttvF6v/+tIMq1btzYLFy40W7duNSNHjjTNmjUz3333nTHGmHfffddIMvv27TP79u0zffr0MZdeeqmprKxs+H9g/KLx48ebFi1amPz8fPPll1+aDz74wMybN8/s37/fJCYmmmuvvdZs3LjRFBQUmJSUFJOVleV/bVZWlomOjjbZ2dmmuLjYzJ8/30gyGRkZZsqUKWbLli3moYceMo0aNTKlpaXGGGNKSkr858yrr75qNm3aZG677TYTHR1t9uzZY4wJPGeM+e/5mZ6ebtatW2fWr19vUlNTzZAhQ/yzvPDCCyYxMdEsWrTIfPXVV2bRokUmNjbW5OfnG2OM+f77703Lli3NkCFDzBdffGGWLl1qzjjjDCPJfPrppyfl3zpUETuotX79+pnu3bv/7DavvPKKiYuL8z//MXaKi4tNcnKyGTlypKmurjbGGHPw4EHTpEkT89FHHwXsY+jQoeaGG26o/wNA0PTr18+MGjXK//z//u//TLt27fzngjHGzJkzxzRr1sxUVVX5X5Oamhqwzb333mtSU1P9z38aO2+99ZYJCwszmzdvrnGGvLw8I8l89tlnPztrVVWViY6ONkuXLvUvk2Tuv/9+//P9+/cbSWbFihXGmP/+4CoqKjJdunQxgwYNMj6f7xf+VXAyeL1e43Q6zbx5845a9+yzz5oWLVqY/fv3+5ctX77chIWFmbKyMmPMf2LntNNO85+XxhjTrl07c8EFF/ifHzlyxDRt2tS89NJLxpj/xs7UqVP92xw+fNi0bt3aTJs2zRhTc+xIMl9++aX/NXPmzDFut9v//MwzzzQLFy4MOIaHHnrIpKWlGWOMeeaZZ0xcXJz54Ycf/OuffvppYscYwz07qJMePXoEPP/HP/6h3NxcFRcXy+v16siRIzp48KAOHDigJk2aSJJ++OEHXXDBBRoyZEjAWw5ffvmlDhw4oEsuuSRgn4cOHVL37t0b/FgQPEVFRUpLS5PD4fAv69u3r/bv36+dO3eqTZs2kqTevXsHbJOWlqbHHntMVVVVCg8PD9jnZ599ptatW+vss88+5tdt3LixunTpErCsvLxc999/v9577z3t3r1bVVVVOnDggHbs2BGw3U9f17RpU8XExGj37t0B21xyySU677zz9PLLLx81H4KjqKhIPp9PAwYMqHFd165dA25y79u3r6qrq7V582a53W5JUseOHQPu7XK73erUqZP/eXh4uOLi4o46H376x6gjIiLUs2dPFRUVHXPWJk2a6Mwzz/Q/T0xM9O+zsrJS27Zt09ChQzVs2DD/NkeOHPHf5FxUVKQuXbooMjKyxhl+zYgd1MlPvyls375dV111le644w5NmTJFsbGx+vDDDzV06FAdOnTIHztOp1Pp6elatmyZ7rnnHv3mN7+RJP/74suXL/cv+xF/MA91FRUVVattfhpPkpSVlaXvvvtOTzzxhE477TQ5nU6lpaXp0KFDAds1atQo4LnD4VB1dXXAsiuvvFKLFi3Spk2b1Llz5+M8EtSn2pwXv6Sm/+5rcz7Ux9cx//8etR+/X86bN++ov/tIWP8yblDGcVu/fr2qq6v12GOPqXfv3jr77LP1zTffHLVdWFiYnn/+efXo0UMXX3yxf5uf3iR61llnBTySk5NP9uGgATVu3FhVVVX+56mpqVqzZk3AzcarV69WdHS0Wrdu7V9WWFgYsJ+1a9eqbdu2NX5z79Kli3bu3KktW7bUabbVq1dr5MiRuuKKK9SxY0c5nU7t2bOnTvv40dSpU5WVlaUBAwZo06ZNx7UP1K+2bdsqKipKBQUFR61LTU3V559/rsrKSv+y1atXKywsTO3atTvhr7127Vr/fz5y5IjWr1+v1NTU49qX2+1WUlKSvvrqq6O+X6akpEj6z/H885//1MGDB2uc4deM2MFxO+uss3T48GHNnj1bX331lZ5//nnNnTu3xm3Dw8P14osvqmvXrurfv7/KysoUHR2tcePG6e6779aCBQu0bds2bdiwQbNnz9aCBQtO8tGgIZ1++ukqLCzU9u3btWfPHt15550qLS3VXXfdpeLiYr3xxhuaPHmyxowZE/B2wY4dOzRmzBht3rxZL730kmbPnq1Ro0bV+DX69eunCy+8UIMGDdLbb7+tkpISrVixQitXrvzZ2dq2bavnn39eRUVFKiwsVGZm5gldDXj00UeVmZmp/v37q7i4+Lj3g/oRGRmpe++9V+PHj9df/vIXbdu2TWvXrtX8+fOVmZmpyMhIZWVl6YsvvtC7776ru+66SzfeeKP/LawTMWfOHC1evFjFxcXKzs7Wvn37dOuttx73/h544AHl5uZq1qxZ2rJlizZu3Ki8vDzNmDFDkjRkyBA5HA4NGzZMmzZt0ptvvqlHH330hI/DBsQOjlvXrl01Y8YMTZs2TZ06ddKLL76o3NzcY24fERGhl156SR07dlT//v21e/duPfTQQ8rJyVFubq5SU1N12WWXafny5f7/pwI7jBs3TuHh4erQoYNatmypw4cP680339THH3+srl276o9//KOGDh2q+++/P+B1N910k3744Qedd955ys7O1qhRozR8+PBjfp1Fixbp3HPP1Q033KAOHTpo/PjxAVeUajJ//nzt27dP55xzjm688UaNHDlSrVq1OqHjnTlzpq6//nr179+/zleaUP9ycnI0duxYTZo0Sampqfr973+v3bt3q0mTJnrrrbe0d+9enXvuubruuus0YMAAPfnkk/XydadOnaqpU6eqa9eu+vDDD7VkyRLFx8cf9/5uu+02Pffcc8rLy1Pnzp3Vr18/5efn+79fNmvWTEuXLtXGjRvVvXt33XfffZo2bVq9HMupzmF+eh0ZAEJEQ33qMoBfH67sAAAAqxE7AADAaryNBQAArMaVHQAAYDViBwAAWI3YAQAAViN2AACA1YgdAABgNWIHAABYjdgBAABWI3YAAIDV/h/53fc2W9Hk8QAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Graph comparison of model metrics\n",
    "plt.bar(accuracies.keys(), [np.mean(accuracies[key]) for key in accuracies])\n",
    "plt.ylim((0, 100))\n",
    "plt.ylabel(\"Accuracy\")\n",
    "plt.savefig(\"Keyword_Accuracy_Plot.png\")\n",
    "plt.show()\n",
    "\n",
    "plt.bar(ious.keys(), [np.mean(ious[key]) for key in ious])\n",
    "plt.ylim((0, 100))\n",
    "plt.ylabel(\"IOU\")\n",
    "plt.savefig(\"Keyword_IOU_Plot.png\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}