speaker-diarization/pyannote_sd_compare.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e9eceebf-c70d-446e-bf1d-f5847a479494",
   "metadata": {},
   "source": [
    "# Pyannote speaker diarization pipeline results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb27be40-d96a-4e57-a25b-b510b20737f8",
   "metadata": {},
   "source": [
    "This notebook presents the results of the [Pyannote.audio 2.0](https://github.com/pyannote/pyannote-audio) pipeline, an open-source toolkit written in Python for speaker diarization. Pyannote.audio is based on PyTorch machine learning framework, and it comes with a pre-trained [model](https://huggingface.co/pyannote/speaker-diarization) of speaker diarization, which we have used here."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3640a050-2d30-4e01-bf8f-f94803a73c2e",
   "metadata": {},
   "source": [
    "Steps of the pipeline:\n",
    "\n",
    "1. Speaker segmentation via pyannote/segmentation\n",
    "2. Econding of speaker segments as embeddings using a model by Speechbrain (ECAPA-TDNN)\n",
    "3. Clustering segments based on the embedding vectors using agglomerative clustering by default"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d63853e-9467-4bf4-a653-b7139d98116a",
   "metadata": {},
   "source": [
    "The pipeline will be compared using four metrics (diarization error rate [DER], diarization error rate with skipped overlap [DER skip], coverage, and purity) as described [here](https://pyannote.github.io/pyannote-metrics/reference.html#diarization)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34250320-9a5b-4481-a29c-93ad1eeb0f05",
   "metadata": {},
   "outputs": [],
   "source": [
    "from rttm import read_rttm\n",
    "from pyannote.core import Annotation, Timeline, Segment\n",
    "from pyannote.metrics.diarization import DiarizationErrorRate, DiarizationCoverage, DiarizationPurity\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "ca93683a-4d14-48ef-a4a3-3ce3dcc2ba3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "RESULTS = [\"pyannote\"]\n",
    "FILE_INDICES = list(range(16))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "eefc545f-2c6c-4091-88a1-6ff999d9ded2",
   "metadata": {},
   "outputs": [],
   "source": [
    "diarization_der = DiarizationErrorRate(collar=0.25)\n",
    "diarization_der_skip = DiarizationErrorRate(collar=0.25, skip_overlap=True)\n",
    "diarization_coverage = DiarizationCoverage(collar=0.25)\n",
    "diarization_purity = DiarizationPurity(collar=0.25)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "3119dcd3-7f17-4072-b17f-19a411849759",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_rttm_annotation(rttm_seq):\n",
    "    annotation = Annotation()\n",
    "    for seg in rttm_seq.sequence:\n",
    "        annotation[Segment(seg.tbeg, seg.tbeg+seg.tdur)] = seg.name\n",
    "\n",
    "    return annotation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "5676c114-c3c1-4a51-98b0-e73127c61c58",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/evaviviani/opt/anaconda3/lib/python3.9/site-packages/pyannote/metrics/utils.py:183: UserWarning: 'uem' was approximated by the union of 'reference' and 'hypothesis' extents.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "results_df = {\"metric\": [], \"method\": [], \"value\": []}\n",
    "\n",
    "method_labels = {\n",
    "    \"pyannote\": \"pa\"\n",
    "}\n",
    "\n",
    "for res_dir in RESULTS:\n",
    "    for i in FILE_INDICES:\n",
    "        res_rttm_seq = read_rttm(os.path.join('results',res_dir,f\"{method_labels[res_dir]}_ami_micro_test_sample_{i}.rttm\"))\n",
    "        ref_rttm_seq = read_rttm(os.path.join(\n",
    "            \"results\", \"reference\", f\"ref_ami_micro_test_sample_{i}.rttm\"))\n",
    "        res_annotation = convert_rttm_annotation(res_rttm_seq)\n",
    "        ref_annotation = convert_rttm_annotation(ref_rttm_seq)\n",
    "        der = diarization_der(ref_annotation, res_annotation)\n",
    "        der_skip = diarization_der_skip(ref_annotation, res_annotation)\n",
    "        coverage = diarization_coverage(\n",
    "            ref_annotation, res_annotation)\n",
    "        purity = diarization_purity(\n",
    "            ref_annotation, res_annotation)\n",
    "\n",
    "        results_df[\"metric\"] = results_df[\"metric\"] + \\\n",
    "            [\"der\", \"der_skip\", \"coverage\", \"purity\"]\n",
    "        results_df[\"method\"] = results_df[\"method\"] + \\\n",
    "            [method_labels[res_dir]] *4\n",
    "        results_df[\"value\"] = results_df[\"value\"] + \\\n",
    "            [der, der_skip, coverage, purity]\n",
    "\n",
    "results_df = pd.DataFrame(results_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "4c456c28-e0df-4e40-bea7-875857ad9f50",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>method</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>der</td>\n",
       "      <td>pa</td>\n",
       "      <td>0.416895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>der_skip</td>\n",
       "      <td>pa</td>\n",
       "      <td>0.310801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>coverage</td>\n",
       "      <td>pa</td>\n",
       "      <td>0.679236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>purity</td>\n",
       "      <td>pa</td>\n",
       "      <td>0.726872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>der</td>\n",
       "      <td>pa</td>\n",
       "      <td>0.266115</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     metric method     value\n",
       "0       der     pa  0.416895\n",
       "1  der_skip     pa  0.310801\n",
       "2  coverage     pa  0.679236\n",
       "3    purity     pa  0.726872\n",
       "4       der     pa  0.266115"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4441f648-066f-4886-b25e-e302d74a67b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df.to_csv(\"pyannote_results.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "00bdb023-ce59-48e2-ba97-9f9da11c7d6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def select_min_max(group):\n",
    "    if pd.DataFrame(group).reset_index()[\"metric\"].apply(lambda x: x in [\"der\", \"der_skip\"]).all():\n",
    "        return np.min(group)\n",
    "    else:\n",
    "        return np.max(group)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "76e2db1a-9f9f-482c-9fe7-9d3890bd5846",
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_results_df = results_df.groupby(\n",
    "    [\"metric\", \"method\"]).aggregate(np.mean)\n",
    "mean_best_results_dict = mean_results_df.groupby(\n",
    "    [ \"metric\"]).aggregate(select_min_max).to_dict()[\"value\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "328a36fc-baf0-4617-b39b-0267948640c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1gAAADQCAYAAAAalMCAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA6bklEQVR4nO3deZycV33v+c+vtu7qfdfSi1q7LHmRZXnBWLYx+GI2+5IVZwhxEnAyieEGAjh5cYchMJOJcSZ3CEsC1yEQwo0hcMM1CeCw2HIsbGPZlq3F2veWet+3Ws/8UaVW712trqerl+/79eqX63nq1FO/lvt01+855/yOOecQERERERGRufPlOgAREREREZGlQgmWiIiIiIhIlijBEhERERERyRIlWCIiIiIiIlmiBEtERERERCRLlGCJiIiIiIhkiRIsmZaZfcrMPprrOERERERkcma208z+Ov34TjO7NdcxLWdKsCRrzCyQ6xhEssmrGwzpP37/Osn5e83sT7L9fiKLlf6uiMzMzALOub3OuQ+lT90JKMHKISVYMoGZfcLMjprZs8Dm9Ln1ZvYjM3vJzP7DzLakz3/NzP7WzF4APpvLuEVyKRsfBJ1zTzjn/iIb8Yh4zczeZ2avmdmrZvYNM2s0s5+lz/3UzBrMrNTMzpiZL/2aQjM7Z2bBTP+umNlNZvacmb1iZj83s0t/lwrM7NtmdsjM/sXMXjCznenn/lP6NS+b2T+bWVHO/qFEMpDuP4fN7Jtm9rqZfSf9M37azKrSbXaa2dPpx59K97s9wDcu3bgzs0bg94EPm9k+M9tlZqfMLJh+XcnoY/GGEiwZw8xuAN4DbAfeDtyYfuorwAedczcAHwW+NOpldcCtzrmPzGOoIp7I9g0GM7sj/UduX/oDYvG4529Mn19vZg+Y2RfGXXtvOp53evudi2TOzLYB/xW4yzl3HfBfgM8DX3fOXQt8E/hr51wPsA+4I/3SdwJPOudiZP535TCwyzl3PfBJ4M/Tbf4A6HLObQX+D+CGdGxV6dje4pzbAewF9PdJFoPNwJecc1cBvaR+xqezldTP+f2XTjjnTgN/C/w359x259x/AE8D70g3eQ/wP9N9UDyioXcZbxfwL865QQAzewLIJzXU/M9mdqld3qjX/LNzLjGvUYp4YNwNhgDwMvASqQ+Cv++cO2ZmN5P6IHhX+mWXPghO1Qc+Cvyhc25P+i768Kj3u5XUh9L7nHNnzWzXuNc2AjcB64GnzGyDc24Ykdy7i9Tv/nYA51ynmb0B+KX089/g8k2HbwG/DjxFqn99Kd0XMv27Ugp83cw2Ag64dOf9NuBz6fc/YGavpc/fQuqD5570tUPAc9n4pkU8ds45tyf9+B+BD03XGHjCOTeUwXUfAz4OfA/4beADVxyhZEQJlmTCB3Q757ZP8fzAPMYi4iUvbjDsAf7KzL5J6q7h+fR1riKVuP0n59yFKV77bedcEjhmZieBLaRGA0QWkyeAPzezClKjTD8DCsn878pngKecc+9OT396eob3M+DHo+/qiywSbpLjOJdnnOWPez6jz1/pG3yNZnYn4HfOHZhLkDIzTRGU8Z4B/rOZhdNTmd4FDAKnzOxXASzlulwGKTKPRm4wjPq6atTz0/6BS6+pej8QJnVHfUv6qYukRrOun+7lMxyL5MrPgF81s0qAdPL0c1IjVAD/G/AfAM65fuBFUqNN/+qcSzjnesn870op0JR+/MCo83uAX0u/fitwTfr888AbzWxD+rlCM9s0t29XZF40pEeCAX4DeBY4TXr6K/DLGV6nDyged+4fgP8B/P0cY5QMKMGSMZxzL5OazvEq8ENSfxQh9cfyd83sVeAgcF9uIhTxVNZvMJjZeufcfufcI6T606UEq5vUnPj/J31XcTK/amY+M1sPrAOOXMH3JJJ1zrmDwP8N7E7/Xfgr4IPAb6en6v0mqXVZl3wLeG/6v5dk+nfls6T6ySuMnXnzJaDazA4B/1f6Gj3OuTZSidg/pWN5jsv9TmQhOwL8oZm9DpQDfwP8GfA5M9sLZLoc4/vAuy8VuUif+2b6mv+U5ZhlEuacboiKiFxiZp8AfgtoBc6SWof1XVJ/6FaRWv/xuHPu02b2NVJ35L8zzfU+D7wJSJL6APgA8Abgo865d5pZA6mbGb9DatrgTufcQ+lrDwM7gRLgI865CaXdRZYrM/MDQefccPomxE+Azc65aI5DE5m19PTXf3XOXe3R9X+F1Hrf3/Ti+jKWEiwRkQUok+RNZDlLjzI/ReqmhwEPO+d+mNuoRK6MlwlW+kbf24C3O+eOZvv6MpFnRS7M7KukyrG2TvbDYqlV3p8jVQp8EHggPT1NREREZFrOuT5SI7wii166vLono1fOuQ96cV2ZmmcjWGZ2O9AP/MMUCdbbSc3XfjtwM/A559zNngQjIuIxM/ttxq45AdjjnPvDXMQjIiIiueHpFMHphjvN7MvA0865f0ofHwHudM5d9CwgERERERERD+VyH6xa4Nyo4/PpcxMSLDN7EHgQYOvWrTccPHhwXgIUWWBs5ibphuozIqA+IzJb6jMiszNpn1kUZdqdc19xzu10zu0Mh8O5DkdkwVOfEZkd9RmR2VGfEZlaLhOsJqB+1HEdlzcSFBERERERWXRymWA9AbwvvWnnLaQ2B9T6KxERERERWbS8LNP+T8CdQJWZnQf+T1J7VeCc+1vgB6QqCB4nVab9t72KRUREREREZD54lmA55+6f4XkHqHyxiIiIiIgsGYuiyIWIiIiIiMhioARLREREREQkS5RgiYiIiIiIZIkSLBERERERkSxRgiUiIiIiIpIlnlURlMWl57XXOP+Nb0AySeWdd7LyXe8a83zbT39K209+gvl8+PLzafid3yFcW0vv/v00ffvbuHgcCwSoe897KN62LUffhcj8manPtPzwh3Q8/TT4/QSLi2n4wAfIq6oC4PhnP8vAiRMUbtrEhj/+4xxELyIiIl5RgiW4ZJJzX/86Gx9+mGBFBUc++UlKd+wgXFs70qbi1lupfvObAeh++WWavvlNNnz84wSKi1n/kY8QKi9n6Nw5jj/6KNf89V/n6lsRmReZ9JmCNWuo/vSn8eXl0faTn9D0+OOse+ghAGre8Q6SkQjtTz2Vq29BREREPKIpgsLAiRPkrVhBXk0NvkCA8ltuoeell8a08YfDI4+TkQiYAVDQ2EiovByA/Lo6ktEoyVhs/oIXyYFM+kzx1q348vIAKNywgVhn58hzJdu2jelTIiKyPPQMRhmOJXIdhnhMI1hCrKuLUEXFyHGwooLBEycmtGv78Y9p+dGPcPE4G//0Tyc83/3iixQ0NuILBj2NVyTXMu0zl3Ts3k3JtdfOR2giIrIADUXj/Gx/E539EXxmbF9bybb6iplfOE5n/zCvnekkEkuwfkUJG1aVzvoasUSSpo4BAn5jdUUhvvRNc8keJViSseq776b67rvp/PnPaf5f/4vG3/u9keeGzp+n6VvfYuPHP57DCEUWno49exg4dYpNn/hErkMREZEcOXC2k87+CABJ53jlVAeN1cUU5md+UzoaT/DjV5uIxlMjYK09QwQDPtZUF2d8jcFInB+9cpaBSByAmtIwd19XpyQryzRFUAiWlxMdNX0p1tlJMD3tbzLlt9xC96jpUNHOTk5+7nM0/t7vkbdihaexiiwEmfaZ3gMHaH7iCdZ/+MMa2RURWcb6hsYun3DO0T88+ZKK7oEI+061c7ipeySZglRCNfoY4Gx7/6ziOHaxZyS5unTNi52Ds7qGzEwJllC4bh2R5mYira0k43G6nn+e0h07xrQZbm4eedyzbx/5K1cCEB8Y4MRf/iWrf+3XKNq0aV7jFsmVTPrM4OnTnP37v2f9hz9MsHT2UzhERGTpqK8qGnMcDgWoKsmf0K6td4h/e+ks+8928uLxVn7yWhPOOQCKw6EJ7YvDs7t5F08kJ5yLTXJO5kZTBAXz+6l/3/s4/uijuGSSyttvJ1xXx4XvfpeCtWsp27GDth//mL6DBzG/H39hIWsefBBIrcuKtLTQ/L3v0fy97wGw4eMf1wdKWdIy6TNNjz9OcniYU5//PAChykrWf+QjABz5zGeIXLxIYniY/R/6EGve/36t0RIRWcI2riol6RynWnopyAty7ZoK/L6J4xzHLvSQTCdUAB19w7T2DLGirIDSghDXrKngwNkunHNUl+RzVe3UM44ms35lCUcudJNIpt6jMC9AbWXh3L45mcDcqP+Ji8HOnTvd3r17cx2GSC5c0QRp9RlZxtRnJhGNJxiKJigJB7FlvO7iXHs/LT1DVBbl0VhTvKz/LUZRn8mx54+2cOxiz5hz91xfT3XJ5cqzw9E4kXiS0oKJI1qZ6B6IcKK5l2DAx8ZVpYRDGm+Zg0n7jP5FRURElokTzb384ngr8UTqw9ld19RSNItF9vOpfzhGU+cAxflBVpUXZDUBOniuk5dPto8ct/cNc+OGmqxdX+RKbakt40xbH9F4atre6vKCMckVQH4oQP6V5VYAlBXmccP66rmEKTNQgiUiIrIMROOJkeQKUvvxvHKqnV1XrcpxZBM1dw/ys/1NI9OYNqws5Q2bs1dE6UhT95jjYxd72LGuGr9Po1iSW2WFedx7YyPn2vvJDwWo0/S9RUlFLkRERJaBoWhiwgL38ZXNFopD57pGkiuAE809DExRce1KjE+k/D5DMwRloQiHAmxaXUZDVZHKpy9SSrBERESWgZJwcMKajfoFend8dHIF4IBkFpeMX7OmcszCiasbKvRBVha9aHziTZRs6x2Ksv9sJ8cu9nj+XouZpgiKiIgsA2bGm66uZd/pdvqGYtRXFrKtoSLXYU1qc20ZLd2DXMqpaisKZ12OejrrVpRQUZSXLnKRP2m5bJHFIukcLxxt5URLLz6DbfUVXNdYmfX36egb5sl950ZugJxo7uGt2+tVIGYSSrBERESWieJwcEGuuRqvoaqIt15fz7n2AYrDQdatKM76e5QV5lFWmJf164rMt1MtfRxvTlUeTDh47UwHqysmFseYq6MXesaMLrf1Xi4hL2MpwRIREZEp9Q5FOd3aRyjgZ92KYkIB/7y8b3VJOOsfEEWWoq6ByMRz/ZF56T8avZqcp2uwzOweMztiZsfN7E8meb7BzJ4ys1fM7DUze7uX8YiIiEjmugci/NtLZ3n1dAcvHm/lyX3nJ6yPEpGUpHM0dQxwormH4Vhi3t53VfnYESQzY6UHo0pbassI+C+nDjWlYao1vXZSno1gmZkf+CJwN3AeeNHMnnDOHRrV7L8C33bO/Y2ZbQV+ADR6FZOIiIhkbvxC9u6BCM1dg9Qu0OIYIrn09IELNHUOAJAXbOee7fWUXOFmwMceeYRoe/u0baKdnZBIgN/P1SWlRONJDAgF/Zz7/sQxlFBVFRsffviK4gEoL8rj3p1rONPeT37Qz5rqIo1gTcHLKYI3AcedcycBzOxx4D5gdILlgJL041LggofxiIiIyCxM9uFpoX+ecs7ROxSjIC9A0J/diTpJ5zh6oYe2niGqS/PZtLpM1QcFgLbeoZHkCiASS3C4qZubNl7ZBtbR9nYizc2ZNU4koL2NS2VgksDESYPZUZgfZGtd+bRtnHO8erqDIxe68ft8XNdYycZVpR5FtDB5mWDVAudGHZ8Hbh7X5lPAv5vZB4FC4C2TXcjMHgQeBGhoaMh6oCJLjfqMyOyoz0xu0+pSTjT3Eo2npjtVl+SzsnzhLmjvHYzy1IEmeodiBP0+btm0gsaa7BXIeOlEG4fTmxSfbuujZzDKzRuztwHyYqI+M1YiMXHqbDx55WXMQ1VVM7aJtrXhEgnM7ydUXZ2Va2bDmbZ+9p/tTB8lef5oC9Ul+cuqqEyui1zcD3zNOff/mtkbgG+Y2dXOuTE/kc65rwBfAdi5c6cmf4vMQH1GZHbUZyZXEg5x741rONveT17AT31V4YIesXn5ZDu96c2TY4kkLxxrpb6qEL/PN5IkzqVIx/Hm3jHHJ5p7l22CpT4zVk1ZmLLCPLrTBSd8ZhmN2iSSDp9NHC3OZCrfwY99jEhzM6HqarY9+uiVBe6Btt6hSc4NK8HKkiagftRxXfrcaL8L3APgnHvOzPKBKqDVw7hEREQkQ+FQgM2ry3IdRkZ6h6JjjqPxBMPRBIebOjjc1I0DNqws4eaNNVe0diQ/6Kd/1Jq0/FCu71PLQuEz463b6zh2MVXgYm1NMRVFUxeASCSTPH+0lVOtfeQFfOxYV836lSVTth/NJRJ0PPss0Y4OAKIdHbTv3k3lrl2Yz9P6dRmpLgmPjPRePre8imF4+X/hRWCjma01sxDwHuCJcW3OAm8GMLOrgHygzcOYREREZImqryoac1xRlEfvUIxD57tIOodzjmMXezjT1n9F19+xrmpkBM9nxo618zPlShaHUMDPtvoKblhXPW1yBXC4qZuTLb045xiOJXjuaAsDkdiM7+ESCU594QucfewxXCzV3sVinH3sMU59/vO4xPxVL5zKmuoirmmoIBTwEQ4FuGXTimU1egUejmA55+Jm9hDwJOAHvuqcO2hmnwb2OueeAP4Y+O9m9mFSBS8ecM4t+2FmERERmb1r11RiBk0dA5QV5rF9beWkyVRqGtfs12atqS6muiRMe98wVcX5FORpBEuuTEff2DIUzjk6+yIU5gWneEX6dc8+S/fevZM+1713L5179lB5++1Zi/NKmBnb11axfRnfgPD0N4Nz7gekSq+PPvfJUY8PAW/0MgYRERHJrdmWnA5VVMx4zclKTvt9xvbGKrY3Xv5gt6qsACN1F/eS1RVXXma+IC9AQ17RzA1FprGiLMyZtr6RY7/PMppG17F797TPt+/enfMES3Jf5EJERESWuNmWnM64bQbKi/LYtXUVB891kUg6ttaVUVMaztr1Ra7ExlWl9A/HONncS37Iz4611Rmt6Yt2dk7/fHpdVi4MRuLEEklKr3Dvr6VECZaIiMgy0z8c41RrH6GAj7U1xXOqrJeJXJecXlNdzJrq7JVrF5krnxk3rKvmhnUz/6yPFqqoIDZNEhWqrJxraFfkpZNtvH6+G+cc1SX53HVNree/VxYyJVgiIiLLSPdAhB++co54uhre0Qs9vH1HA36fd+XXF3PJaZH5MhiJ0z0Qoaokf8rkpPKOOxg4dmzKa1TdcYdX4U2pqz/CoXNdI8dtvcMcaermmjW5SfYWAiVYIiIiy8ixiz0jyRWkEq6LXQPUVWpdkUiunGju5fmjLSSdI+D38aZtqyfd1Lty1y569+2btNBF2c6dVNx223yEO0bfuO0RUudmroi4lOW+WL6IiIjMm8n2f7qSPaGyxSUStO/ePWFPH5dMzvBKkYUvEkvw/NEWvr/3DL841jqy4fVoSed46WQbyXQh7XgiycunJi8KYz4fax96iDUf+AAWTFUctGCQNR/4AGs/+MGc7IO1sryAUGDs+47fMmG5UYIlIiKyjGxaXTpm+lFlcT6rJrlT7qVLO7Ishj19RObi50eaOXaxh+6BCEcudPPCsdYJbZJJRzQ29md9KBqf8prm91N5++0j661ClZVU3n57zjYZDgX83H1dHQ1VRawoC/PGLSuXfYKlKYIiIiLLSEk4xL03ruFMWz+hgI811UUjm+d6rWcwys+PNNPeO0xNaZhtbccW/J4+IlfKOUdTx8CYc+fHHQME/D7qq4o42355z7a1P/omB7/bO+31o21tI/89+LGPzRjPZFsbZEtFUT53bFvtybUXIyVYIiIiy0w4FGBLbdm8v++l5AqgtWeIsp/8lOm2VdWePrKYmRnFBSF6By+vUZqqhPkbt6yk/HwXnX0RVpaHSXy/h0hLS0bv47K8tYHMnRIsERER8ZxzbiS5GtHTPe1rcrmnj0g23LKxhmdev8hwNEFBXoCbNtRM2i7g93HtqKp7x6qrYYaR5SvZnFvmhxIsERER8ZyZUVMaprVn6PLJ0nIYmHoaVLb29OkfjhHwWUYbuYpk04qyAn7p5nUMDMcoCgczno7r1VQ+mR8qciEiIiLz4tbNK1lZVoDfZ6wqL6D+7rumbT/XPX3iiSQ/fe08//LCKb7z/CleOtE2p+uJXAm/zygpCF3xWsdLRWFk8dCtHBEREZkXxeEgV9WVEbzoIxTwEb75DZQd3O/Znj7HLvZwoWsQSH1IPXS+izXVxVSV5M/puiLzoTddFKatd5jqknxu3bKSkvDka7hkYVGCJSIiIvPiYtcgTx+4wKX78ec7B7jv9/93Sl94nrNf+xouFsOCQRoeeICK226bc9np3kk2QO0ZjGYlwdp/poPDF7oJ+Hxcu6aC9StL53xNkdGeO9pCW3rdYlvvMM8daeGt2+tzHJVkQlMERUREZF6cbu1j9GSnSCxBc2/Esz196irH7sXj9xmrK+a+59fZtj72ne5gOJqgfzjGc0da6B6IzPm6IqO1jSsKM/5YFi4lWCIiIjIvCvImTpyZ7Fy21FYU8sYtK6kpDVNbUchbrq0jnIVCFy2jC3UADsYW7xDJgprSsSOtK0rDOYpEZktTBEVERMRTxx55hGh7O37n2ByJk0imxrGCAR+t/xqgFe82TV23ooR1K0rmFP94VcUTpxhWTnJOZC5u3byS54600NY7RHVJmDdsXpHrkCRDSrBERETEU9H29pGNUAOM/fAxfmLdYtg0tbGmmM7+CEcudOP3+biusUIJlmRdUX6Qu6+ry3UYcgWUYImIiIinMtngdDFtmmpm3LC+mh3rqkaORUQuUYIlIiIiY7SnF9Nnq5z5Ut00VYmVSOaSzrH/TCfnO/opCYfYvraK4nAw12F5QgmWiIiIAJBIJvnZ/gs0d6f2jlpRFubN19Tiz0JFPxFZ3g6c7eS1Mx0AdPZH6BqI8K6da5bkjQr9xhQREREATrf2jyRXAC3dQ5xu7c9hRCKyVJxrH/u7pGcwSt9QLEfReMvTBMvM7jGzI2Z23Mz+ZIo2v2Zmh8zsoJn9Dy/jERERkakNx+IZnRMRma2SgtCY44DfR9jDbRpyybPvysz8wBeBu4HzwItm9oRz7tCoNhuBPwXe6JzrMrMar+IRERGR6dVXFfHq6Y6RMup+n1FfVTTDq0QkWzr6htl7oo3ewSj1VUXsXF9NwL80Jpxdv7aKrv4IPYNRgn4fN22sIbhEvrfxvEwbbwKOO+dOApjZ48B9wKFRbT4AfNE51wXgnGv1MB4RERGZRkk4xFu313O4qRuALbVllIRD079IRLIi6Ry7D15gIJIaNT52sYeg38cN66tzHFl2FOUHedfONfQNxQjnBZZscgXeJli1wLlRx+eBm8e12QRgZnsAP/Ap59yPxl/IzB4EHgRoaGjwJFiRpUR9RmR21GcuqyzO541bVuY6DFng1Geyr28oNpJcXTJ6TeRSYGYTpgouRblOHQPARuBO4H7gv5tZ2fhGzrmvOOd2Oud2VlcvjSxexEvqMyKzoz4jMjvqM9lXlB8gL+gfc66iSBtYL0ZeJlhNQP2o47r0udHOA08452LOuVPAUVIJl4iIiIhITnX2D7P/bCdn2/txznn6Xn6fjzduWUlhuvDDyrICrl9b6el7ije8nCL4IrDRzNaSSqzeA/zGuDbfIzVy9fdmVkVqyuBJD2MSEREREZnR2bY+njl0kUtp1cZVpdyyaYWn71lbUci7b15LPOmW9Bqlpc6z/3POuTjwEPAk8DrwbefcQTP7tJndm272JNBhZoeAp4CPOec6vIpJRERE5l8i6Tjb3s/p1j7iiWSuwxHJyKHzXYweszre3MtwLOH5+5qZkqtFbsYRLDNbAfw5sNo59zYz2wq8wTn3dzO91jn3A+AH4859ctRjB3wk/SUiIiJLTCKZ5EevnKOzPwJAcTjI265vmLDWRGTB83iKoCwdmaTHXyM10rQ6fXwU+COP4hEREZEl5Gxb/0hyBalKaSdaenMYkUhmttaVY6OO168sJV83BiQDmazBqnLOfdvM/hRSU//MzPvxUREREVn04smJd/01TVAWg4bqYt62I8iFrkFKw0Ftui0ZyyTBGjCzSkhNQzWzW4AeT6MSERERTwxF4xw638VgJM7ammLqKr390FhfVcS+0+0MR1P3ZoN+H2trSjx9T5FsqSzOp7JYpdJldjJJsD4CPAGsT28IXA38iqdRiYiISNY55/jxq+fpGYwCcLq1jzu2rqKhutiz98wP+nn7jgaOX+wlkXRsWFVCcTjo2fuJiOTajAmWc+5lM7sD2AwYcMQ5F/M8MhEREcmq9r7hkeTqkhMtvZ4mWACFeUGua9R+PiKyPGRSRfB9407tMDOcc//gUUwiIiLigckq9+UHvdwSU0Rk+cnkt+qNox7nA28GXgaUYImIiCwiJeEQW2rLONzUDUA4FGBbQ3lugxIRWWIymSL4wdHHZlYGPO5VQCIiIuKdGzfUsHFVKQOROCtKwwS0oamISFZdybyAAWBttgMRERGR+VFWmEdZYV6uwxARWZIyWYP1fdIl2kltTLwV+LaXQYmIiIiIiCxGmYxg/eWox3HgjHPuvEfxiIiIiIiILFqZrMHaPR+BiIiISHYMRuL4DPJDqhAoIjLfpvzNa2Z9XJ4aOOYpwDnntA27TDAcS7D/TAc9g1FWVxRyVW0ZZpbrsEREloVE0vHs6xc5296PmbF5dSk3bqjJdVgiIsvKlAmWc87bXQfliiWSjr6hKMXhEH7fwkpenjl0gZbuIQAudg0Siye1uaSIyDw52dLL2fZ+AJxzHG7qpr6yiJXlBTmOTERk+ch47oCZ1ZDaBwsA59xZTyISEsnUwOFkyVNL9yDPHLrIcCxBfsjPHVtXU1Manu8QJzUUjY8kV5ecbutTgiUiMk96B6MTzvUMRZVgiYjMoxk3vzCze83sGHAK2A2cBn7ocVzL1mtnOvj2z0/wrT3Heelk24TnXzjWynAsAcBwNMELx1rnO8QpBf0+guP2UynM0/x/EZH5UldZOObYZ8bq8sIpWouIiBcy2V3wM8AtwFHn3FrgzcDznka1TDV3D/Lq6Q7iiSSJpOPQua6RqR6X9A7Fxhz3DU28W5krAb+Pneur8aXXXOUH/Vy/tirHUYmILB8rygrYddUqqkvyWVlWwF3X1FIcDuY6LBGRZSWT4YWYc67DzHxm5nPOPWVm/5/XgS1HHX3DE8519g3TUFU0clxfWTgm6aqvLJrwmlzasKqUuspCeodiVBbn4fdlksOLiEi2NNYU01ijZdQiIrmSSYLVbWZFwH8A3zSzVmDA27CWp5VlE+fIjz/3hs0rCIcCtPUOUVMaXpDrm/JDAZUGFhEREZFlKZNPwU8BpcB/Ad6bfvxpL4NariqL83nDphUcONdJ0sHWuvIJC5NDAT83bcxdyd14Iskvjrdypq2fovwAN66v0eJpEREREZG0TOZvBYB/B54GioFvOec6vAxqOXLOcaK5h+buQTatLuO+G9ewpbYs12FNcOBsJyeae4knknQPRNl96ALxRDLXYYmIiIiILAgzJljOuT9zzm0D/hBYBew2s59kcnEzu8fMjpjZcTP7k2na/bKZOTPbmXHkS8z+s538/EgLp1r7eOlEGz8/0pLrkCbV0jO2DHs0nkq0REREREQksxGsS1qBZqADmHGOmpn5gS8CbwO2Aveb2dZJ2hWTmn74wixiWXKON/eOOT7T2kdsAY4MVZeM3XMr6PdRWhjKUTQiIiIiIgtLJvtg/YGZPQ38FKgEPuCcuzaDa98EHHfOnXTORYHHgfsmafcZ4BFgYgm9ZSQ/6B9zHAz4RsqdLyTXrKmgsaYYM6M4HGTX1lUT9r7KtUQySSS9V5iIiIiIyHzKpMhFPfBHzrl9s7x2LXBu1PF54ObRDcxsB1DvnPs3M/vYVBcysweBBwEaGhpmGcbicF1jJbsPXiCRdBiwfW0Vft/CS7CCfh+7rlrFrqtyHcnkjl7o5uWT7cQSSWorCrntqpWEAv6ZX7jELIc+MxfOOVp6hhiMxKmtKCQvuPx+RmQs9RmR2VGfEZnajAmWc+5PvXhjM/MBfwU8kEEMXwG+ArBz507nRTy5VltRyC/dvJbWniHKivIoCWva3WwNRGL84ngbzqV+RJo6B3j9fPeCLGXvteXQZ+bi2cPNnG7tA1KVOd+6vY6ywrwcR5U9iWSSV093cqFrgPLCPLavraQwT5vNTkd9RmR21GdEpubl3K4mUqNfl9Slz11SDFwNPG1mp4FbgCeWc6GL/FCAhupiJVdXqGcgOpJcXdLVH8lRNLJQdQ9ERpIrgGg8waHzXTmMKPteOdXBwXOddPVHONnSyzOHLuY6JBERkWXDy91gXwQ2mtlaUonVe4DfuPSkc64HqLp0nF7n9VHn3F4PY5JJHHvkEaLt7dO2iXZ2QiIBfj+hiooZrxmqqmLjww9nK8SMVJXkEwr4iMYvFwdZXaE9umSsyYrHxOILr6DMXJzv6B9z3N47zHA0rg3ARURE5oFnf22dc3Ezewh4EvADX3XOHTSzTwN7nXNPePXeMjvR9nYizc2ZNU4kMm87z0IBP2+6upZXTrUzHI2zdkUJG1eV5josWWCqivOpKMqjMz26abDkfk5KwiH6hmIjx/lBP8FluBZRREQkFzy9nemc+wHwg3HnPjlF2zu9jEWmFqqqmrFNtK0Nl0hgfj+h6uqsXNMLNaVh3rq9fuaGsmyZGXdfV8fRCz0MRuI01hRTUxqe+YWLyA3rq+kditI3FCMU8HPzphULsmiOiIjIUqT5IstUS/cgfUMxVlcUZjSV7+DHPkakuZlQdTXbHn10HiK8cknnONXSR89glNqKAlaUaZqgjBUK+Lm6YeaprotVaUGI+25spG8oRkFegMAC20pBRERkKVOCtQy9cKyFoxd6APD7jLdcW7ek7uDvGVUh7uC5Tt64ZSXrVpTkOCqR+WVmlBSoYI6IiMh8023NZWYgEuNYOrkCSCQdB852TtneJRK0795NtKMDgGhHB+27d+OSC7MowHA0zplRFeIADjd15yYYEREREVl2NIK1zCSSjvGbVSSSk29f4RIJTn3hC3TvvVzY0cVinH3sMXr37WPtQw9h/vldON87FGX/mU4GIjHWVBezeXXZmOfNDMxgVLl2rT0RERERkfmiEaxlpiQcYlX52DVJm1ZPXkGt49lnxyRXo3Xv3Uvnnj1Zj286Sef4yWtNnGzppaV7iF8ca+XYxZ4xbfKCfjaP+n7MjKvrl+5aG5GpDEbixCcpSS8iIiLe0gjWMnTnttUcb+6lbyhKfVURK6coAtGxe/e012nfvZvK22/3IsRJdfQNMzAcG3PuTFvfhBLbN26oob6yiO7BKKvLC7QORZaVSCzB7kMXaOkeIuj3ccP66iVXhl5ERGQhU4K1DAX8PrbUls3YLto59dosYGRd1nwpzAtiZrhR0/+K8oOTtl1ZXsDKclUPlOVn/9lOWrqHgNSmyr841kpdZSFhbTIsIiIyLzRFUKYUqph+al2osnKeIkkpyAtw/dpKfJZaU1VaEOKaNZr+JzJaz0BkzHHSuTGbDouIiIi3dEtTplR5xx0MHDs25fNVd9wxj9GkbKuvYP2KEoaiCcoKQ6miFiIyoraykAtdgyPH+UE/lcV5OYxIRERkeVGCJVOq3LWL3n37Ji10UbZzJxW33ZaDqCA/FCBf050E6B2McqFrYKR4ixJu2Ly6jFg8yanWPgrzAmxfW4Xfp8kKIiIi80WfUmVK5vOx9qGH6Nyzh7Nf+xouFsOCQRoeeICK227DPPjQ1jsYpW8oRk1ZmKBfHwplak2dAzx94ALJ9Jq8TatLuXnjihxHlXtmxjVrKrlmzfxO4RUREZEUJVjCsUceIdrePn2jSxsLJ5M0f//7NH//+9M2D1VVsfHhh3HOcbFrkJ7BKKsrCimdpqLf/jMdvHq6AweEAn7ecm0tlcX5s/xuZLk4dK5rJLkCOHaxl+vWVGp0U0RERHJKn0SWkEgswcFzXfQMRqirLGLDypKMpkxF29uJNDdn9B4ukci4LcCLx9s4cqEbADvZzpu2raa2snDS2F870zmyCXI0nuC1Mx286erajN9LlpfRyRUAzjHFntkiIiIi80YJVo519A1zuq2Pwrwg61eWzGla3KW9bwDOdwwQjSfYlsEmu6GqqhnbRDs7IZEAv3/G6oKXrjkcS3B01EbAzjkOnOucPMGKJyZ8YB6OJWZ8H1m+Nq8uo61naCQpX1NdTEGefqWJiIhIbunTSA5d6BzgZwcujOzrdKatj7dur7+iaw1EYiPJ1SWnWvsySrA2PvzwrN7rbFsfrzd1Ywbb6iomTZgAhqNxGJc0TRh1SCsJh6gpDdPac/l7WL+iZFZxyfLSWJNKqJo6U0UuGmuKcx2SiIiIiBKsXDp6oWfMprmtPUN09g9TUTT7dUehgB+/z0iMmiNV4MFalI6+YZ45dHFk1KC15wLvvKGBssKJZaDzQwEaa4o51doHgAFX1ZZPee07t63m9aYuegdj1FcWslYJlsygpjRMTWl43t83k3WLVzLqO9ubHdkSTyQ5erGH3sEodZWF1FUW5SQOERGRpUAJVg75fRPXR11pOeWg38eOdVXsPdGOc468oJ/rGrNfRex8xwCjx6CcczR1DkyaYAHcumUlqysK6U0XuZjuw3Be0M/2xpmnK4rk2mzWLTLLdYu58MyhizR1DgBw7GIPb9i0gg2rSnMclYiIyOKkBCuHttaXc75zgHgiVaGvsbp42ip7M9lSW059VRF9gzGqSvIJeFDmvCQcnHBuuph9ZqzTSJQsMRmtW2xrwyUSmN9PqLo6K9f0wkAkNpJcXXKsuUcJloiIyBVSgpVDlcX53HdjI02dAxTkBVhdXjDnaxbmBSnMm5gEZcuammLOdw5wJj3tb8OqUmorJl+DJbJUTTeVzyUSdDz7LOe+/vXUFEGfjxXvfCeVu3Z5snfcXAV8PnxmY9ZHag86ERGRK6cEK8cK8gJsXER3in1m7LpqFTesq8YMwtpzSGSESyQ49YUv0L137+VzsRhnH3uM3n37WPvQQ5jfn8MIJ8oL+tlaX86Bs51AaurytdqkWERE5Irp07FcEZXDllwbisaJxBJTrv+bD845ugYi5AcDFOQF6Hj22THJ1Wjde/fSuWcPlbffPs9Rzuz6tVU0VBXRMxhlVXmBbpyIiIjMgad/Rc3sHuBzgB94zDn3F+Oe/wjwfiAOtAG/45w742VMIrL4vXKqnYPnunDOUVWSz11X15IXnN+RoeFonJ/sb6KrP4KZcXV9OeHdu6d9Tfvu3QsywYLUlOXK4tlXMBUREZGxPJtob2Z+4IvA24CtwP1mtnVcs1eAnc65a4HvAJ/1Kh4RWfw6+oZ59vWLPHPoAolkqjhMe+8wh5u65z2Wg+e76OqP4Jyjo3eIJ/edY7B1htLtHR1ZjyOeSHK+o5/O/uGsX1tERERmz8sRrJuA4865kwBm9jhwH3DoUgPn3FOj2j8PvNfDeERkEWvqGOCpgxfoHohwrn2AnsEoa2tSFSr7hqLzHk//UAxIbV3Q2R8BoDdYQDFdU74mVJndtU29g1GefPUcw9EEAJtXl3HTxpqsvoeIiIjMjpelomqBc6OOz6fPTeV3gR9O9oSZPWhme81sb1tbWxZDFFmalmKfOdyUmhJYlB8g4Dd6B2NEYqnEoqFq/jfGXVNdTCKZpGsglVwF/T56tlw/7Wuq7rgjqzEcPNc1klwBHLnQTW8Oks2lYCn2GREvqc+ITG1B1OI1s/cCO4FHJ3veOfcV59xO59zO6gz2k5GFq2cwyqmWXgYisVyHsqQtxT5jltqY2+/zsbammLLCEDVlYW7dvIKG6uJ5j6expphbN6+kJByivCiPdStL6Nl0HYPrx8+ETinbuZOK227LagzDsfiEc5eSTpmdpdhnRLykPiMyNS+nCDYB9aOO69LnxjCztwCfAO5wzkU8jEdy7HBTFy8eT93l8plxx7ZV1FXO/8iDLE5b68s529bHqbY+Bobj1FcVcvtVq3JSRfDYI48QbU+tt3pHLDEmqQkFfMTDhfiHBjDAARQWMXTuHIem2T8rUVJGy72/RSjg4+qGiowKTqxfWcr5jsubBJcVhqiaY6EK5xxJ5/AvwD27REREFgMvE6wXgY1mtpZUYvUe4DdGNzCz64EvA/c451o9jEVyLJF0vHr68gL/pHPsO92hBEsytrKsgMqSfDoHIlSXhCkJB3n2cDPvvGHNvMcSbW8n0tw8cjw+xRv9i9UABvqJDPRPe83IYJTm7kEALnYN8u6b185YGbGhqoi7rq7ldFsfBXkBrqotGxnpuxJn2vp48Xgbw9E4dVVF3Lp5BaHAwtq3S0REZKHzLMFyzsXN7CHgSVJl2r/qnDtoZp8G9jrnniA1JbAI+Of0h4Kzzrl7vYpJcsc5RyzhxpyLxpM5ikYWq4HhONUl4ZHjrv4IiaTD77vypOJKhKqqpn1+OBrH9fSAS4DPT35lxbSJz3A0TixcMnIcSyS52DVIY83MUx9rKwuprSzMPPgpRGIJ9hxuJpFM9dNz7f0cCIfYsW7671VERETG8nQfLOfcD4AfjDv3yVGP3+Ll+8vCEfD7WFdTzImW3pFzG1eV5jAiWYxqSsM0dV6eEldVkj/vyRXAxmmm+l3SNxQjGk9QUZQ346jSgbOdHDo1tsR7UX5wTjHOVs9gdCS5ukSl30VERGbP0wRLZLSbN62gojifzv5hVpUVsHZFycwvEhnllk0reP5oCy09Q1QV53PzpoVbkrw4HAQyS5I2rS6lqXOA1p4hDNhcW0ZVyfxu+ltelEco4BszsryyrGBeYxAREVkKlGAtMLFEkqB/aS4u9/uMLbVluQ5DFrGCvAB3XTPdbg+LUyjg563b6+keiBAM+CjMm9/RK0iVmb9z22peOtnOQCRGY3UxV9WVz3scIiIii50SrAWivXeYPUea6R2MUlmcz66rVqXvgIvIcpGLioijrSgr4O07GnIag4iIyGK3NIdKFqE9hy/SO5jaILSjb5gXj6uoooiIiIjIYqMEawGIxhP0Do3deLejT4vLRUREREQWG00RXABCAT+VxfljkiotLhcRkaWk57XXOP+Nb0AySeWdd7LyXe8a8/z5f/xH+l5/HYBkNEq8t5frvvxlAI5/9rMMnDhB4aZNbPjjP5732EVyYS59Jtrezpm/+zuinZ0YsP6jHyWvunq+v4VlSwnWArHrqlX84lgLHf0RVpUVcOPGhVsdTUREZDZcMsm5r3+djQ8/TLCigiOf/CSlO3YQrr1ctKbuve8dedz67//O0JkzI8c173gHyUiE9qeemte4RXJlrn3m9Je/zMp776XkmmtIDA/PaRN6mT0lWAtEcTjIm6+ty3UYIjIPOvqG6R6IsLKsgMJ53u9KJBcGTpwgb8UK8mpSNw/Lb7mFnpdeGvNhcbSu555j1S/90shxybZtI3fqRZaDufSZoaYmXDJJyTXXAODPn99tP0QJlojIrDjn6BmMUpAXIBTwz/r1r53p4NXTHQD4zLjz6tXUVhRmO0yRBSXW1UWoomLkOFhRweCJE5O2jbS3E2lro3jbtvkKT2TBmUufiVy8iL+ggBOf+xzR9PnaX/91zKfSC/NF/9IiIhnqHYryxIun+f7eM3znuZMcv9gzq9fHEkkOnO0cOU46x/4zHdkOU2RR63r+ecpvukkfBkUyNL7PuGSS/iNHqLv/frb82Z8RbW2l45lnchzl8qLfXiIiGXr1VMdIxc9E0vHiiTai8UTGr08mHcmkG3MulkhmNUaRhShYXk608/LNhVhnJ8HyyTey7nr+ecpvuWW+QhNZkObSZ0IVFRQ0NJBXU4P5/ZTecAODp097HbKMogRLRCRDfcNjt1OIJ5IMRzNPsPKCftZUF485t2l1WTZCE1nQCtetI9LcTKS1lWQ8Ttfzz1O6Y8eEdsMXLpAYGKBw48YcRCmycMylzxSsW0dicJBYby8AfYcOTbl2S7yhNVgiIhmqryoas51CWWGI4vDsilTcumUlK8rCdA1EWF1eSH1VUbbDFFlwzO+n/n3v4/ijj+KSSSpvv51wXR0XvvtdCtaupSz9wbEzfSd+fMWzI5/5DJGLF0kMD7P/Qx9izfvfT8m11+biWxGZF3PpM+bzUXv//Rz7i78A5yhobKTyTW/K1beyLJlzbuZWC8jOnTvd3r17cx2GSC5cUY1V9ZnsSTrHoXNdnO/opyQc4rrGSlUBXNjUZ0RmR31GZHYm7TMawRIRyZDPjKsbKri6oWLmxiIiIrIsaQ2WiIiIiIhIlijBEhERERERyRIlWCIiIiIiIlmiBEtERERERCRLlGCJiIiIiIhkiRIsERERERGRLPE0wTKze8zsiJkdN7M/meT5PDP7Vvr5F8ys0ct4REREREREvORZgmVmfuCLwNuArcD9ZrZ1XLPfBbqccxuA/wY84lU8IiIiIiIiXvNyBOsm4Lhz7qRzLgo8Dtw3rs19wNfTj78DvNnMrmgXcRERERERkVwLeHjtWuDcqOPzwM1TtXHOxc2sB6gE2kc3MrMHgQfTh/1mdsSTiGUmVYz7fyPz6kfOuXsyaag+s2Coz+SW+szioz6TW+ozi4/6TG5N2mfMOefJu5nZrwD3OOfenz7+TeBm59xDo9ocSLc5nz4+kW6jH5QFyMz2Oud25joOkcVCfUZkdtRnRGZHfWZh8nKKYBNQP+q4Ln1u0jZmFgBKgQ4PYxIREREREfGMlwnWi8BGM1trZiHgPcAT49o8AfxW+vGvAD9zXg2piYiIiIiIeMyzNVjpNVUPAU8CfuCrzrmDZvZpYK9z7gng74BvmNlxoJNUEiYL11dyHYDIIqM+IzI76jMis6M+swB5tgZLRERERERkufF0o2EREREREZHlRAmWiIiIiIhIlijBEhERERERyRIlWCIiIiIiIlmiBEsmZWaNZnbYzL5pZq+b2XfMrMDMPmlmL5rZATP7iplZrmMVWQjUZ0RmR31GZHbUZxYPJVgync3Al5xzVwG9wB8AX3DO3eicuxoIA+/MZYAiC4z6jMjsqM+IzI76zCKgBEumc845tyf9+B+B24A3mdkLZrYfuAvYlrPoRBYe9RmR2VGfEZkd9ZlFwLONhmVJGL9JmgO+BOx0zp0zs08B+fMelcjCpT4jMjvqMyKzoz6zCGgES6bTYGZvSD/+DeDZ9ON2MysCfiU3YYksWOozIrOjPiMyO+ozi4BGsGQ6R4A/NLOvAoeAvwHKgQNAM/BiDmMTWYjUZ0RmR31GZHbUZxYBc278SKNIqlIN8K/pBZMiMgP1GZHZUZ8RmR31mcVDUwRFRERERESyRCNYIiIiIiIiWaIRLBERERERkSxRgiUiIiIiIpIlSrBERERERESyRAmWTMvMtpvZ20cdf8rMPjqH683p9SILnfqMyOyoz4jMjvrMwqcES2ayHXj7TI1EZMR21GdEZmM76jMis7Ed9ZkFTQnWMmBmjWZ22My+ZmZHzeybZvYWM9tjZsfM7CYzKzSzr5rZL8zsFTO7z8xCwKeBXzezfWb26+lLbjWzp83spJl9aNT7fMTMDqS//mjU+U+k3/dZYPO8fvMiV0B9RmR21GdEZkd9ZolzzulriX8BjUAcuIZUUv0S8FXAgPuA7wF/Drw33b4MOAoUAg8AXxh1rU8BPwfygCqgAwgCNwD7068pAg4C1486XwCUAMeBj+b630Rf+pruS31GX/qa3Zf6jL70Nbsv9Zml/RVAlotTzrn9AGZ2EPipc86Z2X5SnbwOuHfUHNx8oGGKa/2bcy4CRMysFVgB3Ab8i3NuIP0e/xPYReqXxr845wbT55/w5LsTyT71GZHZUZ8RmR31mSVKCdbyERn1ODnqOEnq5yAB/LJz7sjoF5nZzTNcK4F+jmRpUp8RmR31GZHZUZ9ZorQGSy55EvigmRmAmV2fPt8HFGfw+v8A/rOZFZhZIfDu9Lln0ufDZlYMvCv7oYvkhPqMyOyoz4jMjvrMIqUESy75DKn5uq+lh6k/kz7/FKmFk6MXUk7gnHsZ+BrwC+AF4DHn3Cvp898CXgV+CLzo3bcgMq/UZ0RmR31GZHbUZxYpc6nFcSIiIiIiIjJHGsESERERERHJEiVYIiIiIiIiWaIES0REREREJEuUYImIiIiIiGSJEiwREREREZEsUYIlIiIiIiKSJUqwREREREREsuT/B3QaZTkt61Y3AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 864x216 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "grid = sns.FacetGrid(data=results_df, col=\"metric\")\n",
    "\n",
    "grid.map(sns.stripplot, \"method\", \"value\", color=\"steelblue\",\n",
    "         order=method_labels.values(), alpha=0.5, jitter=0.25, zorder=1)\n",
    "grid.map(sns.pointplot, \"method\", \"value\", color=\"indianred\", join=False,\n",
    "         order=method_labels.values(), estimator=np.mean, capsize=0.1, zorder=2)\n",
    "\n",
    "grid.set_titles(col_template=\"{col_name}\", row_template=\"{row_name}\")\n",
    "grid.set(ylim=(0.0, 1.0), yticks=np.linspace(0.0, 1.0, 6))\n",
    "\n",
    "best_pos = {\"der\": (0, 0.9), \"der_skip\": (0, 0.9),\n",
    "            \"coverage\": (0, 0.1), \"purity\": (0, 0.1)}\n",
    "\n",
    "for col in grid.axes:\n",
    "    for row in col:\n",
    "        metric = row.title.get_text()\n",
    "        best = round(mean_best_results_dict[metric], 2)\n",
    "        row.annotate(text=best, xy=best_pos[metric],\n",
    "                     ha='center', va='center', color=\"indianred\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}