new-setup-tests.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nanogpt import GPT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/inar/.local/lib/python3.11/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, GPT2LMHeadModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"gpt2\")\n",
    "model_hf = GPT2LMHeadModel.from_pretrained(\"gpt2\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NanoGPT, loading the pretrained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading weights from pretrained gpt: gpt2\n",
      "forcing vocab_size=50257, block_size=1024, bias=True\n",
      "number of parameters: 123.65M\n"
     ]
    }
   ],
   "source": [
    "model = GPT.from_pretrained('gpt2') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compare with HF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "test_text =  \"Tom and Lily were playing with their toys in the living room. They liked to build towers and bridges with their blocks and cars. \"\n",
    "inputs = tokenizer(test_text, return_tensors=\"pt\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 55.3 ms, sys: 18.3 ms, total: 73.6 ms\n",
      "Wall time: 45.4 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "outputs_hf = model_hf(**inputs, labels=inputs[\"input_ids\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 54.5 ms, sys: 16.2 ms, total: 70.7 ms\n",
      "Wall time: 45.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "outputs = model(inputs.input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([1, 27, 50257]), torch.Size([1, 27, 50257]))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs_hf.logits.shape, outputs[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.allclose(outputs[0], outputs_hf.logits)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (225798 > 1024). Running this sequence through the model will result in indexing errors\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import random\n",
    "\n",
    "def split_text_into_token_chunks(text, max_length, tokenizer):\n",
    "    # Tokenize the text\n",
    "    tokens = tokenizer.tokenize(text)\n",
    "    # Split the tokens into chunks that do not exceed max_length\n",
    "    chunks = []\n",
    "    current_chunk = []\n",
    "    current_length = 0\n",
    "    for token in tokens:\n",
    "        current_chunk.append(token)\n",
    "        current_length += 1\n",
    "        if current_length == max_length:\n",
    "            chunks.append(tokenizer.convert_tokens_to_string(current_chunk))\n",
    "            current_chunk = []\n",
    "            current_length = 0\n",
    "    # Don't forget to add the last chunk if it's not empty\n",
    "    if current_chunk:\n",
    "        chunks.append(tokenizer.convert_tokens_to_string(current_chunk))\n",
    "    return chunks\n",
    "\n",
    "# Function to fetch text from a URL\n",
    "def fetch_text_from_url(url):\n",
    "    response = requests.get(url)\n",
    "    # Check if the request was successful\n",
    "    if response.status_code == 200:\n",
    "        return response.text\n",
    "    else:\n",
    "        raise Exception(f\"Failed to retrieve text from {url}, status code: {response.status_code}\")\n",
    "\n",
    "# URLs to fetch - replace these URLs with the actual Project Gutenberg URLs\n",
    "urls = [\n",
    "    'http://www.gutenberg.org/files/1342/1342-0.txt',  # Example: Pride and Prejudice by Jane Austen\n",
    "    'http://www.gutenberg.org/cache/epub/11/pg11.txt',  # Example: Alice's Adventures in Wonderland by Lewis Carroll\n",
    "]\n",
    "\n",
    "\n",
    "max_length = 512 #tokenizer.model_max_length\n",
    "\n",
    "# List to hold all the text chunks\n",
    "text_chunks = []\n",
    "\n",
    "# Fetch texts and split into chunks\n",
    "for url in urls:\n",
    "    text = fetch_text_from_url(url)\n",
    "    # Optionally, clean and preprocess the text as needed\n",
    "    # ...\n",
    "    # Split the text into chunks suitable for GPT-2\n",
    "    chunks = split_text_into_token_chunks(text, max_length, tokenizer)\n",
    "    text_chunks.extend(chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = tokenizer(text_chunks[111], return_tensors=\"pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 512])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs.input_ids.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 657 ms, sys: 817 ms, total: 1.47 s\n",
      "Wall time: 569 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "outputs_hf = model_hf(**inputs, labels=inputs[\"input_ids\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 554 ms, sys: 656 ms, total: 1.21 s\n",
      "Wall time: 545 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "outputs = model(inputs.input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([1, 512, 50257]), torch.Size([1, 512, 50257]))"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs_hf.logits.shape, outputs[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.allclose(outputs_hf.logits, outputs[0], atol=1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index: (tensor(0), tensor(328), tensor(2129)), HF Value: 103.21697998046875, My Value: 103.2179946899414, Diff: -0.00101470947265625\n",
      "Index: (tensor(0), tensor(328), tensor(3422)), HF Value: 101.6489028930664, My Value: 101.64990997314453, Diff: -0.001007080078125\n",
      "Index: (tensor(0), tensor(328), tensor(3857)), HF Value: 103.56383514404297, My Value: 103.5648422241211, Diff: -0.001007080078125\n",
      "Index: (tensor(0), tensor(328), tensor(4200)), HF Value: 101.34343719482422, My Value: 101.3444595336914, Diff: -0.0010223388671875\n",
      "Index: (tensor(0), tensor(328), tensor(4201)), HF Value: 102.64205932617188, My Value: 102.64306640625, Diff: -0.001007080078125\n",
      "Index: (tensor(0), tensor(328), tensor(4395)), HF Value: 102.17903137207031, My Value: 102.18004608154297, Diff: -0.00101470947265625\n",
      "Index: (tensor(0), tensor(328), tensor(10254)), HF Value: 105.78755950927734, My Value: 105.78858184814453, Diff: -0.0010223388671875\n",
      "Index: (tensor(0), tensor(328), tensor(11056)), HF Value: 104.9476089477539, My Value: 104.94861602783203, Diff: -0.001007080078125\n",
      "Index: (tensor(0), tensor(328), tensor(11271)), HF Value: 103.5008316040039, My Value: 103.50184631347656, Diff: -0.00101470947265625\n",
      "Index: (tensor(0), tensor(328), tensor(11681)), HF Value: 102.28570556640625, My Value: 102.28671264648438, Diff: -0.001007080078125\n",
      "Index: (tensor(0), tensor(328), tensor(11747)), HF Value: 103.37384796142578, My Value: 103.3748550415039, Diff: -0.001007080078125\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "# Assuming outputs_hf.logits and outputs[0] are your two tensors\n",
    "difference = torch.abs(outputs_hf.logits - outputs[0])\n",
    "\n",
    "# You can use a threshold to find all elements with a difference above this threshold\n",
    "threshold = 1e-3  # You can adjust this threshold\n",
    "large_diffs = difference > threshold\n",
    "\n",
    "# Now, get the indices of the largest differences\n",
    "largest_diff_indices = torch.where(large_diffs)\n",
    "\n",
    "counter = 0\n",
    "# Inspect values\n",
    "for idx in zip(*largest_diff_indices):\n",
    "    hf_value = outputs_hf.logits[idx].item()\n",
    "    my_value = outputs[0][idx].item()\n",
    "    print(f\"Index: {idx}, HF Value: {hf_value}, My Value: {my_value}, Diff: {hf_value - my_value}\")\n",
    "    counter+=1\n",
    "    if counter > 10:\n",
    "        break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x3488fccd0>]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABPf0lEQVR4nO3deVxU5f4H8M+wowKKKIugormjCLib+xZu7WmZWZatVqbVTb2p3Ra93bJFS1tMcyn7dV3ytrgrariCmOCSKCgKiAuyKDvn9wcyzjBn9jMzz8Dn/XrxesGcZ848c5g553ue5fuoJEmSQERERCQQF0dXgIiIiKgmBihEREQkHAYoREREJBwGKERERCQcBihEREQkHAYoREREJBwGKERERCQcBihEREQkHDdHV8ASlZWVyMzMhI+PD1QqlaOrQ0RERCaQJAkFBQUICQmBi4vhNhKnDFAyMzMRFhbm6GoQERGRBTIyMhAaGmqwjFMGKD4+PgCq3qCvr6+Da0NERESmyM/PR1hYmPo6bohTBijV3Tq+vr4MUIiIiJyMKcMzOEiWiIiIhMMAhYiIiITDAIWIiIiEwwCFiIiIhMMAhYiIiITDAIWIiIiEwwCFiIiIhMMAhYiIiITDAIWIiIiEwwCFiIiIhMMAhYiIiITDAIWIiIiEwwBFRnFZBb7Zcw6pOYWOrgoREVGdxABFxuc7zuD9309i6MI4R1eFiIioTmKAIiPxQq6jq0BERFSnMUAhIiIi4TBAISIiIuEwQCEiIiLhMEAhIiIi4TBAISIiIuEwQCEiIiLhMEAhIiIi4TBAISIiIuEwQCEiIiLhMEAhIiIi4TBAISIiIuEwQCEiIiLhMEAhIiIi4TBAISIiIuEwQCEiIiLhMEAhIiIi4TBAkaGCytFVICIiqtMYoBAREZFwGKAQERGRcBigEBERkXAYoBAREZFwGKAQERGRcBigEBERkXAYoBAREZFwGKAQERGRcBigEBERkXAYoBAREZFwGKAQERGRcBigEBERkXAYoBAREZFwGKAQERGRcBigEBERkXAYoBAREZFwGKDIUKkcXQMiIqK6jQEKERERCYcBChEREQmHAQoREREJhwEKERERCYcBChEREQmHAQoREREJhwEKERERCYcBChEREQnH7ABlz549GDNmDEJCQqBSqbBx40at7evXr8eIESMQEBAAlUqFpKQknX0MHDgQKpVK62f8+PGWvgciIiKqZcwOUG7evInIyEgsXrxY7/a+fftiwYIFBvczZcoUZGVlqX+++uorc6tCREREtZSbuU+IjY1FbGys3u0TJ04EAKSnpxvcT7169RAUFGTuyxMREVEd4LAxKGvWrEFAQAA6deqE119/HQUFBXrLlpSUID8/X+uHiIiIai+zW1CUMGHCBISHhyMoKAjJycmYOXMmjh07hm3btsmWnz9/Pt555x0715KIiIgcxSEBypQpU9S/R0REoE2bNujWrRsSExMRHR2tU37mzJmYPn26+u/8/HyEhYXZpa5ERERkf0JMM46Ojoa7uzvOnDkju93T0xO+vr5aP0RERFR7CRGgpKSkoKysDMHBwY6uChEREQnA7C6ewsJCpKamqv9OS0tDUlIS/P390bx5c1y/fh0XLlxAZmYmAOD06dMAgKCgIAQFBeHs2bNYs2YNRo4ciYCAAJw4cQIzZsxAVFQU+vbtq9Dbso5K5egaEBER1W1mt6AcOXIEUVFRiIqKAgBMnz4dUVFRmDNnDgBg06ZNiIqKwqhRowAA48ePR1RUFJYuXQoA8PDwwI4dOzBixAi0a9cOr7zyCoYPH47t27fD1dVVqfdFRERETkwlSZLk6EqYKz8/H35+fsjLy7PJeJQJ3x7An6nXAADpC0Ypvn8iIqK6yJzrtxBjUIiIiIg0MUAhIiIi4TBAISIiIuEwQCEiIiLhMEAhIiIi4TBAISIiIuEwQCEiIiLhMEAhIiIi4TBAISIiIuEwQCEiIiLhMEAhIiIi4TBAISIiIuEwQCEiIiLhMEAhIiIi4TBAkaGCytFVICIiqtPcHF0BIiIiEkdeURlSLuXBy8MV0c0bOawebEEhIiIitVNZ+Xjs24N44+djDq0HAxQiIiISDgMUIiIiEg4DFCIiIhIOAxQiIiISDgMUIiIiEg4DFCIiIhIOAxQiIiISDgMUIiIiEg4DFCIiIhIOAxQiIiISDgMUIiIiEg4DFCIiIhIOAxQiIiISDgMUE+QVlWHNwfO4frPU0VUhIiKqExigyFCptP+e/lMSZm9IxtPfH3ZMhYiIiOoYBigm2HEqBwBw9MINx1aEiIiojmCAQkRERMJhgGJEwvlcR1eBiIiozmGAIkOS7vz+4JJ4x1WEyAlk3ijC3f/eia/3nHV0VYioFmGAQkRW+c+W07iYW4QPfj/l6KoQUS3CAIWIrFJeKRkvRERkJgYoREREJBwGKERERCQcBihEREQkHAYoREREJBwGKE6guKwC3+49h7NXCh1dFSIiIrtggOIEPt9xBu/9dhJDPo5zdFWIdKiMFyEiMhsDlBoqKyVUSmJNmzzCbLZERFTHMEDRUFkpYcSnexB/9pqjq0JERE7gSkEJPt56GhnXbzm6KrUOAxQNOQUlOJPDcR5ERGSaqT8kYtHOVIz7ar+jq1LrMEAhIiKy0MG06wCAzLxiB9ek9mGAokHF0X5ERERCYICigfEJkfkY2BORLTBAqYU2J2eh5wfbcTj9uqOrQkREZBEGKLXQ86sTcTm/BE8sO+ToqhAREVmEAUotVlEpVj4XIiIiUzFAISIiIuEwQCEiIiLhMEAhIqtwEg8R2QIDFE080xIREQmBAYomjiklIiISAgOUWqy0otLRVSAiIrIIAxRNtbCLZ8PRi46uAhERkdkYoNRyK+LPO+R1067exM2Scoe8NhEROT8GKKS4lMw8DPpoN/p9uAsAsPt0DiavOIzL+VztszZScTEeIrIBBigaVLWxj8cBdpzMAQBcv1kKAHhy+WHsPJWDf25MdmS1iIjIBKLMFzE7QNmzZw/GjBmDkJAQqFQqbNy4UWv7+vXrMWLECAQEBEClUiEpKUlnHyUlJXj55ZcREBCA+vXrY+zYsbh40fFjJSQT/i17/r6Ccg4+tciVghJHV4GIiJyE2QHKzZs3ERkZicWLF+vd3rdvXyxYsEDvPqZNm4YNGzZg7dq12LdvHwoLCzF69GhUVFSYWx27e+K7Q/h8Z6re7Tn5xfhydyquFvJiTNYrLCmHJIlyP0NEZD9u5j4hNjYWsbGxerdPnDgRAJCeni67PS8vD8uWLcOqVaswdOhQAMDq1asRFhaG7du3Y8SIEeZWye5+PpKB6cPaym574rtDOJVdgN2nr+D/nutt55qRpuRLeci4fguxnYMdXRWLnLtSiMEfx2FguyZY8VQPR1eHiMiu7D4GJSEhAWVlZRg+fLj6sZCQEERERCA+Pl72OSUlJcjPz9f6sQUlxqCcyi4AABxKu271vqpxZIxlRi/ahxfWJOJYxg1HV8Uiaw9nAAB2n77i4JoQEdmf3QOU7OxseHh4oFGjRlqPBwYGIjs7W/Y58+fPh5+fn/onLCzMHlWtHQTqHnDUZI+zVwod88ICkiQJRaXKdqUygCYiWxBmFo8kSXqnK86cORN5eXnqn4yMDDvXzrHECTHI2T3z/RF0mLMZF3NvOboqREQG2T1ACQoKQmlpKXJzc7Uez8nJQWBgoOxzPD094evrq/VDJnJAswXvqMW141TVFPD/Jjh+1hwRkSF2D1BiYmLg7u6Obdu2qR/LyspCcnIy+vTpY+/qWIQXYMvwuBERkanMnsVTWFiI1NQ702zT0tKQlJQEf39/NG/eHNevX8eFCxeQmZkJADh9+jSAqpaToKAg+Pn54emnn8aMGTPQuHFj+Pv74/XXX0fnzp3Vs3qIiET3+Y4zKC6rwJv3tHd0VYhkpeYUoqmvJ3y93B1dFYuY3YJy5MgRREVFISoqCgAwffp0REVFYc6cOQCATZs2ISoqCqNGjQIAjB8/HlFRUVi6dKl6H5988gnuu+8+PPLII+jbty/q1auH//3vf3B1dVXiPdlFZaWEjOvsxzcHU6JTbVFcVoGF2/7Gl7vPcgkHElJKZh6GLoxDz/d3OLoqFjO7BWXgwIEGE0c9+eSTePLJJw3uw8vLC4sWLcKiRYvMfXmbMuf6+dr/JeGXpEwsfCQSD0SH2q5SRKKrg3FnpcY5sLScmaVJPNXpCYrKxE+Aqo8ws3iczS9JVV1YX+zSn1VWCAJNMyYy5I/jWYj72zlyvvBrRWR7DFAswK4Kw3h4xFFWUYmkjBuoqBT7ino5vxgvrEnEpO8OOboqRCQIBihWEvu0r19BcRnmbUpBwvlcnW2pOQXYdCxT8TVgGLeYR4njNXP9cdz3xZ/4cMspBfZmO9cKSx1dBSISDAOU2k5Pc8bHW//Givh0PLhEd3mBoQv34JUfj2L7yRxb145srDrfyVdx57QeZxeFdXj4iGyPAYoTsEXLQ2qO8fTvxy/lKfqa7Pohe8q4fgs3btm+ZYafaxKRKS3gB85dwxs/H7PL98QSZs/iqc0sOs/wVsppXSssgZ+3O9xc616cruRFVYlFNpWWnVeMfh/uAgCkLxjl4NoQiWn81wcAAC4qFf79UBcH10ZX3Tsz24gkSfj1r0ydxxTZtyJ7qZ22nbiMPvN3mL16dGpOAWLe2477vvzTRjUjRzp28YZN96/0+CwiR7ogaE4vBigaLDnlVD8n7u8rmPrDUa1t+UXlVtfJag44keqb5WSLO+0pK48gM68Yjy87aNbzNhy9BABIvpSveJ3IfM7cTcJZfUS2wQBFgzVTMZMVHq+hjyRJKK9gYqiayqw8JoUl5fhoy2mczGLAQsbV9vaT4rIKLNx6Gn/ZuCWKyBAGKBr2n71mUjlH5pR49JsD6LNgJ4o1sgOWVVRic3IWrhWWmLSPs1cKcdXEsnXFgj9OYvGuVMR+ttfRVSEnUxvbT5bsPovPd6Zi7GLDXaCFJeUY99V+LP8zzU41I1PVhl5IBigayk0MPLI11t5Iu3oT+89eQ0GJfbpzDpy7jpyCEhy9cEP92FdxZ/H86kTc+4Xx8RRZeUUY8nEcTmUX2LCWzuf4Rfu0gBE5g9Mmnh9W/JmGg2nX8c7/Tti4RqTp0+1/4/v4dINlakF8wlk8Snj0mwMOff3NKdkAgIu5Rboba/SPpzhyzIWDbjWd9k7CTsfrwrVbOJx+HfdFNYOri/kvKuIQDFv/z532M6Wwm6XOu84LUNUabsln3pHSr97Ep9vPAAAm9WmpyD5F/A4DbEHR4ukmzuG4VliCV9ceRXzqVdntL65JsGjchVnnVWaSNVtZRaVW95uoNP+1/f+zCzN+PoYfDl1wWH1EPUHWdP7aTXy2/Qzyi8rUjzlL3UWWd6sMJeX2/d4cy7iBjnM249u954wXFkihnVrrRSDOFVkAIQ29HV0FtX/9egK/JGXisW/lZ6fk3irTmdZszNpDFzBl5RElqicca68Rx2S6eDYnZ+P5VQnI07gYGXP3v3ei45zNeoOU7/alIfazvSaPF7I1zQRNh82cql1NM9gZ/NFu7D2j3IJ/ebfK8MHvJ3EiU4zBy6M/34dPtv+NmeuPO7oqtcb1m6WI/NdW9Jm/066v+491f6GkvBLv/XbSrq/rTBw9Q40BiqAyTJiXXlhi3h3HWzypqhmb8lxaXonnVydgc0o2PrvdnGqKy/klqJSAM5flM/X+69cTOJmVj0U7xVgFu+u/tim6v3NXb2LiMuUW/Hvnfyn4es85jPxcjMHL1WPNzM27Q/odSa86ltduGs9mmpVXxBw0dQgDFAc4eiEX6VdvmlzeqhiWX2aLPLfqTkuTLWY8lZQ7dqq4s3RLpAjScmKIiJl0a6Plf6ah9/ydWLBZ7IUvlVZRKeHguWu4KdO18+rao7hwzfoka6KeDxig2FnG9Vu4/8t4DPxot862kvIKzP/jJA6eM226s2iSL+Vh12n9Cww66ktgyevuOq1cNwWZx553yNcKS1BpQdoASdA5Eqk5Bdh56rLdXs+eX+nqmUI1F76s7b7bl4ZxXx/ApO90WyZ/ScrEMysPyz6vNtybMkCxszM5+qfvLduXhq/izmHc1we0Tn8H9TQnixb0jl60D08tP2xy65AkSQbHd5zIzEdWnszMJBKKqHdf+rz+8zE8tCQeB85dQ8x72y0al6V58rf3+5ckCZuOZeLMZd1zydCFezB5xREkZdywb6WckLNcwNcerhq8fuR8LgDdz5spC786KwYoAkm7cufCbk63gmhNzKau6/DurycR+c5WbDtx544vv7gMu07n4NyVQoz8fC96mzhwztGDuWoDRx5C5T/D+q8+/024iCPnczH1h0QAwI5T+lv9lHbmcoHVs7x2n76CV348imGf7NFbxtqMyPw6kQgYoGhRNqS2phn4cp5C4x4EOtPUvAh9dzv75Pzf74yif+DLeDy1/DBeXJNo17oZ4iQ3WrXe7tM5mLb2qFmzqgyx5g7akqduScnGsE/2YNxX+y1/YQDH7bSsBjk3UbshzcEARVD2/HB9uv1vZN5QritFX0xUVlGJTccyDU6xrW6udPZMt6b8/zKu38Kra4/abR0nTc7SvK3pyeWHsTEpEx9tOe3oqmgx9Rbg/w5nAJCf0u6MFm49jZX7zzu6Gk7v0o0iDPpot9HMsPoo8VUWrRW+GgMUgTjqmvHp9jN2yYZ75HwuXvnxKB6x8A6yoLgM2XnFxguaQISGpedXJ+CXpEyMXrRPdruoJ41q6VdvYsEfpxTP6WIseFJqXJI13zelB/IWFJfhwSXxTpM07OyVQny+M9VhScPWKpBUUJQWhvm/n0Ta1ZuYuynFYXUQ4Xwoh6nuBVVWYfzLY1Jftokn0vMyU9UycotQWFKOBp7mfUyMveTZK6ZPsdbUed5WAMCh2UNQUSlhlZPfvSk9uC2noBgf/HYSE3q1QPeW/oruW87YxfuQXyxmVktTvj/WsHbvBcVl8PFyV/+9/M90JJzPRcL5XDzTr5VO+f8mXMSl3CK8OrSNxa8pSZJiY7Xkprza01vrj6O5fz30uSvA4n2I0opo7Urs+ojy/qzBFhQtyoaRr/98DB/WmLOv5Ifmvd9OIu+W4f749Gu3sDk5y6I7vg1HL6HzvC16t38fn67InYy5jmXkYcI3B/Hl7rN2f22Rzd6QjI1JmXh4qWUtVKZ++iVJQmpOoaLBidJ3cC//eNSs8porlN8qNfN9WVD3/9TopjJ2s/H6z8fwyfa/Le4OLKuoxMjP9+HVteYdF5GlK5D/QwRbUu5MEvhk298622sGlaK3rCqJAYoNbT+Zgy93nzV5UJ8lwcs+PWv1VMsrKsPzqxOx46RlMxUkCbLdKjn5xZi7KQVvrT9uszuAag8uicdXcdrByLkaU5lN+cp+s+ccBn+0GzkF2u/H2PibY04yZdPahE01T4Sfbv9bNgBdtDMVQxfGWfQaC/44hXkObMrWpBm0f7evasD2vE0p6Dhni80zxVo6xiqvqMyiy9P+s9dwMisfvySZtzyGPdTlGXg1z52f7TA9a7W1ap4HRcQARYtt2sQ0o2Klm93+vfmUSaP6rcmL8NvxLJ3HNPueLXlP5oxbSDifi/l/WJ898v3fT+Kcxkqg1dKvGe5yMnXadE1lFZUoKL4TnIqUotvYNeFkVj4+3X5GdnmEhTJ3eaYoKa/A0rizWBGfbtU4EqUOo+ZufjpSNYB1xe2BipoDcU9k5uNAjeSJWnlQTAwZHHkdFueTR5oqLfgwx/2tTBLJ2RuSFdmPLTFAsYMrty/Gu07l4BkDSaEsGbRl6sXzRpHxdS5Mte/MVZ3ulXKNOwFT3sW6xIuK1cdcZVammU/NKcCEbw8YvcsetjAOnedtxfWbpVgadxbd399h1evamuYKvfkKTeXVpHkuLjcwRkS0i+nIz/di/NcHlE0aaOM3aW0Q5+yNGll5RUZbFEX7nJnq3wql+lciRb6tMUCxk9LySjy1Qj4lsT2sPqDMWJGyiko8vuwg/pugHWBk599pLrQ2EZUxjm6JeOb7I/gz9ZrB2UiSdKePPP7sVSz445TJyfcqK6syhZqyYKSSdpzKwQ8H7T+myFz2+O8fStcNPpWcim8LuwwknHPyeMOgjOu3tM45kiSh9/yd6P+fXcgvVj7QdkaWLOcgAgYodvL098aDE4F6ALRoBgR/JGfrbLfkbkvzvdr7bUuw7oSdZeZUZ33/15oPbz9xGR9vPY3/Jl7EKz8ele1a07RqfzpGL9qrDnyUmDY5a8NxSJKkbvWzF3P/H7dKyzFm0T48/u1BlFrYIqbdTaMrJdO0Aaman/+C4jKdcQWVlRKOZdxAqYFWI3O+Q3Jlz1wuUPQGyJJz0flrN/Ht3nMoKrXtDYqmk1n56PfhLgyWWdsMkB8/52yMfTRMOb9EvbsNS5xwUgEDFHuQgL1nDA9m/XbvOWw4esnql4o3MmjWWr/9pfwguzQzVnbWx9AJ/pYCJ8zNyVkY+J9dOK5Qki25+j6z8ggW7UzFm//9y6R9vP1LCpIv5eOz7eYPrDN0AXpr3XFM/eHObI+Pt4qVGA0AfjyUgeOX8rAv9SrmbjLel553qwxrD13QGrBubPB6+lXzWrBu3CpF53lbMfjj3VqPL91zFvd+8Sf2WDB2oKC4DNtPGF/8r+bUfUd00Qz6aDfe++2kXT8vW1Kqbpgya0EgojiNL3leUZnBriFRByozQLGD/XpWJ96nEbS899tJ2TLmeuzbg4rsxxxifrTvqDko1pK7w+dXJyL92i1MWXnE7IFt+kpbetwqKiWtQcpKd6lVDxittmhnqtX7XLk/HWP0JKQzlyRJKCm/855/PJRhoHSVF9Yk4K31x/GKnunHpp2fDRc6cK6qWyjjunZX0Ld703TKmtra9fSKIwbHrRUUlxls6amsrFqQ0x7Xn+peBLnuMTlf7k7FzPXHHdZl6+iuYgAosXI8nJzisgoknL+OCgHen7WYqE2Drf6f12/KD1B9fNlBpC8YZZsXtcCLaxKwWaYLxxSXcuX75/VlmrTnV6fm2A9rukI0x9o4wis/HsWmY8q0YtlzmuGcX0yfWlx14ai6op6WWbHXEvFnq24S9M2AUDq3xPSfkrBwXFer91PzYl8zOB62cA+y84vxeK/mss+fsvIIdpzKwayR7Y2+ljlJGZU4Xh9urmppCWjgYfW+LGHtOSg7rxg/WJkHaq4Z3wtTTf3hKLafNN7q5gzYgkJqvx/PhtxYKmNTKlUqlWwmWgCImCuf6K1mMChJklWzJGydOdQaSt6pmRqcSJJktAsj5ZJ1K94a80vSJYumt98qq8CIT/bg/d9OKF8pM1gTyK5XoLtWztIa+YCqA+bNyfIXpOqVmtcYGfz8/m8nEDF3C4Z/EoetKZbdpFjKHmNWjI1TOpWdb3bA/tSKw/jcyrwlSnTr11RbghOAAQoJ4r3fTqKkzLTmTqsv95KyLTiJF3LRd8FOi1ufrFXdfK8ZB83dlILId7Yq9hp7z5g3fuLrPWfx6tok3PfFnyaV1+yCWJdwEacvF+Abma4RR9LXTWLPLs4/U69ZNF5A8wK9+oDuEhHVx/rvy4V4dlWC5RW0gDXdT6a05Kw6cB5t//kHthkYy3PPp3vRw8w0ACezbBvgEwMUh7uYe0urP110cneVSpygl+2z38VIAnD0wg313wVWpmyfvOIwLt0owvOr75zYjSV/A2zT/1xN6VVmJy47ZFZyvQ9+tzxXQ4WRKZESxJrxZmwQtlwLWs2Hal5oT2Tm439WdOXV3L/mzLN/bjQ+qFiUMZOFJeVWjbGSJODt2+/3pTWJSlXLruzxvxDk362DAYqD3f3vXWj3z82OroZBpUZS2VvyBbJ3muXnNe4KcwqKtZYIeM7KO0a5lp9X1yYZfM7VAvtO463Jkv+ZvrFUlnj/t5MWfwbkghNDeR4SzhsftGnK8dBXZP4f1QPclYmacvKLMfLzvWatJ1Sz/lcc/PlSwq3SckTM3YLu7223eB/Xbt45DjrnMTsEuZIk4ZId8+dYkzFckyg3AAxQyKj/bDmtzu2gr0lVs2XFlA/3aZm1SK7fUu4CWNNmjX71cxaupqyPsRYwueNRcy0ha1g6YNHWaygZsjklGy//IH8BNvb52Xvmis5ie61m/Y4PfpefCffgEssWT6w5vkBf18quU8qkHq/W4wPzMw7XrNkn2y1bjkAkZy5XrfZd4MCVk/eeuYJv9pyzeBzZu7+eRN8FO/H97SUUbC3XwnOoKC1mNTFAIZNYuh6NPtWzKjTN+cW0tSGMreBsjLmJ1gy5catUdmCxJlPuZnedtmwxR03mnGRWxKfbbayBvoy4RzNuYOX+dPzfYe1pwmlGusf0He+v95yzqH76/H37AmmtXJnPqzU3qNcKbRfIi8SU/EjmXljLTQjKU3MK1d/ZicsO4f3fTxrNY1XtcI1ZV9/9WdV1faeVjczBAIXMYsrMBkuj8dQc0y4Ib64zLZGZPXT91zajZd6XubOveUf21HLLs4BWBzfGbvKeXH5I/fsNK4M8c+jLA1RaXok5v6TgzXV/ac3CsibdvjnjZDSdyi4wOntF38dapapaQHFJnLIBkj7VFz1RWdM98PDSeHV+qGk/JRksa8l5JvKdrXjNwH6z8oowdGEcur+v3a1kajfNw0tNa61bti8Nq2QGK9uTqK0mmpgHhUwy7qsDeH14W2xJ0R0Jn5pTiH2pdy5Clp6gik2cxVMXnc4uQLsgH9ltOQUlyDEhP8vu08p2RShJqT7vx5cdQv82ARY9V65FyZTZS7dKK8yebnr+2i2sPXQBD0SHwsPNxe4Xi6MXcvHvzadw4Nx1dArxNVq+oLgM7q4u8HJ3Vbwuml1nh9Nz8fiyg0h8e5jF+9txUn9r5M3SCmw4egmfjOsqe6ulxNT74rIKg8fpSkEJ3v3V8BT60vJKPLw0Hh1D/LS6cG21po6osQpbUMgkVwtL8Nb647Lbhn2yR2vGwfFLyqSDr81qpiY35lap4X74pIwbig5i1eeAntYQUZzMysdXCnb1TFx2yHghC1R/n77ZW1VXay8QOWYOit124rI6821Kpu5FWXO16cKScnSetxU9NFoVTAmobpWW4/8OZ6hbtRLO52LIx7v1JsvT9OZ/jxktUzP3UvXqvFqZdw1cz22VSbb925t1WlwqK++8nimzkvb8fQXHLubhxxqJ4JRK0ugsGKAQ1QLPrkrANRsHKBKAs1eUGZfhjE5lK5/3wpYBn6GF8oxdmrdq5Aw5cTuAyTdxOn518DJvU1X33WPfHMQT3x3Cg0vicfbKTUz67hCSNW5i1ifqJivbbqAVpNrSuLO4rNFy2P8/u3RWGD9yPtekOpviy92psjlk5PxcY7mI0opKPGRi9w8AvWnqv91nn25EUbCLR4MgM6uIhCRJwOwNpg1kro3+sU6+BVFUveabPxtISX8crxrTc/pygc6SBaM11mWquRSFOWpmyD2Upj1IdeE2ZWYzXcy9hS92VWXxfbxXC6Pl5eKLBAWCpWQbZ38uq6jEuSs3FVkZXQlsQaFa4611fynSbGvvVN+m0jcbxl7O5CizLo6cvxVac8cZ5d0qw+cKLMgoikIrEx+K6GaJ2Mk0D5y7ptWaZKnnVyVgxKd7TFqA0x7YgkK1xtrDGegc6odV+89jzpiOFu/H3qm+TTXFwKq29mBp3hRTLljGZmzUZv/ZannWXRFV5/hxZP4SU9iyjcCaHEMbj14yezDy+K8PAACWP9XdotesHqhcvXaTNVmMlcQWFKpVZm9IxqnsAjz2zUFHV0VRPydcxCmZ5Hb2tCLesmmR/zIyY6G2eumHRJNy4GTesP8K2dYOyi230WwSZ2Esn8qXu8/KPr7qwHl8tPW07LZqNYN1c2Z4GRp3BAD7zlzFjVul+D4+3S6D6q3FFhQiJ2BNbhClHFMojXZd8dtfWTabKWIta7rUVu1Px9u/pChYG8fQtwK7MUWlFegwx7LlSd42YR0ka8zUM9Oy2r83n3KqWZZsQSEispFLDmgdMYUps2Sqra0x1dWU4GT0or1m18merAnQvt3rvDNpnCk4ARigaBH0ZoeIyGHWH9WdBmyM0rNN8ouVzXysrzVQ35ITmtPrPzZxZpB1rWf2vRjp60UqMrJSt60xQCEicpArBSUmJe4ShaO6rKb/ZDxxmzne+K/8chma05YXaowVMXUtHk1PrbB8+YrnVyeqf3fk2DN7rsQsh2NQiIhsxFhqckcPfDbXHgsu1ErYflJ3iQ1bs3bqt8hLS9Qk6ro8bEHRIOo/iYick7P1+RuTXMveD4mNAQoREZnkcPp144XI6ZgzaNqeGKAQEZFJnKnbgpwfAxQNnMVDRER10WQrBvXaCgMUIiKiOm7nKfG6eRigEBERkXAYoBAREZFwGKAQERGRcBigEBERkXAYoGgQdeVRIiKiuoYBChEREQnH7ABlz549GDNmDEJCQqBSqbBx40at7ZIkYd68eQgJCYG3tzcGDhyIlBTt5bkHDhwIlUql9TN+/Hir3ggRERHVHmYHKDdv3kRkZCQWL14su/3DDz/EwoULsXjxYhw+fBhBQUEYNmwYCgq0F8WaMmUKsrKy1D9fffWVZe+AiIiIah2zVzOOjY1FbGys7DZJkvDpp59i9uzZeOCBBwAA33//PQIDA/HDDz/gueeeU5etV68egoKCLKw2ERER1WaKjkFJS0tDdnY2hg8frn7M09MTAwYMQHx8vFbZNWvWICAgAJ06dcLrr7+u08KiqaSkBPn5+Vo/tqDicsZERERCMLsFxZDs7GwAQGBgoNbjgYGBOH/+vPrvCRMmIDw8HEFBQUhOTsbMmTNx7NgxbNu2TXa/8+fPxzvvvKNkVWVxFg8REZEYFA1QqtVsiZAkSeuxKVOmqH+PiIhAmzZt0K1bNyQmJiI6OlpnfzNnzsT06dPVf+fn5yMsLMwGNSciIiIRKNrFUz2mpLolpVpOTo5Oq4qm6OhouLu748yZM7LbPT094evrq/VDREREtZeiAUp1t41mV01paSni4uLQp08fvc9LSUlBWVkZgoODlawOEREROSmzu3gKCwuRmpqq/jstLQ1JSUnw9/dH8+bNMW3aNHzwwQdo06YN2rRpgw8++AD16tXDY489BgA4e/Ys1qxZg5EjRyIgIAAnTpzAjBkzEBUVhb59+yr3zoiIiMhpmR2gHDlyBIMGDVL/XT02ZNKkSVixYgXefPNNFBUV4cUXX0Rubi569uyJrVu3wsfHBwDg4eGBHTt24LPPPkNhYSHCwsIwatQozJ07F66urgq9LSIiInJmKskJp67k5+fDz88PeXl5io5HOXDuGsZ/fUCx/RERETmz9AWjFN2fOddvrsVDREREwmGAQkRERMJhgEJERETCYYBCREREwmGAQkRERMJhgKKhReN6jq4CERERgQGKlmA/b0dXgYiIiMAAhYiIiATEAIWIiIiEwwCFiIiIhMMAhYiIiITDAIWIiIiEwwCFiIiIhMMAhYiIiITDAIWIiIiEwwCFiIiIhMMAhYiIiITDAIWIiIiEwwCFiIiIhMMApYYmPp6OrgIREVGdxwCFiIiIhMMAhYiIiITDAIWIiIiEwwClhspKydFVICIiqvMYoNRw7Wapo6tARERU5zFAISIiIuEwQCEiIiLhMEAhIiIi4TBAISIiIuEwQCEiIiLhMEAhIiIi4TBAISIiIuEwQCEiIiLhMEAhIiIi4TBAISIiIuEwQDGiR0t/R1eBiIiozmGAYsSEXs0dXQUiIqI6hwEKERERCYcBChEREQmHAQoREREJhwEKERERCYcBChEREQmHAYoRkuToGhAREdU9DFCIiIhIOAxQiIiISDgMUIiIiEg4DFCIiIhIOAxQjJDAUbJERET2xgCFiIiIhMMAxQgVVI6uAhERUZ3DAMUIQ108Lw5srf59VJdge1SHiIioTmCAYoRmojZXF+3WlGlD29q5NkRERHUDAxQzzBrZQetvD7c7h48dQURERMphgGIGFe5067wxop1jK0NERFSLuTm6As6kYT13PNW3JR7t0Ryhjby1tsW0aIRf/8pyUM2IiIhqFwYoRkgS8PHDkTiYdg1jI0OgUqkQ5l9Pp1znZn5Y90JvNGnghWdWHsbflwsdUFsiIqLagV08JngwJhQfPhQJN1fdw9WonjsAoH2wL2Ja+KN543rYMq2/vatIRERUq7AFxUoHZw1FaUUlGnjeOZQqle6Q2aEdArH95GV7Vo2IiMhpsQXFSh5uLlrBiT7P9m9lh9oQERHVDgxQ7CSqeUNHV4GIiMhpMECxg/oernB3dcHZD0Y6uipEREROgQGKEZWS9asZx781BEBVJtrZIzugVUB9q/dJRERUmzFAMaJxAw+r9+F3e6YPAEzp3wornuph9T6JiIhqM7MDlD179mDMmDEICanKCbJx40at7ZIkYd68eQgJCYG3tzcGDhyIlJQUrTIlJSV4+eWXERAQgPr162Ps2LG4ePGiVW/EViJDG1r0vNe4Tg8REZHFzA5Qbt68icjISCxevFh2+4cffoiFCxdi8eLFOHz4MIKCgjBs2DAUFBSoy0ybNg0bNmzA2rVrsW/fPhQWFmL06NGoqKiw/J0IJtjPy9FVICIiclpm50GJjY1FbGys7DZJkvDpp59i9uzZeOCBBwAA33//PQIDA/HDDz/gueeeQ15eHpYtW4ZVq1Zh6NChAIDVq1cjLCwM27dvx4gRI6x4O0RERFQbKDoGJS0tDdnZ2Rg+fLj6MU9PTwwYMADx8fEAgISEBJSVlWmVCQkJQUREhLpMTSUlJcjPz9f6EV2Aj/VjV4iIiOoqRQOU7OxsAEBgYKDW44GBgept2dnZ8PDwQKNGjfSWqWn+/Pnw8/NT/4SFhSlZbYPk0tubYlC7pnhhYGt88Vi0wjUyLoTdS0RE5ORsMounZqp3SZJk07+bWmbmzJnIy8tT/2RkZChWV2P8vN2NF5KhUqnwj3vaY1SXYKNlh3cMNFrGHAPaNVF0f0RERPamaIASFBQEADotITk5OepWlaCgIJSWliI3N1dvmZo8PT3h6+ur9WMr90c1s9m+q0mwPreKMYdnD7X5axAREdmKogFKeHg4goKCsG3bNvVjpaWliIuLQ58+fQAAMTExcHd31yqTlZWF5ORkdRlH8nJ3jtQwb4xoZ3B7Ex9PO9XEsA7Bvtj3j0GOrgYRETkZs2fxFBYWIjU1Vf13WloakpKS4O/vj+bNm2PatGn44IMP0KZNG7Rp0wYffPAB6tWrh8ceewwA4Ofnh6effhozZsxA48aN4e/vj9dffx2dO3dWz+oRxbG5w40XUkCrJg0AyK90HOznhay8Yp3HXxp0F/KKyvD1nnMyzzLcnWZP7q4qBDQQI1giIiLnYXaAcuTIEQwadOeOePr06QCASZMmYcWKFXjzzTdRVFSEF198Ebm5uejZsye2bt0KHx8f9XM++eQTuLm54ZFHHkFRURGGDBmCFStWwNXVVYG3ZB3NzPaWjj8xRlUjgLj7rgAsjTsrW3bWyA7oGe6PHh/sAACENvLGzNgO6m3yAUrVm3BzUaG80vbdSYY83quFQ1+fiIick9kBysCBAyEZWJ9GpVJh3rx5mDdvnt4yXl5eWLRoERYtWmTuy9cKNcegGBuT0tT3zqycib1amDTwFgCOzxuB0vJKRP5rq+z27dMHYOjCOJP2ZamHY0JRWlFp09cgIqLax+wAhcQ3uH3VYGNvD1d4e+hvlQq38aKFn4yLNDp7i4iISI5zjAi1IwUWLzbLD8/0NKu8Kdf7rmENLauMwlxuV7ZmlxYREZExDFAcLKZlI4PblR4HY88kbvYO9oiIqPZggCKoZ/u3wuS+4ejXJkDR/Y6JDFF0f6ZgLw8REZmLY1AcwMfrTquIq0ol29Iwa2QHm7x2fU/7/8sZnxARkbnYglKDPbK8+tf3wBePReObJ7pZvNZPTT3D/U0qF6TRxePqYtvQobrlxM3VBS8MbG3Wc7u3bIS/5tknD83g9k2xY8YANHBA8Ea6zP2sEFHtxADFQUZ1CcYwC9bgsWTAaXWg0Ny/ntnPtYZmy9A/7mlv1nPbBvrA18sdHYJtt6wBUJUr5quJMWjdpAFeGnSXVfs6OGuIQrWyDw+FgmOl3dfV9stNEJH4xDxDkaL2vzUEzw1ohTXP9LSou6W6peXhmFCdbRtf6otfX77byhrq9+vLd+PertaNm9EMjj4ZF6m1rV2QD9wVulAH+jrXKtJrppg3g0wpafNHOuR1ici5MEARgK07lYL8vDAztgPCZFpQNrxofP2j/W8NxvKnussmiOsa1hARzfwUqaemkIbeAKqCo3oe1nW9hPl7q3+/P0o3yKoLHojWbZXo3tK0bkGlKZEb5/FezRWoiXO4q2kDR1eByCEYoNTwQHTVBax9kI+RkrVDVPNGeLJPS53Hpw1tgy6hflj9dE809fXCoHZN7Zp07em7wxXbl6FuMXtOhe7hoIAAEC8XjbWtbu/d11mhmlgv+Z0RNtt3s4beWPxYlM32TyQyBig19GrVGHFvDMQvU/s6rA7P9W+l81j1dOOxVnZ3mKpbC39smno37taY5tzGjndyXu6OX5dJaS4uwPbp/fHyYOvGulhCtKnehlrdjA1UlwuoHeW1oW3RwNMNk/sqF1BrCg+ob/exY0Si4LQFGS0a2zYFvCHH5gyHXz3d5GwrJ/dAcVmlwdT1SvjnqA44mVWAPq0b62wLaeiNX1++G6MX7TNpXx1DbDvA1Rnd1dQHM4a3w6KdqcYLy+gU4ouUzHyDZbzdXVFUVmHR/q31Vmx7LPjjlM32n75glM32bYkxkVXdnl7utrnX+/dDXWyyXyJnwBYUwcgFJ0BVv72tgxMAeKZfK3z8SCRc9ExBjmjmhyAjg0F3vz4QPz3bC20DDXeTNWvobXC7rbi73nlvkXZcFiC0kXl3wkM76M7y+nZSN4ODTH293PDjs70M7jeggQd+e8VwF4t/fQ/TKllDr1a6gW1dYKuewqY+njbas3Hv3RfhsNcmAhigCKGjAlNpbZ3TRJOx7oKWAfXR08CF6p2xnfDHq/0wvnuYRa8f7OeFlo3r4cMHTbu7jG7RUOtvN5c7H/tZI+/M8LFlN8j9Uc0w20DyvXfGdjJpPyqoDI4F2jT1bqNrMb1/f2d0CqnqYvlyQjRaN9FtMZwzuqPFQYohd99lPDOyuYGcIzVu4LgAwtZqYzernDmjOwrXMkdVGKAIoImPJ/b9YxCS5gwz+7n/HNUBzRp6Y2asaZln5S5u5l6YrR1YGtHMDx2CfU1+3Wf6affv92rVGLvfGITeMt1Q1apnHL0+vC2C/byx7x+DZBO/aWb1VVLNQdafjOuKRnou+G0DGyg2KNvQMV38WBSe7d8KwzRaZkZ2DsaOGQN1yt4X1QwfPxyp8zgAxEYE6X2NBp76L2rpC0ZhtQmLYzbwdMOMYW2NlnO0X1++W/G1smpSAYpNgzeXr5cbpg1to/N4Ez2tOn7e7jg8eyge61k7Z1i9Plz8z2RtwwBFEKGN6qFhPfPvWJ/p1wp/vjUY4Rp3wR5utv23WnpS3jytH5Y+Ho2YFlULJJo6K6h1kwY49e49Oo831NMdBgBjuoQg+Z0RmDq46gQb2qgefG0UjMiZbGQWUtwbA+1TkduCfL0wuksIZo3soLf7rpqvV9XQNM3p2dUGtG2CuWPkW3uOzR0OVxfH5JR5foB89tk3RrRTojqyNAf6KjEbTF+LlburC9abkA4AAL6aGKPz2HQLg71B7ZvKtngZ6nZq4uOJgFraqsTFT+2PAUot0cDTDe+M7YS5YzqaHUCYOwX1xUGWpSJvH+SLeyLu5FKRzPjGyzU3+3i54+fne8vmcmnRuJ5Vqev73mXbsRSaA7Hlum0imsl3+wU0qLqIhQfU13nMECXSx+96fSC+n9zD4B205vgee3orVj5T8YsDW+OPV/uhXaCP2bN/DH1+xnXT7p5UYomMLydE690W3dzwqufVRnTSbd16ZYhuK4gpXFQqtJVp2dP3ta0+XpP7trTo9ZRm6mexsQnfH8D2+apIFwOUWmRSn5Z4yoLpjs8PbIWmPp4mX8R8vBw7+Uvz4ty9pT+imjfCpql9sfyp7tj4Ul8sfTza6hT5SrUEWMrTzVW2eb167SbNU++zGtPS5YLN6cPaKrJIZPVxd3VRYcu0/rK5TEIb1cMTvVtY/VpKrYmlUqnQIdgXW17rj3ky43z6tG6MlZN7IDJUd9qzoQvc+/crP4C0Z7g/HozWTiSoGbj2lhnXZW4Qbu7/xtfLHYlvD0OKCbleqmcyWdISbAuT7w7HzhkDjJYb3cX01A2mHm9H5jyqTRigEJr6eOHgrCFmr5djb/99vjeeG9BK64JcrUtoQwxq1xRdwxpqtdLICW1kfPaQqwrY9lp/LH+yu8X1NZW+i3FEMz+cevce2fEpSt/NLZG5e2/W8M5g1Zr97+2CfPTmMvnXvRHwtHE3oyZr1hSq5+GK/m2bYOnEGIzvHoalj985DhN66r+Y11zks4W/dakJHu0RBpVKhY8fkR/3A1TN4KrpX/fqBl09DCwcakng7l/fwyGroCuhVZMG2Dytn8Ey5kwweKSb/oH9/xx1ZxzgjFoyXqVfG+OD2m2JAQoBUCb9uK11a+mPmbEdrJ5d8PUT3TC0Q1Oj/fptAn3QyUguly8nRMPNRYVWMjNhqv1gwsBQfUx5r90UuFsLl6m/t4crjs0ZrjWWx5ZCG3nj91cMX0zk9FGgOy7YzxsLHuyC9kF3/t9TzUio10WmBcaQmmOQgv3kg2bNbtCaQcLwjoG4P0p3CYO1U3rJzsyytYcNXLwdQemEfpIEvDZM/nvQKsBxubNs5YGoZvjmCd2g2J6cMywmi4UH2GYK5913BWBf6lU8JLOgoGjCA+rj20nKtIyM7ByMwe2bys60OPfBSNwqqzDaLGxtGnp/jSZ1Lw/r7zk0g1V9eXmM78P854zrFqZO7ifCgEQXlQo+Xm4oKC5XfN+GkkH6ebsjr6gMgG5LjaYuoX6yNxYuLsrkTNL3L4xu0RAnsrSTBf44pZfBlht78HJ3QXFZJQDgsZ7N9QZ9mszJxSRBkp31N/+Bzng4JhQr4tPVjxmaoh8Z1hDHMm5oPebj6YaCEvnP2d43B6Hfh7tMrqdSFo7ravfXrIktKHVMTAt/fPxwJNa9YNqsADnVd+yaKbiXPB6NLydE4917Te+b1zfOw1Frj1gaJni5u+o0E3u5u8LFRWVyn3Wwn/aslUd7GJ6qqflqbq4qvHtvJ8we2QFNfZxrRWWRSZBM/kxYElAN7dBU9nFDwZ21M/TM+Yzre0udm/np1L1368Z2zcUkRzMgcdeoi9wNQM9wfxycNQQ7jIxR6d+2idHXbeDpphNItgn0wT9HdcDnj+qeyxY/GoXhHbWTML55j/7ZZnKLvNYVDFDqoAdjQtVTfS3h6+WOlHdGaA1A8/Fyx8jOwWbduU3o1Rx3NW2AlzRmBYUH1Ddr0JqtzDKQVM2QN0a0w8B2TQzmCpET5l8PSx+Pwaqne+D3V/rhQZnVh7XUOOdO7N0SUzQHy2psj2re0Ky6OJsWt0/gn43viucG6I5PMsSRLTX6WvHuuT0TR27tq19ectwaYZqirTh/AMAzViwGenj2UJ3HxkSG4E0zppS/MaIdAn29jHahdjfjfdbMnP1Mv1YYG6l7LmtYzx1fO6DrxBkXnWQXD1lEiUFzvl7u2D69Ksj5YtdZq/enpDZG0vTr89Ig08ct9GsTgL1nruKJPlWDMe8xM6gx5H9T78brPx/DA9HN0K+N8btAZ1Y9tfbers1wb9dm+CrunMX7EmEo1pwxHRHdohEGt9dtYdGcwVbzztqU1hUR3h8A3B/dDN/uS7PouX7e7vBwdUFpRaX6sUWPRiElM0/9t7FWB0M5lDQ9O6AVPt72NwDjwWy/NgH48KEuRpMu2io5pDFuDm7hsgRbUIgM8LVhptBlk7rj91f64TEj3Tly3DW6x+S6kSKa+WHztP54tr/5+U8MDfg1lSnjan575W68qDG1XfPiaVbDhhnn3bg3BmLemI7m7N2mnuzTEk18PPF4rzszhup5uOGRbmFGE57VzI0i4uVnUDvlg2OVCrJvtlOIH3q18od/fQ880bul3uc/FBOK1k10W6dGdNJd+8rT7U4Li7HPpEqlwiPdwtAltKGRkrpcXFR68wtp8rVhiodXhrTR6Wp2NAYoRAZ4ubsi0LfqxHFv1xAcmDkEwzrqnsgs4eHmgo4hvhbNoHJ3u/McpfNOLH5Mf8IwU5nyljqF+OFNO09tb9G4Pp60IFeQuUwdKzJvbCccnDnEJuseyTHlImgKzQB0dBf90/rlEr3Z0tpneyPx7WF6j39sRBA+ejhS9jtn6f/A2KrtpozNcXNR4Scji3wCwKtD2+Lv92JNrpupVk7ugenD2soGbo7ELh4iI7a+NgC7TuVgWMdA1Pd0Q9/WjbHtxGVHV8tmlFhl2p5jO6ydBaW0H6b0RJumPuj+/naTyhtbekAJSx+PQUpmHga1kx+Ye1/XEDTx8URZhaQ1G0Ufzeu7uQGtl7uL1ctO9GndGLtPX7FqH2ar8aF+sk9L3Ns1xOhF3ZSM2e6uLmjVpAGe6N0CK/efN1hWX/A1ukswfv0ry8Az7/zTWjWpD0jAuas38edbg9XfeVG6AKuxBYWEMPT2AnaipMnW5OftjvuimgmVrKp6cUhrBhuKRrMlyNEzQszVIdgH7YN80L9tE/RpHYAmPp7qrjIRcmTcExGEGcPb6W2t6xzaELNHdURnjeR7+v4FLRvXtyoknNS7JcL86+GfozrgPw91wZIJ0fAx47ulArDwka5W1MDQnk33UEwookxcgsCYkZ2DLajBHdun98e/TVzdvdqW1/rj6NvDFLkhsRVxzrhUpy15PBrnrtxE20CxmhhF1feuAKS8M0KooMlSn4yLxN6/r2pl6RwbGYJV+8+jd+vG+HqP5YNeraV5QY9u3hCJF27IlnNzdcHvr/TTugNdObkHvtuXjqf6tnRIHgtLuGmk968ZzKx/sQ/O5hSiZ6vGSKqRx8MUzRp64z8PdUH32/lSnul3Z8bViE5BaDXrd5P3ZU53jNKtAodnD0V2XrHeTMrmCm3krZ5NZGnCzNZNGqhzwOjj4aa9b3dXF70rrIvC+c9uVCu4u7qgnZ37q51dbQhOAOD+qFDcH6Wd4M/L3RX/u73Wjz0DFLmEe9U+fzQK838/haf7ybda1eyqCW1UD3MUHpDr5e6KUZ2Dcau0XGfJBls3z0c3b6QemGvOa8W0aISE87mY2LsF+sisjgzYp5vLUtX1v/d21t4mPp6KjeUxV7SelAFygc3Gl/pix8nLKC6rwJmcQgxoK9+9J7LacYYjUoBo/a/OTKnF/mzlvq4h2JiUqbNAZkhDb0zo2Rz1PFy1ZnAAVQHHFwZWHLYXEepgzriflZN7IPlSniJLMgD2X5bj/57rjYLiMkUGo48yMKAYqMpZtCJe9/H4twYj4/ots7qUuoY1RNewhmbWUCwMUIhIaNX5Yh7todxaL5+M64p37o2An8w08vfv76zY64ioY7CvTqp6c3PwmBMj1Pd0Q0+ZlZhFo6/xzNVFpdhMucUamWWruww1lwcZGxmCSknCaz8d03peSENvhBgZKyL6TYElGKAQmen+6FAs2pmKASakwVaCCOvSONKSx2MQn3rVpLTjplKpVLLBSV3zaI8wvD68HRobyblSWxj6Lr0yuA12nbqCSzeKlH1Njd81W39WPt0Tiedz0ad1Y63t90eF6gQo+jRr6G3WopaylRIYZ/EQmcnP2x2HZg8VYjGtuqCBpxuGdwrSm5bcli3+jSxcLNERNLtdptwegDpEJhutpvkPdLFpcGKLGSK2+nc39fXCvn8MwlQzskFbo4GnG/q3bWJwQUhjpvQLN7pulzlEW9WeLShEFnC2abC2Nqqz4b51ewrz90bG9SJFUnt/NbEb3vzvMbw2rK0CNbOfe7s2Q2RoQ52BtEoxdiFb8VT3291yyl087UG0C3RkqDIzhZwVAxQisl6N87pmU/rSx6PRIdhwtk0lrXiqBz7actqypu8a2gX54Jepd1u9n6fvDscyC9eesVRLG+ZfMRb7DWzXFAP1JIWzlrkxhGZxweIPo4z1xGgOmvVyM32hVn2CfcVKdc8AhYis5m9gEOE9EfZtXWndpAGWPB5j19c05u3RVdON7R2kyDF0kTa1BeHBmFB8uzdNdkFDpex9c5BW/pi3Ytujb+sA4Vo5HOHPtwbj4vVbiNSYpePiUjWuKq+ozOL9vhXbHoWl5Xg4JtR4YTtggEJEaN2kAZr4eJo95uLLCdFYl3ARM4Y7VxeII7SzcIVsU9nzuu3r5Y59/xhk02Ch5orEz/ZrJXS+FHtq1tBbdnyPt7urVQFKo/oe+EKBtbiUwgCF6DbR1nSxJ3dXF+x/azBczLzgjOwcrE7T7ShyqzmT7dmzJaNnuL/e4ET07NN1fRaeNfjNJiIAsGo2gSPMG9MRmXnF6GRkNVkyXfWU1+oVvEX28/O9sWT3WcxVOFtvXdC8cT3jhQTAAIWIFBfRzA8J53PhqWflVSU82bf2LJRoT/+6txMeXLIfrw3V7ZYLaOCJY3OGw9vD+gGXtta9pT+6P6lMdlpNzjzEZUr/Vnj31xMY0SlQdvu6F3pj1f7zmDWyg51rZhkGKESCc8YMkV88Fo3Fu85gUu+Wjq6KOGx84TN19zEt/PH3e7Hw0BM8+jlR7hfSNrlvS/Rp3Rh3NZXv9opp4Y+YFsoHdbbCAIWIFBfk54X37qvdKeOdmb7gRFTO3KphTyqVyq5T+m3NuT6lREREZmBw47wYoBARkcPZY20kDqh2LgxQiG7rEe48fbNENTl7ArOdMwbY/DWeub1GkT29M7YTAOCFga3t/trOjmNQiG6LDGuIDS/2sckCZ0RkmD1WVDZlVllEM2XXv5nUpyXuiQhCUx/T35+nmwtKyivRW2Ol47qIAQqRBs21LYjIvga0bYK4v69gZOcgrcftEbxUG94xEJ+Mi0SnEOUClUAz17jZPn0Adp3OwSPdwhSrgzNigEJEZAeG1itSgpe78/fYL34sCrtOX8GQ22v8fPNEN6w+cB5zR9svGZtKpcL9UY5diybMvx6e4BR9BihERPYwuH1TPNmnJbqEKtuF8NXEGMz//SQ+Gx+l6H4dwcfLHWMjQ9R/D+sYiGEd5ZOOmapZQ+fImkq6GKAQEdmBi4sK824PmFTSiE5BGNEpyHjBOsrbwxVH3x4GV1eV0w8krmsYoBAJjouNEVmnUX3bdq+RbTh/pyURERHVOgxQiIiISDgMUIiIiEg4DFCIiIhIOAxQiIiISDgMUIiIiEg4DFCIiIhIOAxQiIiISDgMUIgE169NEwCAn7e7g2tCRGQ/zCRLJLg3RrRDeJP6GHx7ATUiorqAAQqR4Lw9XDGxVwtHV4OIyK5s0sVTUFCAadOmoUWLFvD29kafPn1w+PBh9fYnn3wSKpVK66dXr162qAoRERE5IZu0oDzzzDNITk7GqlWrEBISgtWrV2Po0KE4ceIEmjVrBgC45557sHz5cvVzPDy4mBMRERFVUbwFpaioCOvWrcOHH36I/v3746677sK8efMQHh6OJUuWqMt5enoiKChI/ePv7690VYiIiMhJKR6glJeXo6KiAl5eXlqPe3t7Y9++feq/d+/ejaZNm6Jt27aYMmUKcnJy9O6zpKQE+fn5Wj9ERERUeykeoPj4+KB379549913kZmZiYqKCqxevRoHDx5EVlYWACA2NhZr1qzBzp078fHHH+Pw4cMYPHgwSkpKZPc5f/58+Pn5qX/CwsKUrjYREREJRCVJkqT0Ts+ePYvJkydjz549cHV1RXR0NNq2bYvExEScOHFCp3xWVhZatGiBtWvX4oEHHtDZXlJSohW85OfnIywsDHl5efD19VW6+kRERGQD+fn58PPzM+n6bZNBsq1bt0ZcXBxu3ryJ/Px8BAcHY9y4cQgPD5ctHxwcjBYtWuDMmTOy2z09PeHp6WmLqhIREZGAbJpJtn79+ggODkZubi62bNmCe++9V7bctWvXkJGRgeDgYFtWh4iIiJyETVpQtmzZAkmS0K5dO6SmpuKNN95Au3bt8NRTT6GwsBDz5s3Dgw8+iODgYKSnp2PWrFkICAjA/fffb4vqEBERkZOxSYCSl5eHmTNn4uLFi/D398eDDz6I999/H+7u7igvL8fx48excuVK3LhxA8HBwRg0aBB++ukn+Pj42KI6RERE5GRsMkjW1swZZENERERiMOf6zdWMiYiISDgMUIiIiEg4TrmacXWvFDPKEhEROY/q67Ypo0ucMkApKCgAAGaUJSIickIFBQXw8/MzWMYpB8lWVlYiMzMTPj4+UKlUiu67OkttRkYGB+DaEI+z7fEY2wePs33wONuHrY+zJEkoKChASEgIXFwMjzJxyhYUFxcXhIaG2vQ1fH19+SWwAx5n2+Mxtg8eZ/vgcbYPWx5nYy0n1ThIloiIiITDAIWIiIiEwwClBk9PT8ydO5eLE9oYj7Pt8RjbB4+zffA424dIx9kpB8kSERFR7cYWFCIiIhIOAxQiIiISDgMUIiIiEg4DFCIiIhIOAxQNX375JcLDw+Hl5YWYmBjs3bvX0VUSxp49ezBmzBiEhIRApVJh48aNWtslScK8efMQEhICb29vDBw4ECkpKVplSkpK8PLLLyMgIAD169fH2LFjcfHiRa0yubm5mDhxIvz8/ODn54eJEyfixo0bWmUuXLiAMWPGoH79+ggICMArr7yC0tJSW7xtu5o/fz66d+8OHx8fNG3aFPfddx9Onz6tVYbH2XpLlixBly5d1ImoevfujT/++EO9ncdYefPnz4dKpcK0adPUj/E4W2/evHlQqVRaP0FBQertTn+MJZIkSZLWrl0rubu7S99884104sQJ6dVXX5Xq168vnT9/3tFVE8Lvv/8uzZ49W1q3bp0EQNqwYYPW9gULFkg+Pj7SunXrpOPHj0vjxo2TgoODpfz8fHWZ559/XmrWrJm0bds2KTExURo0aJAUGRkplZeXq8vcc889UkREhBQfHy/Fx8dLERER0ujRo9Xby8vLpYiICGnQoEFSYmKitG3bNikkJESaOnWqzY+BrY0YMUJavny5lJycLCUlJUmjRo2SmjdvLhUWFqrL8Dhbb9OmTdJvv/0mnT59Wjp9+rQ0a9Ysyd3dXUpOTpYkicdYaYcOHZJatmwpdenSRXr11VfVj/M4W2/u3LlSp06dpKysLPVPTk6OeruzH2MGKLf16NFDev7557Uea9++vfTWW285qEbiqhmgVFZWSkFBQdKCBQvUjxUXF0t+fn7S0qVLJUmSpBs3bkju7u7S2rVr1WUuXbokubi4SJs3b5YkSZJOnDghAZAOHDigLrN//34JgHTq1ClJkqoCJRcXF+nSpUvqMj/++KPk6ekp5eXl2eT9OkpOTo4EQIqLi5MkicfZlho1aiR9++23PMYKKygokNq0aSNt27ZNGjBggDpA4XFWxty5c6XIyEjZbbXhGLOLB0BpaSkSEhIwfPhwrceHDx+O+Ph4B9XKeaSlpSE7O1vr+Hl6emLAgAHq45eQkICysjKtMiEhIYiIiFCX2b9/P/z8/NCzZ091mV69esHPz0+rTEREBEJCQtRlRowYgZKSEiQkJNj0fdpbXl4eAMDf3x8Aj7MtVFRUYO3atbh58yZ69+7NY6ywl156CaNGjcLQoUO1HudxVs6ZM2cQEhKC8PBwjB8/HufOnQNQO46xUy4WqLSrV6+ioqICgYGBWo8HBgYiOzvbQbVyHtXHSO74nT9/Xl3Gw8MDjRo10ilT/fzs7Gw0bdpUZ/9NmzbVKlPzdRo1agQPD49a9b+SJAnTp0/H3XffjYiICAA8zko6fvw4evfujeLiYjRo0AAbNmxAx44d1SdcHmPrrV27FomJiTh8+LDONn6WldGzZ0+sXLkSbdu2xeXLl/Hee++hT58+SElJqRXHmAGKBpVKpfW3JEk6j5F+lhy/mmXkyltSxtlNnToVf/31F/bt26ezjcfZeu3atUNSUhJu3LiBdevWYdKkSYiLi1Nv5zG2TkZGBl599VVs3boVXl5eesvxOFsnNjZW/Xvnzp3Ru3dvtG7dGt9//z169eoFwLmPMbt4AAQEBMDV1VUn0svJydGJCklX9ahxQ8cvKCgIpaWlyM3NNVjm8uXLOvu/cuWKVpmar5Obm4uysrJa8796+eWXsWnTJuzatQuhoaHqx3mclePh4YG77roL3bp1w/z58xEZGYnPPvuMx1ghCQkJyMnJQUxMDNzc3ODm5oa4uDh8/vnncHNzU78/Hmdl1a9fH507d8aZM2dqxWeZAQqqTlYxMTHYtm2b1uPbtm1Dnz59HFQr5xEeHo6goCCt41daWoq4uDj18YuJiYG7u7tWmaysLCQnJ6vL9O7dG3l5eTh06JC6zMGDB5GXl6dVJjk5GVlZWeoyW7duhaenJ2JiYmz6Pm1NkiRMnToV69evx86dOxEeHq61ncfZdiRJQklJCY+xQoYMGYLjx48jKSlJ/dOtWzdMmDABSUlJaNWqFY+zDZSUlODkyZMIDg6uHZ9li4fX1jLV04yXLVsmnThxQpo2bZpUv359KT093dFVE0JBQYF09OhR6ejRoxIAaeHChdLRo0fV07AXLFgg+fn5SevXr5eOHz8uPfroo7LT2UJDQ6Xt27dLiYmJ0uDBg2Wns3Xp0kXav3+/tH//fqlz586y09mGDBkiJSYmStu3b5dCQ0NrxZTBF154QfLz85N2796tNW3w1q1b6jI8ztabOXOmtGfPHiktLU3666+/pFmzZkkuLi7S1q1bJUniMbYVzVk8ksTjrIQZM2ZIu3fvls6dOycdOHBAGj16tOTj46O+bjn7MWaAouGLL76QWrRoIXl4eEjR0dHq6Z0kSbt27ZIA6PxMmjRJkqSqKW1z586VgoKCJE9PT6l///7S8ePHtfZRVFQkTZ06VfL395e8vb2l0aNHSxcuXNAqc+3aNWnChAmSj4+P5OPjI02YMEHKzc3VKnP+/Hlp1KhRkre3t+Tv7y9NnTpVKi4utuXbtwu54wtAWr58uboMj7P1Jk+erP6eN2nSRBoyZIg6OJEkHmNbqRmg8Dhbrzqvibu7uxQSEiI98MADUkpKinq7sx9jlSRJkuXtL0RERETK4xgUIiIiEg4DFCIiIhIOAxQiIiISDgMUIiIiEg4DFCIiIhIOAxQiIiISDgMUIiIiEg4DFCIiIhIOAxQiIiISDgMUIiIiEg4DFCIiIhIOAxQiIiISzv8DogwAUeR2tD4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(outputs_hf.logits[0,328].detach().numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_probability_distribution(logits):\n",
    "    probabilities = F.softmax(logits, dim=-1)\n",
    "    return probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\r'"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(inputs.input_ids[0,328])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'. I hope your plans in favour of the ----shire will\\r\\nnot be affected by his being in the neighbourhood.â\\x80\\x9d\\r\\n\\r\\nâ\\x80\\x9cOh no--it is not for _me_ to be driven away by Mr. Darcy. If _he_\\r\\nwishes to avoid seeing _me_ he must go. We are not on friendly terms,\\r\\nand it always gives me pain to meet him, but I have no reason for\\r\\navoiding _him_ but what I might proclaim to all the world--a sense of\\r\\nvery great ill-usage, and most painful regrets at his being what he is.\\r\\nHis father, Miss Bennet, the late Mr. Darcy, was one of the best men\\r\\nthat ever breathed, and the truest friend I ever had; and I can never be\\r\\nin company with this Mr. Darcy without being grieved to the soul by a\\r\\nthousand tender recollections. His behaviour to myself has been\\r\\nscandalous; but I verily believe I could forgive him anything and\\r\\neverything, rather than his disappointing the hopes and disgracing the\\r\\nmemory of his father.â\\x80\\x9d\\r\\n\\r\\nElizabeth found the interest of the subject increase, and listened with\\r\\nall her heart; but the delicacy of it prevented further inquiry.\\r\\n\\r\\nMr. Wickham began to speak on more general topics, Meryton, the\\r\\nneighbourhood, the society, appearing highly pleased with all that he\\r\\nhad yet seen, and speaking of the latter, especially, with gentle but\\r\\nvery intelligible gallantry.\\r\\n\\r\\nâ\\x80\\x9cIt was the prospect of constant society, and good society,â\\x80\\x9d he added,\\r\\nâ\\x80\\x9cwhich was my chief inducement to enter the ----shire. I know it to be a\\r\\nmost respectable, agreeable corps; and my friend Denny tempted me\\r\\nfurther by his account of their present quarters, and the very great\\r\\nattentions and excellent acquaintance Meryton had procured them.\\r\\nSociety, I own, is necessary to me. I have been a disappointed man, and\\r\\nmy spirits will not bear solitude. I _must_ have employment and society.\\r\\nA military life is not what I was intended for, but circumstances have\\r\\nnow'"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_chunks[111]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token: \n",
      "          Probability: 0.9245\n",
      "Token: <|endoftext|> Probability: 0.0074\n",
      "Token: \n",
      "\n",
      "         Probability: 0.0027\n",
      "Token: all        Probability: 0.0020\n",
      "Token: â          Probability: 0.0020\n",
      "Token: �          Probability: 0.0012\n",
      "Token: ,          Probability: 0.0010\n",
      "Token: \n",
      "          Probability: 0.0010\n",
      "Token: -          Probability: 0.0009\n",
      "Token: Â          Probability: 0.0007\n"
     ]
    }
   ],
   "source": [
    "# Get the top 20 probabilities and indices for the logits at a specific position\n",
    "topk_values, topk_indices = torch.topk(get_probability_distribution(outputs_hf.logits[0, 328].detach()), 10)\n",
    "\n",
    "# Convert topk_indices to tokens using the tokenizer\n",
    "topk_tokens = [tokenizer.decode([idx]) for idx in topk_indices.tolist()]\n",
    "\n",
    "# Print the tokens and their probabilities\n",
    "for token, prob in zip(topk_tokens, topk_values.tolist()):\n",
    "    print(f\"Token: {token :10} Probability: {prob:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\\n', '<|endoftext|>', '\\n\\n', 'all', 'â', '�', ',', '\\n\\xa0', '-', 'Â']"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topk_tokens\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Intermediate results\n",
    "\n",
    "- nanoGPT matches HF\n",
    "- test text quality! Gutenberg is full of weird tokens."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}