mnist_no_labels.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import torchvision\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size_train = 512\n",
    "batch_size_test = 1000\n",
    "torch.manual_seed(0); np.random.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "transform = torchvision.transforms.Compose([\n",
    "            torchvision.transforms.ToTensor(),\n",
    "            torchvision.transforms.Normalize((0.1307,), (0.3081,))])\n",
    "\n",
    "ds_train = torchvision.datasets.MNIST('~/datasets/mnist', train=True, transform=transform)\n",
    "ds_test = torchvision.datasets.MNIST('~/datasets/mnist', train=False, transform=transform)\n",
    "dl_train = torch.utils.data.DataLoader(ds_train, batch_size=batch_size_train, shuffle=True)\n",
    "dl_test = torch.utils.data.DataLoader(ds_test, batch_size=batch_size_test, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([512, 1, 28, 28]), torch.Size([512]))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x, y = next(iter(dl_train))\n",
    "x.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/work2/08258/akumar01/maverick2/env/lib64/python3.6/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /pytorch/c10/core/TensorImpl.h:1156.)\n",
      "  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([512, 1, 28, 28]) torch.Size([512, 10])\n"
     ]
    }
   ],
   "source": [
    "def Network():\n",
    "    return nn.Sequential(\n",
    "        torch.nn.Conv2d(1, 2, (3,3), padding=1),\n",
    "        torch.nn.ReLU(),\n",
    "        torch.nn.MaxPool2d((2,2)),\n",
    "        torch.nn.Conv2d(2, 4, (3,3), padding=1),\n",
    "        torch.nn.ReLU(),\n",
    "        torch.nn.MaxPool2d((2,2)),\n",
    "        torch.nn.Conv2d(4, 8, (3,3), padding=1),\n",
    "        torch.nn.ReLU(),\n",
    "        torch.nn.MaxPool2d((2,2)),\n",
    "        torch.nn.Conv2d(8, 16, (3,3), padding=1),\n",
    "        torch.nn.ReLU(),\n",
    "        torch.nn.MaxPool2d((2,2)),\n",
    "        torch.nn.Flatten(-3, -1),\n",
    "        torch.nn.Linear(16, 10)\n",
    "    )\n",
    "    \n",
    "net = Network()\n",
    "print(x.shape, net(x).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = Network() \n",
    "loss_function = torch.nn.CrossEntropyLoss()\n",
    "opt = torch.optim.Adam(net.parameters(), lr=1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_ds_stats(net, loader):\n",
    "    losses = []\n",
    "    accs = []\n",
    "    for batch_idx, (X_batch, Y_batch) in enumerate(loader):\n",
    "        Y_batch_pred = net(X_batch)\n",
    "        loss = loss_function(Y_batch_pred, Y_batch)\n",
    "        losses.append(loss.item())\n",
    "        Y_batch_pred_idx = Y_batch_pred.argmax(dim=-1)\n",
    "        acc = (Y_batch_pred_idx==Y_batch).sum().item()/len(Y_batch)\n",
    "        accs.append(acc)\n",
    "    return np.average(losses), np.average(accs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_confusion_mat(x, y, yp):\n",
    "    yp = yp.argmax(dim=-1)\n",
    "    mat = torch.zeros(10, 10)\n",
    "    for yi, ypi in zip(y, yp):\n",
    "        mat[yi, ypi] += 1.\n",
    "    return mat\n",
    "def show_confusion_mat(mat):\n",
    "    mat = mat.cpu().numpy()\n",
    "    mat = mat/mat.sum()\n",
    "    plt.imshow(mat)\n",
    "    plt.colorbar()\n",
    "    plt.ylabel('Labels'); plt.xlabel('Predicted')\n",
    "def batch_cossim(a, b=None):\n",
    "    if b is None:\n",
    "        b = a[:, None]\n",
    "    else:\n",
    "        b = b[:, None]\n",
    "    a = a[None]\n",
    "    return torch.cosine_similarity(a, b, dim=-1)\n",
    "def batch_l2(a, b=None):\n",
    "    if b is None:\n",
    "        b = a[:, None]\n",
    "    else:\n",
    "        b = b[:, None]\n",
    "    a = a[None]\n",
    "    return (a-b).norm(dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEGCAYAAAAaODK0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAXrklEQVR4nO3dfbRddX3n8feHBBISkafQmZpgyVpQO8EnNAKVyihRDGOHTDswBYcO09JJnVUsPnRa8A9Qxn9oWdCuyjimPAxVRmAiXc1oxiBSR3Q0JjwIJpHxCgwEcAlJioEKSW4+88feF2+v956zDzn7nrPP/rzW2st99v7t/fveLP362/v3sGWbiIhRd9CgA4iImA1JdhHRCkl2EdEKSXYR0QpJdhHRCnMHHcBkh2ie57Nw0GHECDts2f6+33P31ua0GV7kBfb4JR3IPd77roXesXO8Utl7H3xpg+2VB1JfvwxVspvPQk7RikGHESPsHbe92Pd73vPG+X2/Z102+qsHfI8dO8f5zobXVio75xd/sOiAK+yToUp2ETH8DOyn/y3kuiXZRURPjNnrao+xwyTJLiJ6lpZdRIw8Y8YbOM00yS4ierafJLuIGHEGxhuY7GodICRppaSHJY1JurTOuiJi9uzHlbZhUlvLTtIc4DrgPcB2YJOkdba31lVnRNTPwN4GvrOrs2V3MjBm+xHbe4BbgVU11hcRs8CY8YrbMKnznd1i4IlJv7cDp0wtJGk1sBpgPgtqDCci+sIwPlx5rJKBd1DYXgOsAXi1jmrgP2FEuxQzKJqnzmT3JHDspN9LymMR0WhinANaS2Ag6kx2m4ATJC2lSHLnAe+vsb6ImAVFB0WS3cts75N0MbABmAPcaHtLXfVFxOwoxtkl2f0jttcD6+usIyJm3/607CJi1KVlFxGtYMR4A7/okGQXET3LY2xEjDwj9njOoMPoWZJdRPSkGFScx9iIobbgoD013LU5H9zplyZ2UDQvPUfEQNli3AdV2rrptgycpNMl3Sdpn6Rzppy7UNIPyu3CbnWlZRcRPdvfh5ZdxWXgHgf+PfBHU649CrgCWE7xZH1vee2umepLsouInhQdFH1JHS8vAwcgaWIZuJeTne3HynNT1x54L/AV2zvL818BVgKfn6myJLuI6EmPHRSLJG2e9HtNudIRVFwGbgbTXbu40wVJdhHRs/Hq4+yetb28zliqSgdFRPRkYgZFla2LA1kGrudrk+wiomf7fVClrYuXl4GTdAjFMnDrKoawAThT0pGSjgTOLI/NKMkuInpSLARw4C072/uAiWXgtgG3294i6UpJZwNIepuk7cC5wGckbSmv3Qn8Z4qEuQm4cqKzYiZ5ZxcRPTFib5+mi023DJztyyftb6J4RJ3u2huBG6vWlWQXET2xqTRgeNgk2UVEj9SXQcWzLckuInpi0rKLiJbI4p0RMfKMsnhnRIy+4lOKzUsdzYs4IgYsH8mOiBYwVJkdMXSS7CKiZ2nZRcTIs5WWXUSMvqKDIl8Xi4iRpwwqjhh2b1/wg77fcwNv7fs9h1nRQZF3dhHRAplBEREjLzMoIqI1evjgztBIsouIntiwd3+SXUSMuOIxNskuIlogMygiYuQ1dehJbW1RScdK+jtJWyVtkXRJXXVFxGxSvz6lOKvqbNntAz5q+z5JhwH3SvqK7a011hkRsyDfoJjE9tPA0+X+bknbgMVAkl1EgxW9sZkbOy1JxwEnARunObcaWA0wnwWzEU5EHIAMKp6BpFcBXwA+ZPsnU8/bXgOsAXi1jnLd8UTEgctj7BSSDqZIdLfYvqPOuiJidjS1N7a2ZCdJwA3ANtvX1FVPRMy+YetpraLOlt1pwG8DD0l6oDz2Mdvra6wzImpmi31Jdj9j+xvQwAf7iOiqiY+xzUvPETFQE+/sqmzdSFop6WFJY5Iuneb8PEm3lec3liM7kHSwpJslPSRpm6TLutWVZBcRPetHspM0B7gOOAtYBpwvadmUYhcBu2wfD1wLXFUePxeYZ/sNwFuB359IhDNJsouInkyMs+tDy+5kYMz2I7b3ALcCq6aUWQXcXO6vBVaUnZ8GFkqaCxwK7AF+bmjbZEl2EdGz/ajSBiyStHnStnrSbRYDT0z6vb08xnRlbO8DngOOpkh8L1DM0nocuNr2zk4xZ9WTaJXjD35x0CE0ng37qi/e+azt5TWEcTIwDrwGOBK4R9Jdth+Z6YK07CKiZ316jH0SOHbS7yXlsWnLlI+shwM7gPcDX7a91/aPgW8CHZNqkl1E9KSP7+w2ASdIWirpEOA8YN2UMuuAC8v9c4C7bZvi0fUMAEkLgVOB73eqLI+xEdEz92Gcne19ki4GNgBzgBttb5F0JbDZ9jqKWViflTQG7KRIiFD04t4kaQvFeN6bbD/Yqb4ku4joWb8WAihnVK2fcuzySfsvUgwzmXrd89Md7yTJLiJ6YjdzBkWSXUT0SIznU4oR0Qb9eGc325LsIqInWc8uItrBxXu7pkmyi4ieZVn2iBh5TgdFRLRFHmMjohXSGxsRI89OsouIlsjQk4hohbyzi4iRZ8T+9MZGRBs0sGGXZBcRPUoHRUS0RgObdkl2EdGztOwihtyiOQsHHULjGdi/P8kuIkadgbTsIqINMs4uItohyS4iRp/SQRERLZGWXUSMPIPTGxsR7dC8ZFf7bF5JcyTdL+mLddcVEbPEFbchUinZSTpN0sJy/wJJ10j6pYp1XAJse6UBRsQQGtVkB3wa+AdJbwI+CvwQ+OtuF0laArwPuP4VRxgRw2ViUHGVbYhUTXb7bBtYBXzK9nXAYRWu+3Pgj4H9MxWQtFrSZkmb9/JSxXAiYpDsatswqZrsdku6DLgA+JKkg4CDO10g6deBH9u+t1M522tsL7e9/GDmVQwnIgZqv6ptXUhaKelhSWOSLp3m/DxJt5XnN0o6btK5N0r6lqQtkh6SNL9TXVWT3W8BLwEX2f4RsAT4sy7XnAacLekx4FbgDEmfq1hfRAwxudrW8R7SHOA64CxgGXC+pGVTil0E7LJ9PHAtcFV57Vzgc8AHbJ8IvBPY26m+SsnO9o9sX2P7nvL347Y7vrOzfZntJbaPA84D7rZ9QZX6ImKIVe2c6P4YezIwZvsR23soGkWrppRZBdxc7q8FVkgScCbwoO3vAtjeYXu8U2Udx9lJ2j1DyCru71d3+2siYtT0rfNhMfDEpN/bgVNmKmN7n6TngKOBXwYsaQNwDHCr7T/tVFnHZGe7SidEV7a/BnytH/eKiCFQvfNhkaTNk36vsb2mDxHMBX4NeBvwD8BXJd1r+6udLqhE0q8BJ9i+SdIi4DDbjx5oxBHRQDOOr/g5z9pePsO5J4FjJ/1eUh6brsz28j3d4cAOilbg120/CyBpPfAWYMZkV3VQ8RXAnwCXlYcOoXg5GBFt079xdpuAEyQtlXQIxbv9dVPKrAMuLPfPoXj3b2AD8AZJC8ok+M+BrZ0qq9qy+w3gJOA+ANtPSerLI25ENE+3ntYqyndwF1MkrjnAjba3SLoS2Gx7HXAD8FlJY8BOioSI7V2SrqFImAbW2/5Sp/qqJrs9ti0Vf+LE1LGIaKk+DRi2vR5YP+XY5ZP2XwTOneHaz9HDE2bVcXa3S/oMcISk/wDcBfxV1UoiIgatUsvO9tWS3gP8hKLL93LbX6k1sogavPXef9P3ey7i//b9nsOuH4+xs62X9eweAg6laMA+VE84ETH0TKWpYMOmam/s7wHfAX6Tokfk25J+t87AImKINXCJp6otu/8EnGR7B4Cko4H/A9xYV2ARMbxG+TF2B7B70u/d5bGIaKNRS3aSPlLujgEbJf0txZ+5Cniw5tgiYliNWrLjZwt0/rDcJvxtPeFExLCrsnzTMOq2EMAnZiuQiGiQBvbGVnpnJ+kYiuXVTwReXg3U9hk1xRURQ6yJLbuqMyhuAb4PLAU+ATxGMSctItqogUNPqia7o23fAOy1/b9t/y6QVl1EG1Vckn3YWn9Vh55MrO3+tKT3AU8BR9UTUkQMvSFLZFVUTXaflHQ4xTdj/xJ4NfChuoKKiOGm6ot3Do2qCwF8sdx9DngXgKQP1RRTRETfVX1nN52PdC8SESOpgR0Uvax6MlXzBtpExIEbws6HKg4k2TXwz42Ivmjg//oP5Luxh9YSUUQMv1FLdv36bmxEjA4xwr2xEREva+E7u4hoqyS7iGiFJLuI4bbhzTf1/Z7/ltP6fs9hl8fYiGiHJLuIGHlOb2xEtEVadhHRBnlnFxHtkGQXESNvCFc0qeJAlniKiBYS/VuWXdJKSQ9LGpN06TTn50m6rTy/UdJxU86/VtLzkv6oW121JjtJR0haK+n7krZJ+tU664uI2dGPZCdpDnAdcBawDDhf0rIpxS4Cdtk+HrgWuGrK+WuA/1Ul5rpbdn8BfNn2rwBvArbVXF9EzIb+LN55MjBm+xHbe4BbgVVTyqwCbi731wIrJAlA0r8CHgW2VAm5tmRXfrPidOAGANt7bP99XfVFxCyqnuwWSdo8aVs96S6LgScm/d5eHmO6Mrb3UXwa4mhJrwL+hOLTrpXU2UGxFHgGuEnSm4B7gUtsvzC5UPnHrwaYz4Iaw4mIvuht1ZNnbS+vIYqPA9fafr5s6HVV52PsXOAtwKdtnwS8APzcC0jba2wvt738YObVGE5E9E1/HmOfBI6d9HtJeWzaMpLmAocDO4BTgD+V9BjFlw4/JuniTpXV2bLbDmy3vbH8vZZpkl1ENE+fpottAk6QtJQiqZ0HvH9KmXXAhcC3gHOAu20beMfLsUgfB563/alOldXWsrP9I+AJSa8rD60AttZVX0TMnn70xpbv4C4GNlB0Xt5ue4ukKyWdXRa7geId3RjFFw1fcYOp7kHFHwRukXQI8AjwOzXXFxF16+OgYtvrgfVTjl0+af9F4Nwu9/h4lbpqTXa2HwDqeDkZEYPUwBkUmS4WET2ZmEHRNEl2EdEz7W9etkuyi4jeNHQhgCS7iOhZHmMjoh2S7CKG2wIdPOgQRkJadhHRDkl2ETHy8nWxiGiDjLOLiPZw87Jdkl1E9Cwtu4gYfRlUHBFtkQ6KiGiFJLuIGH0mHRQR0Q7poIiIdkiyi4hRl0HFEdEOdhbvjIiWaF6uS7KLiN7lMTYiRp+BPMZGRCs0L9cl2UVE7/IYGxGtkN7YiBh9WfUkYvjd9dMjBh1C4xWDipuX7ZLsIqJ3DVz15KBBBxARzSO70tb1PtJKSQ9LGpN06TTn50m6rTy/UdJx5fH3SLpX0kPlf57Rra4ku4jojXvYOpA0B7gOOAtYBpwvadmUYhcBu2wfD1wLXFUefxb4l7bfAFwIfLZb2El2EdGjYm5sla2Lk4Ex24/Y3gPcCqyaUmYVcHO5vxZYIUm277f9VHl8C3CopHmdKkuyi4je2dU2WCRp86Rt9aS7LAaemPR7e3mM6crY3gc8Bxw9pcy/Bu6z/VKnkNNBERG96e0j2c/aXl5XKJJOpHi0PbNb2bTsIqJ31Vt2nTwJHDvp95Ly2LRlJM0FDgd2lL+XAH8D/DvbP+xWWa3JTtKHJW2R9D1Jn5c0v876ImKW9KGDAtgEnCBpqaRDgPOAdVPKrKPogAA4B7jbtiUdAXwJuNT2N6uEXFuyk7QY+ENgue3XA3Mo/piIaDjt319p66R8B3cxsAHYBtxue4ukKyWdXRa7ATha0hjwEWBieMrFwPHA5ZIeKLdf6FRf3e/s5lL0kuwFFgBPdSkfEcPO9G1Qse31wPopxy6ftP8icO40130S+GQvddXWsrP9JHA18DjwNPCc7TunlpO0eqKnZi8dO1MiYgiIagOKh21KWZ2PsUdSjJFZCrwGWCjpgqnlbK+xvdz28oPpOEwmIoZFfzooZlWdHRTvBh61/YztvcAdwNtrrC8iZksDk12d7+weB06VtAD4KbAC2FxjfRExG/r4zm421ZbsbG+UtBa4D9gH3A+sqau+iJg93Xpah1GtvbG2rwCuqLOOiJhtw/eIWkWmi0VEb0ySXUS0RPOeYpPsIqJ3wzaGrooku4joXZJdRIw8G8ab9xybZBetMvbSPx10CKMhLbuIaIUku4gYeQa6f19i6CTZRUSPDM47u4gYdSYdFBHREnlnFxGtkGQXEaMvCwFERBsYyBJPEdEKadlFxOjLdLGIaAODM84uIlohMygiohXyzi4iRp6d3tiIaIm07CJi9BmPjw86iJ4l2UVEb7LEU0S0RgOHnhw06AAiolkMeL8rbd1IWinpYUljki6d5vw8SbeV5zdKOm7SucvK4w9Lem+3upLsIqI3LhfvrLJ1IGkOcB1wFrAMOF/SsinFLgJ22T4euBa4qrx2GXAecCKwEvgv5f1mlGQXET3z+HilrYuTgTHbj9jeA9wKrJpSZhVwc7m/FlghSeXxW22/ZPtRYKy834yG6p3dbnY9e5fX/r8KRRcBz9YdTx81Kd4mxQo9xnvXiTVG0t0w/Nv+0oHeYDe7NtzltYsqFp8vafOk32tsryn3FwNPTDq3HThlyvUvl7G9T9JzwNHl8W9PuXZxp0CGKtnZPqZKOUmbbS+vO55+aVK8TYoVmhVvk2LtxPbKQcfwSuQxNiIG5Ung2Em/l5THpi0jaS5wOLCj4rX/SJJdRAzKJuAESUslHULR4bBuSpl1wIXl/jnA3bZdHj+v7K1dCpwAfKdTZUP1GNuDNd2LDJUmxdukWKFZ8TYp1tqV7+AuBjYAc4AbbW+RdCWw2fY64Abgs5LGgJ0UCZGy3O3AVmAf8Ae2O/aIyA2c4xYR0as8xkZEKyTZRUQrNC7ZdZteMiwkHSvp7yRtlbRF0iWDjqkKSXMk3S/pi4OOpRNJR0haK+n7krZJ+tVBx9SJpA+X/z34nqTPS5o/6JjaplHJruL0kmGxD/io7WXAqcAfDHGsk10CbBt0EBX8BfBl278CvIkhjlnSYuAPgeW2X0/xMv68wUbVPo1KdlSbXjIUbD9t+75yfzfF/xg7jvAeNElLgPcB1w86lk4kHQ6cTtFTh+09tv9+oEF1Nxc4tBwrtgB4asDxtE7Tkt1000uGOoEAlCs1nARsHHAo3fw58MfAsK/fsxR4BripfOS+XtLCQQc1E9tPAlcDjwNPA8/ZvnOwUbVP05Jd40h6FfAF4EO2fzLoeGYi6deBH9u+d9CxVDAXeAvwadsnAS8Aw/z+9kiKJ5ClwGuAhZIuGGxU7dO0ZNfzFJFBknQwRaK7xfYdg46ni9OAsyU9RvF64AxJnxtsSDPaDmy3PdFSXkuR/IbVu4FHbT9jey9wB/D2AcfUOk1LdlWmlwyFchmaG4Bttq8ZdDzd2L7M9hLbx1H8u95teyhbH7Z/BDwh6XXloRUUI+mH1ePAqZIWlP+9WMEQd6iMqkZNF5tpesmAw5rJacBvAw9JeqA89jHb6wcX0kj5IHBL+X96jwC/M+B4ZmR7o6S1wH0UvfT3k6ljsy7TxSKiFZr2GBsR8Yok2UVEKyTZRUQrJNlFRCsk2UVEKyTZtYCkcUkPlCtu/A9JCw7gXv9N0jnl/vWdFjeQ9E5JPQ+elfSYpKpfr4qoJMmuHX5q+83liht7gA9MPllOTu+Z7d+z3Wkw7zvJTIEYEkl27XMPcHzZ6rpH0jpga7mO3Z9J2iTpQUm/D8VMEEmfKtcQvAv4hYkbSfqapOXl/kpJ90n6rqSvlosffAD4cNmqfIekYyR9oaxjk6TTymuPlnRnud7b9YBm+d8kWqBRMyjiwJQtuLOAL5eH3gK83vajklZTrMbxNknzgG9KupNitZbXUawf+E8opmXdOOW+xwB/BZxe3uso2zsl/VfgedtXl+X+O3Ct7W9Iei3FTJh/BlwBfMP2lZLeB1xU6z9EtFKSXTscOmnK2j0Uc3bfDnzH9qPl8TOBN068j6P4PucJFOvGfb78ctNTku6e5v6nAl+fuJftnTPE8W5gWTE9FIBXl6vCnA78ZnntlyTtemV/ZsTMkuza4ae23zz5QJlwXph8CPig7Q1Tyv2LPsZxEHCq7ReniSWiVnlnFxM2AP+xXJYKSb9cLoj5deC3ynd6vwi8a5prvw2cruJjxUg6qjy+GzhsUrk7KSbwU5Z7c7n7deD95bGzgCP79UdFTEiyiwnXU7yPu0/S94DPULT8/wb4QXnur4FvTb3Q9jPAauAOSd8FbitP/U/gNyY6KCi/w1B2gGzlZ73Cn6BIllsoHmcfr+lvjBbLqicR0Qpp2UVEKyTZRUQrJNlFRCsk2UVEKyTZRUQrJNlFRCsk2UVEK/x/VvkfQsMDfz4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_confusion_mat(calc_confusion_mat(x, y, net(x)))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda:0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b00094d6331e4bb58fb47984b672dd9f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/118 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "667c18c768fc4dbe803d8d3a9294e992",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/118 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57217b391d104124849b97157a50aefa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/118 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b8d40a6a0f1a467d931470b81dc6e28e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/118 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "defe1ecebc074181990a71bfc655339a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/118 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "net = Network().to(device)\n",
    "loss_function = torch.nn.CrossEntropyLoss()\n",
    "opt = torch.optim.Adam(net.parameters(), lr=1e-2)\n",
    "\n",
    "for _ in range(5):\n",
    "    loop = tqdm(dl_train)\n",
    "    for x, y in loop:\n",
    "        x, y = x.to(device), y.to(device)\n",
    "        yp = net(x)\n",
    "        loss = batch_cossim(yp).mean()\n",
    "        opt.zero_grad()\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        loop.set_postfix({'loss': loss.item()})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEGCAYAAAAaODK0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAYH0lEQVR4nO3df7RdZX3n8feHGxJCkB8m1CUJSkZ+jFFb0YggLVPBH2HsmGmLS3RwGIuT2lWsv2Yc6KwFyvIfWhbYGRnHlB9DkRGcSFczmhKKlBEpxoSAYBKxISAk4EBIBiECSe79zB97B2+vN/fuwz37nn3O/rzW2otz9nnO83zvXZdvnr2f/TyPbBMRMegO6HUAERHTIckuIlohyS4iWiHJLiJaIckuIlphRq8DGG3m0MGefeBhvQ6jMr/4Yq9D6LnZr6+n3uc31VNvP/GhB3e9zhee38me3bs0lTre+845fnrHcKWy99z/4mrbS6bSXrc0KtnNPvAwTjnm3F6HUdnwTx7qdQg99/ob6vkT2vTWvbXU2092n/q2rte5/q7/MuU6nt4xzA9Wv6ZS2aFX/+O8KTfYJY1KdhHRfAZGGOl1GB1LsouIjhizx9UuY5skyS4iOpaeXUQMPGOG+3CaaZJdRHRshCS7iBhwBob7MNnV+lCxpCWSHpS0WdIFdbYVEdNnBFc6mqS2np2kIeBK4N3AVmCtpJW2N9bVZkTUz8CePrxnV2fP7iRgs+0ttncDNwJLa2wvIqaBMcMVjyap857dfOCxUe+3Am8fW0jSMmAZwEEzDq0xnIjoCsNws/JYJT1fCMD2ctuLbS+eOdT9uYAR0V3FDIpqR5PU2bPbBhw96v2C8lxE9DUxzJTWEuiJOpPdWuA4SQspktzZwIdrbC8ipkExQJFk9xLbeyWdD6wGhoBrbG+oq72ImB7Fc3ZJdv+E7VXAqjrbiIjpN5KeXUQMuvTsIqIVjBju/YMcHUuyi4iO5TI2IgaeEbs91OswOpZkFxEdKR4qzmXslBx3wk5Wrf5m1+t971Fv7nqdUcjGODB0/OtqqXfmLWu7Xqf8i67UkwGKiBh4thh2enYR0QIj6dlFxKArBij6L3X0X8QR0VMZoIiI1hjOc3YRMej6dQZF/0UcET034gMqHZOZbFMuSbMk3VR+vkbSMeX5AyVdJ+kBSZskXThZW0l2EdGRYiGAAyodExm1KdeZwCLgQ5IWjSl2HrDT9rHAFcCl5fkPALNsvwl4K/CH+xLh/iTZRURHjNjjoUrHJKpsyrUUuK58vQI4Q5Iocu4cSTOA2cBu4OcTNZZkFxEdsWHYB1Q6gHmS1o06lo2qarxNueaPae6lMrb3As8AcykS3y7gCeBR4DLbOyaKOwMUEdEhdfJQ8Xbbi2sI4iRgGDgKOAK4U9Jttrfs7wvp2UVER0xHPbuJVNmU66Uy5SXrYcDTFPvZ3GJ7j+0ngbuACZNqkl1EdKwbAxSM2pRL0kyKTblWjimzEji3fH0WcLttU1y6ng4gaQ5wMvDjiRrLZWxEdMSoK4t37m9TLkmXAOtsrwSuBq6XtBnYQZEQoRjFvVbSBkDAtbbvn6i9JLuI6EixlWJ3Usd4m3LZvmjU6xcoHjMZ+73nxjs/kSS7iOhQNsmOiBYwVJod0TRJdhHRsfTsImLg2UrPLiIGXzFAkd3FImLgZQ+KKdv4+JG89fN/1PV6D11Szw5YP39N939985bf3fU667R92Sm11NtPv4fhnzxUS727l7yt63X6rqn/XosBityzi4gW6MfFO5PsIqIj3ZpBMd2S7CKiY9lwJyIGng17RpLsImLAFZexSXYR0QKZQRERA69fHz2prS8q6WhJfy9po6QNkj5ZV1sRMZ3Uta0Up1OdPbu9wGdtr5f0CuAeSX9ne2ONbUbENOhgD4rGqC3Z2X6CYucfbD8raRPFTkFJdhF9rBiNzdzYcZWb154IrBnns2XAMoADDzliOsKJiCno14eKa7+olnQI8E3gU7Z/ZRNb28ttL7a9eMZBc+oOJyK6YKTcTnGyo0lq7dlJOpAi0d1g++Y624qI6dGvo7G1JTtJotgZaJPty+tqJyKmX9NGWquos2d3KvAR4AFJ95Xn/rTcTSgi+pQt9ibZ/ZLt70HDLtojoityGRsRAy/37CKiNZLsImLg9etzdkl2EdGxpj1DV0Urkt3sLTtqqXfmLfVstNJPds2v549+Xi211qOuTYdedceTXa/zgBeHp1yHDXuzeGdEtEEuYyNi4OWeXUS0hpPsIqINMkAREQPPzj27iGgFMZzR2Ihog9yzi4iBl7mxEdEOLu7b9Zv+u/COiJ7r1rLskpZIelDSZkkXjPP5LEk3lZ+vKfez2ffZr0u6u9yq9QFJB03UVnp2EdERd2mAQtIQcCXwbmArsFbSyjHbrZ4H7LR9rKSzgUuBD0qaAXwN+IjtH0qaC+yZqL307CKiY3a1YxInAZttb7G9G7gRWDqmzFLguvL1CuCMcsuH9wD32/5hEY+ftj3hxN8ku4jomK1KxyTmA4+Ner+1PDduGdt7gWeAucDxgCWtlrRe0ucmayyXsRHRkaLXVnk0dp6kdaPeL7e9vAthzAB+E3gb8AvgO5Lusf2dib4QEdGRDh492W578X4+2wYcPer9gvLceGW2lvfpDgOepugFftf2dgBJq4C3APtNdrmMjYiOdeme3VrgOEkLJc0EzgZWjimzEji3fH0WcLttA6uBN0k6uEyC/wLYyATSs4uIjhgx0oXRWNt7JZ1PkbiGgGtsb5B0CbDO9kqKvaevl7QZ2EGRELG9U9LlFAnTwCrb356ovSS7iOhYt54pLveRXjXm3EWjXr8AfGA/3/0axeMnlSTZRURnOhugaIwku4joXB9OF0uyi4iOpWc3RTO272Le8ru7Xu/U91OK/fnSv7m6lnqvuPj1tdRbh0Mf3VtLvcM/6f7udfaLU68DGBlJsouIQWcgPbuIaIN+XOIpyS4iOpdkFxGDr9Ik/8ZJsouIzqVnFxEDz+CMxkZEO/Rfsqt91RNJQ5LulfStutuKiGniikeDVEp2kk6VNKd8fY6kyyW9tmIbnwQ2vdwAI6KBBjXZAV8BfiHpN4DPAg8BfzXZlyQtAN4HXPWyI4yIZtn3UHGVo0GqJru95YJ5S4Ev274SeEWF730J+Bwwsr8CkpZJWidp3R6mPpUlIurXpcU7p1XVZPespAuBc4BvSzoAOHCiL0j6HeBJ2/dMVM72ctuLbS8+kFkVw4mInhpRtaNBqia7DwIvAufZ/hnFWvF/Psl3TgXeL+kRii3STpdUeaG9iGguudrRJJWSne2f2b7c9p3l+0dtT3jPzvaFthfYPoZiKeXbbZ8z5YgjoreqDk40LNlN+JydpGcZP2QBtn1oLVFFRIM1b/ChigmTne0qgxCTsn0HcEc36oqIBmhYr62Kyg8VS/pNSR8tX8+TtLC+sCKi0UYqHg1SabqYpIuBxcAJwLXATIpdfU6tL7SIaKQBX7zzd4ETgfUAth+X1JVL3IjoP00baa2iarLbbdtS8SPumzoWES3Vh8mu6j27b0j6KnC4pH8P3Ab8ZX1hRUR0V6Wene3LJL0b+DlwPHCR7b/rdjB7581h+++f0u1qedUdT3a9Tqhn96d+85//7A9qqXce3d9lri6zt+yopd4m74o3yJexAA8Asyk6sA/UE05ENJ5p3FSwKqou8fQx4AfA7wFnAd+XVM8/6RHRfIM2g2KU/wicaPtpAElzgX8ArqkrsIhorkG+jH0aeHbU+2fLcxHRRoOW7CR9pny5GVgj6W8ofsylwP01xxYRTTVoyY5fLtD5UHns8zf1hBMRTdfE5ZuqmGwhgC9MVyAR0Uf6cDS26tzYIymWV38DcNC+87ZPrymuiGiwfuzZVZ1BcQPwY2Ah8AXgEWBtTTFFRNP14aMnVZPdXNtXA3ts/x/bfwCkVxfRRhWXZG9a76/qoyd7yv8+Iel9wOPAK+sJKSIar2GJrIqqye6Lkg6j2DP2vwKHAp+qK6iIaDY1bGHOKqpuuPMt28/Y/pHtd9p+K/C6mmOLiAEnaYmkByVtlnTBOJ/PknRT+fkaSceM+fw1kp6T9B8ma6vysuzj+MzkRSJiIHVhgELSEHAlcCawCPiQpEVjip0H7LR9LHAFcOmYzy8H/rZKyFNJdv33oE1ETF33BihOAjbb3mJ7N8X+0kvHlFkKXFe+XgGcIUkAkv418DCwoUrYU0l2fXiLMiK6onrPbp6kdaOOZaNqmQ88Nur91vIc45WxvRd4Bpgr6RDgP1E8ClfJVPaNnV21kYgYMNW7OtttL64hgs8DV9h+ruzoTWpa9o2NiMEhujYauw04etT7BeW58cpslTQDOIxixaW3A2dJ+jPgcGBE0gu2v7y/xjpZqTgi4qV7dl2wFjiu3IN6G3A28OExZVYC5wJ3UywcfLttA7+1r4CkzwPPTZToIMkuIl6OLiQ723slnQ+sBoaAa2xvkHQJsM72SuBq4HpJm4EdFAnxZUmyi4jOdWl40vYqYNWYcxeNev0C8IFJ6vh8lbYaleyG55gdJ+2ZvGCHXnVH16uMUm07t9VSa3RL0+a9VtGoZBcRfSLJLiIGnvtzbmySXUR0Lj27iGiD3LOLiHZIsouIgdfAJderSLKLiI6I/ryMncqqJ5OSdLikFZJ+LGmTpFPqbC8ipscg70Hxcv0FcIvtsyTNBA6uub2ImA4NS2RV1Jbsyj0rTgP+HUC5ON/uutqLiGnUh8muzsvYhcBTwLWS7pV0laQ5YwtJWrZvYb/h53bVGE5EdEWfbqVYZ7KbAbwF+IrtE4FdwK9sqGF7ue3FthcPHfIruTAimmiAN8l+ObYCW22vKd+voEh+EdHnNFLtaJLakp3tnwGPSTqhPHUGsLGu9iJi+vTjZWzdo7GfAG4oR2K3AB+tub2IqFsDL1GrqDXZ2b4PqGOzjYjopSS7iBh0/TqDIskuIjqmkf7Ldkl2EdGZ3LOLiLbIZWxEtEOS3dTM+ukvOP5j67peb3aqqs+mzx1RS73Hf6yWamsx/JOHeh3CtEvPLiLaIckuIgZedheLiDbIc3YR0R7uv2yXZBcRHUvPLiIGXx4qjoi2yABFRLRCkl1EDD6TAYqIaIcMUEREOyTZRcSgy0PFEdEOdhbvjIiW6L9cV+u+sRExoLq1laKkJZIelLRZ0gXjfD5L0k3l52skHVOef7ekeyQ9UP739MnaSrKLiM4YGHG1YwKShoArgTOBRcCHJC0aU+w8YKftY4ErgEvL89uBf2X7TcC5wPWThZ1kFxGdc8VjYicBm21vsb0buBFYOqbMUuC68vUK4AxJsn2v7cfL8xuA2ZJmTdRYkl1EdKyDy9h5ktaNOpaNqmY+8Nio91vLc4xXxvZe4Blg7pgyvw+st/3iRDFngCIiOtbBaOx224tri0N6A8Wl7XsmK5ueXUR0puol7OT5cBtw9Kj3C8pz45aRNAM4DHi6fL8A+Gvg39qedCOQRvXsdh81h59+/B1dr/e1F/9D1+sEGDr+dV2vs982b5m1bWavQ+i5n36h+3+zAP/shv/b9Tr1yPemXgeg7syNXQscJ2khRVI7G/jwmDIrKQYg7gbOAm63bUmHA98GLrB9V5XG0rOLiM6NVDwmUN6DOx9YDWwCvmF7g6RLJL2/LHY1MFfSZuAzwL7HU84HjgUuknRfefzaRO01qmcXEf2hSz07bK8CVo05d9Go1y8AHxjne18EvthJW0l2EdGZrFQcEe2QubER0RZZvDMiBl42yY6I1ujDnl2tj55I+rSkDZJ+JOnrkg6qs72ImCbdeah4WtWW7CTNB/4EWGz7jcAQxUODEdHnNDJS6WiSui9jZ1CsRrAHOBh4fJLyEdF0ZtIHhpuotp6d7W3AZcCjwBPAM7ZvHVtO0rJ9KyIM79pVVzgR0SXCyNWOJqnzMvYIirWoFgJHAXMknTO2nO3lthfbXjw0Z05d4UREN9nVjgapc4DiXcDDtp+yvQe4GahnxnRETK8+THZ13rN7FDhZ0sHA88AZwLoa24uI6dCn9+xqS3a210haAawH9gL3Asvrai8ipk/TRlqrqHU01vbFwMV1thER0615l6hVZAZFRHTGJNlFREv031Vskl1EdK5pz9BVkWQXEZ1LsouIgWfDcP9dxzYq2c18fFdtO4HVod92AqvDnG399y98t7367j211FvH39ck+0h3UlF36plGjUp2EdEnkuwiYuAZyB4UETH4DM49u4gYdCYDFBHRErlnFxGtkGQXEYMvCwFERBsYyBJPEdEK6dlFxODLdLGIaAOD85xdRLRCZlBERCvknl1EDDw7o7ER0RLp2UXE4DMeHu51EB1LsouIzmSJp4hojT589OSAXgcQEf3FgEdc6ZiMpCWSHpS0WdIF43w+S9JN5edrJB0z6rMLy/MPSnrvZG0l2UVEZ1wu3lnlmICkIeBK4ExgEfAhSYvGFDsP2Gn7WOAK4NLyu4uAs4E3AEuA/1bWt19JdhHRMQ8PVzomcRKw2fYW27uBG4GlY8osBa4rX68AzpCk8vyNtl+0/TCwuaxvvxp1z+5Zdm6/zSt+WqHoPGB73fF0UT/F21msX11RXyTV9P53+7eVfwe9jxVeO9UKnmXn6tu8Yl7F4gdJWjfq/XLby8vX84HHRn22FXj7mO+/VMb2XknPAHPL898f8935EwXSqGRn+8gq5SSts7247ni6pZ/i7adYob/i7adYJ2J7Sa9jeDlyGRsRvbINOHrU+wXluXHLSJoBHAY8XfG7/0SSXUT0ylrgOEkLJc2kGHBYOabMSuDc8vVZwO22XZ4/uxytXQgcB/xgosYadRnbgeWTF2mUfoq3n2KF/oq3n2KtXXkP7nxgNTAEXGN7g6RLgHW2VwJXA9dL2gzsoEiIlOW+AWwE9gJ/bHvCERG5D+e4RUR0KpexEdEKSXYR0Qp9l+wmm17SFJKOlvT3kjZK2iDpk72OqQpJQ5LulfStXscyEUmHS1oh6ceSNkk6pdcxTUTSp8u/gx9J+rqkg3odU9v0VbKrOL2kKfYCn7W9CDgZ+OMGxzraJ4FNvQ6igr8AbrH9z4HfoMExS5oP/Amw2PYbKW7Gn93bqNqnr5Id1aaXNILtJ2yvL18/S/E/44RPePeapAXA+4Creh3LRCQdBpxGMVKH7d22/19Pg5rcDGB2+azYwcDjPY6ndfot2Y03vaTRCQSgXKnhRGBNj0OZzJeAzwFNX79nIfAUcG15yX2VpDm9Dmp/bG8DLgMeBZ4AnrF9a2+jap9+S3Z9R9IhwDeBT9n+ea/j2R9JvwM8afueXsdSwQzgLcBXbJ8I7AKafP/2CIorkIXAUcAcSef0Nqr26bdk1/EUkV6SdCBForvB9s29jmcSpwLvl/QIxe2B0yV9rbch7ddWYKvtfT3lFRTJr6neBTxs+ynbe4CbgXf0OKbW6bdkV2V6SSOUy9BcDWyyfXmv45mM7QttL7B9DMXv9Xbbjex92P4Z8JikE8pTZ1A8Sd9UjwInSzq4/Ls4gwYPqAyqvpoutr/pJT0Oa39OBT4CPCDpvvLcn9pe1buQBsongBvKf/S2AB/tcTz7ZXuNpBXAeopR+nvJ1LFpl+liEdEK/XYZGxHxsiTZRUQrJNlFRCsk2UVEKyTZRUQrJNm1gKRhSfeVK278L0kHT6Gu/yHprPL1VRMtbiDptyV1/PCspEckVd29KqKSJLt2eN72m8sVN3YDHx/9YTk5vWO2P2Z7ood5f5vMFIiGSLJrnzuBY8te152SVgIby3Xs/lzSWkn3S/pDKGaCSPpyuYbgbcCv7atI0h2SFpevl0haL+mHkr5TLn7wceDTZa/ytyQdKembZRtrJZ1afneupFvL9d6uAjTNv5Nogb6aQRFTU/bgzgRuKU+9BXij7YclLaNYjeNtkmYBd0m6lWK1lhMo1g98FcW0rGvG1Hsk8JfAaWVdr7S9Q9J/B56zfVlZ7n8CV9j+nqTXUMyEeT1wMfA925dIeh9wXq2/iGilJLt2mD1qytqdFHN23wH8wPbD5fn3AL++734cxf6cx1GsG/f1cuemxyXdPk79JwPf3VeX7R37ieNdwKJieigAh5arwpwG/F753W9L2vnyfsyI/Uuya4fnbb959Iky4ewafQr4hO3VY8r9yy7GcQBwsu0Xxoklola5Zxf7rAb+qFyWCknHlwtifhf4YHlP79XAO8f57veB01RsVoykV5bnnwVeMarcrRQT+CnLvbl8+V3gw+W5M4EjuvVDReyTZBf7XEVxP269pB8BX6Xo+f818I/lZ38F3D32i7afApYBN0v6IXBT+dH/Bn533wAF5T4M5QDIRn45KvwFimS5geJy9tGafsZosax6EhGtkJ5dRLRCkl1EtEKSXUS0QpJdRLRCkl1EtEKSXUS0QpJdRLTC/wfKKkK0U/NzTAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_confusion_mat(calc_confusion_mat(x, y, net(x)))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.320777940750122, 0.12009999999999998)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc_ds_stats(net, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c03e9c324139481894b6107e2d6abc95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b25bae4e1bc347cdb230ce8c1391ba97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/118 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.2664697542786598, 0.9155000000000001)\n",
      "1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7493b3d952d463b982eed5987616a78",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/118 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.15614763796329498, 0.9506999999999998)\n",
      "2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2bc2de004d3f46c98716ef819009596a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/118 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.15456263944506646, 0.9498)\n",
      "3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69adae735dbe4880b2b377cdc0235897",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/118 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.13556534424424171, 0.9571000000000002)\n",
      "4\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48bc32518340471485a4fff7477961a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/118 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.13374587297439575, 0.9549)\n"
     ]
    }
   ],
   "source": [
    "losses = []\n",
    "for epoch_idx in tqdm(range(5)):\n",
    "    print(epoch_idx)\n",
    "    for batch_idx, (X_batch, Y_batch) in tqdm(enumerate(train_loader), total=len(train_loader)):\n",
    "#         X_batch, Y_batch = X, Y\n",
    "        Y_batch_pred = net(X_batch)\n",
    "        loss = loss_function(Y_batch_pred, Y_batch)\n",
    "\n",
    "        opt.zero_grad()\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "\n",
    "        losses.append(loss.item())\n",
    "        \n",
    "#     plt.plot(losses);plt.show() \n",
    "    print(calc_ds_stats(net, test_loader))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.13374583050608635, 0.9549)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc_ds_stats(net, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "394d66e9c3084056a47e74048108c214",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e529f7ad7fb44ebd91fe64e380ddb08e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/118 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2.35796172618866, 0.1032)\n"
     ]
    }
   ],
   "source": [
    "net = Network() \n",
    "loss_function = torch.nn.CrossEntropyLoss()\n",
    "opt = torch.optim.Adam(net.parameters(), lr=1e-2)\n",
    "\n",
    "losses = []\n",
    "for epoch_idx in tqdm(range(1)):\n",
    "    print(epoch_idx)\n",
    "    for batch_idx, (X_batch, Y_batch) in tqdm(enumerate(train_loader), total=len(train_loader)):\n",
    "        Y_batch_pred = net(X_batch)\n",
    "        \n",
    "        # change label to the hard prediction\n",
    "        Y_batch = Y_batch_pred.argmax(dim=-1).detach()\n",
    "        # change random labels to random numbers\n",
    "        Y_random = torch.randint(low=0, high=10, size=Y_batch.shape)\n",
    "        Y_random_mask = torch.rand(size=Y_batch.shape)<0.5\n",
    "        Y_batch[Y_random_mask] = Y_random[Y_random_mask]\n",
    "        \n",
    "        loss = loss_function(Y_batch_pred, Y_batch)\n",
    "\n",
    "        opt.zero_grad()\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "\n",
    "        losses.append(loss.item())\n",
    "        \n",
    "#     plt.plot(losses);plt.show() \n",
    "    print(calc_ds_stats(net, test_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7bc4f5b9ee824af9878de9571a4c77c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d63ce4d6d5cc464cba0c5056ebd45c16",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/118 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "net = Network() \n",
    "loss_function = torch.nn.KLDivLoss()\n",
    "opt = torch.optim.Adam(net.parameters(), lr=1e-2)\n",
    "\n",
    "losses = []\n",
    "for epoch_idx in tqdm(range(1)):\n",
    "    print(epoch_idx)\n",
    "    for batch_idx, (X_batch, Y_batch) in tqdm(enumerate(train_loader), total=len(train_loader)):\n",
    "        Y_batch_pred = net(X_batch)\n",
    "        \n",
    "        # change label to the hard prediction\n",
    "        Y_batch_pred_hard = Y_batch_pred.argmax(dim=-1).detach()\n",
    "        \n",
    "        Y_random = torch.randint(low=0, high=10, size=Y_batch.shape)\n",
    "        Y_random_mask = torch.rand(size=Y_batch.shape)<0.5\n",
    "        Y_batch_pred_hard[Y_random_mask] = Y_random[Y_random_mask]\n",
    "        \n",
    "        \n",
    "        Y_batch = torch.ones_like(Y_batch_pred)\n",
    "        Y_batch[torch.arange(len(Y_batch)), Y_batch_pred_hard] += 1.\n",
    "        Y_batch = Y_batch.softmax(dim=-1)\n",
    "        \n",
    "        loss = loss_function(Y_batch_pred.log(), Y_batch)\n",
    "        \n",
    "\n",
    "        opt.zero_grad()\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "\n",
    "        losses.append(loss.item())\n",
    "        \n",
    "#     plt.plot(losses);plt.show() \n",
    "#     print(calc_ds_stats(net, test_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(net, loader):\n",
    "    idxs_seen = [[] for _ in range(10)]\n",
    "    for batch_idx, (X_batch, Y_batch) in enumerate(loader):\n",
    "        Y_batch_pred = net(X_batch)\n",
    "        Y_batch_pred_idx = Y_batch_pred.argmax(dim=-1)\n",
    "        \n",
    "        for i in range(10):\n",
    "            idxs_seen[i].append(Y_batch_pred_idx[Y_batch==i])\n",
    "        if batch_idx==10:\n",
    "            break\n",
    "    for i in range(10):\n",
    "        idxs_seen[i] = torch.cat(idxs_seen[i], dim=0)\n",
    "    return idxs_seen\n",
    "idxs_seen = test(net, train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAALICAYAAABiqwZ2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABn5ElEQVR4nO39e7ytdV3vf7/em8VB5RSwUlkcliXqpsLDXiE76ic3UDcCiaWZbDU0ktvHVh96gym67RGW5aFUalcaSomYIJEpoZWGWNpWYiGIt6C5ItysBcgCAUEUQT73H9d3La81mXPNMdcchznmfD0fj+sxr8N3XNfnO8Y1PvMzrsMYqSokSZIkdf7LpAOQJEmSlhILZEmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpB4LZEmSJKnHAlkkuTfJjw1pXW9I8r42vjZJJVk1pHUf1GLdaRjrW8B2H53kX5Lck+Qd49z2UpDkzUluT3LrItYx8D42zP1Rmhbm4Xm3ax42D4+VBfICJHlxki8nuS/JrUnenWTvBTz+xiTHDjGe7a4vyVFJHmo7+r1JNia5KMlP99tV1e5VdcM82zoqycb5Yqqq36+q3xi8F9vd5jb9q6r/22L9wTDWvwCnAbcDe1bVGTMXJnl/kjePOaaxSHIQcAZwaFU9ZpblQ9vHZms7yHPbCoDL2/vyq8N8j2npMQ+bh83DD1u+FPLw77b35YNJzhq4c0uYBfKAkpwBvA34TWAv4AjgYOBTSXaZZGzzuLmqdgf2oIv5q8Bnkxwz7A0N6wjFEnQwcF2tzF/VOQi4o6pu206bse1jc7gAuBrYF/hfwMVJVo9p2xoj8/D8zMPL0jTk4Q3Aa4GPj2l7o1dVDvMMwJ7AvcDzZszfHdgM/Hqbfj/w5t7yo4CNbfx84CHgu21drwXWAkX3yfhm4BbgNb3HL2h9s8S9tf2M+X8CrO9NF/D4Nn48cB1wD7AJeA3wqLadh9q27gX2B84CLgY+CHwb+I0274NtXUPtX299q1qb/YFLgG/RvTlf2lvXWcBFwAdaX74CrNvOa/wzwJXA3e3vz/RifAD4fovj2Fkeu7UfvRhfAtwE3Am8DPhp4FrgLuBPeo/9ceDTwB10R0f+Cti7t/xpdMXfPcBfAx+e8ZydCFzT1vt/gMN6y17XXsN7gK8Bx8zR973a87QZ+AbwRroPz8fOeN3fP6R9bF/g79o+cyXwZuBzM9u2/ab/3P/dLNt5AnA/sEdv3meBl006bzgMd8A8bB42Dy/JPDxjmx8Ezpp0vhhKzpl0ANMwAMcBD25JCDOWnQdc0Ma3vkHb9DY7LXBj/43dexNfQJf8fqq9OY7dkfXNEttcb5qj25vtUW26/6a5Bfi5Nv4jwNPmWhdd8nsAeHZ7Iz+C2RPzUPrHwxPzvwB/BuwGPKWt++hebN+j+0ezE/AW4AtzPE/70CXQFwGrgJPb9L6zxTnL47cu78X4nhbXL7Q4Pgr8KLAGuA14Rmv/eODngV2B1a1PZ7dlu9AlylcBOwO/TJektmzrqW1dT299PKU9Z7sCT6T7x7B/L64fnyP+DwAfozvysBb4d+DU7e1Di9zHLmzDI4FDW5wPS8wDPve/BFw/Y96fAP970nnDYbgD5uFZ14V5mJnLMQ8Pso8NLQ/P2OayKZC9xGIw+wG3V9WDsyy7pS1fjDdV1Xeq6svAX9IlhlG6GQiw9yzLHgAOTbJnVd1ZVV+cZ12fr6qPVtVDVfXdOdoMvX9JDgSOBF5XVd+rqmuA9wG/1mv2uar6RHXXyp0PPHmO1Z0AfL2qzq+qB6vqArrTU7+4iBB/t8X1SeA7dP+8b6uqTXRHOJ8KUFUbqupTVXV/VW0G3gk8o63jCLp/FH9cVQ9U1UeAf+tt4zTgz6vqiqr6QVWdR3c09QjgB3QJ+tAkO1fVjVX1HzODbDfaPB94fVXdU1U3Au+g+ye1GLPuY217zwF+u6ruq6rr6IqbHbU73dGmvrvp/sloeTEPz808PDvz8Hjy8LJkgTyY24H95ri267Ft+WLc1Bv/Bt0pq1FaQ/fp8K5Zlj2H7tP+N5L8c5L/Ps+6bppn+cw2w+rf/sC3quqeGete05vu3+17H7DbHK/h/u2xfTPXtVDf7I1/d5bp3WHrndkXJtmU5Nt0n763/KPfH9hU7WN5038uDwbOSHLXlgE4kO5oxQbg1XRHcG5r25jted+P7qhIv/+L7TvMvY+tpvtn0+/HIPvQXO6lO/Xetyfd6UwtL+bhuZmHZ2ceHk8eXpYskAfzebpPhL/cn5lkd+CZwGVt1nfoTldsMfNu02J2B/bGD6L71LeY9c3nl4AvVtV3Zi6oqiur6iS601Afpbt+bHvbGiSGUfTvZmCfJP0jhQfRXeu1UDfTJbm+HV3XQv0+XT9/qqr2BF5I94kfuqNia5Kk177/XN4E/F5V7d0bHtmOvFBVH6qqn6XrW9Hd3DTT7XRHq/r9H0bf59rHNtOdJj+gN+9A5jbf/vUV4Mdm7AdPbvO1vJiHzcOjYh5eXB5eliyQB1BVdwNvAv53kuOS7JxkLV3S2kh32gi6i/SPT7JPksfQfXLs+yYw2/cK/laSRyb5CbqbCj68yPU9TDprkvw23U0cb5ilzS5JXpBkr6p6gO7i/Yd629o3yV6DbG+Gofevqm6iuxniLUl2S3IYcCrdJ/+F+gTwhCT/I8mqJL9Kd03WpTuwroXag+4o6N1J1tDdnb/F5+lO0b2ixXUScHhv+XuBlyV5ent9H5XkhCR7JHlikqOT7Ep37d2Wmzy20U57XgT8XnvcwcDp7MDzOMg+1rb3EeCstk88iW1Px8603X28qv6dbj/67bYf/BJwGPA3C41fS5t5eOu2zMPDZx5eRB5u2905yW50deWqtj+M9buyh80CeUBV9Xa6He0P6RLWFXSfHI+pqvtbs/OBL9FdoP9JfpiAtngL8MZ2GuY1vfn/THf372XAH7brpRazvr79k2y54/lKuhs0juptY6YXATemO830MuAFrf9fpbvJ44a2vYWcnhtV/06mu5nhZuBv6a6n+qcFxAVAVd1BdxfyGXR3Mb8WOLGqFnvKdhBvortD+m66r8f5SC+u79MdLTuV7hTZC+n+Wdzflq8HXkp3U9qddM/xi9vDdwXeSndk4la6I1GvnyOGV9IdRboB+BzwIeAvFtCHhe5jr6C7Y/tWun3ggi19msW5dNfv3ZXko3O0eT6wju45eCvw3HYdoZYZ87B5eETMw4vPw++l+wBwMt3XbX6XxV9DPVHZ9rIajVM7+vGfwM5z3HgibSPJFcB7quovJx3LsCR5G/CYqjpl0rFo5TEPa6HMwyuDR5ClJSzJM5I8pp3aO4Xu8oF/mHRci5HkSUkOa6cCD6c7MvO3k45LkmZjHl6Zlusv7kjLxRPprk17FN2pt+dW1S2TDWnR9qA7nbc/3bVt76D7/k9JWorMwyuQl1hIkiRJPV5iIUmSJPUsiUss9ttvv1q7du2kw5Cksbnqqqtur6rVk46jz1wsaaWZKxcviQJ57dq1rF+/ftJhSNLYJJn5q2ETZy6WtNLMlYu9xEKSJEnqsUCWJEmSeiyQJUmSpJ4lcQ2yNClrz/z42Ld541tPGPs2JWkpMxdrqfEIsiRJktRjgSxJkiT1WCBLkiRJPRbIkjQlkuyd5OIkX01yfZL/nmSfJJ9K8vX290da2yT54yQbklyb5GmTjl+SpoUFsiRNjz8C/qGqngQ8GbgeOBO4rKoOAS5r0wDPBA5pw2nAu8cfriRNJwtkSZoCSfYC/h/gXICq+n5V3QWcBJzXmp0HPLuNnwR8oDpfAPZO8tixBi1JU8oCWZKmw+OAzcBfJrk6yfuSPAp4dFXd0trcCjy6ja8Bbuo9fmObt40kpyVZn2T95s2bRxi+JE0PC2RJmg6rgKcB766qpwLf4YeXUwBQVQXUQlZaVedU1bqqWrd69eqhBStJ08wCWZKmw0ZgY1Vd0aYvpiuYv7nl0on297a2fBNwYO/xB7R5kqR5WCBL0hSoqluBm5I8sc06BrgOuAQ4pc07BfhYG78E+LX2bRZHAHf3LsWQJG2HPzUtSdPjlcBfJdkFuAF4Cd2BjouSnAp8A3hea/sJ4HhgA3BfaytJGoAFsiRNiaq6Blg3y6JjZmlbwMtHHZMkLUdeYiFJkiT1WCBLkiRJPRbIkiRJUo8FsiRJktRjgSxJkiT1WCBLkiRJPRbIkiRJUo8FsiRJktQzcIGcZKckVye5tE0/LskVSTYk+XD7ZSeS7NqmN7Tla0cUuyRJkjR0CzmC/Crg+t7024B3VdXjgTuBU9v8U4E72/x3tXaSJEnSVBioQE5yAHAC8L42HeBo4OLW5Dzg2W38pDZNW35May9JkiQteYMeQT4beC3wUJveF7irqh5s0xuBNW18DXATQFt+d2u/jSSnJVmfZP3mzZt3LHpJkiRpyOYtkJOcCNxWVVcNc8NVdU5VrauqdatXrx7mqiVJkqQdtmqANkcCz0pyPLAbsCfwR8DeSVa1o8QHAJta+03AgcDGJKuAvYA7hh65JEmSNALzHkGuqtdX1QFVtRZ4PvDpqnoBcDnw3NbsFOBjbfySNk1b/umqqqFGLUmSJI3IYr4H+XXA6Uk20F1jfG6bfy6wb5t/OnDm4kKUJEmSxmeQSyy2qqrPAJ9p4zcAh8/S5nvArwwhNkmSJGns/CU9SZIkqccCWZIkSeqxQJakKZFkpyRXJ7m0TT8uyRVJNiT5cJJd2vxd2/SGtnztRAOXpCljgSxJ0+NVwPW96bcB76qqxwN3Aqe2+acCd7b572rtJEkDskCWpCmQ5ADgBOB9bTrA0cDFrcl5wLPb+Eltmrb8mNZekjQAC2RJmg5nA68FHmrT+wJ3tR9rAtgIrGnja4CbANryu1t7SdIALJAlaYlLciJwW1VdNYJ1n5ZkfZL1mzdvHvbqJWkqWSBL0tJ3JPCsJDcCF9JdWvFHwN5Jtnyf/QHApja+CTgQoC3fC7hjthVX1TlVta6q1q1evXp0PZCkKWKBLElLXFW9vqoOqKq1wPOBT1fVC4DLgee2ZqcAH2vjl7Rp2vJPV1WNMWRJmmoWyJI0vV4HnJ5kA901xue2+ecC+7b5pwNnTig+SZpKC/qpaUnSZFXVZ4DPtPEbgMNnafM94FfGGpgkLSMeQZYkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6LJAlSZKkHgtkSZIkqWfeAjnJgUkuT3Jdkq8keVWbv0+STyX5evv7I21+kvxxkg1Jrk3ytFF3QpIkSRqWQY4gPwicUVWHAkcAL09yKHAmcFlVHQJc1qYBngkc0obTgHcPPWpJkiRpROYtkKvqlqr6Yhu/B7geWAOcBJzXmp0HPLuNnwR8oDpfAPZO8thhBy5JkiSNwoKuQU6yFngqcAXw6Kq6pS26FXh0G18D3NR72MY2b+a6TkuyPsn6zZs3LzRuSZIkaSQGLpCT7A78DfDqqvp2f1lVFVAL2XBVnVNV66pq3erVqxfyUEmSJGlkBiqQk+xMVxz/VVV9pM3+5pZLJ9rf29r8TcCBvYcf0OZJknaQN0xL0vgM8i0WAc4Frq+qd/YWXQKc0sZPAT7Wm/9rLTkfAdzduxRDkrRjvGFaksZk1QBtjgReBHw5yTVt3huAtwIXJTkV+AbwvLbsE8DxwAbgPuAlwwxYklaidqDhljZ+T5L+DdNHtWbnAZ8BXkfvhmngC0n2TvJYD1hI0vzmLZCr6nNA5lh8zCztC3j5IuOSJM1hkTdMWyBL0jz8JT1JmiLDvmHabxSSpIezQJakKTGKG6b9RiFJejgLZEmaAt4wLUnjM8hNepKkyfOGaUkaEwtkSZoC3jAtSePjJRaSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9IymQkxyX5GtJNiQ5cxTbkCRtn7lYknbM0AvkJDsBfwo8EzgUODnJocPejiRpbuZiSdpxoziCfDiwoapuqKrvAxcCJ41gO5KkuZmLJWkHrRrBOtcAN/WmNwJPn9koyWnAaW3y3iRfG0Eso7AfcPukgxgD+zkieds4t7aVr+fSc/CI128uXh7s54hMIBf7Wi5Ns+biURTIA6mqc4BzJrX9HZVkfVWtm3Qco2Y/lxf7qbmYi5c2+7l8rIQ+wvLp5ygusdgEHNibPqDNkySNj7lYknbQKArkK4FDkjwuyS7A84FLRrAdSdLczMWStIOGfolFVT2Y5BXAPwI7AX9RVV8Z9nYmaOpORe4g+7m82M8Vxly8bNjP5WMl9BGWST9TVZOOQZIkSVoy/CU9SZIkqccCWZIkSeqxQJ5Fkt2S/FuSLyX5SpI3zdHueUmua20+NO44F2uQfiY5KMnlSa5Ocm2S4ycR6zAk2an149JZlu2a5MPtJ3mvSLJ2AiEu2jx9PL3tr9cmuSzJqL+Hd2S2189em+ckqSRT/3VDK5W5eJs2yyIXr4Q8DObiGW2mMhdbIM/ufuDoqnoy8BTguCRH9BskOQR4PXBkVf0E8OpxBzkE8/YTeCNwUVU9le4u+D8bb4hD9Srg+jmWnQrcWVWPB94FTObnPBZve328GlhXVYcBFwNvH1tUw7e9fpJkj9bmirFFpFEwF//QcsnFKyEPg7kYmO5cbIE8i+rc2yZ3bsPMuxlfCvxpVd3ZHnPbGEMcigH7WcCebXwv4OYxhTdUSQ4ATgDeN0eTk4Dz2vjFwDFJMo7YhmW+PlbV5VV1X5v8At334k6dAV5LgN+l++f6vbEEpZEwF2/bjCnPxSshD4O5eIapzcUWyHNopw2uAW4DPlVVMz/9PAF4QpJ/TfKFJMeNPcghGKCfZwEvTLIR+ATwyvFGODRnA68FHppj+daf5a2qB4G7gX3HEtnwnM32+9h3KvD3I41mdM5mO/1M8jTgwKr6+DiD0miYi7c6i+nPxWez/PMwmIuB6c/FFshzqKofVNVT6D7ZHZ7kJ2c0WQUcAhwFnAy8N8ne44xxGAbo58nA+6vqAOB44PwkU7XfJDkRuK2qrpp0LKOykD4meSGwDviDkQc2ZPP1s+2b7wTOGGtgGhlz8VZTnYtXQh4Gc3Fv+dTn4ql5c01KVd0FXA7MPCqxEbikqh6oqv8E/p0uSU+l7fTzVOCi1ubzwG7AfmMNbvGOBJ6V5EbgQuDoJB+c0Wbrz/ImWUV3CvOOcQa5SIP0kSTHAv8LeFZV3T/eEIdivn7uAfwk8JnW5gjgkmm7OUQPZy6e+ly8EvIwmIu3mP5cXFUOMwZgNbB3G38E8FngxBltjgPOa+P70Z0W2nfSsY+gn38PvLiN/1e6694y6dgX0eejgEtnmf9y4D1t/Pl0N8NMPN4h9/GpwH8Ah0w6xlH2c0abz9DdDDPxeB126DU2F/+wzbLJxSshD8/TT3PxFAweQZ7dY4HLk1wLXEl3PdilSX4nybNam38E7khyHd2n/d+sqmn7pDtIP88AXprkS8AFdAl6Wfz84ox+ngvsm2QDcDpw5uQiG54ZffwDYHfgr5Nck+SSCYY2VDP6qeXDXLzMc/FKyMNgLp5G/tS0JEmS1OMRZEmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpB4LZEmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpB4LZEmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpB4LZJHk3iQ/NqR1vSHJ+9r42iSVZNWQ1n1Qi3WnYaxvAdt9dJJ/SXJPkneMc9tLQZI3J7k9ya2LWMfA+9gw90dpWpiH592uedg8PFYWyAuQ5MVJvpzkviS3Jnl3kr0X8Pgbkxw7xHi2u74kRyV5qO3o9ybZmOSiJD/db1dVu1fVDfNs66gkG+eLqap+v6p+Y/BebHeb2/Svqv5vi/UHw1j/ApwG3A7sWVVnzFyY5P1J3jzmmMYiyUHAGcChVfWYWZYPbR+bre18z22SH01yQZKbk9yd5F+TPH1hvdQ0MQ+bh83DD1s+0Tzc2lyeZHOSbyf5UpKTBu/h0mSBPKAkZwBvA34T2As4AjgY+FSSXSYZ2zxurqrdgT3oYv4q8Nkkxwx7Q8M6QrEEHQxcV1U16UAm4CDgjqq6bTttxraPzWJ34ErgvwH7AOcBH0+y+xi2rTEzD8/PPLwsLfU8DPAq4LFVtSfdh5kPJnnsmLY9GlXlMM8A7AncCzxvxvzdgc3Ar7fp9wNv7i0/CtjYxs8HHgK+29b1WmAtUHQ7083ALcBreo9f0PpmiXtr+xnz/wRY35su4PFt/HjgOuAeYBPwGuBRbTsPtW3dC+wPnAVcDHwQ+DbwG23eB9u6htq/3vpWtTb7A5cA3wI2AC/tress4CLgA60vXwHWbec1/hm6Quvu9vdnejE+AHy/xXHsLI/d2o9ejC8BbgLuBF4G/DRwLXAX8Ce9x/448GngDrqjI38F7N1b/jTg6taHvwY+POM5OxG4pq33/wCH9Za9rr2G9wBfA46Zo+97tedpM/AN4I10H56PnfG6v39I+9i+wN+1feZK4M3A52a2bftN/7n/uwHfr98G/tuk84bDcAfMw+Zh8/BU5GHgcOB7wOGTzhuLyjmTDmAaBuA44MEtCWHGsvOAC9r41jdom95mpwVu7L+xe2/iC+iS30+1N8exO7K+WWKb601zdHuzPapN9980twA/18Z/BHjaXOuiS34PAM9ub+RHMHtiHkr/eHhi/hfgz4DdgKe0dR/di+17dP9odgLeAnxhjudpH7oE+iJgFXBym953tjhnefzW5b0Y39Pi+oUWx0eBHwXWALcBz2jtHw/8PLArsLr16ey2bBe6RPkqYGfgl+mS1JZtPbWt6+mtj6e052xX4Il0/xj278X143PE/wHgY3RHHtYC/w6cur19aJH72IVteCRwaIvzYYl5kOd+lu0+pT3fe006bzgMd8A8POu6MA8zcznm4UH2saHnYeDS9jwX8A/Af5l03ljM4CUWg9kPuL2qHpxl2S1t+WK8qaq+U1VfBv6SLjGM0s1AgL1nWfYAcGiSPavqzqr64jzr+nxVfbSqHqqq787RZuj9S3IgcCTwuqr6XlVdA7wP+LVes89V1Sequ1bufODJc6zuBODrVXV+VT1YVRfQnZ76xUWE+Lstrk8C36H7531bVW0CPkuXVKmqDVX1qaq6v6o2A+8EntHWcQTdP4o/rqoHquojwL/1tnEa8OdVdUVV/aCqzgPub4/7AV2CPjTJzlV1Y1X9x8wg2402zwdeX1X3VNWNwDvo/kktxqz7WNvec4Dfrqr7quo6uuJm0ZLsSfc6v6mq7h7GOrWkmIfnZh6enXl4jHm4qk6kK/CPBz5ZVQ8tdp2TZIE8mNuB/ea4tuuxbfli3NQb/wbdKatRWkP3Ce+uWZY9h27n/kaSf07y3+dZ103zLJ/ZZlj92x/4VlXdM2Pda3rT/bt97wN2m+M13L89tm/muhbqm73x784yvTtsvTP7wiSbknyb7jTpln/0+wObqrqP5k3/uTwYOCPJXVsG4EC6oxUbgFfTHcG5rW1jtud9P7qjIv3+L7bvMPc+tprun02/H4PsQ9uV5BF0pwu/UFVvWez6tCSZh+dmHp6deXiMeRigfYj4e+AXkjxrGOucFAvkwXye7hPhL/dnthuBnglc1mZ9h+50xRYz7zYtZndgb/wguk99i1nffH4J+GJVfWfmgqq6sqpOojsN9VG668e2t61BYhhF/24G9kmyx4x1bxogntnWdfCMeTu6roX6fbp+/lR1Nze8kO4TP3RHxdYkSa99/7m8Cfi9qtq7NzyyHXmhqj5UVT9L17eiu7lpptvpjlb1+z+Mvs+1j22mO01+QG/egcxt3v0rya50++pG4P+zsDA1RczD5uFRMQ8vMg/PYhXdtd1TywJ5AO107ZuA/53kuCQ7J1lLl7Q20p02gu4i/eOT7JPkMXSfHPu+Ccz2vYK/leSRSX6C7qaCDy9yfQ+Tzpokv013E8cbZmmzS5IXJNmrqh6gu3h/yymSbwL7JtlrkO3NMPT+VdVNdDdDvCXJbkkOA06l++S/UJ8AnpDkfyRZleRX6a7JunQH1rVQe9Dd+HB3kjV0d+dv8Xm6U3SvaHGdRHfzwxbvBV6W5Ont9X1UkhOS7JHkiUmObsXj9/jhTR7baKc9LwJ+rz3uYOB0duB5HGQfa9v7CHBW2yeexLanY2fa7j6eZGe6G5S+C5wy7af0NDfz8NZtmYeHzzy8uDz8pCTPTPKI9r58IfD/AP+80PiXEgvkAVXV2+l2tD+kS1hX0H1yPKaq7m/Nzge+RHeB/if5YQLa4i3AG9tpmNf05v8z3d2/lwF/2K6XWsz6+vZPsuWO5yvpbtA4qreNmV4E3JjuNNPLgBe0/n+V7iaPG9r2FnJ6blT9O5nuZoabgb+lu57qnxYQFwBVdQfdXchn0N3F/FrgxKpa7CnbQbyJ7g7pu4GP0yWtLXF9n+5o2al0p8heSPfP4v62fD3wUro7le+ke45f3B6+K/BWuiMTt9IdiXr9HDG8ku4o0g3A54APAX+xgD4sdB97Bd0d27fS7QMXbOnTLM6lu37vriQfnWX5z9C9dr8A3JUffg/ozy0gfk0J87B5eETMw4vLw6FdRkJ3dPpVwK8OcO38kpZtL6vROLWjH/8J7DzHjSfSNpJcAbynqv5y0rEMS5K3AY+pqlMmHYtWHvOwFso8vDJ4BFlawpI8I8lj2qm9U4DD6L4+Z2q103GHtVOBh9MdmfnbScclSbMxD69My/UXd6Tl4ol016Y9iu7U23Or6pbJhrRoe9Cdztuf7tq2d9B9/6ckLUXm4RXISywkaUok2Zvue2Z/ku7O8l+n+3WuD9NdB3oj3S/N3ZkkwB/RfV3YfcCLp/2aQEkaFy+xkKTp8UfAP1TVk+h+cOF64Ezgsqo6hO4GrDNb22cCh7ThNODd4w9XkqbTkjiCvN9++9XatWsnHYYkjc1VV111e1WtHrR9+2qva4Af6/9oQZKv0d2tfkuSxwKfqaonJvnzNn7BzHZzbcNcLGmlmSsXD3QN8qhP661du5b169cP2hdJmnpJZv5q2HweR/cVSn+Z5MnAVXRfp/ToXtF7K/DoNr6GbX8da2ObN2eBbC6WtNLMlYsHvcTC03qSNFmr6L6r9d1V9VS670w9s9+gHVle0GnBJKclWZ9k/ebNm4cWrCRNs3kL5HZa7/+h+6Joqur7VXUXcBJwXmt2HvDsNn4S8IHqfAHYu532kyTtuI3Axqq6ok1fTFcwf3NLjm1/b2vLN7Htz8cewCw/XVtV51TVuqpat3r1wFd8SNKyNsgR5P5pvauTvC/Jo1j4ab1teNRCkgZXVbcCNyV5Ypt1DHAdcAmw5cv9T+GHX9V0CfBr7XtOjwDuXgZfTSVJYzHINchbTuu9sqquSPJHzHJaL8mCTutV1TnAOQDr1q2b/J2CWpHWnvnxsW/zxreeMPZtatl4JfBXSXah+z7Wl9Ad6LgoyanAN4DntbafoLsXZAPd/SAvGX+40mDMxVpqBimQZzutdybttF7vzukFndaTJC1MVV0DrJtl0TGztC3g5aOOSZKWo3kvsfC0niRJklaSQX9q2tN6kiRJWhEGKpA9rSdJkqSVwp+aliRJknoskCVJkqQeC2RJkiSpxwJZkiRJ6rFAliRJknoskCVJkqQeC2RJkiSpxwJZkiRJ6rFAliRJknoskCVJkqQeC2RJkiSpxwJZkiRJ6rFAliRJknoskCVJkqQeC2RJkiSpxwJZkiRJ6rFAliRJknoskCVJkqQeC2RJkiSpxwJZkiRJ6rFAliRJknoskCVpSiTZKcnVSS5t049LckWSDUk+nGSXNn/XNr2hLV870cAlacoMXCCbmCVp4l4FXN+bfhvwrqp6PHAncGqbfypwZ5v/rtZOkjSghRxBNjFL0oQkOQA4AXhfmw5wNHBxa3Ie8Ow2flKbpi0/prWXJA1goALZxCxJE3c28FrgoTa9L3BXVT3YpjcCa9r4GuAmgLb87tb+YZKclmR9kvWbN28eUeiSNF0GPYJ8NiNIzJKk+SU5Ebitqq4a9rqr6pyqWldV61avXj3s1UvSVJq3QB5VYvaohSQN7EjgWUluBC6kO4P3R8DeSVa1NgcAm9r4JuBAgLZ8L+COcQYsSdNskCPII0nMHrWQpMFU1eur6oCqWgs8H/h0Vb0AuBx4bmt2CvCxNn5Jm6Yt/3RV1RhDlqSpNm+BbGKWpCXrdcDpSTbQXcp2bpt/LrBvm386cOaE4pOkqbRq/iZzeh1wYZI3A1ezbWI+vyXmb9EV1ZKkIaiqzwCfaeM3AIfP0uZ7wK+MNTBJWkYWVCCbmCVJkrTc+Ut6kiRJUo8FsiRJktRjgSxJkiT1WCBLkiRJPRbIkiRJUo8FsiRJktRjgSxJkiT1WCBLkiRJPRbIkiRJUo8FsiRJktRjgSxJkiT1WCBLkiRJPRbIkiRJUo8FsiRJktRjgSxJkiT1WCBLkiRJPRbIkiRJUo8FsiRJktRjgSxJkiT1WCBLkiRJPRbIkiRJUo8FsiRJktRjgSxJUyDJgUkuT3Jdkq8keVWbv0+STyX5evv7I21+kvxxkg1Jrk3ytMn2QJKmx7wFsklZkpaEB4EzqupQ4Ajg5UkOBc4ELquqQ4DL2jTAM4FD2nAa8O7xhyxJ02mQI8gmZUmasKq6paq+2MbvAa4H1gAnAee1ZucBz27jJwEfqM4XgL2TPHa8UUvSdJq3QDYpS9LSkmQt8FTgCuDRVXVLW3Qr8Og2vga4qfewjW3ezHWdlmR9kvWbN28eXdCSNEUWdA2ySVmSJivJ7sDfAK+uqm/3l1VVAbWQ9VXVOVW1rqrWrV69eoiRStL0GrhANilL0mQl2ZkuD/9VVX2kzf7mlrN07e9tbf4m4MDeww9o8yRJ8xioQDYpS9JkJQlwLnB9Vb2zt+gS4JQ2fgrwsd78X2s3Th8B3N076ydJ2o5BvsXCpCxJk3ck8CLg6CTXtOF44K3Azyf5OnBsmwb4BHADsAF4L/A/JxCzJE2lVQO02ZKUv5zkmjbvDXRJ+KIkpwLfAJ7Xln0COJ4uKd8HvGSYAUvSSlRVnwMyx+JjZmlfwMtHGpQkLVPzFsgmZUmSJK0k/pKeJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPWMpEBOclySryXZkOTMUWxDkrR95mJJ2jFDL5CT7AT8KfBM4FDg5CSHDns7kqS5mYslaceN4gjy4cCGqrqhqr4PXAicNILtSJLmZi6WpB20agTrXAPc1JveCDx9ZqMkpwGntcl7k3xtBLGMwn7A7ZMOYgzs54jkbePc2la+nkvPwSNev7l4ebCfIzKBXOxruTTNmotHUSAPpKrOAc6Z1PZ3VJL1VbVu0nGMmv1cXuyn5mIuXtrs5/KxEvoIy6efo7jEYhNwYG/6gDZPkjQ+5mJJ2kGjKJCvBA5J8rgkuwDPBy4ZwXYkSXMzF0vSDhr6JRZV9WCSVwD/COwE/EVVfWXY25mgqTsVuYPs5/JiP1cYc/GyYT+Xj5XQR1gm/UxVTToGSZIkacnwl/QkSZKkHgtkSZIkqccCeRZJdkvyb0m+lOQrSd40R7vnJbmutfnQuONcrEH6meSgJJcnuTrJtUmOn0Ssw5Bkp9aPS2dZtmuSD7ef5L0iydoJhLho8/Tx9La/XpvksiSj/h7ekdleP3ttnpOkkkz91w2tVObibdosi1y8EvIwmItntJnKXGyBPLv7gaOr6snAU4DjkhzRb5DkEOD1wJFV9RPAq8cd5BDM20/gjcBFVfVUurvg/2y8IQ7Vq4Dr51h2KnBnVT0eeBcwmZ/zWLzt9fFqYF1VHQZcDLx9bFEN3/b6SZI9WpsrxhaRRsFc/EPLJRevhDwM5mJgunOxBfIsqnNvm9y5DTPvZnwp8KdVdWd7zG1jDHEoBuxnAXu28b2Am8cU3lAlOQA4AXjfHE1OAs5r4xcDxyTJOGIblvn6WFWXV9V9bfILdN+LO3UGeC0Bfpfun+v3xhKURsJcvG0zpjwXr4Q8DObiGaY2F1sgz6GdNrgGuA34VFXN/PTzBOAJSf41yReSHDf2IIdggH6eBbwwyUbgE8Arxxvh0JwNvBZ4aI7lW3+Wt6oeBO4G9h1LZMNzNtvvY9+pwN+PNJrROZvt9DPJ04ADq+rj4wxKo2Eu3uospj8Xn83yz8NgLgamPxdbIM+hqn5QVU+h+2R3eJKfnNFkFXAIcBRwMvDeJHuPM8ZhGKCfJwPvr6oDgOOB85NM1X6T5ETgtqq6atKxjMpC+pjkhcA64A9GHtiQzdfPtm++EzhjrIFpZMzFW011Ll4JeRjMxb3lU5+Lp+bNNSlVdRdwOTDzqMRG4JKqeqCq/hP4d7okPZW2089TgYtam88DuwH7jTW4xTsSeFaSG4ELgaOTfHBGm60/y5tkFd0pzDvGGeQiDdJHkhwL/C/gWVV1/3hDHIr5+rkH8JPAZ1qbI4BLpu3mED2cuXjqc/FKyMNgLt5i+nNxVTnMGIDVwN5t/BHAZ4ETZ7Q5Djivje9Hd1po30nHPoJ+/j3w4jb+X+mue8ukY19En48CLp1l/suB97Tx59PdDDPxeIfcx6cC/wEcMukYR9nPGW0+Q3czzMTjddih19hc/MM2yyYXr4Q8PE8/zcVTMHgEeXaPBS5Pci1wJd31YJcm+Z0kz2pt/hG4I8l1dJ/2f7Oqpu2T7iD9PAN4aZIvARfQJehl8fOLM/p5LrBvkg3A6cCZk4tseGb08Q+A3YG/TnJNkksmGNpQzeinlg9z8TLPxSshD4O5eBr5U9OSJElSj0eQJUmSpB4LZEmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpB4LZEmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpB4LZEmSJKnHAlmSJEnqsUCWJEmSeiyQRZJ7k/zYkNb1hiTva+Nrk1SSVUNa90Et1p2Gsb4FbPfRSf4lyT1J3jHObS8FSd6c5PYkty5iHQPvY8PcH6VpYR6ed7vmYfPwWFkgL0CSFyf5cpL7ktya5N1J9l7A429McuwQ49nu+pIcleShtqPfm2RjkouS/HS/XVXtXlU3zLOto5JsnC+mqvr9qvqNwXux3W1u07+q+r8t1h8MY/0LcBpwO7BnVZ0xc2GS9yd585hjGoskBwFnAIdW1WNmWT60fWy2tgt5bpM8oxUCy/K1UMc8bB42Dz9s+cTzcNtPvtuL4ZOD93BpskAeUJIzgLcBvwnsBRwBHAx8Kskuk4xtHjdX1e7AHnQxfxX4bJJjhr2hYR2hWIIOBq6rqpp0IBNwEHBHVd22nTZj28fmkmRn4I+AK8a1TY2feXh+5uFlaSryMPCLrbDevap+YYzbHY2qcphnAPYE7gWeN2P+7sBm4Nfb9PuBN/eWHwVsbOPnAw8B323rei2wFii6T8Y3A7cAr+k9fkHrmyXure1nzP8TYH1vuoDHt/HjgeuAe4BNwGuAR7XtPNS2dS+wP3AWcDHwQeDbwG+0eR9s6xpq/3rrW9Xa7A9cAnwL2AC8tLeus4CLgA+0vnwFWLed1/hngCuBu9vfn+nF+ADw/RbHsbM8dms/ejG+BLgJuBN4GfDTwLXAXcCf9B7748CngTvojo78FbB3b/nTgKtbH/4a+PCM5+xE4Jq23v8DHNZb9rr2Gt4DfA04Zo6+79Wep83AN4A30n14PnbG6/7+Ie1j+wJ/1/aZK4E3A5+b2bbtN/3n/u+28/qdCbydGfuUw/IZMA+bh83DSzYPAzfO9rpM8zDxAKZhAI4DHtySEGYsOw+4oI1vfYO26W122pk7UO9NfAFd8vup9uY4dkfWN0tsc71pjm5vtke16f6b5hbg59r4jwBPm2tddMnvAeDZ7Y38CGZPzEPpHw9PzP8C/BmwG/CUtu6je7F9j+4fzU7AW4AvzPE87UOXQF8ErAJObtP7zhbnLI/furwX43taXL/Q4vgo8KPAGuA24Bmt/eOBnwd2BVa3Pp3dlu1ClyhfBewM/DJdktqyrae2dT299fGU9pztCjyR7h/D/r24fnyO+D8AfIzuyMNa4N+BU7e3Dy1yH7uwDY8EDm1xPiwxD/LctzYHt5h3H6S9w3QOmIdnXRfmYWYuxzw8yD427Dx8I/DN9vp/EnjypHPGYgcvsRjMfsDtVfXgLMtuacsX401V9Z2q+jLwl3SJYZRuBgLsPcuyB4BDk+xZVXdW1RfnWdfnq+qjVfVQVX13jjZD71+SA4EjgddV1feq6hrgfcCv9Zp9rqo+Ud21cucDT55jdScAX6+q86vqwaq6gO701C8uIsTfbXF9EvgO3T/v26pqE/BZuqRKVW2oqk9V1f1VtRl4J/CMto4j6P5R/HFVPVBVHwH+rbeN04A/r6orquoHVXUecH973A/oEvShSXauqhur6j9mBtlutHk+8PqquqeqbgTeQfdPajFm3cfa9p4D/HZV3VdV19EVN4vxx8BvVdW9i1yPljbz8NzMw7MzD48vD7+ArrA/GLgc+MeF3BuwFFkgD+Z2YL85ru16bFu+GDf1xr9Bd8pqlNbQfTq8a5Zlz6H7tP+NJP+c5L/Ps66b5lk+s82w+rc/8K2qumfGutf0pvt3+94H7DbHa7h/e2zfzHUt1Dd749+dZXp32Hpn9oVJNiX5Nt1p0i3/6PcHNlV1H8+b/nN5MHBGkru2DMCBdEcrNgCvpjuCc1vbxmzP+350R0X6/V9s32HufWw13T+bfj8G2YdmleQXgT2q6sM7ug5NDfPw3MzDszMPjyEPA1TVv1bVd1vB/Za2zZ9bzDonzQJ5MJ+n+0T4y/2ZSXYHnglc1mZ9h+50xRYz7zYtZndgb/wguk99i1nffH4J+GJVfWfmgqq6sqpOojsN9VG668e2t61BYhhF/24G9kmyx4x1bxogntnWdfCMeTu6roX6fbp+/lRV7Qm8kO4TP3RHxdYkSa99/7m8Cfi9qtq7NzyyHXmhqj5UVT9L17eiu7lpptvpjlb1+z+Mvs+1j22mO01+QG/egcxtvv3rGGBd+zaDW4FfBV6d5GMLDVhLnnnYPDwq5uHF5eG5HpN5Wy1hFsgDqKq7gTcB/zvJcUl2TrKWLmltpDttBN1F+scn2SfJY+g+OfZ9E5jtewV/K8kjk/wE3U0FW46G7ej6HiadNUl+m+4mjjfM0maXJC9IsldVPUB38f5DvW3tm2SvQbY3w9D7V1U30d0M8ZYkuyU5DDiV7pP/Qn0CeEKS/5FkVZJfpbsm69IdWNdC7UF348PdSdbQ3Z2/xefpTtG9osV1EnB4b/l7gZcleXp7fR+V5IQkeyR5YpKjk+xKd+3dlps8ttFOe14E/F573MHA6ezA8zjIPta29xHgrLZPPIltT8fONN8+/lvAE+iufXwK3c1C76Xbz7SMmIe3bss8PHzm4UXk4XTfjX1k23d3S/KbdEfF/3Wh8S8lFsgDqqq30+1of0iXsK6g++R4TFXd35qdD3yJ7mL1T/LDBLTFW4A3ttMwr+nN/2e6u38vA/6wXS+1mPX17Z9kyx3PV9LdoHFUbxszvQi4Md1pppfRXVdEVX2V7iaPG9r2FnJ6blT9O5numqebgb+lu57qnxYQFwBVdQfdXchn0N3F/FrgxKpa7CnbQbyJ7g7pu4GP0yWtLXF9n+5o2al0p6teSPfP4v62fD3wUro7le+ke45f3B6+K/BWuiMTt9IdiXr9HDG8ku4o0g3A54APAX+xgD4sdB97Bd0d27fS7QMXbOnTLM6lu37vriQfnbmwXa9365aB7h/Qd6rqWwuIX1PCPGweHhHz8CLyMN0HjHfT9X8T3Q21z2yv6dTKtpfVaJza0Y//BHae48YTaRtJrgDeU1V/OelYhiXJ24DHVNUpk45FK495WAtlHl4ZPIIsLWHpfh3uMe3U3inAYcA/TDquxUjypCSHtVOBh9MdmfnbScclSbMxD69My/UXd6Tl4ol016Y9iu7U23Or6pbJhrRoe9Cdztuf7tq2d9B9/6ckLUXm4RXISywkSZKkHi+xkCRJknqWxCUW++23X61du3bSYUjS2Fx11VW3V9XqScfRZy6WtNLMlYuXRIG8du1a1q9fP+kwJGlsksz81bCJMxdLWmnmysVeYiFJkiT1WCBLkiRJPRbIkiRJUs+SuAZZmpS1Z3587Nu88a0njH2bkrSUmYu11HgEWZIkSeqxQJYkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJakKZFkpyRXJ7m0TT8uyRVJNiT5cJJd2vxd2/SGtnztRAOXpCljgSxJ0+NVwPW96bcB76qqxwN3Aqe2+acCd7b572rtJEkDskCWpCmQ5ADgBOB9bTrA0cDFrcl5wLPb+Eltmrb8mNZekjQAC2RJmg5nA68FHmrT+wJ3VdWDbXojsKaNrwFuAmjL727tJUkDsECWpCUuyYnAbVV11QjWfVqS9UnWb968edirl6SpZIEsSUvfkcCzktwIXEh3acUfAXsnWdXaHABsauObgAMB2vK9gDtmW3FVnVNV66pq3erVq0fXA0maIhbIkrTEVdXrq+qAqloLPB/4dFW9ALgceG5rdgrwsTZ+SZumLf90VdUYQ5akqWaBLEnT63XA6Uk20F1jfG6bfy6wb5t/OnDmhOKTpKm0av4mnSQ7AeuBTVV1YpLH0Z3q2xe4CnhRVX0/ya7AB4D/RndK71er6sahRy5JK1BVfQb4TBu/ATh8ljbfA35lrIFJ0jKykCPIfv+mJEmSlr2BCmS/f1OSJEkrxaBHkM9myN+/6VcLSZIkaSmat0Ae1fdv+tVCkiRJWooGuUlvy/dvHg/sBuxJ7/s321Hi2b5/c+N8378pSZIkLTXzHkH2+zclSZK0kizme5D9/k1JkiQtOwN/DzL4/ZuSJEla/vwlPUmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpB4LZEmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpB4LZEmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpB4LZEmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpB4LZEmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpB4LZEmSJKnHAlmSpkCSA5NcnuS6JF9J8qo2f58kn0ry9fb3R9r8JPnjJBuSXJvkaZPtgSRNDwtkSZoODwJnVNWhwBHAy5McCpwJXFZVhwCXtWmAZwKHtOE04N3jD1mSppMFsiRNgaq6paq+2MbvAa4H1gAnAee1ZucBz27jJwEfqM4XgL2TPHa8UUvSdLJAlqQpk2Qt8FTgCuDRVXVLW3Qr8Og2vga4qfewjW3ezHWdlmR9kvWbN28eXdCSNEXmLZC97k2Slo4kuwN/A7y6qr7dX1ZVBdRC1ldV51TVuqpat3r16iFGKknTa5AjyF73JklLQJKd6Yrjv6qqj7TZ39xy6UT7e1ubvwk4sPfwA9o8SdI85i2Qve5NkiYvSYBzgeur6p29RZcAp7TxU4CP9eb/WjurdwRwd+9SDEnSdqxaSONFXve2TWJOchrdEWYOOuighcYtSSvNkcCLgC8nuabNewPwVuCiJKcC3wCe15Z9Ajge2ADcB7xkrNFK0hQbuECeed1bdzCjU1WVZMHXvQHnAKxbt25Bj5WklaaqPgdkjsXHzNK+gJePNChJWqYG+hYLr3uTJEnSSjHIt1h43ZskSZJWjEEusfC6N0mSJK0Y8xbIXvcmSZKklcRf0pMkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6RlIgJzkuydeSbEhy5ii2IUnaPnOxJO2YoRfISXYC/hR4JnAocHKSQ4e9HUnS3MzFkrTjRnEE+XBgQ1XdUFXfBy4EThrBdiRJczMXS9IOWjWCda4BbupNbwSePrNRktOA09rkvUm+NoJYRmE/4PZJBzEG9nNE8rZxbm0rX8+l5+ARr99cvDzYzxGZQC72tVyaZs3FoyiQB1JV5wDnTGr7OyrJ+qpaN+k4Rs1+Li/2U3MxFy9t9nP5WAl9hOXTz1FcYrEJOLA3fUCbJ0kaH3OxJO2gURTIVwKHJHlckl2A5wOXjGA7kqS5mYslaQcN/RKLqnowySuAfwR2Av6iqr4y7O1M0NSditxB9nN5sZ8rjLl42bCfy8dK6CMsk36mqiYdgyRJkrRk+Et6kiRJUo8FsiRJktRjgTyLJLsl+bckX0rylSRvmqPd85Jc19p8aNxxLtYg/UxyUJLLk1yd5Nokx08i1mFIslPrx6WzLNs1yYfbT/JekWTtBEJctHn6eHrbX69NclmSUX8P78hsr5+9Ns9JUkmm/uuGVipz8TZtlkUuXgl5GMzFM9pMZS62QJ7d/cDRVfVk4CnAcUmO6DdIcgjweuDIqvoJ4NXjDnII5u0n8Ebgoqp6Kt1d8H823hCH6lXA9XMsOxW4s6oeD7wLmMzPeSze9vp4NbCuqg4DLgbePraohm97/STJHq3NFWOLSKNgLv6h5ZKLV0IeBnMxMN252AJ5FtW5t03u3IaZdzO+FPjTqrqzPea2MYY4FAP2s4A92/hewM1jCm+okhwAnAC8b44mJwHntfGLgWOSZByxDct8fayqy6vqvjb5BbrvxZ06A7yWAL9L98/1e2MJSiNhLt62GVOei1dCHgZz8QxTm4stkOfQThtcA9wGfKqqZn76eQLwhCT/muQLSY4be5BDMEA/zwJemGQj8AngleONcGjOBl4LPDTH8q0/y1tVDwJ3A/uOJbLhOZvt97HvVODvRxrN6JzNdvqZ5GnAgVX18XEGpdEwF291FtOfi89m+edhMBcD05+LLZDnUFU/qKqn0H2yOzzJT85osgo4BDgKOBl4b5K9xxnjMAzQz5OB91fVAcDxwPlJpmq/SXIicFtVXTXpWEZlIX1M8kJgHfAHIw9syObrZ9s33wmcMdbANDLm4q2mOhevhDwM5uLe8qnPxVPz5pqUqroLuByYeVRiI3BJVT1QVf8J/Dtdkp5K2+nnqcBFrc3ngd2A/cYa3OIdCTwryY3AhcDRST44o83Wn+VNsoruFOYd4wxykQbpI0mOBf4X8Kyqun+8IQ7FfP3cA/hJ4DOtzRHAJdN2c4gezlw89bl4JeRhMBdvMf25uKocZgzAamDvNv4I4LPAiTPaHAec18b3ozsttO+kYx9BP/8eeHEb/690171l0rEvos9HAZfOMv/lwHva+PPpboaZeLxD7uNTgf8ADpl0jKPs54w2n6G7GWbi8Trs0GtsLv5hm2WTi1dCHp6nn+biKRg8gjy7xwKXJ7kWuJLuerBLk/xOkme1Nv8I3JHkOrpP+79ZVdP2SXeQfp4BvDTJl4AL6BL0svj5xRn9PBfYN8kG4HTgzMlFNjwz+vgHwO7AXye5JsklEwxtqGb0U8uHuXiZ5+KVkIfBXDyN/KlpSZIkqccjyJIkSVKPBbIkSZLUY4EsSZIk9VggS5IkST0WyJIkSVKPBbIkSZLUY4EsSZIk9VggS5IkST0WyJIkSVKPBbIkSZLUY4EsSZIk9VggS5IkST0WyCLJvUl+bEjrekOS97XxtUkqyaohrfugFutOw1jfArb76CT/kuSeJO8Y57aXgiRvTnJ7klsXsY6B97Fh7o/StDAPz7td87B5eKwskBcgyYuTfDnJfUluTfLuJHsv4PE3Jjl2iPFsd31JjkryUNvR702yMclFSX66366qdq+qG+bZ1lFJNs4XU1X9flX9xuC92O42t+lfVf3fFusPhrH+BTgNuB3Ys6rOmLkwyfuTvHnMMY1FkoOAM4BDq+oxsywf2j42W9tBn9skr0ryn0m+k+T6JE8YrIeaNuZh87B5+GHLJ5qHex+a+kMledjrNE0skAfUXui3Ab8J7AUcARwMfCrJLpOMbR43V9XuwB50MX8V+GySY4a9oWEdoViCDgauq6qadCATcBBwR1Xdtp02Y9vHZpPkN4BTgROA3YET6f6RapkxD8/PPLwsLek83PvQtHuL4aeAh4C/GfW2R6qqHOYZgD2Be4HnzZi/O7AZ+PU2/X7gzb3lRwEb2/j5dDvMd9u6XgusBYruk/HNwC3Aa3qPX9D6Zol7a/sZ8/8EWN+bLuDxbfx44DrgHmAT8BrgUW07D7Vt3QvsD5wFXAx8EPg28Btt3gfbuobav976VrU2+wOXAN8CNgAv7a3rLOAi4AOtL18B1m3nNf4Z4Erg7vb3Z3oxPgB8v8Vx7CyP3dqPXowvAW4C7gReBvw0cC1wF/Anvcf+OPBp4A66ou6vgL17y58GXN368NfAh2c8ZycC17T1/h/gsN6y17XX8B7ga8Axc/R9r/Y8bQa+AbyR7sPzsTNe9/cPaR/bF/i7ts9cCbwZ+NzMtm2/6T/3fzfLdv5Le55n7ZvD8hkwD5uHzcNLMg/Pst3fBi6fdM5YdM6ZdADTMADHAQ9uSQgzlp0HXNDGt75B2/Q2Oy1wY/+N3XsTX0CX/H6qvTmO3ZH1zRLbXG+ao9ub7VFtuv+muQX4uTb+I8DT5loXXfJ7AHh2eyM/gtkT81D6x8MT878AfwbsBjylrfvoXmzfo/tHsxPwFuALczxP+9Al0BcBq4CT2/S+s8U5y+O3Lu/F+J4W1y+0OD4K/CiwBrgNeEZr/3jg54FdgdWtT2e3ZbvQJcpXATsDv0yXpLZs66ltXU9vfTylPWe7Ak+k+8ewfy+uH58j/g8AH6M78rAW+Hfg1O3tQ4vcxy5swyOBQ1ucD0vMAz73B7X2r2rr+U/gTcB/mXTecBjugHl41nVhHmbmcszDg+xjQ8vDM7YZ4D+AF086Zyx28BKLwewH3F5VD86y7Ja2fDHeVFXfqaovA39JlxhG6Wa6nXjvWZY9AByaZM+qurOqvjjPuj5fVR+tqoeq6rtztBl6/5IcCBwJvK6qvldV1wDvA36t1+xzVfWJ6q6VOx948hyrOwH4elWdX1UPVtUFdKenfnERIf5ui+uTwHfo/nnfVlWbgM/SJVWqakNVfaqq7q+qzcA7gWe0dRxB94/ij6vqgar6CPBvvW2cBvx5VV1RVT+oqvOA+9vjfkCXoA9NsnNV3VhV/zEzyHajzfOB11fVPVV1I/AOun9SizHrPta29xzgt6vqvqq6jq642VEHtL+/QPeP//9Ft3+duoh1amkyD8/NPDw78/B48nDfzwKPpjurMdUskAdzO7DfHNd2PZbFX+94U2/8G3SnrEZpDd2nw7tmWfYcuk/730jyz0n++zzrumme5TPbDKt/+wPfqqp7Zqx7TW+6f7fvfcBuc7yG+7fH9s1c10J9szf+3Vmmd4etd2ZfmGRTkm/TnSbd8o9+f2BTtY/lTf+5PBg4I8ldWwbgQLqjFRuAV9MdwbmtbWO2530/uqMi/f4vtu8w9z62mu6fTb8fg+xDc9lSDLy9qu5q/1j+nG4f1vJiHp6beXh25uHx5OG+U4C/qap7h7S+ibFAHszn6T4R/nJ/ZpLdgWcCl7VZ36E7XbHFzLtNi9kd2Bs/iO5T32LWN59fAr5YVd+ZuaCqrqyqk+hOQ32U7vqx7W1rkBhG0b+bgX2S7DFj3ZsGiGe2dR08Y96Ormuhfp+unz9VVXsCL6T7xA/dUbE1SdJr338ubwJ+r6r27g2PbEdeqKoPVdXP0vWt6G5umul2uqNV/f4Po+9z7WOb6U6TH9CbdyBzm2//+hrd6c5+ux19X2hpMw+bh0fFPLy4PAxAkkcAv8LwjkZPlAXyAKrqbrrrGv93kuOS7JxkLV3S2kh32gi6i/SPT7JPksfQfXLs+yYw2/cK/laSRyb5CbqbCj68yPU9TDprkvw23U0cb5ilzS5JXpBkr6p6gO7i/Yd629o3yV6DbG+Gofevqm6iuxniLUl2S3IY3Wn1D+5AfJ8AnpDkfyRZleRX6a7JunQH1rVQe9Dd+HB3kjV0d+dv8Xm6U3SvaHGdBBzeW/5e4GVJnt5e30clOSHJHkmemOToJLvSXXu35SaPbbTTnhcBv9cedzBwOjvwPA6yj7XtfQQ4q+0TT2Lb07EzbXcfr6r76Pan17b4D6A75TmO105jZB7eui3z8PCZhxeRh3t+ie668csXGvdSZIE8oKp6O92O9od0CesKfnj3/P2t2fnAl+gu0P8kP0xAW7wFeGM7DfOa3vx/prv79zLgD9v1UotZX9/+Sbbc8Xwl3XWaR/W2MdOLgBvbaaaXAS9o/f8q3U0eN7TtLeT03Kj6dzLdzQw3A39Ldz3VPy0gLgCq6g66u5DPoLuL+bXAiVU1jq8KexPdHdJ3Ax+nS1pb4vo+3dGyU+lOkb2Q7p/F/W35euCldHcq30n3HL+4PXxX4K10RyZupTsS9fo5Yngl3VGkG4DPAR8C/mIBfVjoPvYKuju2b6XbBy7Y0qdZnEt3/d5dST66nfXdS7cffH4H4teUMA+bh0fEPLz4PAzd5RXnz7gcZWplmfRjKrWjH/8J7DzHjSfSNpJcAbynqv5y0rEMS5K3AY+pqlMmHYtWHvOwFso8vDJ4BFlawpI8I8lj2qm9U4DDgH+YdFyLkeRJSQ5rpwIPpzsy87eTjkuSZmMeXpmW6y/uSMvFE+muTXsU3am351bVLZMNadH2oDudtz/dtW3voPv+T0laiszDK5CXWEiSJEk9XmIhSZIk9SyJSyz222+/Wrt27aTDkKSxueqqq26vqtWTjqPPXCxppZkrFy+JAnnt2rWsX79+0mFI0tgkmfmrYRNnLpa00syVi73EQpIkSeqxQJYkSZJ6LJAlSZKkniVxDbI0KWvP/PjYt3njW08Y+zYlaSkzF2up8QiyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1DNwgZxkpyRXJ7m0TT8uyRVJNiT5cJJd2vxd2/SGtnztiGKXJEmShm4hR5BfBVzfm34b8K6qejxwJ3Bqm38qcGeb/67WTpIkSZoKAxXISQ4ATgDe16YDHA1c3JqcBzy7jZ/UpmnLj2ntJUmL4Jk8SRqPQY8gnw28FnioTe8L3FVVD7bpjcCaNr4GuAmgLb+7td9GktOSrE+yfvPmzTsWvSStLJ7Jk6QxmLdATnIicFtVXTXMDVfVOVW1rqrWrV69epirlqRlxzN5kjQ+qwZocyTwrCTHA7sBewJ/BOydZFU7SnwAsKm13wQcCGxMsgrYC7hj6JFL0spyNt2ZvD3a9MBn8pJsOZN3+8yVJjkNOA3goIMOGlXskjRV5j2CXFWvr6oDqmot8Hzg01X1AuBy4Lmt2SnAx9r4JW2atvzTVVVDjVqSVpBRnckDz+ZJ0mwGOYI8l9cBFyZ5M3A1cG6bfy5wfpINwLfoimpJ0o7zTJ4kjdGCCuSq+gzwmTZ+A3D4LG2+B/zKEGKTJNGdyQNeD5DkKOA1VfWCJH9Nd6buQmY/k/d5PJMnSQvmL+lJ0vR6HXB6O2O3L9ueydu3zT8dOHNC8UnSVFrMJRaSpDHzTJ4kjZ5HkCVJkqQeC2RJkiSpxwJZkiRJ6rFAliRJknoskCVJkqQeC2RJkiSpxwJZkiRJ6rFAliRJknoskCVJkqQeC2RJkiSpxwJZkiRJ6rFAliRJknoskCVJkqQeC2RJkiSpxwJZkiRJ6rFAliRJknoskCVJkqQeC2RJkiSpxwJZkiRJ6rFAliRJknoskCVJkqQeC2RJkiSpxwJZkiRJ6pm3QE5yYJLLk1yX5CtJXtXm75PkU0m+3v7+SJufJH+cZEOSa5M8bdSdkCRJkoZlkCPIDwJnVNWhwBHAy5McCpwJXFZVhwCXtWmAZwKHtOE04N1Dj1qSVhgPVkjS+MxbIFfVLVX1xTZ+D3A9sAY4CTivNTsPeHYbPwn4QHW+AOyd5LHDDlySVhgPVkjSmCzoGuQka4GnAlcAj66qW9qiW4FHt/E1wE29h21s82au67Qk65Os37x580LjlqQVxYMVkjQ+AxfISXYH/gZ4dVV9u7+sqgqohWy4qs6pqnVVtW716tULeagkrWgerJCk0RqoQE6yM11x/FdV9ZE2+5tbjka0v7e1+ZuAA3sPP6DNkyQtkgcrJGn0BvkWiwDnAtdX1Tt7iy4BTmnjpwAf683/tXaDyBHA3b2jG5KkHeTBCkkaj0GOIB8JvAg4Osk1bTgeeCvw80m+DhzbpgE+AdwAbADeC/zP4YctSSuLByskaXxWzdegqj4HZI7Fx8zSvoCXLzIuSdK2thys+HKSa9q8N9AdnLgoyanAN4DntWWfAI6nO1hxH/CSsUYrSVNs3gJZkjR5HqyQpPHxp6YlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6LJAlSZKkHgtkSZIkqccCWZIkSeqxQJYkSZJ6RlIgJzkuydeSbEhy5ii2IUnaPnOxJO2YoRfISXYC/hR4JnAocHKSQ4e9HUnS3MzFkrTjRnEE+XBgQ1XdUFXfBy4EThrBdiRJczMXS9IOWjWCda4BbupNbwSePrNRktOA09rkvUm+NoJYRmE/4PZJBzEG9nNE8rZxbm0rX8+l5+ARr99cvDzYzxGZQC72tVyaZs3FoyiQB1JV5wDnTGr7OyrJ+qpaN+k4Rs1+Li/2U3MxFy9t9nP5WAl9hOXTz1FcYrEJOLA3fUCbJ0kaH3OxJO2gURTIVwKHJHlckl2A5wOXjGA7kqS5mYslaQcN/RKLqnowySuAfwR2Av6iqr4y7O1M0NSditxB9nN5sZ8rjLl42bCfy8dK6CMsk36mqiYdgyRJkrRk+Et6kiRJUo8FsiRJktRjgTyLJLsl+bckX0rylSRvmqPd85Jc19p8aNxxLtYg/UxyUJLLk1yd5Nokx08i1mFIslPrx6WzLNs1yYfbT/JekWTtBEJctHn6eHrbX69NclmSUX8P78hsr5+9Ns9JUkmm/uuGVipz8TZtlkUuXgl5GMzFM9pMZS62QJ7d/cDRVfVk4CnAcUmO6DdIcgjweuDIqvoJ4NXjDnII5u0n8Ebgoqp6Kt1d8H823hCH6lXA9XMsOxW4s6oeD7wLmMzPeSze9vp4NbCuqg4DLgbePraohm97/STJHq3NFWOLSKNgLv6h5ZKLV0IeBnMxMN252AJ5FtW5t03u3IaZdzO+FPjTqrqzPea2MYY4FAP2s4A92/hewM1jCm+okhwAnAC8b44mJwHntfGLgWOSZByxDct8fayqy6vqvjb5BbrvxZ06A7yWAL9L98/1e2MJSiNhLt62GVOei1dCHgZz8QxTm4stkOfQThtcA9wGfKqqZn76eQLwhCT/muQLSY4be5BDMEA/zwJemGQj8AngleONcGjOBl4LPDTH8q0/y1tVDwJ3A/uOJbLhOZvt97HvVODvRxrN6JzNdvqZ5GnAgVX18XEGpdEwF291FtOfi89m+edhMBcD05+LLZDnUFU/qKqn0H2yOzzJT85osgo4BDgKOBl4b5K9xxnjMAzQz5OB91fVAcDxwPlJpmq/SXIicFtVXTXpWEZlIX1M8kJgHfAHIw9syObrZ9s33wmcMdbANDLm4q2mOhevhDwM5uLe8qnPxVPz5pqUqroLuByYeVRiI3BJVT1QVf8J/Dtdkp5K2+nnqcBFrc3ngd2A/cYa3OIdCTwryY3AhcDRST44o83Wn+VNsoruFOYd4wxykQbpI0mOBf4X8Kyqun+8IQ7FfP3cA/hJ4DOtzRHAJdN2c4gezlw89bl4JeRhMBdvMf25uKocZgzAamDvNv4I4LPAiTPaHAec18b3ozsttO+kYx9BP/8eeHEb/690171l0rEvos9HAZfOMv/lwHva+PPpboaZeLxD7uNTgf8ADpl0jKPs54w2n6G7GWbi8Trs0GtsLv5hm2WTi1dCHp6nn+biKRg8gjy7xwKXJ7kWuJLuerBLk/xOkme1Nv8I3JHkOrpP+79ZVdP2SXeQfp4BvDTJl4AL6BL0svj5xRn9PBfYN8kG4HTgzMlFNjwz+vgHwO7AXye5JsklEwxtqGb0U8uHuXiZ5+KVkIfBXDyN/KlpSZIkqccjyJIkSVKPBbIkSZLUY4EsSZIk9VggS5IkST0WyJIkSVKPBbIkSZLUY4EsSZIk9VggS5IkST0WyJIkSVKPBbIkSZLUY4EsSZIk9VggS5IkST0WyCLJvUl+bEjrekOS97XxtUkqyaohrfugFutOw1jfArb76CT/kuSeJO8Y57aXgiRvTnJ7klsXsY6B97Fh7o/StDAPz7td87B5eKwskBcgyYuTfDnJfUluTfLuJHsv4PE3Jjl2iPFsd31JjkryUNvR702yMclFSX66366qdq+qG+bZ1lFJNs4XU1X9flX9xuC92O42t+lfVf3fFusPhrH+BTgNuB3Ys6rOmLkwyfuTvHnMMY1FkoOAM4BDq+oxsywf2j42W9tBntskT0ny2SR3t+3/1uA91LQxD5uHzcMPW74U8vDPJPm39gHm2iQ/O3gPlyYL5AElOQN4G/CbwF7AEcDBwKeS7DLJ2OZxc1XtDuxBF/NXgc8mOWbYGxrWEYol6GDguqqqSQcyAQcBd1TVbdtpM7Z9bA4fAv4F2Ad4BvA/kzxrTNvWGJmH52ceXpaWdB5Osg/wd8AfAHsDbwf+LsmPjHrbI1VVDvMMwJ7AvcDzZszfHdgM/Hqbfj/w5t7yo4CNbfx84CHgu21drwXWAkX3yfhm4BbgNb3HL2h9s8S9tf2M+X8CrO9NF/D4Nn48cB1wD7AJeA3wqLadh9q27gX2B84CLgY+CHwb+I0274NtXUPtX299q1qb/YFLgG8BG4CX9tZ1FnAR8IHWl68A67bzGv8McCVwd/v7M70YHwC+3+I4dpbHbu1HL8aXADcBdwIvA34auBa4C/iT3mN/HPg0cAfd0ZG/AvbuLX8acHXrw18DH57xnJ0IXNPW+3+Aw3rLXtdew3uArwHHzNH3vdrztBn4BvBGug/Px8543d8/pH1sX7pk+u32XL8Z+NzMtm2/6T/3fzdH/PfRHVnZMv3XwOsnnTcchjtgHjYPm4eXZB5u/f/KjHn/Dpw66byxqJwz6QCmYQCOAx7ckhBmLDsPuKCNb32Dtultdlrgxv4bu/cmvoAu+f1Ue3McuyPrmyW2ud40R7c326PadP9Ncwvwc238R4CnzbUuuuT3APDs9kZ+BLMn5qH0j4cn5n8B/gzYDXhKW/fRvdi+R/ePZifgLcAX5nie9qFLoC8CVgEnt+l9Z4tzlsdvXd6L8T0trl9ocXwU+FFgDXAb8IzW/vHAzwO7Aqtbn85uy3ahS5SvAnYGfpkuSW3Z1lPbup7e+nhKe852BZ5I949h/15cPz5H/B8APkZ35GEtvcQ22+s+hH3swjY8Eji0xfmwxDzIc9/a/D7w1vYcPRHYCPz0pPOGw3AHzMOzrgvzMDOXYx4eZB8bWh6mK5CvmzHv68C7Jp03FjN4icVg9gNur6oHZ1l2S1u+GG+qqu9U1ZeBv6RLDKN0MxC6UyEzPQAcmmTPqrqzqr44z7o+X1UfraqHquq7c7QZev+SHAgcCbyuqr5XVdcA7wN+rdfsc1X1iequlTsfePIcqzsB+HpVnV9VD1bVBXSnp35xESH+bovrk8B36P5531ZVm4DP0iVVqmpDVX2qqu6vqs3AO+kuE4DuNNkq4I+r6oGq+gjwb71tnAb8eVVdUVU/qKrzgPvb435Al6APTbJzVd1YVf8xM8h2o83z6Y643lNVNwLvoPsntRiz7mNte88Bfruq7quq6+iKm8W4FHgu3VGWrwLnVtWVi1ynlh7z8NzMw7MzD48nD38e2D/JyUl2TnIK3VH5Ry5inRNngTyY24H95ri267Ft+WLc1Bv/Bt0pq1FaQ/fp8K5Zlj2H7tP+N5L8c5L/Ps+6bppn+cw2w+rf/sC3quqeGete05vu3+17H7DbHK/h/u2xfTPXtVDf7I1/d5bp3WHrndkXJtmU5Nt0p0m3/KPfH9hU7eN4038uDwbOSHLXlgE4kO5oxQbg1XRHcG5r25jted+P7qhIv/+L7TvMvY+tpvtn0+/HIPvQrNq1b/8A/A7dkaIDgf93kv+5o+vUkmUenpt5eHbm4THk4aq6AzgJOJ3uOT4O+Ce6s3lTywJ5MJ+n+0T4y/2ZSXYHnglc1mZ9h20/Mc2827SY3YG98YPoPvUtZn3z+SXgi1X1nZkLqurKqjqJ7jTUR+muH9vetgaJYRT9uxnYJ8keM9a9aYB4ZlvXwTPm7ei6Fur36fr5U1W1J/BCuk/80B0VW5Mkvfb95/Im4Peqau/e8Mh25IWq+lBV/Sxd34ru5qaZbqc7WtXv/zD6Ptc+tpnuNPkBvXkHMrf59q8fA35QVR9oR5020p02PH6hAWvJMw+bh0fFPLy4PExV/XNV/XRV7UN35PtJbHukfepYIA+gqu4G3gT87yTHtVMIa+mS1ka600bQXaR/fJJ9kjyG7pNj3zfp/qHP9FtJHpnkJ+huKvjwItf3MOmsSfLbdDdxvGGWNrskeUGSvarqAbqL9x/qbWvfJHsNsr0Zht6/qrqJ7maItyTZLclhwKl0n/wX6hPAE5L8jySrkvwq3TVZl+7AuhZqD7obH+5Osobu7vwtPk93iu4VLa6TgMN7y98LvCzJ09vr+6gkJyTZI8kTkxydZFe6a++23OSxjXba8yLg99rjDqY7CrDg53GQfaxt7yPAWW2feBLbno6dab59/N/bpv9Hkv/S9qNfpbsRR8uIeXjrtszDw2ceXlweJslT23tyT+APgZuq6h8XGv9SYoE8oKp6O92O9od0CesKuk+Ox1TV/a3Z+cCX6C7Q/yQ/TEBbvAV4YzsN85re/H+mu/v3MuAP2/VSi1lf3/5JttzxfCXdDRpH9bYx04uAG9OdZnoZ8ILW/6/S3eRxQ9veQk7Pjap/J9PdzHAz8Ld011P90wLiAraeHjqR7nsm76C7U/vEqlrsKdtBvInuDum7gY/TJa0tcX2f7mjZqXSnyF5I98/i/rZ8PfBSujuV76R7jl/cHr4r3Y1rt9Od4vxR4PVzxPBKuqNINwCfo/vatL9YQB8Wuo+9gu6O7Vvp9oELtvRpFufSXb93V5KPzlxYVd+me47+v3TPwTXA/4/ujmwtM+Zh8/CImIcXkYeb19L18ya6S55+aQGxL0nZ9rIajVM7+vGfwM5z3HgibSPJFcB7quovJx3LsCR5G/CYqjpl0rFo5TEPa6HMwyuDR5ClJSzJM5I8pp3aOwU4jO6mtKmV5ElJDmunAg+nOzLzt5OOS5JmYx5emZbrL+5Iy8UT6a5NexTdqbfnVtUtkw1p0fagO523P921be+g+/5PSVqKzMMrkJdYSJIkST1eYiFJkiT1LIlLLPbbb79au3btpMOQpLG56qqrbq+q1ZOOo89cLGmlmSsXL4kCee3ataxfv37SYUjS2CSZ+athE2culrTSzJWLvcRCkiRJ6rFAliRJknqWxCUW0qSsPfPjY9/mjW89YezblKSlzFyspcYjyJIkSVKPBbIkSZLUY4EsSZIk9VggS5IkST0WyJIkSVKPBbIkSZLUY4EsSZIk9VggS5IkST0WyJIkSVKPBbIkSZLUY4EsSZIk9VggS5IkST0WyJIkSVKPBbIkSZLUY4EsSZIk9VggS5IkST0WyJIkSVKPBbIkSZLUY4EsSZIk9VggS5IkST0WyJIkSVKPBbIkSZLUY4EsSVMiyU5Jrk5yaZt+XJIrkmxI8uEku7T5u7bpDW352okGLklTZuAC2cQsSRP3KuD63vTbgHdV1eOBO4FT2/xTgTvb/He1dpKkAS3kCLKJWZImJMkBwAnA+9p0gKOBi1uT84Bnt/GT2jRt+TGtvSRpAAMVyCZmSZq4s4HXAg+16X2Bu6rqwTa9EVjTxtcANwG05Xe39g+T5LQk65Os37x584hCl6TpMugR5LMZcmI2KUvSYJKcCNxWVVcNe91VdU5VrauqdatXrx726iVpKs1bII8qMZuUJWlgRwLPSnIjcCHdGbw/AvZOsqq1OQDY1MY3AQcCtOV7AXeMM2BJmmaDHEE2MUvSBFXV66vqgKpaCzwf+HRVvQC4HHhua3YK8LE2fkmbpi3/dFXVGEOWpKk2b4FsYpakJet1wOlJNtBdynZum38usG+bfzpw5oTik6SptGr+JnN6HXBhkjcDV7NtYj6/JeZv0RXVkqQhqKrPAJ9p4zcAh8/S5nvAr4w1MElaRhZUIJuYJUmStNz5S3qSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk98xbISQ5McnmS65J8Jcmr2vx9knwqydfb3x9p85Pkj5NsSHJtkqeNuhOStNyZiyVpfAY5gvwgcEZVHQocAbw8yaHAmcBlVXUIcFmbBngmcEgbTgPePfSoJWnlMRdL0pjMWyBX1S1V9cU2fg9wPbAGOAk4rzU7D3h2Gz8J+EB1vgDsneSxww5cklYSc7Ekjc+CrkFOshZ4KnAF8OiquqUtuhV4dBtfA9zUe9jGNm/muk5Lsj7J+s2bNy80bklasczFkjRaAxfISXYH/gZ4dVV9u7+sqgqohWy4qs6pqnVVtW716tULeagkrVjmYkkavYEK5CQ70yXkv6qqj7TZ39xyuq79va3N3wQc2Hv4AW2eJGkRzMWSNB6DfItFgHOB66vqnb1FlwCntPFTgI/15v9au4P6CODu3uk/SdIOMBdL0visGqDNkcCLgC8nuabNewPwVuCiJKcC3wCe15Z9Ajge2ADcB7xkmAFL0gplLpakMZm3QK6qzwGZY/Exs7Qv4OWLjEuS1GMulqTx8Zf0JEmSpB4LZEmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpB4LZEmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpB4LZEmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpB4LZEmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpB4LZEmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpB4LZEmSJKnHAlmSJEnqsUCWJEmSeiyQJUmSpJ6RFMhJjkvytSQbkpw5im1IkrbPXCxJO2boBXKSnYA/BZ4JHAqcnOTQYW9HkjQ3c7Ek7bhRHEE+HNhQVTdU1feBC4GTRrAdSdLczMWStINWjWCda4CbetMbgafPbJTkNOC0Nnlvkq+NIJZR2A+4fdJBjIH9HJG8bZxb28rXc+k5eMTrNxcvD/ZzRCaQi30tl6ZZc/EoCuSBVNU5wDmT2v6OSrK+qtZNOo5Rs5/Li/3UXMzFS5v9XD5WQh9h+fRzFJdYbAIO7E0f0OZJksbHXCxJO2gUBfKVwCFJHpdkF+D5wCUj2I4kaW7mYknaQUO/xKKqHkzyCuAfgZ2Av6iqrwx7OxM0dacid5D9XF7s5wpjLl427OfysRL6CMukn6mqSccgSZIkLRn+kp4kSZLUY4EsSZIk9VggzyLJbkn+LcmXknwlyZvmaPe8JNe1Nh8ad5yLNUg/kxyU5PIkVye5Nsnxk4h1GJLs1Ppx6SzLdk3y4faTvFckWTuBEBdtnj6e3vbXa5NclmTU38M7MtvrZ6/Nc5JUkqn/uqGVyly8TZtlkYtXQh4Gc/GMNlOZiy2QZ3c/cHRVPRl4CnBckiP6DZIcArweOLKqfgJ49biDHIJ5+wm8Ebioqp5Kdxf8n403xKF6FXD9HMtOBe6sqscD7wIm83Mei7e9Pl4NrKuqw4CLgbePLarh214/SbJHa3PF2CLSKJiLf2i55OKVkIfBXAxMdy62QJ5Fde5tkzu3YebdjC8F/rSq7myPuW2MIQ7FgP0sYM82vhdw85jCG6okBwAnAO+bo8lJwHlt/GLgmCQZR2zDMl8fq+ryqrqvTX6B7ntxp84AryXA79L9c/3eWILSSJiLt23GlOfilZCHwVw8w9TmYgvkObTTBtcAtwGfqqqZn36eADwhyb8m+UKS48Ye5BAM0M+zgBcm2Qh8AnjleCMcmrOB1wIPzbF868/yVtWDwN3AvmOJbHjOZvt97DsV+PuRRjM6Z7OdfiZ5GnBgVX18nEFpNMzFW53F9Ofis1n+eRjMxcD052IL5DlU1Q+q6il0n+wOT/KTM5qsAg4BjgJOBt6bZO9xxjgMA/TzZOD9VXUAcDxwfpKp2m+SnAjcVlVXTTqWUVlIH5O8EFgH/MHIAxuy+frZ9s13AmeMNTCNjLl4q6nOxSshD4O5uLd86nPx1Ly5JqWq7gIuB2YeldgIXFJVD1TVfwL/Tpekp9J2+nkqcFFr83lgN2C/sQa3eEcCz0pyI3AhcHSSD85os/VneZOsojuFecc4g1ykQfpIkmOB/wU8q6ruH2+IQzFfP/cAfhL4TGtzBHDJtN0cooczF099Ll4JeRjMxVtMfy6uKocZA7Aa2LuNPwL4LHDijDbHAee18f3oTgvtO+nYR9DPvwde3Mb/K911b5l07Ivo81HApbPMfznwnjb+fLqbYSYe75D7+FTgP4BDJh3jKPs5o81n6G6GmXi8Djv0GpuLf9hm2eTilZCH5+mnuXgKBo8gz+6xwOVJrgWupLse7NIkv5PkWa3NPwJ3JLmO7tP+b1bVtH3SHaSfZwAvTfIl4AK6BL0sfn5xRj/PBfZNsgE4HThzcpENz4w+/gGwO/DXSa5JcskEQxuqGf3U8mEuXua5eCXkYTAXTyN/alqSJEnq8QiyJEmS1GOBLEmSJPVYIEuSJEk9FsiSJElSjwWyJEmS1GOBLEmSJPVYIEuSJEk9/3/pOvYlC0GF+gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x720 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10, 10))\n",
    "for i in range(10):\n",
    "    plt.subplot(5, 2, i+1)\n",
    "    plt.hist(idxs_seen[i].numpy())\n",
    "    plt.title(f'Output Distribution of Images of Digit {i}')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Problem\n",
    "Can we create an MNIST classifier with no labels? \n",
    "Instead of using human labels, let us use the most up to date hard prediction of the neural network.\n",
    "\n",
    "Will this allow it to at least segment the dataset in someway, maybe classifying different classes with different outputs (not necessarily the right index)?\n",
    "\n",
    "However, this will surely get stuck in a local optima where it classifies EVERYTHING as a 5 or something.\n",
    "\n",
    "So, can we add random noise to some percentage of the hard predicted labels, forcing it to have diversity in its outputs?\n",
    "\n",
    "Or can we add random noise to the actual hard prediction distribution??\n",
    "\n",
    "# Result\n",
    "No we can't, it always predicts the same class.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}