svr_tester.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from forgebox.imports import *\n",
    "from transformers import AutoModel, AutoTokenizer, AutoConfig\n",
    "from tqdm.notebook import tqdm\n",
    "from typing import List, Type\n",
    "from sklearn.svm import SVR\n",
    "from forgebox.ftorch.cuda import CudaHandler\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import pickle\n",
    "import random\n",
    "from torch.utils.data.dataloader import DataLoader, default_collate\n",
    "from copy import deepcopy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_LAUNCH_BLOCKING\"]=\"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "WEIGHTS = Path(\"/nvme/GCI/public/lit/weights\")"
   ]
  },
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPIAAACFCAYAAABhT6XdAAAgAElEQVR4Ae1dB3QU19VeJKEuRBeYJgmQAFHUQDTRRDfFYFGERBVCEh2DwZVi4w62wfRieu+mGLAB01RBINGMEzuJHcdO4hrHcZw4//ef7ylXHg2z2l1YghBzz5kzuzOv3ve+d+8rc68FTqaU8RPxUNNwVG3SAg9FdUBgm64IjIxB667d0b9fX3SJi8Pz06eic+cuaN66PSJat8PkZ+fh6RdexsOPxmPpunV4c9lSzH/pRTz51JOYPGUyZj3zjMOl/PNf/oIff/xHifF++c9/8MUXX+I///lPieHu5ct7UcZvv/sOP//8s0PV/tvffsA333xbYpyvv/kGf/z8T/j3v/9dYji+tKf9mM6nn36G77773mZ6DPD3v/8dn/3xc/zf//2fXeHtDcT+w7qzrbT00z//idEp4zHzqTnax0W/n5rzPJLTJqn/v/zyi6qLo3yXxCzyw1n3/YffRbXGzVGtSXPUju6Ehu26IygyBlEduyC+fz907NwFT05IRbuOndG0VTs0iY5B+tNz8OwrC9FrYDxeXboUc+c/j1lPPoEpU6dg7NhknDx1ylnFM9MxOXDXOcABhiBevW4jRiSn4ez5TMM8tUA2DODAQ6cDmXnPeXWhksi1ItuiXqsOqNMiGs3axqJ3zx6IaN0W8QPj0TSmHepHxqB+eDSGpE3ChCeeQWyP3pj27FykT56CkaPHYNjwJGzestmB6phBTQ7cew4cfveYAjBBPP/lBVYLVOqBzJLvPvwu4pNT0TCmPeq2iEZodBs0bdUGIVExaNKqHYIjWqFei2j1rmX3hxHTow+axLRHl34D0b3/o5jw2Awcff+EVSaYL0wOlFYOXLiYh/Ubt4D3kqYRh44cxcatO5xSjbsikZ1SMjMRkwMmB+zmgAlku1llBjQ5UHo5YAK59LaNWTKTA3ZzwASy3awyA5ocKL0cMIFcetvGLJnJAbs5YALZblaZAU0OlF4OmEAuvW1jlszkgN0cuGtA/mzfIQzuHIfaXlXRzbM5Jvn3Rm3XyqjlVh4vV6+L5TWCsapGMJ6rWgc1XcujjmsVTK3QD509w1DLqyoSunTF5/sO2V0RM6DJgQeZA3cFyCfnzEc7Lz8EuPiiq1czpPv1RKjbQwhy80Zqxep4pXpdNPfwRriHN16tHogx/tUQ6OaNxm61MN6vF7p4hqm4Hb39cWrOCw9y+5h1NzlgFwecDuRP9x1CJ29/VHDxQGevMKT4dUUjt4fg7+KGqZVqYnXN+mji4Y2KXl7qaurpjbU162NCxQBUcHFTYB7n1w2xno1UGl19KoLS3SSTAyYHrHPA6UAe3qUbarr6IMa9Aab790eIW00EuHpgfMUAvFUjCMEe3qhawR+PTZuGSRMnooqfHxp4emFpQDBS/KujmqsHGrnVwgz/RxDtHoQarj4YHdfdeg3MNyYHTA7A6UCu6VkJ7bxCMNG/N5q710U9Nx8M9quCxQFBaOnjj8DqAZiYNgFndxzF2Z1HkZaSinrVqiPGx1+FGeBbGXXcfBDuHqjSaO3eADU8K5lNZXLA5EAJHHA6kNt5hYKqcYR7IPzKuWGkfzVsrRWCKE9fBFTwx+gRo3D1vWysGjIXa4fNx5X3szF8WJJ6F+Pli221QjGsQjX4lnNDS/dgpPp1U9K9hDqYr+4hB/iN7/+CfvrpJ7u/G+d36M765vjvP/7otLS0fPrhh787NV2nA/mZioOUJK7q6qkkMVemQz29UcXHF+ljUpB3KguLH3kCjaoEIaxKMJYPmo28U9kYO3wUqnj7gHNmzqMpmZkGJfPTFQdpeXBf/mbH4kfkS1eucWr5N2/fqT5e//5vf3NquiUlxo/gV65dr+rDT/XGjZ+CjZu3GXZMht2xa68q4+THZpWUrOG7d4+9h6kznlSfBfIj/XkvvIIvvvzSMOzR4+9j4tTHi8K+vOBN/O2HHwzDZmblIHXiNFUuGjHQEg0E8Oslvmf9mO9Lr72Ob7/9Thvslt9s4znPv6TCG33V9K9//Qtr3t6A8VOmF6X7+qKlVst4SwYlPLAcOfJuCa8df0UpStW4u4+/AmQXv0qoVakyEocOQ975HHzw7DbEVm4Kv/LeqFDeG52rtMDZ53fh4rlsDI0fjFoVK6GrXyWsqBGMLt4VUNvNB63dGzpekFIWg43MTvHaG285tWRrN2xW6X799TdOTbekxJ578VWV56yn5yoAT5/1jPq/cNGSYtF+//s/YNrjT6l3AvhiAWz82b33gIpLIC1btRazn3tR/eeA+Je//rVY7P3vHFLvCBKC8MVXF6r/HDwoVYU44L36+iL1jmXipR8YOADwOdOicYAnn52n/hPY1gYGpv/+yQ+K0l29dr1kqe4c0J5/6TX1ngMT03167vyifAjyOyFL/QahcCaYPcqVU9J0W60QtXpd3dcPj/Trj8sf5OKbDz/HhZVH0a92O/iX90HF8r4YWK8jLq09od5RMj/c62FU9/FFVx9/bK0Zgod9K8HNUu5O6lgq4pYVIF+5el11Pkoo1onETiog+/iT36ln/A6XYCAI33v/pAIFJbe9xLQZl6DVdnJKXaa7d//BoqSoSgvwtJrJth271fN3Dh0pCsuP+RmWQKI05G8tkGnmiM/02gMHBz7/4PTZorS0P2huiOVlHVX6OiBnZueo5xxgtCaB9uwrHKz27HtHm5zDvy2BQQ3gTDD38a2EZTWC0d7HH9V8fDDs0UHIPnUeW5IXYM+IN/HJ2SvIXHIQ8aFdMLRRN+SueBcfn87HrsSF2J76BrJPnsPg/gNQzdsHHX38VVo9fCraVTGaVzlw8LAa6TiaciSltQbpcJIIOwY7BK03UBV7c8lypTZR0lAKaOk3v/0YlDQcRdm4i5auwGef/VEbRP3mXPHI0eNq1GVYhmNcIWtAPvbeCVUOpr1kxWrk5F6UKHbdRSLTFhU7JyUg1bt3Dr1bTBJJYgVXrqr6MD+CkSBzxE7UKwsLpRXz09LNj36jOirrQCKPWY6vvvpa/SefHQHyX//6lQL/9p17tNkoEzoECssuRPWbz9imWmIZCC72BSG29Y0Pb6q/lPKMpwWyDFR61fj6jQ9VWE4hjGjBm2+p92w/IyBTpebzT3V9RwYs9pk7IQVkAfO5c+fvJC0Vd0vNEPT3q4IAXz/07tELR7bsx97Jy9C6SmO0rhGGdYPn48us3yBj/h5kvbQPX2R+hFUD5qBlQGPEVmmK/TNW4dDmvejRrTtq+vphYIUq2FjTtmrNUY5GzsgsNh4lBO/8r2U+GSdMl3Ac9dnYDMuOKkQLD3zGiwARNZHh2dGE2GGoZjIcAfL4k88WxRNgGgF5+eq3VTh2cKpdUl6O3vaSAJnlZ/xn571QNHflb0pLIdqOYhkZjoOW1FmvBkp4ozvj8DIi8oV1NyJHgWyUBp9RXWcd1qzfVBSE6w58ZjSHFTX6n//8Z1F4+WEEZIZjWiwv20yIgySfX7qcL4+K7hcvXVbvyEdKZobT81TUda2aLwmwzzDOnVARkOMHDcWPmrnE7SZKVbiyuwfCmoQh50wmDj+5Du0rhsG1nAtcLC5oVqU+Dk9ahe8//jO+++2X2J+6FE0qB8LFUg6u5VzRqVIzHJ29AVmnzyM0JBRVyrtjoF9lm8X58OZHasSnuiRWMQkwdi5eQpRAZBobWFQcSlMBqRbIb7y1THV6UReZxqEjhfaYKAWEuPDDNLXqHq078pmMtHogU3uQ91JediKWg9LcXhIgc/721deF0o95ScfTSjRRK0VKMg/mRfDZu/rMMnOQNCLOla1JXSMgf/6nL3Dt+o1bLmvmccgnqQMBLSRzdi3w5N26TVsVn2ktVU9GQGYYajOsJ3lKPsogTUGhLxvbkeFY73/84yerQJY09Yb4qNkwL17ff2+fNVB9PfhfAdlZIGaCc6vWQd3yHqgREICN6zbg7Jv7MCq0N3zcPFHRzRc9glojf/n7+O765/j2+ue4tPgoutSLhr+bD3zdvJDSpB/OL3kHb69ei+rVqiGwvIc6m21UeP0zNqSAQt5RfSaTZLQWaawfGakWM5wWyExDK9H4n6BmOHYCITYkn+kbmao1pTrLxYthZLGL6iwlIzt4SWZkOdqfz8wudjFNIQEyO6yWWBYOYJxeCFHVZRk++s1v5dEt95LyE2nFQdCIZJ5s9M4IyEtXFEpSlkl70WSuEVHKMdy6DcUNMnIg0g7W2riyYEbVX0/WgHz9xs2i1W9tud47cas1V5k7Z+dcUMlbk8gctJgW2/zc+UxwLk5ecwCQPErqB/qy6/9bnAliJr6+ZgMk+1dDHR9fxLRsiR1vb8GpF3aiT1Bb9Apui0MTV+HLCx/j8MRVODp1Lb68+DEOpC5Dt6AY9A+OxZkFu7F1zUZER0airo+vOpvNI5z2EFcUjx0/oea8HD3JNGGSrHJSLRQpqU3zT18UMloPZM6nqJpz24OdUdITqclRlM+oOpVEeiAzrHQCxmf6XHnVb4WI+iv58q7ttALky/lXbsletkKYN4mdTdJh5+c8UDuPZxhb+TG+NYlcEqCMgPzJ736vysRyaS/uGetp34HCVWlKX/1gTd6xXEbEtuM7AklPRkAWTYp95+Sp08oONgEnGhv7l5AM6tr5ujUgMw7bSPgvd6rVssKur5fkY8/d4gx1WptRM3cvLAwIxMRKNeDn6obY9u1xYMdeZL3+DvKXHFfA3TthKWICmqBtzWY4OG2VenZ58THkLD6EvVt3om3rNvB3c8O0yg9hQfUgNCnvqc3C8DcXoNjBySCOcpTE7OScJ/KZAJnvKEH1xPcMpwWyqMx8zoGBCyU7d+9V4QTIMtLKIo8+XflvBGS+yy+4quar2kHn9JlzEg3s7JQm2uu3H39S9F6AbCRx2MFYdq2mwPKy7FpJwHoK2G3lR6Cz8xkR06R6bURGQDYKZ/SMgGI9CCYjkJP3fG+kmnKhku+oAuvJCMiybsFFLy0xPtuI9ScRdLLtxnnzHz79TF2yWEathc+obmuJkpjty0GUgxfrQyBrB2dteHt/O/1ASCUXd0R7+mJpjSAlTYP9K6FVVEvs3bADn1/+GKef2YbONSNQ0d0XlTz80K12S5ybtwt/yv8Eu97eguiIKNT3r4iJlQLUkc0ID18wTVvExQ82GOfAWpLGEiDLosPnOvWNIy3jC5B58ob/2Xk4ygpRRedzATIBwAZmJy6JrAFZ4rBj5Fy4qNJievaOzgJkbmPoiZ3OSPuQcJQoImkIYHtIpiralV7GE+nEdQUjul0gkyfkN+Nrt5a0eXB7iWG4j6slTotKahvpG9q6yMBvtF9MjYP5cO2FZeFvWxfBSuJaAHco9HWQrTPpd9ryO/Lb6UDu6NkE1V290aC8F7bUClELVVx97tKpE04dfg8n5m1Dx+rN4e/uq8AcVyMSH7y0CycPHkOH2Fi1Us2z2ZseaohgN0+VVpxXM5t1YgciU7UqFMEokkeAfPBw4UIGR3EBC+fL0kjC0C///BeVHufUWuL2FvMRIPOdND4X3IQ4ghNIHGkJYj2QKVWp+mqlL+PKKG/v4pMAmRqD1IfpnDmXocq5eNlKVSSmx/k5F2+0xPkm66Odd2vf638zHMNzUZF1IjFfATjnl0Z0O0DmtIZ5UYOShTyjtAVU1BS0ax/SVtSijMgIyJu37lB5cpqjJVktF42DYObAob9kUYuHPfhO5r0EMevCPLUk21JXr93QPnb4t9OB/JT/o+ob5EouHmjp6YslNYKQ4F8NvuXd0btXLxzbcxhHn1qHzrWilDR+b+4WvLv7ILp36wa/8u4YUbGaihPp4YNKLp7o7tUCs/wH2qzYB2fOKUaR0QQr51SyrE8GCpDZ6WROwg5ChnPUZkdjOAEyM5RFLIKFkl67OKMFMlcemQavbbv2gBJCBgbZj9QDWVY7GYdh2NCiLXBgsJcEyEyHc1caPZftGA4i2m0y2YrhIMZOtmLNOlVnxjVSPY3KwHpw0CCvOBBxVVxWkvnfGjkKZK4VsFzMh3NgDkDaS7ulyDw3bdmuwrLNt27fVXTYgwO5kcrNOEZAlrZkvlyt5hFYmaLw2SkrB0Kk3tbmyJS8HNiZBvnFfkJ+8T8HwTslpwM5wrc+Rvt2QZxXU3VWmt8mr6lZH0MqVEVgpcroEdcN727ah10zlmHvEytxeMMedO3UBcGVKiOpQjV1rDPW2x9VXDyVZZFRfl3Q3CfYZj3ZwQgIMkYugpSA43/tlgtXjHlQhIAm8DkvkgbQApnzbgG4pCmr228tW1WsTFSdpEMzLDshR3eRWnogM/KNDz8qFofxmL9WqhTLxOCPADm/4EqR9sF02An182Z2aNZZAMJwrL/RAReDrIoeEfSU7ozPi+mRz9q5eFHg//5wFMisj6RvdGeeeuKZblknYRy2R0mSXICsX2Ckqs1BUcsnTlHs0VqkH+k1H5aV6Qp4hW/se7INqq+PI/+dDuSeHTujmXsdzPYfjDbuDRHg6oUWHt7gkc3O3hVQ078iBjzyCLJPnEfW+2fRp08f1PT3V2ezt9UMQVN3b1Rz9UJ7z0bgBxhhbrXANO0ldiZbHvo4OuqJnZ7M1as+DOeIF0F2cqrlAmB9Pkb/ua3DOfudNijzZDqcUpRE1Eq4xWOvFLaWFsvLumpVemth/5fPqX05clrNWtnIT9bPGWlp87idPqKNb/Tb6UA+s2M3arl5ooFrDcz0H4C2XiGgmh2nPqIIRi+/yvD28ESP7j3QNS4Ovh4e6FehipLaHb0rqLDtvUKVYYFAt+qo7eaJczt2G5X9tp5xK4Ej7YGDR4rAxo4vczx+FWOSyYH7jQNOBzIZsOGp2ajp5oVojyBl6qeNewNUdfVAP9/KWF4zGD19KyLA20dd/XwrKUN8vX0qo4qLhzJKMNYvDuHu9VDLzQubn57tVJ5yYYRqEqUv1TDtPJqquNH2hlMLYCZmcuAucOCuAJnlfH/7LnTv2Bkd/ZtifqUkZb6nuqs7+AHE7tohaOHujUgPb+yrHYo4b39Uc3FHmHsdPF8xEe38GqNbx844tWPXXagylAP0kx+cUR8PELxc/OGpndKmIt6VypuJlkkO3DUgl0lumZUyOVBKOWACuZQ2jFkskwOOcMAEsiPcMsOaHCilHDCBXEobxiyWyQFHOGAC2RFumWFNDpRSDphALqUNYxbL5IAjHDCB7Ai3zLAmB0opB+4akI+cOY8xw9MwN2wY9tZLxf7ANIwOH4CweXMRsO1tRM56Ei83Ho7DgRPwaqPhaJUwCtU3r0b1DSsQPuNxJA9PQ+bBWy0ylFI+msUyOXBPOXBXgLxs13481TwRbXsNQs9HRuP1kJHYH5yONQ3HYlCHBAQveBFRj83A802SsDR4BHr2HYmQxQtRY/NqBfQR7Yfh0ej+aDNhGg6+UdyEzT3llpm5yYFSygGnA/lI3iXMjEjCppBUxA5ORuhL89Hl4WFYGzwGewPTsCokGd37D0dUSjpmhA3GoPZDEPrGAgRsXIngN19FQquBWNJwFEa2SUDIskWITZuKqwfv3LpnKeW/WSyTA07hgNOBvCBxFnY0SMOO0AmIHTQKQSsWo/HyxRjYbhDWN0jB3uB0LAgehratu6Nzyx4Inj8XD21ciZDFr6N39wRsDUnDlgbjMKL1EISuXobgVUsw5jn7v891ClfMREwO3GcccDqQ94SkY2dwKhY2GI6ozr0R9PrLqLV+BcLmPIvxrZOwtf447AtMQ2pQT4QkJ6P6plUK7N0S07EgZDh2B6dhRfBI9GzeFaHz56HW2qWovc7YfMx9xmuzuCYH7hoHnA7kQ4HjsaTBCPSJG4xOzTqhTdI4PLRuOapvWYOox2ZicqNHcCBoPOaFJSL8mWfVvDhi9FjMaTwM7wSmY1dwKlLDH0VMTDf0jhuCBq++iICtxc2j3DVumAk7zAF7TRI5nLAuAr9Ks/ejFkfK5EhYXZFu+WtvWvzO2dY347ckbuOB04H8dvBoDO06EuEvv4wRUQMxs8lgRIyfhJqbVqHO2mWImfwY5gbH46mwoYicMROhj03D7GZJSorvC07D1LB4REyehi6PjMBbjceif/9k1Fmy0EY1Sv9rNh4/mzS9MTrWVvZ6Y6RBCdq9FvNM5DW/aqPRBj3R/BG/PxcLIIxD441Ghh1owK8kb5J8TyukYpmEpoVohojtrSdaIqG5JQnLO+2JGeWrj2vrv9O9MY5tl4QWc+cgZOUSjIkehLUNkjG98aOIfOJJBeTglW+h6+CxGNWsH1oNHY2xrROwtUEqdgSnYm5YIlpMnKxWsLvGJ6v58iuNRyAuMd1WPUr9ezYsv4EWA/XOKrCY+nmQvTGSt2JChyaFaGZHvjmnySOtJKf9cgEwTeXStrjY0tK3DQ3uiZVRth1BqifxcsF8CGCaTmJYvWdKWpmRQYZmhFhG+RaehgzvlJzujbHuc8+qveCQNSuQFN4f24PHYWf9NIyJjkez+c+rveKqy19Hs7je6B/aGZvrjsb+wHQsbDIK7YYlo9665ai3egk69k/CzpDx2BOUileCBt9pPe95/LICZLHbTIN0InUoldg52YFpFpdECcn/BA0NFxIsRkCw1jBMm3EptWixUsjIGyPtRzMvvRE7MTao9awhdt30/rUEVGKoz57y28sLll28ZNAyjRD5JgOB3r6ahLH37nRvjDzUQSCGpKSgTWhrvBo8VIFxQ/2xGNBxCEIXLVRgbjZtGuaGDMGBQO4vJ6Nnr0TUW74INTasQMPZT6F5eDtMD+ynBgIujtlDtIVkemN88LwxZmRmq4GCZpy0xP8EOP11CdFdC62syiAkz8XrB70ukjh4UMqL0UYj44E0lMj0bXmmZHrUEDgo6fP9/R8+VWkYGeuTstlzL3LiRteqzvDGWHP9CrSc/gQmhQ9B624DMCBumAIqAbssZAzieiUgcPmbiJr+OOY3ScLG4GT0bzcQjRe8ogAe+sZr6N8tEf2j+qJ9UirmhQ3H7qBUm3XhPEPMtHIk11pB1JpOJSPF/5OEI4NFvdJa0RQbzmwsqliiZjG81swsG52qFcNxZBdTuPxvemMsbDojINhsVIMAYl9a643RIJh6RNPA2jawFo59R9RerfTXhjcqP/sMLyNiH2E/EGJfY//REzUAlpHeOO+EioDsLB9QIXOfxfi2w7EhZBxih4xBo9nPYGDHIdgWlIID9dLwZshItE5MRlTaRDwRNgTJEQMR9tILCsQPrVyEvt0S8FrDRHUgJHTJG+iemI43ghNt1tH0xvirEXcOVhzh2UEeBG+MRp2DrlkIHqrzRgtejEOVn2sMDENe0XyxNTICMuNQYBgR58raqYQYd9SvZYiHE6Z/J+R0b4yJLQdiXYMUbA8Zj9jBoxG87E2EvfoSxrQcpA567KqfhtmNEtCmc1/0jeiFFjNnofa65ai/fBE6x4/B8tBkbGqQghFtCg+ENFr8Ojr3TbKrjuzA2oUNRhIGmt4YUbQgpJ0z6hlLddSa98fS6o1RXweWU7QiupyxRgSiXHQKYOQmRuLqgewoL8SBAjU2qu9c/BLvEyyDNcku+du6O90b47Z6ydgdnIq1IWPRssdABC59A7XXrUCLGTPwdMRwtTrNjyiG1+mMBgkJqL5xJequXYr2E6ZjXuhQ7K6fhvVUtyN6gyCuuW4ZAlaW7OlQKml6YxROFN45x6NUknkZ/RBJx2VH58JPWfDGqK011WQZvI38YWnD0r0QBy4aiSefeBn5UWYcPZD5jLy0JpHJX6rXWqJ3CeG/3LkgRxDTCOSdkNO9MfJrpjX1R2NwVH+0atoO4eMnqVVsnuBqPXo85jYcor54eq5xIlo8M1sd9mj62HRMbT5YrV7vqZeK6Y3j0TI8Fu36JyJo8QL1tZStSpreGG/1/yuuTrgCK1QWvTFK3ThgiWcRuvdxhLhfTXDt2rPfMJoRkAlASlgjolpN9VpPdAzAxTfuTdM7CQceDiD67Sp9PFv/nX4gZFNQMlJiR6Ll07MxNLI/0qKHoMUTT6gDIUFUn3kUMzgBT4clIPLxJ9Bw9tOYEJ2IzQ3GqS+kqHbHjhyPzn0S8WKzkeieNB51Vhg71tZWzvTG+OB6Y5R+wEVNgpFSTj/FYhiChi5axUOixOOdq9OMSz9TRmQEZJH8Wm+OjGvkmZJO9Zi3aEeSR8GVqypfDiR3Qk4H8uPhwxA58wmErnhLHQhZ3TAZo5v0QejLL6gz1/yiqV/3JIxvOhBRY8cjvkMCNoeMU2esF4WMQETSaHDlmgdCNoWkYXZYIlpPnmGzjqY3xgfXGyM7h3he5HRCq4HoO44cFNHv227bsVsBavfeWwdEpmEEZNnVsMczpeyUnD2fWVQkbpdyp4USWe9HuSiQnT+cDuSQqZPV2eqQtcsxMnIAtgWPw/YGaXg0dgiacItp0yrUXvgiWrXrgW4tumJl4AhldGBZ6Bh065OE4DXLELhqCboMHIWdIRPU0c3HgwfYrI4sJlCdMb0xrlGdUr9NJgckypo3Rh7ukDkn68YVe+1F5wNC9ELJsAQP+UEAy8IY1WHx2inh5W4EZEpX2fLkAFKSZ0qR0qIxMF85hELvnXdKTgdyrU2rUG/NUoTOmI5OjWOxODhJLX4tazgKD/cfjQZL31RbTY1T0zAzeIAC8fqGYzGkUxIaLngZ3Idu8MI8RMbE4bn6g7EjeBz2ht7qeU9fcTJVTu1Io5reGIvPm8uqN0bxaSztrr/TA6WWuJItIGJYgpr7uFxJtkZGQGZYSlV7PVPmF1wt5t2Te9fcfnIGOR3IXY8fQpO5szGiVTxiOvVBYkw8tjVIVSe4FoSORHR8olKxIx+fiRebDFer1AlNe6Pp/OfUF1L8/rhLv+HoGdULnfslYWHDETib8LrddaVaZXpjfLC9MdrbWbiF5KjnTGtpc/5tr2dK+h/T7ydbS9fe504H8oFrV5HUZRRWN0pBbPwoNJsxAynRg7ArKBV7gtPwQpPhaI1o0RQAABozSURBVJE+AdETp+KZsARMbTwQ4c/MVkcza7y9FJ3ix+C50AQMbz0ETV9fgAH9xuKjd36dV9hbMWvhuN3AEdj0xmiNQ+bz+5EDTgcymZDxyg5sC0lXpn7qL3ldrVpPb5GAbfVTsaN+KqZGDENMn0FICO+LtqlTEKjmxW+hbcpEvNQoSVkSUaZ+Vi3FKyedo3pI45jeGIUT5r0sceCuAJkM+vRQLtKeeQ7B/Jrp7eWIGTkOr4UOx4F66dheLxkD6rZHw779UW3tEjVnbv7sbExqFg+e/NrVeAKeS3ka+z4uPsdzFuPp6Nz0xugsbprplAYO3DUgl4bKmWUwOfCgcMAE8oPS0mY9yzQHTCCX6eY1K/egcMAE8oPS0mY9yzQHTCCX6eY1K/egcMAE8oPS0mY9yzQHTCCX6eY1K/egcOCuAXnX9i3o2LEDPDw8YLFYDK/q3uXxdOsaCK/uCU/XckVhGKdDhw7YsmXLg9IOZj1NDtwRB+4KkN+YOwuNq7nB1cUYwAS2SzkLmlf3QuawULzRuS5qepcvArIW+NOnGxs3u6Nam5FNDpQxDjgdyO/s2oYu9dzRsZ47/NzLKcBqgSm/vd1cMKZZFfwyPRK/H9cUvYIroJwVyW1K5jLW68zqOJ0DTgfy5MFx6BHsjm5B7mhYxRUersZSObiiO04MDsEvs6Lxy7QIrOpWF15uxsDv2KGD0ytuJmhyoCxxwOlA7tvYD73quyswt67tDn9Pl2IqM6Uu1erUFtXw7cTm+OWJlvhlZjQujQhFqxrexebKlN6u5Syo6e9Vlnhu1sXkgNM54HQgE8RydQ9yR72KLsXUawKzsqcrjg9qiF8ejyoE8hMt8d3EcDzbuiYqergWAz6ldLMAV6dX3EzQORxwxAMhP1ZxBtEel72mcejFsSTTP9ryOOLxURuvpN+OlLWkdGy9u6tA7lnfHVEPucHLzVI0//Ut74KBIRXx6bimhWo1JTKvGZHIH9kEdfzcigE/wLccOtQtb6sepf49LZjQ9M6D5o2RH9DTQge/Aac1Dlra2Hfg0G211zfffKsseUhatPJBE09GdPjdY0UO1ZjvU3OeB31EGZG9Hh9pEoh56y+2Kw0LaMlWWdkf9Ono/2dl52qTLPG3070xijSWe5dAd9TyK5TKVKtr+blhXc96+HlaRJE0VkDmXPmxSIxuVgUV3F0U8N1dLAit6qrU9BJrcR+8ZMOxQ+k9/t1p0UuzN0Z2ZnHFQiCtWLOuyKsDzTI5Qt99931RWrS1Raup4iGCHiO0tGffO4rXMnCKjybyn2aTtURje3xOEC1btbbIGR3j6u13MRzzpKFH/aUFsj1lZX/QpyH/aWaXeekdzWnLrf/tdG+MAmC5c+GLc2WC0s3Fgpia3vjrhOZqXqwALBKZ98ejkDEsBI2reCqp7O/hgrZ1ClV1fcHvt/9lBciOeCCkVGSH1BqXowdCgoHA4W97SaxcvnPo3aIoX339tUqLoBOfTTQcwTwJBrqNEbp67YZ6TiN5QiIVtfH5zsjjo/hoskejsresUg7tnfUgf1gmmiKyl5zujVEALHeq13GB7gjwc0GAT3m81rGWUqN/mfVflVoL5JnR+GFyC0wMr6aAH1rVDV2D7Aey6Y1xo3IUxs7KDv/3H3+8pR/QjjKNoVMtpQF7SrOff/75lnDWHoh0s8cD4bpNW5UqLa5KJU2xP01PhPYQAceOzUtvr5reJAjcM2fPq6SojvK/kZ1oMdj/ww+FNs3oiI+A1/rHYiLsR0yD4YXokYPPrJnLlXCOlFXiaO80Xs98tO5Xte+t/S5y4uYsb4xc4NJfBHJYdVe0r+2Dq6Ob4N+PRaj9Y+4hF7sei8RPU8OxtU8g6vq5oUNddwVkpmeLqNqIaVKO9qY3xjTQn5FW6tGmMjsJ+UO/vKLC0XevvcQ4vIyIQOM80hZxzsxyGA00RnGpojM8VV89cUDhO/G4eeTocfVfvGBqw1O1Z1jx4ax9p/1t5PExM6vQ5C61DJrU3bl7L/ILrtwyCDpSVm2e/M0BVQYsR6Qx4xYB2VneGFs+5Ab9Ff2QG8ID3DElKgCHH22ID4aElHit6xmI7sF+iKntXpSWvtL6/6Y3xtLtjVHaiyZhCaaXF/zqz4tzSrpP0V98Tvrtx5+oOLv33urORaQnjcSTPvnd71VYPegZTubrdFJnjSjxOZ9nGQloITGAz+fai8CjGxghR8oqceROLYppa6cP8s7W3eneGH3dy0F/VfFyRZva3pgeXQPtavsgrJonwqpav1pU90BEQOEetKRlqyJ8T7VGr3qJWw/TG+O99cbI9qHfKWoDvLT+pelORQsO+S02n2n5lM+M1GWmy3cy92UfEE2DUpP5UC2mdiLpar096PsVtROGW7dhc7FXlPBcqNy6fZda/abJ5c3bd6qwnNPK1pojZdVmQAks0tiRqY6k4XRvjHIEU3vnotXUqACcSwhBTA3vYttL2nD8zcMitf1cEFu3+BFPKXBJd9MbY3HusHMTNOzcpHvpjZEDqUhESl4tcdGKZdNff/7LX1SwkuanIpHpkkWI4JW8BLzkg4BUn7/E47YYw3PaoRcIEkZ/l/n+xbzL6pWjZZX0ROLTS8rtkNO9MeqByf+1fNyQmRiK7yeFY0W3unArZ3xsk9tTPNIZXsMN3YPd0bjar0c8bVXO9MZ4q8VRWdzRHoi4F94YORcWn0unrez7ltS+3IsmwDjH1dOfvviiUIJu2lrsFQFOcHGbixZT6UWCIGE6jKMnOljju2mPPwUeDLGX6Gua8WQ77XbLeifSmGV1+oEQPZD93F0wpFFl/GNquFqtvjkmDBEB3upctT5suXIWBPi6oGO98ugZ7I7YOuVR1avwiKctxpreGG91PkYVkwCyRlz0YcdlR+Tc0h6SqYo9HgiZHrdTZM7JFdnbIS5kUqJShRXtQtLh1hbLzwUoEkFKB+JGK+JPPjtPqa/aBUDGoQsZpsHDKty+MqJLl/PVgKCPK3N+SlSSI2WVfPbuP6jyP3TkqDxy+H7XgRzoXx6nh4YUHcfk9hK/Qa7i6VbsKCalMfeZW9YqlMbcvuJqNT+84LFOW8TNdDYGJY4QtxnY+Hwum/syKtPZl6hPlBjiyIvbKyS6/2A8rcrG56IC0Q+vkMy/uOAmRIlAIHGkZefjxfTkQAgXRaj66iUUndAxnL1HH+VACFfspT4sg8w7Fy9bqYrE9Jg3nZtpiXNB5kfPgvaQIx4IWR5ZoaY/4DshKaf28Ae3tahCE+TkN+nzz/+k6sPtNe0hDdmW0m813fjwpgrPdKjiWyPxK6adpzN9quHkH9tTyN6yMjylP8vPfnI7c2PJ0zZCJKSdd72UHdW0slKp+WGEOgAyPRK5SY3Ugpc2LMFa1dsFPAkme9C8t6rlBs6xbZHpjbFwy42jOg8tsHOxc2gXlf7X3hg5aLAcan6q85DIdzykYS9xIGZ9mB4HJzpuk0UtnuTSkux1U8IynDg/Z3jtnjbn4Cwb06RfZK0HR/6WLS2mzZVpCUutROvFkQOkVlNwpKyyD3672orU2+lAFosgri7l1AGQzQ8HFe4VywGQmdH4eWoERjetAp67FjDz4wjuNfPzRy2QCezGAba/fiIjZdRkw/AyvTEWnzezE2vPPZNH1AD0xxalc1i7U/qx8wqf2cEJFu1cnHFnPT23KIyE1d4dVSW/+OLLorMCTIfA1p4ak/KyfLJnLPlR+9Ge9GJY7gPLe6M766UlrlTLWQWGp7bHKZ1W8kt4e8pKCcw8WA85mSbxHb07Hcg070M1mcDsGuiHm8lhxc9U8yTXrGgci2+Iun6FVkEYvrIXV6rLq3PVWiDziOekwXF218v0xvgnyMkla0yjyksJI+qotXC2nrMD2+uB0FZajrxnub/6yroaLGlxPsu5vH6Akfe3e+dWUUlquDZde8uqjXM7v50OZNrqoprMzxGXxtVRJ7VuOVM9K1p9i5zcrKqSyJwb16/sqha4tCCW3ycObL+duhnGMb0xGrLFfHifc8DpQCY/npk1Ay2qe+EP/FRR881xMUBPj8ThgQ1AA3z8PrlN7eIqtYB49UuznMpi0xujU9lpJlZKOHBXgMy65R/fh1+WjsMvT7e7VbVWH0pE45OUMHQL9ENY9fLqwwoBb7/G/ng8oStOOlESa/ltemPUcsP8XRY4cNeAXBaYY9bB5MD9wgETyPdLS5nlNDlQAgdMIJfAHPOVyYH7hQMmkO+XljLLaXKgBA6YQC6BOeYrkwP3CwdMIN8vLWWW0+RACRwwgVwCc8xXJgfuFw7cNSD/7Zs8fHrjDZzeEodh+8Nhud4OlpwEWM6l4kbeFPzm0lRsuzAZlrMpsFzuAUtODBatbYYLh+Px0cVp+Oyjxfj294Wfpt0vzDTLaXLgXnHgrgD5z7/biY8uTsbNnInIPfwo5p6Ig+VaJ1hyB8NyLgUbcyfhvYuTMCN7PCznxsJS0AMN87pg18GeyD89GjdzJ6nr2vFEZH645l7xxszX5MB9wwGnA/n7r3Jx/dxYXDk5AtfOjMHVrHScyU3B7ILBsFx6BJbzo2E5Pw7VM8bBcj4FlqwkdCsYgK2XRiE/Jx1XM8bhyqkRuHpqFG5kpSMrNxWffJN53zDULKjJgXvBAacD+dqBdBxf3BwHlzTHu4ub4+yyaFza1w8Xs8bh7bzRsFweAEvOMAVgS+4wjLw8DKdyU3HxRBJy13XA+4ua4dCSwuvMonDkbuuO31z+1aj4vWCSmafJgdLOAacDeebOluh2NBKNT0Wi3fFITNoVjk2rw5GzJQ7ZZ0dj0aURsBR0UVdUQR+8d2Eccg7H4+iaVnhxUwv0PxyBFicjEXEiEokHwrFsXQuc2tCxtPPRLJ/JgXvKAacD2XK9beHC1rWOsNxoD8vVNqh0NhoblzfFhZ29kZUzDhNzeqNHRiw2XUzC9RMjcHppFCbubA7LpRhYbrSB5Xps4cXfmS0xb0PTe8okM3PrHLDXJBENP4jJWOupOecNv7e211sjv1W+ExM71kpM81FaqyHWwjnrufOBnN8dluxhsJwfCUvGcFgu94SloC2iT0TinZWRKHg/CVknhuHcgb7IO5eMy5vi8OqmcFjyWsNypSMsOUP+G3ckLBcHwlLQAZbLMc6q7z1Lh41KSxD2+A5ypJC0rUwrE9aMxjmSlr1h+cH+yrXri0zv0FIGzeIYdVxnemNkHWmZhPXVG6Bn2W15QNTWjyZxaYxPLIPQ8ge/VdcT7W8xP6Prxoe/2mhjPA4I5IuY4mUcGuLXWybR50G+ieliscapD2Prv9O9MVoyRsCSkYL2mWmwnE+FJXMELPldFRgXb4jA5e098WH2BNzMGo/rH4xC9vIYhJxtBcuVWFguxMNyPhlNMtIQksH4o2G51BeWG+1s1aPUv2djsdOwIzqTxPgeAfO/IjE4R1M+BLAYDKRPKS050xtjRmZ20cBBPopnCcnPHg+IElYc0TEd2jFjuQk6/hf71BKWtrz4XDwlau80/SNEaylikZQmpghIMXVEW2ElaSO0AMo8eDnivkfy5t3p3hgt58bh+ZwJKMibim25kxSoLbmDYPmwHebub4NLe/rio7wp+OjSVFw/l4JTmzvCcqktLFfiYMkYhdSsdGRcnIrMi1NRm2Dm3vO1+3+OXFaALCCgzWyRwJTQ9LXFjqj1q+Qsb4ySJwFx7r/+q/RAdsQDIqWkvqy0W8ZnBK6WaI2T+dqiy/mF9r/WvL2hWFBKaKZrzUUqByAOImLt9baBHBjUAHTgduTI7Vm4L1ZqjgznUrE0dyIKLk3FOxcmo1ZGKixZw5F6JR6nskbjxtkUfJiRqq4b51Nx9WwytueNQvjlQUoav5I9Hll5U5CXNwVdMtJhyRqBtgUD9NkY/qd9JJqr5YhI5lN1YmeSDieRaOiMrjNpDZGWFt9cshz0hEBJo/e2R88BHLFpH5qNSiNzRsbqOFekA7HnX3pNhWU4xhWyBmTaYGY5mDZN9Bo5H5M0jO4ikenMjJYfKRWoppUlb4z0IMm60VYW24nA0AKZvOW0hZfWJDD5JVYqxVsjn7Ff6AHL5+LIXMtn5sVByhbRFDI9KOo1IzEfbM0cMM0tMw+2O+93BGQB87lzha4pbRW6pPcPnU9DQmY61uZOwtSsCaiWkYr5WanIyUjBxfcSkLGzB46vb4ejb8fg1MYOyN3fHxdPj8DhrFT0zUxD2Pk0LM6ZgIU5E5RqPipzHI7mppaUpXpH1UYsHHKEM70xlh1vjNrGNwLynXhAlLS56MWBgAOhEOfkBBcHZdrFpi8pOoBzZD2CAw7TuH7jpiRbdBc/UQQvJbNTgOwsb4xbcyfB/XwaLGfHqeuZ7PG4kj0e+dt74uiS5nh+fVPEHwxH3yMtMG53C6xaHoaM5a2Qf3gQMi5MRgvGPVMYd1BmOk5lpePakcFFlbf2w/TG+KuBdUooSjB2DK1BdvH4oLVAyU5KSWTv6jPTtCahOFemilgSiWcGR7wxatMzAvKdeECUtMUhG7UqIUmXddZfJXl05DyboJc5M/mu1xSoPXJRjPziCrtTgOwsEJMBF7PHY+bFEeoUV8e8oXg/cxyydz+M7avD0eH9SFguxMCS367wutwGluyWmLIrHCdWt8LlY8OwKycdlkvxKv67ueORc2QQMt6KEt6WeGcH1jNMXJywA5BEldH75mUDsrHE04RkpHcRwjkgw2lXTWWVUm92lao1VSuWixfjyWKX2DSmak+JYo04arPTaC+tVwhRrelUXEsiYahGClHlZhnor8galZQfVVvG5wKREck82egdn92uN0ZtekZAFsmm9QKhjcMys+7WiAtpDMMBTWujmtMVTruWrlgDrlDTtC4XpmRh7OZHxe2GS/rCZ6ZJoBpNxdZv3KLypOM60h0D2ZkgZoGu7B+AIzmjEVrQA6vzRyH7vaE4+VY4+hyJgCWfi1pdYLkwsPDcNVekud+c1xpztoYjZ30n5GSn4rGCeEy9/CjyMsbi7MYOmLOpmfCoxLvpjbE4e9ih2Ok4iJDuV2+M2loZAfl2PSAy3es3PlSA4mBsa5tIyiEDh37wlPe0eU0NkS5YuVZDQGtduYow4IKh0B0D+ccff5S0nHLPW9EGeR+MwJYz8cjPm4gL23tgw/ImsFxpDQv3mLk9dY7nrFPV4pb6kOJaLILOROHYoua4cj4FH2SPwYmMkSg4moDNayNR7pxticxRj3McGQUpiSmtxC+T+H7iCMlG0xPfM65WIsuKI59z/s3RmSoT/1MlJVHK8D8XqkoivUSWsFQ1ucgmozzT0vqDonM1jvzaS+tnSCSykXQoK94YhVe8GwH5djwgMi06MiffeX362R+12ZT4m1qf9IkSAwLgFhXDsh+SGFe26+gY7g+ffqYuWZmntsNn9h5okfydfiDk4qIoXDk2DFcz03AjZwLOb+yEKTuaQ5344j7xuRSsyJ2IvRcmY2J2OiznxhRK6ayWWLuiKS4dT8D17PG4mp2Ogt198fKWCFjyWkl5rd5Nb4xl1xujttGNgEx1mGDkIC3ah8TRe2uU5/SQIQO/0SDIcAQ63a3qV6I5vyU4OQALcbrDsPrpFd/L1ItlkwU0xi/pEpVb0rd1dz6QF0eDnx9yn/ij7Ak4s7kTEg9Far5HHoeLeVNw+dJUrOM+M79Hzu8Jy4XWWLo+HBePDMLNC5Pw0YVJKNjZG3N2RsFypY2teqgNezLG9MZYyKqy5o1ROoARkPnOEQ+IVKFF5aX/J2t0Kb9AgU2rpTGsbGmJK1U+k8VFvUM5kbSyQMitT86z9Re3C9l/uXXKdyWtmxiV1+lAvrKnHz48P04B+ebFycg9Eo9573UuPDv9X4nMraUtuZMwJovfIyerwyB18zph9zs9UXBubOGBkbwpuP7+cCw/2g2WfNsHQkxvjGXbG6N0XmtAttcDIlVbWU3m2QACUH+JBOZCJxcjCTBOrbgDINMV7vtrt6Hoj1mmRwQtw8oJOMbXnxiT+sj9jufIkpCz7jdOj8X1o0Nx7VA8rr07RH1bfPb8KDxfMBSW/P6wZA6HJWNs4YmvjGRYLgxGfMGj2HJxBPLOjMaV48Nw7fAgXDs8GNeOJyEjYwzWXx5js3hUW0xvjIW+oNlxeDxQrzLe794Y2QkEyFyv0JM9HhApEcmfki6q1ELclqNEFpDyTo+WBJ6e6BhPtvgkfa6E0wezLSoCss5/ta148t7pEvnSwXScXBqJE281x4klETi9shVy9vVDQVYq1vN75IJehR9D8Njmpf5IKhiM07mpuHxyODI2dMSJZYwbjvffCsepZVHI2d4dvyv4dS4iBbd25xyFCwxGjJY4Rude2enJfO22koT/+ptvlPdCo/mPhJE750+cf+nnavLe6M5tHTro1m59GIWz9Yx5Mp2y7o3RFh/YBtq9clvh7XlPSU6pz7stYhj2AXv6i6207H3vdCDnfnoUDc9Ew5LbCpaMlhi7PxybVzbHhbc7KInLLSnL9Q6w3IhF+NXeyLg4HpcODsTxZZF4YWNzxJyMhCW7FSxZLdHraCSWrm2Ov9w8Zm99bIbj1gFHVR6nE7Cx48t+c2ZWjs00zAAmB0obB5wOZFZw+hcrCr94yu8Ey+U2CDoTjRVrWyBzV09kZY7Fc2d7Y+J7sdiek4gr7yXixIpopO2JKAS/OizSBZb8zurQyPQ/OPdrIc5rODei9OWqJec6ogZxoeGnn34qbW1klsfkgE0O3BUgM9ctX2Wgw415sFzpq/aQOx+LwKFVUbh2bBiuHEvA5X391WeM+Zu74RV+j0wJTkmdOwgu2SPR4fp8bPnmtM0K3E4A0xvj7XDNjFOaOXDXgFyaK22WzeRAWeOACeSy1qJmfR5IDphAfiCb3ax0WeOACeSy1qJmfR5IDphAfiCb3ax0WeOACeSy1qJmfR5IDphAfiCb3ax0WeOACeSy1qJmfR5IDphAfiCb3ax0WePA/wNV2qLjuTKOVAAAAABJRU5ErkJggg=="
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image.png](attachment:image.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AgainstCLS_bs_02_154406\n",
      "0\n",
      "AgainstCLS_bs_01_233421\n",
      "30\n",
      "AgainstCLS_bs_02_092503\n",
      "12\n",
      "AgainstCLS_bs_01_222958\n",
      "30\n",
      "AgainstCLS_bs_01_233917\n",
      "30\n",
      "AgainstCLS_bs_01_201849\n",
      "30\n",
      "AgainstCLS_bs_01_200924\n",
      "35\n",
      "AgainstCLS_bs_02_154538\n",
      "0\n",
      "AgainstCLS_bs_02_102534\n",
      "30\n",
      "AgainstCLS_bs_02_092507\n",
      "30\n",
      "AgainstCLS_bs_01_223109\n",
      "30\n"
     ]
    }
   ],
   "source": [
    "for slug in WEIGHTS.ls():\n",
    "    if \"AgainstCLS_bs_0\" in slug:\n",
    "        print(slug)\n",
    "        print(len(list(f for f in (WEIGHTS/slug).iterdir() if \"tar.gz\" in f.name or \".pkl\" in f.name)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_configs(tag, save):\n",
    "    config = AutoConfig.from_pretrained(tag)\n",
    "    tokenizer = AutoTokenizer.from_pretrained(tag)\n",
    "    tokenizer.save_pretrained(save/\"tokenizer\")\n",
    "    config.save_pretrained(save/\"config\")\n",
    "    return tokenizer, config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> 2 cuda devices found >>>\n",
      "Device 0: \n",
      "\tname:Tesla V100-PCIE-32GB\n",
      "\tused:20375MB\tfree:12135MB\n",
      "Device 1: \n",
      "\tname:Tesla V100-PCIE-32GB\n",
      "\tused:23231MB\tfree:9279MB\n",
      "cuda stats refreshed\n",
      "Found the most idle GPU: cuda:0, 12135 MB Mem remained\n"
     ]
    }
   ],
   "source": [
    "DEV = CudaHandler().idle().device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(\"train.csv\")\n",
    "test_df = pd.read_csv(\"test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "DO_PAIR = True\n",
    "BATCH_SIZE = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BaseDataset(Dataset):\n",
    "    def __init__(self, df, tokenizer):\n",
    "        self.df = df\n",
    "        self.indices = df.index\n",
    "        self.tokenizer = tokenizer\n",
    "\n",
    "    def __len__(self,): return len(self.df)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        row_ = dict(self.df.loc[self.indices[idx]])\n",
    "        row = dict((k, v) for k, v in row_.items()\n",
    "                   if k in [\"excerpt\", \"id\", \"target\"])\n",
    "        return row\n",
    "\n",
    "    def collate(self, batch):\n",
    "        batch = default_collate(batch)\n",
    "        batch[\"excerpt\"] = self.tokenizer(batch[\"excerpt\"],\n",
    "                                          truncation=True,\n",
    "                                          return_tensors='pt',\n",
    "                                          return_token_type_ids=False,\n",
    "                                          padding=True,\n",
    "                                          max_length=self.tokenizer.max_len_sentences_pair\n",
    "                                          )\n",
    "        if 'target' in batch:\n",
    "            batch['target'] = batch['target'].float()\n",
    "        return batch\n",
    "\n",
    "\n",
    "class PairDataset(BaseDataset):\n",
    "    def __init__(self, df, tokenizer):\n",
    "        super().__init__(df, tokenizer)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        row_ = dict(self.df.loc[self.indices[idx]])\n",
    "        row = dict((k, v) for k, v in row_.items()\n",
    "                   if k in [\"excerpt\", \"id\", \"target\"])\n",
    "        extra_text = dict(self.df.loc[random.choice(self.indices)])['excerpt']\n",
    "        row['excerpt'] = str(row['excerpt'])+\"[SEP]\"+str(extra_text)\n",
    "        return row\n",
    "\n",
    "\n",
    "def split_k(df, k: int = 5):\n",
    "    kmap = np.random.choice(range(k), size=len(df))\n",
    "    return dict((ki, dict(train=(kmap != ki), valid=(kmap == ki))) for ki in range(5))\n",
    "\n",
    "\n",
    "LitDataset = PairDataset if DO_PAIR else BaseDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_dl(df, tokenizer, ds_class, istrain = True) -> DataLoader:\n",
    "    \"\"\"\n",
    "    Create dataloader from a dataframe\n",
    "    \"\"\"\n",
    "    ds = ds_class(df, tokenizer)\n",
    "    dl = DataLoader(ds, shuffle=False,\n",
    "                    batch_size=BATCH_SIZE if istrain else BATCH_SIZE*2,num_workers=8,\n",
    "                    collate_fn = ds.collate)\n",
    "    return dl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {'train': array([ True,  True, False, ...,  True,  True,  True]),\n",
       "  'valid': array([False, False,  True, ..., False, False, False])},\n",
       " 1: {'train': array([ True,  True,  True, ...,  True,  True, False]),\n",
       "  'valid': array([False, False, False, ..., False, False,  True])},\n",
       " 2: {'train': array([ True, False,  True, ...,  True,  True,  True]),\n",
       "  'valid': array([False,  True, False, ..., False, False, False])},\n",
       " 3: {'train': array([False,  True,  True, ..., False,  True,  True]),\n",
       "  'valid': array([ True, False, False, ...,  True, False, False])},\n",
       " 4: {'train': array([ True,  True,  True, ...,  True, False,  True]),\n",
       "  'valid': array([False, False, False, ..., False,  True, False])}}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "splits = split_k(train_df, 5)\n",
    "splits\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AttentionHead(nn.Module):\n",
    "    def __init__(self, in_features, hidden_dim):\n",
    "        super().__init__()\n",
    "        self.in_features = in_features\n",
    "        self.middle_features = hidden_dim\n",
    "\n",
    "        self.W = nn.Linear(in_features, hidden_dim)\n",
    "        self.V = nn.Linear(hidden_dim, 1)\n",
    "        self.out_features = hidden_dim\n",
    "\n",
    "    def forward(self, features):\n",
    "        att = torch.tanh(self.W(features))\n",
    "        score = self.V(att)\n",
    "        attention_weights = torch.softmax(score, dim=1)\n",
    "        context_vector = attention_weights * features\n",
    "        context_vector = torch.sum(context_vector, dim=1)\n",
    "\n",
    "        return context_vector\n",
    "\n",
    "\n",
    "class BaseModel(nn.Module):\n",
    "    is_pair = False\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.is_acu_vec = False\n",
    "\n",
    "    def vec_plus(self, vec):\n",
    "        if self.is_acu_vec:\n",
    "            self.vecs.append(vec.clone().cpu().numpy())\n",
    "\n",
    "    def inference(\n",
    "        lit_model, model_weights: Path, data_loader: DataLoader, filename: str\n",
    "    ) -> None:\n",
    "        lit_model.load_state_dict(torch.load(str(model_weights)), strict=False)\n",
    "        lit_model = lit_model.eval()\n",
    "        lit_model = lit_model.cuda(DEV)\n",
    "        results = []\n",
    "        for batch_idx, batch in tqdm(enumerate(data_loader), leave=False, total=len(data_loader)):\n",
    "            ids = batch[\"id\"]\n",
    "            excerpt = batch[\"excerpt\"]\n",
    "            excerpt[\"input_ids\"] = excerpt[\"input_ids\"].cuda(DEV)\n",
    "            excerpt[\"attention_mask\"] = excerpt[\"attention_mask\"].cuda(DEV)\n",
    "            with torch.no_grad():\n",
    "                y_ = lit_model(**excerpt)[:, 0].detach().cpu().numpy()\n",
    "            results.append(pd.DataFrame({\"id\": ids, \"target\": y_}))\n",
    "        pd.concat(results).to_csv(filename, index=False)\n",
    "        lit_model.cpu()\n",
    "\n",
    "    def vec_train_pred(\n",
    "            lit_model, model_weights: Path, data_loader: DataLoader) -> None:\n",
    "        state_dict = torch.load(str(model_weights))\n",
    "        if \"state_dict\" in state_dict:\n",
    "            state_dict = state_dict[\"state_dict\"]\n",
    "        lit_model.load_state_dict(state_dict, strict=False)\n",
    "        lit_model = lit_model.eval()\n",
    "        lit_model = lit_model.cuda(DEV)\n",
    "\n",
    "        lit_model.vecs = []\n",
    "        lit_model.targets = []\n",
    "        lit_model.is_acu_vec = True\n",
    "        for batch_idx, batch in tqdm(enumerate(data_loader), leave=False, total=len(data_loader)):\n",
    "            ids = batch[\"id\"]\n",
    "            excerpt = batch[\"excerpt\"]\n",
    "            excerpt[\"input_ids\"] = excerpt[\"input_ids\"].cuda(DEV)\n",
    "            excerpt[\"attention_mask\"] = excerpt[\"attention_mask\"].cuda(DEV)\n",
    "            y = batch[\"target\"]\n",
    "            lit_model.targets.append(y)\n",
    "            with torch.no_grad():\n",
    "                vec = lit_model.get_vec(excerpt)\n",
    "                lit_model.vec_plus(vec)\n",
    "        lit_model.is_acu_vec = False\n",
    "        lit_model.cpu()\n",
    "        return lit_model.vecs\n",
    "\n",
    "    def vec_test_pred(\n",
    "        lit_model, model_weights: Path, data_loader: DataLoader, filename: str\n",
    "    ) -> None:\n",
    "        lit_model.load_state_dict(torch.load(str(model_weights)), strict=True)\n",
    "        lit_model = lit_model.eval()\n",
    "        lit_model = lit_model.cuda(DEV)\n",
    "        results = []\n",
    "        lit_model.vecs = []\n",
    "        lit_model.is_acu_vec = True\n",
    "        for batch_idx, batch in tqdm(\n",
    "            enumerate(data_loader),\n",
    "            leave=False,\n",
    "            total=len(data_loader)):\n",
    "            \n",
    "            ids = batch[\"id\"]\n",
    "            excerpt = batch[\"excerpt\"]\n",
    "            excerpt[\"input_ids\"] = excerpt[\"input_ids\"].cuda(DEV)\n",
    "            excerpt[\"attention_mask\"] = excerpt[\"attention_mask\"].cuda(DEV)\n",
    "            with torch.no_grad():\n",
    "                y_ = lit_model(**excerpt)[:, 0].detach().cpu().numpy()\n",
    "            results.append(pd.DataFrame({\"id\": ids, \"target\": y_}))\n",
    "        lit_model.is_acu_vec = False\n",
    "        pd.concat(results).to_csv(filename, index=False)\n",
    "        lit_model.cpu()\n",
    "        return lit_model.vecs\n",
    "\n",
    "\n",
    "class PairCLS(BaseModel):\n",
    "    is_pair = True\n",
    "\n",
    "    def __init__(self, base):\n",
    "        super().__init__()\n",
    "        self.base = base\n",
    "        self.config = self.base.config\n",
    "        self.reg = nn.Linear(base.config.hidden_size, 3)\n",
    "\n",
    "    def get_vec(self, excerpt: torch.Tensor) -> torch.Tensor:\n",
    "        return self.base(**excerpt).last_hidden_state[:, 0, :]\n",
    "\n",
    "    def forward(self, **excerpt: torch.Tensor) -> torch.Tensor:\n",
    "        vec = self.get_vec(excerpt)\n",
    "        self.vec_plus(vec)\n",
    "        return self.reg(vec)[:, :1]\n",
    "\n",
    "\n",
    "class AgainstCLS(BaseModel):\n",
    "    def __init__(self, base):\n",
    "        super().__init__()\n",
    "        self.base = base\n",
    "        self.config = self.base.config\n",
    "        self.reg = nn.Linear(base.config.hidden_size, 1)\n",
    "\n",
    "    def get_vec(self, excerpt):\n",
    "        return self.base(**excerpt).last_hidden_state[:, 0, :]\n",
    "\n",
    "    def forward(self, **excerpt: torch.Tensor) -> torch.Tensor:\n",
    "        vec = get_vec(**excerpt)\n",
    "        return self.reg(vec)\n",
    "\n",
    "\n",
    "class CLSReg(BaseModel):\n",
    "    def __init__(self, base):\n",
    "        super().__init__()\n",
    "        self.base = base\n",
    "        self.config = self.base.config\n",
    "        self.reg = nn.Linear(base.config.hidden_size, 1)\n",
    "\n",
    "    def forward(self, **excerpt: torch.Tensor) -> torch.Tensor:\n",
    "        vec = self.get_vec(excerpt)\n",
    "        self.vec_plus(vec)\n",
    "        return self.reg(vec)\n",
    "\n",
    "    def get_vec(self, excerpt: torch.Tensor) -> torch.Tensor:\n",
    "        return self.base(**excerpt).last_hidden_state[:, 0, :]\n",
    "\n",
    "\n",
    "class MeanPooler(BaseModel):\n",
    "    def __init__(self, base):\n",
    "        super().__init__()\n",
    "        self.base = base\n",
    "        self.config = self.base.config\n",
    "        self.reg = nn.Linear(base.config.hidden_size, 1)\n",
    "        self.dropout = nn.Dropout(.2)\n",
    "\n",
    "    def reduce_hidden_state(self, vec, attention_mask):\n",
    "        vec = vec*(attention_mask[..., None])\n",
    "        return vec.sum(dim=1)/(attention_mask.sum(-1)[..., None])\n",
    "\n",
    "    def forward(self, **excerpt: torch.Tensor) -> torch.Tensor:\n",
    "        vec = self.get_vec(excerpt)\n",
    "        self.vec_plus(vec)\n",
    "        return self.reg(vec)\n",
    "\n",
    "    def get_vec(self, excerpt):\n",
    "        attention_mask = excerpt[\"attention_mask\"]\n",
    "        vecs = self.base(**excerpt).last_hidden_state\n",
    "        return self.reduce_hidden_state(vecs, attention_mask)\n",
    "\n",
    "\n",
    "class WithAttnHead(BaseModel):\n",
    "    def __init__(self, base):\n",
    "        super().__init__()\n",
    "        self.base = base\n",
    "        self.config = self.base.config\n",
    "        self.head = AttentionHead(\n",
    "            self.config.hidden_size,\n",
    "            self.config.hidden_size)\n",
    "        self.dout = nn.Dropout(.1)\n",
    "        self.reg = nn.Linear(base.config.hidden_size, 1)\n",
    "\n",
    "    def forward(self, **excerpt: torch.Tensor) -> torch.Tensor:\n",
    "        vec = self.get_vec(excerpt)\n",
    "        self.vec_plus(vec)\n",
    "        return self.reg(self.dout(vec))\n",
    "\n",
    "    def get_vec(self, excerpt: torch.Tensor) -> torch.Tensor:\n",
    "        return self.head(self.base(**excerpt).last_hidden_state)\n",
    "\n",
    "\n",
    "def merge_subs(subs: List[pd.DataFrame]) -> pd.DataFrame:\n",
    "    concat = pd.concat(list(su.set_index(\"id\") for su in subs), 1)\n",
    "    return pd.DataFrame({\"id\": concat.index, \"target\": concat.values.mean(-1)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rmse(y_pred, y_true):\n",
    "    return np.sqrt(mean_squared_error(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def svming(lit_model, model_weights, train_dl, test_dl, weight_idx, fold_idx, C=10, kernel=\"rbf\"):\n",
    "    lit_model.vec_train_pred(model_weights, train_dl)\n",
    "    x = np.concatenate(lit_model.vecs)\n",
    "    y = np.concatenate(lit_model.targets)\n",
    "    svm = SVR(C=C, kernel=kernel, gamma='auto')\n",
    "    svm.fit(x, y)\n",
    "    lit_model.vec_test_pred(\n",
    "        model_weights, test_dl, filename=f\"sub_w{weight_idx}_f{fold_idx}.csv\")\n",
    "    x_test = np.concatenate(lit_model.vecs)\n",
    "    y_test = svm.predict(x_test)\n",
    "\n",
    "    return pd.DataFrame({\"id\": list(test_dl.dataset.df[\"id\"]), \"target\": y_test})\n",
    "\n",
    "\n",
    "def svming_folds(\n",
    "    lit_model, model_weights,\n",
    "    train_dl, weight_idx, C=10, kernel=\"rbf\"\n",
    "):\n",
    "    lit_model.vec_train_pred(model_weights, train_dl)\n",
    "    x = np.concatenate(lit_model.vecs)\n",
    "    y = np.concatenate(lit_model.targets)\n",
    "    print(x[:5,:5])\n",
    "    RMSEs = []\n",
    "    for k, tv in splits.items():\n",
    "        xtrain, ytrain = x[tv[\"train\"]], y[tv[\"train\"]]\n",
    "        xtest, ytest = x[tv[\"valid\"]], y[tv[\"valid\"]]\n",
    "        svm = SVR(C=C, kernel=kernel, gamma='auto')\n",
    "        svm.fit(xtrain, ytrain)\n",
    "        ytest_ = svm.predict(xtest)\n",
    "        with (model_weights.parent/(f\"{model_weights.name}_{k}.tar.gz\")).open(\"wb\") as f:\n",
    "            pickle.dump(svm, f)\n",
    "\n",
    "        RMSEs.append({\"pred\":ytest_,\"fold\":k})\n",
    "    svm = SVR(C=C, kernel=kernel, gamma='auto')\n",
    "    svm.fit(x, y)\n",
    "    with (model_weights.parent/(f\"{model_weights.name}.tar.gz\")).open(\"wb\") as f:\n",
    "        pickle.dump(svm, f)\n",
    "    return RMSEs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pred_svms(tag, save, ModelClass):\n",
    "    save_configs(tag, save)\n",
    "    tokenizer = AutoTokenizer.from_pretrained(tag, use_fast=True)\n",
    "    config = AutoConfig.from_pretrained(save/\"config\")\n",
    "\n",
    "    LitDataset = PairDataset if ModelClass.is_pair else BaseDataset\n",
    "\n",
    "    print(f\"Using datatset class {LitDataset}\")\n",
    "\n",
    "    test_dl = create_dl(test_df, tokenizer=tokenizer,\n",
    "                        ds_class=LitDataset, istrain=False)\n",
    "    train_dl = create_dl(train_df, tokenizer=tokenizer,\n",
    "                         ds_class=LitDataset, istrain=True)\n",
    "\n",
    "    model = AutoModel.from_config(config)\n",
    "    model.pooler = nn.Linear(model.config.hidden_size, 1)\n",
    "\n",
    "    lit_model = ModelClass(model)\n",
    "\n",
    "    WEIGHTS = list(p for p in Path(save).iterdir()\n",
    "                   if p.name[-3:] in [\".h5\", \"kpt\",\"bin\"])\n",
    "    print(WEIGHTS)\n",
    "    svm_preds = []\n",
    "    for i, weight_path in tqdm(list(enumerate(WEIGHTS))):\n",
    "        svm_preds += svming_folds(lit_model, weight_path,\n",
    "                                  train_dl,\n",
    "                                  weight_idx=i)\n",
    "    return pd.DataFrame(svm_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build SVR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using datatset class <class '__main__.BaseDataset'>\n",
      "[PosixPath('/nvme/GCI/public/lit/weights/AgainstCLS_bs_02_102534/epoch-10-val_loss-0.34fd3.h5'), PosixPath('/nvme/GCI/public/lit/weights/AgainstCLS_bs_02_102534/epoch-7-val_loss-0.41fd2.h5'), PosixPath('/nvme/GCI/public/lit/weights/AgainstCLS_bs_02_102534/epoch-7-val_loss-0.41fd0.h5'), PosixPath('/nvme/GCI/public/lit/weights/AgainstCLS_bs_02_102534/epoch-12-val_loss-0.39fd4.h5'), PosixPath('/nvme/GCI/public/lit/weights/AgainstCLS_bs_02_102534/epoch-7-val_loss-0.45fd1.h5')]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0578de54bac496f87aee106329bb831",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/89 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.12503955  0.17452016 -1.019507   -0.56013834  1.1429292 ]\n",
      " [-0.17213349  0.27361295 -1.009174   -0.52047145  1.2328502 ]\n",
      " [-0.18213859  0.25557533 -0.813734   -0.23477428  1.0825307 ]\n",
      " [-0.1810584   0.26151186 -0.80786914 -0.22588404  1.0849662 ]\n",
      " [-0.17913096  0.19482408 -1.0790204  -0.6535414   1.2507541 ]]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/89 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.6747504   0.72313976 -0.7012788  -1.3085343   1.5915488 ]\n",
      " [ 0.64635825  0.7711146  -0.60313773 -1.2180829   1.6681602 ]\n",
      " [ 0.52950925  0.7152321  -0.7374993  -1.218741    1.5057473 ]\n",
      " [ 0.35708237  0.74782324 -0.6837339  -1.3645672   1.1000643 ]\n",
      " [ 0.6409148   0.8033733  -0.59312594 -1.2932904   1.9611179 ]]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/89 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.14918463 -0.5457363   0.20787343 -0.58258057  1.383842  ]\n",
      " [-0.23047647 -0.4631032   0.21476956 -0.49833792  1.3364964 ]\n",
      " [-0.32225847 -0.666155    0.1871406  -0.47943082  1.2276831 ]\n",
      " [-0.3610151  -0.36879575 -0.10963945 -0.51517755  0.6386685 ]\n",
      " [-0.2515041  -0.4653018   0.02596207 -0.70190936  1.4101228 ]]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/89 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.11779767  0.6313695  -0.4589912  -0.4447339   1.7024349 ]\n",
      " [ 0.11148918  0.81869185 -0.43031383 -0.27749237  1.5798119 ]\n",
      " [ 0.04215658  0.6276091  -0.45620155 -0.26740983  1.4082147 ]\n",
      " [ 0.07083755  0.63235736 -0.59320426 -0.22151594  1.1896138 ]\n",
      " [ 0.0457306   0.6711476  -0.52248144 -0.34094188  1.9266204 ]]\n"
     ]
    }
   ],
   "source": [
    "svr_df = pred_svms(\n",
    "    tag = \"roberta-base\",\n",
    "    save = WEIGHTS/\"AgainstCLS_bs_02_102534\",\n",
    "    ModelClass=AgainstCLS,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def score_pred(df):\n",
    "    rmses = []\n",
    "    for fd in range(df.fold.max()+1):\n",
    "        stacked = np.stack(\n",
    "            list(df.query(f\"fold=={fd}\").pred.values)).mean(0)\n",
    "        rmses.append(\n",
    "            rmse(stacked,\n",
    "                 train_df[splits[fd][\"valid\"]][\"target\"].values)\n",
    "        )\n",
    "    return rmses\n",
    "\n",
    "def score_pred2(df):\n",
    "    rmses = []\n",
    "    for fd in range(df.fold.max()+1):\n",
    "        for idx, row in df.query(f\"fold=={fd}\").iterrows():\n",
    "            pred = row[\"pred\"]\n",
    "            rmses.append(\n",
    "            rmse(pred,\n",
    "                 train_df[splits[fd][\"valid\"]][\"target\"].values)\n",
    "        )\n",
    "    return rmses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = score_pred(svr_df)\n",
    "scores, sum(scores)/len(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "score_pred2(svr_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PairCLS_bs2pre_30_113040, 0.36924000923041045"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "CLSReg_bs2pre_29_170744"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['epoch=2-val_loss=0.53fd1.ckpt', 'epoch=7-val_loss=0.40fd0.ckpt']"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(filter(lambda x:x[-4:]==\"ckpt\",(WEIGHTS/\"AgainstCLS_bs_02_092503\").ls()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}