project_2_recommender_and_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "alike-morgan",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from IPython.display import Markdown, display, HTML\n",
    "from collections import defaultdict\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from livelossplot import PlotLosses\n",
    "\n",
    "# Fix the dying kernel problem (only a problem in some installations - you can remove it, if it works without it)\n",
    "import os\n",
    "os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "blessed-knitting",
   "metadata": {},
   "source": [
    "# Load the dataset for recommenders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "victorian-bottom",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>term</th>\n",
       "      <th>length_of_stay_bucket</th>\n",
       "      <th>rate_plan</th>\n",
       "      <th>room_segment</th>\n",
       "      <th>n_people_bucket</th>\n",
       "      <th>weekend_stay</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[260-360]</td>\n",
       "      <td>[5-inf]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[0-160]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>Easter</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[260-360]</td>\n",
       "      <td>[5-inf]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[260-360]</td>\n",
       "      <td>[5-inf]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>HighSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[1-1]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>HighSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[0-160]</td>\n",
       "      <td>[1-1]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>HighSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[1-1]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>HighSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[1-1]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>HighSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>11</td>\n",
       "      <td>9</td>\n",
       "      <td>HighSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>HighSeason</td>\n",
       "      <td>[8-inf]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>11</td>\n",
       "      <td>LowSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[5-inf]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_path = os.path.join(\"data\", \"hotel_data\")\n",
    "\n",
    "interactions_df = pd.read_csv(os.path.join(data_path, \"hotel_data_interactions_df.csv\"), index_col=0)\n",
    "\n",
    "base_item_features = ['term', 'length_of_stay_bucket', 'rate_plan', 'room_segment', 'n_people_bucket', 'weekend_stay']\n",
    "\n",
    "column_values_dict = {\n",
    "    'term': ['WinterVacation', 'Easter', 'OffSeason', 'HighSeason', 'LowSeason', 'MayLongWeekend', 'NewYear', 'Christmas'],\n",
    "    'length_of_stay_bucket': ['[0-1]', '[2-3]', '[4-7]', '[8-inf]'],\n",
    "    'rate_plan': ['Standard', 'Nonref'],\n",
    "    'room_segment': ['[0-160]', '[160-260]', '[260-360]', '[360-500]', '[500-900]'],\n",
    "    'n_people_bucket': ['[1-1]', '[2-2]', '[3-4]', '[5-inf]'],\n",
    "    'weekend_stay': ['True', 'False']\n",
    "}\n",
    "\n",
    "interactions_df.loc[:, 'term'] = pd.Categorical(\n",
    "    interactions_df['term'], categories=column_values_dict['term'])\n",
    "interactions_df.loc[:, 'length_of_stay_bucket'] = pd.Categorical(\n",
    "    interactions_df['length_of_stay_bucket'], categories=column_values_dict['length_of_stay_bucket'])\n",
    "interactions_df.loc[:, 'rate_plan'] = pd.Categorical(\n",
    "    interactions_df['rate_plan'], categories=column_values_dict['rate_plan'])\n",
    "interactions_df.loc[:, 'room_segment'] = pd.Categorical(\n",
    "    interactions_df['room_segment'], categories=column_values_dict['room_segment'])\n",
    "interactions_df.loc[:, 'n_people_bucket'] = pd.Categorical(\n",
    "    interactions_df['n_people_bucket'], categories=column_values_dict['n_people_bucket'])\n",
    "interactions_df.loc[:, 'weekend_stay'] = interactions_df['weekend_stay'].astype('str')\n",
    "interactions_df.loc[:, 'weekend_stay'] = pd.Categorical(\n",
    "    interactions_df['weekend_stay'], categories=column_values_dict['weekend_stay'])\n",
    "\n",
    "display(HTML(interactions_df.head(15).to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "realistic-third",
   "metadata": {},
   "source": [
    "# (Optional) Prepare numerical user features\n",
    "\n",
    "The method below is left here for convenience if you want to experiment with content-based user features as an input for your neural network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "variable-jaguar",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['user_term_WinterVacation', 'user_term_Easter', 'user_term_OffSeason', 'user_term_HighSeason', 'user_term_LowSeason', 'user_term_MayLongWeekend', 'user_term_NewYear', 'user_term_Christmas', 'user_length_of_stay_bucket_[0-1]', 'user_length_of_stay_bucket_[2-3]', 'user_length_of_stay_bucket_[4-7]', 'user_length_of_stay_bucket_[8-inf]', 'user_rate_plan_Standard', 'user_rate_plan_Nonref', 'user_room_segment_[0-160]', 'user_room_segment_[160-260]', 'user_room_segment_[260-360]', 'user_room_segment_[360-500]', 'user_room_segment_[500-900]', 'user_n_people_bucket_[1-1]', 'user_n_people_bucket_[2-2]', 'user_n_people_bucket_[3-4]', 'user_n_people_bucket_[5-inf]', 'user_weekend_stay_True', 'user_weekend_stay_False']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_term_WinterVacation</th>\n",
       "      <th>user_term_Easter</th>\n",
       "      <th>user_term_OffSeason</th>\n",
       "      <th>user_term_HighSeason</th>\n",
       "      <th>user_term_LowSeason</th>\n",
       "      <th>user_term_MayLongWeekend</th>\n",
       "      <th>user_term_NewYear</th>\n",
       "      <th>user_term_Christmas</th>\n",
       "      <th>user_length_of_stay_bucket_[0-1]</th>\n",
       "      <th>user_length_of_stay_bucket_[2-3]</th>\n",
       "      <th>user_length_of_stay_bucket_[4-7]</th>\n",
       "      <th>user_length_of_stay_bucket_[8-inf]</th>\n",
       "      <th>user_rate_plan_Standard</th>\n",
       "      <th>user_rate_plan_Nonref</th>\n",
       "      <th>user_room_segment_[0-160]</th>\n",
       "      <th>user_room_segment_[160-260]</th>\n",
       "      <th>user_room_segment_[260-360]</th>\n",
       "      <th>user_room_segment_[360-500]</th>\n",
       "      <th>user_room_segment_[500-900]</th>\n",
       "      <th>user_n_people_bucket_[1-1]</th>\n",
       "      <th>user_n_people_bucket_[2-2]</th>\n",
       "      <th>user_n_people_bucket_[3-4]</th>\n",
       "      <th>user_n_people_bucket_[5-inf]</th>\n",
       "      <th>user_weekend_stay_True</th>\n",
       "      <th>user_weekend_stay_False</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.681818</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.590909</td>\n",
       "      <td>0.409091</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.863636</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.772727</td>\n",
       "      <td>0.227273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>50</td>\n",
       "      <td>0.043478</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.434783</td>\n",
       "      <td>0.304348</td>\n",
       "      <td>0.217391</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.913043</td>\n",
       "      <td>0.086957</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.260870</td>\n",
       "      <td>0.739130</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.565217</td>\n",
       "      <td>0.434783</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.173913</td>\n",
       "      <td>0.521739</td>\n",
       "      <td>0.304348</td>\n",
       "      <td>0.782609</td>\n",
       "      <td>0.217391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>96</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.681818</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.636364</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.863636</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.590909</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.272727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>115</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371</th>\n",
       "      <td>706</td>\n",
       "      <td>0.095238</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.511905</td>\n",
       "      <td>0.190476</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.047619</td>\n",
       "      <td>0.011905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.297619</td>\n",
       "      <td>0.380952</td>\n",
       "      <td>0.238095</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.976190</td>\n",
       "      <td>0.023810</td>\n",
       "      <td>0.035714</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.107143</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.130952</td>\n",
       "      <td>0.154762</td>\n",
       "      <td>0.583333</td>\n",
       "      <td>0.130952</td>\n",
       "      <td>0.583333</td>\n",
       "      <td>0.416667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1383</th>\n",
       "      <td>1736</td>\n",
       "      <td>0.034483</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.482759</td>\n",
       "      <td>0.206897</td>\n",
       "      <td>0.275862</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.241379</td>\n",
       "      <td>0.551724</td>\n",
       "      <td>0.206897</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.172414</td>\n",
       "      <td>0.827586</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.931034</td>\n",
       "      <td>0.068966</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.379310</td>\n",
       "      <td>0.413793</td>\n",
       "      <td>0.206897</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.448276</td>\n",
       "      <td>0.551724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7301</th>\n",
       "      <td>7779</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def n_to_p(l):\n",
    "    n = sum(l)\n",
    "    return [x / n for x in l] if n > 0 else l\n",
    "\n",
    "def calculate_p(x, values):\n",
    "    counts = [0]*len(values)\n",
    "    for v in x:\n",
    "        counts[values.index(v)] += 1\n",
    "\n",
    "    return n_to_p(counts)\n",
    "\n",
    "def prepare_users_df(interactions_df):\n",
    "\n",
    "    users_df = interactions_df.loc[:, [\"user_id\"]]\n",
    "    users_df = users_df.groupby(\"user_id\").first().reset_index(drop=False)\n",
    "    \n",
    "    user_features = []\n",
    "\n",
    "    for column in base_item_features:\n",
    "\n",
    "        column_values = column_values_dict[column]\n",
    "        df = interactions_df.loc[:, ['user_id', column]]\n",
    "        df = df.groupby('user_id').aggregate(lambda x: list(x)).reset_index(drop=False)\n",
    "\n",
    "        def calc_p(x):\n",
    "            return calculate_p(x, column_values)\n",
    "\n",
    "        df.loc[:, column] = df[column].apply(lambda x: calc_p(x))\n",
    "\n",
    "        p_columns = []\n",
    "        for i in range(len(column_values)):\n",
    "            p_columns.append(\"user_\" + column + \"_\" + column_values[i])\n",
    "            df.loc[:, p_columns[i]] = df[column].apply(lambda x: x[i])\n",
    "            user_features.append(p_columns[i])\n",
    "\n",
    "        users_df = pd.merge(users_df, df.loc[:, ['user_id'] + p_columns], on=[\"user_id\"])\n",
    "    \n",
    "    return users_df, user_features\n",
    "    \n",
    "\n",
    "users_df, user_features = prepare_users_df(interactions_df)\n",
    "\n",
    "print(user_features)\n",
    "\n",
    "display(HTML(users_df.loc[users_df['user_id'].isin([706, 1736, 7779, 96, 1, 50, 115])].head(15).to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "amino-keyboard",
   "metadata": {},
   "source": [
    "# (Optional) Prepare numerical item features\n",
    "\n",
    "The method below is left here for convenience if you want to experiment with content-based item features as an input for your neural network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "formal-munich",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['term_WinterVacation', 'term_Easter', 'term_OffSeason', 'term_HighSeason', 'term_LowSeason', 'term_MayLongWeekend', 'term_NewYear', 'term_Christmas', 'length_of_stay_bucket_[0-1]', 'length_of_stay_bucket_[2-3]', 'length_of_stay_bucket_[4-7]', 'length_of_stay_bucket_[8-inf]', 'rate_plan_Standard', 'rate_plan_Nonref', 'room_segment_[0-160]', 'room_segment_[160-260]', 'room_segment_[260-360]', 'room_segment_[360-500]', 'room_segment_[500-900]', 'n_people_bucket_[1-1]', 'n_people_bucket_[2-2]', 'n_people_bucket_[3-4]', 'n_people_bucket_[5-inf]', 'weekend_stay_True', 'weekend_stay_False']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>term_WinterVacation</th>\n",
       "      <th>term_Easter</th>\n",
       "      <th>term_OffSeason</th>\n",
       "      <th>term_HighSeason</th>\n",
       "      <th>term_LowSeason</th>\n",
       "      <th>term_MayLongWeekend</th>\n",
       "      <th>term_NewYear</th>\n",
       "      <th>term_Christmas</th>\n",
       "      <th>length_of_stay_bucket_[0-1]</th>\n",
       "      <th>length_of_stay_bucket_[2-3]</th>\n",
       "      <th>length_of_stay_bucket_[4-7]</th>\n",
       "      <th>length_of_stay_bucket_[8-inf]</th>\n",
       "      <th>rate_plan_Standard</th>\n",
       "      <th>rate_plan_Nonref</th>\n",
       "      <th>room_segment_[0-160]</th>\n",
       "      <th>room_segment_[160-260]</th>\n",
       "      <th>room_segment_[260-360]</th>\n",
       "      <th>room_segment_[360-500]</th>\n",
       "      <th>room_segment_[500-900]</th>\n",
       "      <th>n_people_bucket_[1-1]</th>\n",
       "      <th>n_people_bucket_[2-2]</th>\n",
       "      <th>n_people_bucket_[3-4]</th>\n",
       "      <th>n_people_bucket_[5-inf]</th>\n",
       "      <th>weekend_stay_True</th>\n",
       "      <th>weekend_stay_False</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def map_items_to_onehot(df):\n",
    "    one_hot = pd.get_dummies(df.loc[:, base_item_features])\n",
    "    df = df.drop(base_item_features, axis = 1)\n",
    "    df = df.join(one_hot)\n",
    "    \n",
    "    return df, list(one_hot.columns)\n",
    "\n",
    "def prepare_items_df(interactions_df):\n",
    "    items_df = interactions_df.loc[:, [\"item_id\"] + base_item_features].drop_duplicates()\n",
    "    \n",
    "    items_df, item_features = map_items_to_onehot(items_df)\n",
    "    \n",
    "    return items_df, item_features\n",
    "\n",
    "\n",
    "items_df, item_features = prepare_items_df(interactions_df)\n",
    "\n",
    "print(item_features)\n",
    "\n",
    "display(HTML(items_df.loc[items_df['item_id'].isin([0, 1, 2, 3, 4, 5, 6])].head(15).to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5bf1d2c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "class NNM(nn.Module):\n",
    "\n",
    "    \n",
    "    def __init__(self, features, drop_out, neurons, seed=6789):\n",
    "        self.seed = torch.manual_seed(seed)\n",
    "        super(NNM, self).__init__()\n",
    "        \n",
    "        self.model = nn.Sequential(\n",
    "            nn.Linear(features, neurons),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(drop_out * 0.5),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(drop_out),\n",
    "            nn.Linear(neurons, 1)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.model(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "figured-imaging",
   "metadata": {},
   "source": [
    "# Neural network recommender\n",
    "\n",
    "<span style=\"color:red\"><font size=\"4\">**Task:**</font></span><br> \n",
    "Code a recommender based on a neural network model. You are free to choose any network architecture you find appropriate. The network can use the interaction vectors for users and items, embeddings of users and items, as well as user and item features (you can use the features you developed in the first project).\n",
    "\n",
    "Remember to keep control over randomness - in the init method add the seed as a parameter and initialize the random seed generator with that seed (both for numpy and pytorch):\n",
    "\n",
    "```python\n",
    "self.seed = seed\n",
    "self.rng = np.random.RandomState(seed=seed)\n",
    "```\n",
    "in the network model:\n",
    "```python\n",
    "self.seed = torch.manual_seed(seed)\n",
    "```\n",
    "\n",
    "You are encouraged to experiment with:\n",
    "  - the number of layers in the network, the number of neurons and different activation functions,\n",
    "  - different optimizers and their parameters,\n",
    "  - batch size and the number of epochs,\n",
    "  - embedding layers,\n",
    "  - content-based features of both users and items."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "unlike-recipient",
   "metadata": {},
   "outputs": [],
   "source": [
    "from recommenders.recommender import Recommender\n",
    "\n",
    "\n",
    "class NNRecommender(Recommender):\n",
    "    \"\"\"\n",
    "    Linear recommender class based on user and item features.\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, seed=6789, n_neg_per_pos=5, epochs=10, batch_size=100, drop_out = 0.5, learning_rate=0.001, neurons=100):\n",
    "        \"\"\"\n",
    "        Initialize base recommender params and variables.\n",
    "        \"\"\"\n",
    "        self.model = None\n",
    "        self.n_neg_per_pos = int(n_neg_per_pos)\n",
    "        self.epochs = int(epochs)\n",
    "        self.batch_size = int(batch_size)\n",
    "        self.drop_out = drop_out\n",
    "        self.learning_rate = learning_rate\n",
    "        self.neurons = int(neurons)\n",
    "        self.recommender_df = pd.DataFrame(columns=['user_id', 'item_id', 'score'])\n",
    "        self.users_df = None\n",
    "        self.user_features = None\n",
    "        self.seed = seed\n",
    "        self.rng = np.random.RandomState(seed=seed)\n",
    "    \n",
    "    def fit(self, interactions_df, users_df, items_df):\n",
    "        \"\"\"\n",
    "        Training of the recommender.\n",
    "        \n",
    "        :param pd.DataFrame interactions_df: DataFrame with recorded interactions between users and items \n",
    "            defined by user_id, item_id and features of the interaction.\n",
    "        :param pd.DataFrame users_df: DataFrame with users and their features defined by user_id and the user feature columns.\n",
    "        :param pd.DataFrame items_df: DataFrame with items and their features defined by item_id and the item feature columns.\n",
    "        \"\"\"\n",
    "        \n",
    "        interactions_df = interactions_df.copy()\n",
    "        \n",
    "        # Prepare users_df and items_df \n",
    "        # (optional - use only if you want to train a hybrid model with content-based features)\n",
    "        \n",
    "        users_df, user_features = prepare_users_df(interactions_df)\n",
    "        \n",
    "        self.users_df = users_df\n",
    "        self.user_features = user_features\n",
    "        \n",
    "        items_df, item_features = prepare_items_df(interactions_df)\n",
    "        items_df = items_df.loc[:, ['item_id'] + item_features]\n",
    "        \n",
    "        # Generate negative interactions\n",
    "        \n",
    "        interactions_df = interactions_df.loc[:, ['user_id', 'item_id']]\n",
    "        \n",
    "        interactions_df.loc[:, 'interacted'] = 1\n",
    "        \n",
    "        negative_interactions = []\n",
    "        \n",
    "        tuples = interactions_df.loc[:, ['user_id', 'item_id']]\n",
    "        \n",
    "        user_id_unique = interactions_df.user_id.unique()\n",
    "        item_id_unique = interactions_df.item_id.unique()\n",
    "        \n",
    "        while len(negative_interactions) < self.n_neg_per_pos * len(interactions_df):\n",
    "            random_user = np.random.choice(user_id_unique)\n",
    "            random_item = np.random.choice(item_id_unique)\n",
    "            \n",
    "            if (random_user, random_item) in tuples:\n",
    "                continue\n",
    "            negative_interactions.append((random_user, random_item, 0))\n",
    "        \n",
    "        interactions_df = pd.concat(\n",
    "            [interactions_df, pd.DataFrame(negative_interactions, columns=['user_id', 'item_id', 'interacted'])])\n",
    "        \n",
    "        # Merge user and item features\n",
    "        # (optional - use only if you want to train a hybrid model with content-based features)\n",
    "        \n",
    "        interactions_df = pd.merge(interactions_df, users_df, on=['user_id'])\n",
    "        interactions_df = pd.merge(interactions_df, items_df, on=['item_id'])\n",
    "        \n",
    "        x = interactions_df.loc[:, user_features + item_features].values\n",
    "        y = interactions_df['interacted'].values\n",
    "        \n",
    "        x_size = len(x[0])\n",
    "        y_size = y.shape[0]\n",
    "        \n",
    "        # Initialize the neural network model\n",
    "        \n",
    "        self.model = NNM(features=x_size, drop_out=self.drop_out, neurons=self.neurons, seed=self.seed)\n",
    "        self.loss = torch.nn.MSELoss()\n",
    "        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.learning_rate)\n",
    "        \n",
    "        # Train the model using an optimizer\n",
    "        for epoch in range(self.epochs):\n",
    "            for b in range((y_size-1)//self.batch_size + 1):\n",
    "                \n",
    "                start = b*self.batch_size\n",
    "                end = start+self.batch_size\n",
    "                \n",
    "                xb = x[start:end]\n",
    "                yb = y[start:end]\n",
    "                \n",
    "                X_train = torch.tensor(xb.astype(np.float32))\n",
    "                Y_train = torch.tensor(yb.astype(np.float32)).reshape(-1, 1)\n",
    "                \n",
    "                Y_predict = self.model(X_train)\n",
    "                \n",
    "                loss = self.loss(Y_predict, Y_train)\n",
    "                loss.backward()\n",
    "                \n",
    "                self.optimizer.step()\n",
    "                self.optimizer.zero_grad()\n",
    "            \n",
    "    \n",
    "    def recommend(self, users_df, items_df, n_recommendations=1):\n",
    "        \"\"\"\n",
    "        Serving of recommendations. Scores items in items_df for each user in users_df and returns \n",
    "        top n_recommendations for each user.\n",
    "        \n",
    "        :param pd.DataFrame users_df: DataFrame with users and their features for which recommendations should be generated.\n",
    "        :param pd.DataFrame items_df: DataFrame with items and their features which should be scored.\n",
    "        :param int n_recommendations: Number of recommendations to be returned for each user.\n",
    "        :return: DataFrame with user_id, item_id and score as columns returning n_recommendations top recommendations \n",
    "            for each user.\n",
    "        :rtype: pd.DataFrame\n",
    "        \"\"\"\n",
    "        \n",
    "        self.recommender_df = self.recommender_df[:0]\n",
    "        \n",
    "        users_df = users_df.loc[:, 'user_id']\n",
    "        users_df = pd.merge(users_df, self.users_df, on=['user_id'], how='left').fillna(0)\n",
    "        \n",
    "        items_df, item_features = prepare_items_df(items_df)\n",
    "        items_df = items_df.loc[:, ['item_id'] + item_features]\n",
    "    \n",
    "        recommendations = pd.DataFrame(columns=['user_id', 'item_id', 'score'])\n",
    "        \n",
    "        for ix, user in users_df.iterrows():\n",
    "            \n",
    "            user['one'] = 1\n",
    "            items_df['one'] = 1\n",
    "            user = user.to_frame().transpose()\n",
    "            \n",
    "            carthesian_user_items = user.merge(items_df, on='one').drop(\"one\", axis=1, errors='ignore')\n",
    "            \n",
    "            cart_list_predict = carthesian_user_items.loc[:, user_features + item_features].values\n",
    "            \n",
    "            self.model.eval() \n",
    "            scores = self.model(torch.tensor(cart_list_predict.astype(np.float32)))\n",
    "            scores = scores.detach().numpy()[:, 0]\n",
    "            \n",
    "            chosen_ids = np.argsort(-scores)[:n_recommendations]\n",
    "            \n",
    "            recommendations = []\n",
    "            for item_id in chosen_ids:\n",
    "                recommendations.append(\n",
    "                    {\n",
    "                        'user_id': user['user_id'].iloc[0],\n",
    "                        'item_id': item_id,\n",
    "                        'score': scores[item_id]\n",
    "                    }\n",
    "                )\n",
    "            \n",
    "            user_recommendations = pd.DataFrame(recommendations)\n",
    "\n",
    "            self.recommender_df = pd.concat([self.recommender_df, user_recommendations])\n",
    "\n",
    "        return self.recommender_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "copyrighted-relative",
   "metadata": {},
   "source": [
    "# Quick test of the recommender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "greatest-canon",
   "metadata": {},
   "outputs": [],
   "source": [
    "items_df = interactions_df.loc[:, ['item_id'] + base_item_features].drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "initial-capital",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit method\n",
    "nn_recommender = NNRecommender()\n",
    "nn_recommender.fit(interactions_df, None, None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "digital-consolidation",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>score</th>\n",
       "      <th>term</th>\n",
       "      <th>length_of_stay_bucket</th>\n",
       "      <th>rate_plan</th>\n",
       "      <th>room_segment</th>\n",
       "      <th>n_people_bucket</th>\n",
       "      <th>weekend_stay</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>32</td>\n",
       "      <td>0.509784</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>54</td>\n",
       "      <td>0.507627</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>50</td>\n",
       "      <td>0.421191</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>22</td>\n",
       "      <td>0.415719</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>21</td>\n",
       "      <td>0.367016</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "      <td>23</td>\n",
       "      <td>0.365034</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.0</td>\n",
       "      <td>60</td>\n",
       "      <td>0.354927</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.0</td>\n",
       "      <td>52</td>\n",
       "      <td>0.352862</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.0</td>\n",
       "      <td>14</td>\n",
       "      <td>0.284460</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.0</td>\n",
       "      <td>75</td>\n",
       "      <td>0.279727</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.782633</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2.0</td>\n",
       "      <td>22</td>\n",
       "      <td>0.698177</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2.0</td>\n",
       "      <td>112</td>\n",
       "      <td>0.618279</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2.0</td>\n",
       "      <td>158</td>\n",
       "      <td>0.612442</td>\n",
       "      <td>LowSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2.0</td>\n",
       "      <td>50</td>\n",
       "      <td>0.585715</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2.0</td>\n",
       "      <td>198</td>\n",
       "      <td>0.523757</td>\n",
       "      <td>LowSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2.0</td>\n",
       "      <td>14</td>\n",
       "      <td>0.517598</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2.0</td>\n",
       "      <td>9</td>\n",
       "      <td>0.490094</td>\n",
       "      <td>HighSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2.0</td>\n",
       "      <td>44</td>\n",
       "      <td>0.436437</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2.0</td>\n",
       "      <td>75</td>\n",
       "      <td>0.433439</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.603164</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>3.0</td>\n",
       "      <td>44</td>\n",
       "      <td>0.523331</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>3.0</td>\n",
       "      <td>21</td>\n",
       "      <td>0.502149</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>3.0</td>\n",
       "      <td>32</td>\n",
       "      <td>0.459086</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>3.0</td>\n",
       "      <td>456</td>\n",
       "      <td>0.450534</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>3.0</td>\n",
       "      <td>228</td>\n",
       "      <td>0.434105</td>\n",
       "      <td>LowSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>3.0</td>\n",
       "      <td>373</td>\n",
       "      <td>0.432582</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>3.0</td>\n",
       "      <td>52</td>\n",
       "      <td>0.421232</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>3.0</td>\n",
       "      <td>59</td>\n",
       "      <td>0.418327</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.405439</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>4.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.782248</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>4.0</td>\n",
       "      <td>14</td>\n",
       "      <td>0.740720</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.732920</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>4.0</td>\n",
       "      <td>22</td>\n",
       "      <td>0.684023</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>4.0</td>\n",
       "      <td>50</td>\n",
       "      <td>0.607851</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>4.0</td>\n",
       "      <td>112</td>\n",
       "      <td>0.602840</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>4.0</td>\n",
       "      <td>124</td>\n",
       "      <td>0.600527</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>4.0</td>\n",
       "      <td>75</td>\n",
       "      <td>0.597428</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>4.0</td>\n",
       "      <td>12</td>\n",
       "      <td>0.567538</td>\n",
       "      <td>LowSeason</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>4.0</td>\n",
       "      <td>69</td>\n",
       "      <td>0.520004</td>\n",
       "      <td>HighSeason</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[3-4]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>5.0</td>\n",
       "      <td>67</td>\n",
       "      <td>0.608276</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0.582900</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[0-160]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>5.0</td>\n",
       "      <td>81</td>\n",
       "      <td>0.540301</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[0-160]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>5.0</td>\n",
       "      <td>458</td>\n",
       "      <td>0.536994</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[0-160]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>5.0</td>\n",
       "      <td>23</td>\n",
       "      <td>0.515021</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>5.0</td>\n",
       "      <td>258</td>\n",
       "      <td>0.507356</td>\n",
       "      <td>HighSeason</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[0-160]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>5.0</td>\n",
       "      <td>475</td>\n",
       "      <td>0.481367</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[0-160]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>5.0</td>\n",
       "      <td>74</td>\n",
       "      <td>0.466428</td>\n",
       "      <td>OffSeason</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[0-160]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>5.0</td>\n",
       "      <td>84</td>\n",
       "      <td>0.464506</td>\n",
       "      <td>HighSeason</td>\n",
       "      <td>[4-7]</td>\n",
       "      <td>Standard</td>\n",
       "      <td>[160-260]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>5.0</td>\n",
       "      <td>504</td>\n",
       "      <td>0.459412</td>\n",
       "      <td>WinterVacation</td>\n",
       "      <td>[2-3]</td>\n",
       "      <td>Nonref</td>\n",
       "      <td>[0-160]</td>\n",
       "      <td>[2-2]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Recommender method\n",
    "\n",
    "recommendations = nn_recommender.recommend(pd.DataFrame([[1], [2], [3], [4], [5]], columns=['user_id']), items_df, 10)\n",
    "\n",
    "recommendations = pd.merge(recommendations, items_df, on='item_id', how='left')\n",
    "display(HTML(recommendations.to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "advanced-eleven",
   "metadata": {},
   "source": [
    "# Tuning method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "strange-alaska",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation_and_testing.testing import evaluate_train_test_split_implicit\n",
    "\n",
    "seed = 6789"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "stable-theta",
   "metadata": {},
   "outputs": [],
   "source": [
    "from hyperopt import hp, fmin, tpe, Trials\n",
    "import traceback\n",
    "\n",
    "def tune_recommender(recommender_class, interactions_df, items_df, \n",
    "                     param_space, max_evals=1, show_progressbar=True, seed=6789):\n",
    "    # Split into train_validation and test sets\n",
    "\n",
    "    shuffle = np.arange(len(interactions_df))\n",
    "    rng = np.random.RandomState(seed=seed)\n",
    "    rng.shuffle(shuffle)\n",
    "    shuffle = list(shuffle)\n",
    "\n",
    "    train_test_split = 0.8\n",
    "    split_index = int(len(interactions_df) * train_test_split)\n",
    "\n",
    "    train_validation = interactions_df.iloc[shuffle[:split_index]]\n",
    "    test = interactions_df.iloc[shuffle[split_index:]]\n",
    "\n",
    "    # Tune\n",
    "\n",
    "    def loss(tuned_params):\n",
    "        recommender = recommender_class(seed=seed, **tuned_params)\n",
    "        hr1, hr3, hr5, hr10, ndcg1, ndcg3, ndcg5, ndcg10 = evaluate_train_test_split_implicit(\n",
    "            recommender, train_validation, items_df, seed=seed)\n",
    "        return -hr10\n",
    "\n",
    "    n_tries = 1\n",
    "    succeded = False\n",
    "    try_id = 0\n",
    "    while not succeded and try_id < n_tries:\n",
    "        try:\n",
    "            trials = Trials()\n",
    "            best_param_set = fmin(loss, space=param_space, algo=tpe.suggest, \n",
    "                                  max_evals=max_evals, show_progressbar=show_progressbar, trials=trials, verbose=True)\n",
    "            succeded = True\n",
    "        except:\n",
    "            traceback.print_exc()\n",
    "            try_id += 1\n",
    "            \n",
    "    if not succeded:\n",
    "        return None\n",
    "        \n",
    "    # Validate\n",
    "    \n",
    "    recommender = recommender_class(seed=seed, **best_param_set)\n",
    "\n",
    "    results = [[recommender_class.__name__] + list(evaluate_train_test_split_implicit(\n",
    "        recommender, {'train': train_validation, 'test': test}, items_df, seed=seed))]\n",
    "\n",
    "    results = pd.DataFrame(results, \n",
    "                           columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "    display(HTML(results.to_html()))\n",
    "    \n",
    "    return best_param_set"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "reliable-switzerland",
   "metadata": {},
   "source": [
    "## Tuning of the recommender\n",
    "\n",
    "<span style=\"color:red\"><font size=\"4\">**Task:**</font></span><br> \n",
    "Tune your model using the code below. You only need to put the class name of your recommender and choose an appropriate parameter space."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "obvious-astrology",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 400/400 [12:44:35<00:00, 114.69s/trial, best loss: -0.25222175201015656]  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NNRecommender</td>\n",
       "      <td>0.057026</td>\n",
       "      <td>0.144603</td>\n",
       "      <td>0.190767</td>\n",
       "      <td>0.256959</td>\n",
       "      <td>0.057026</td>\n",
       "      <td>0.107081</td>\n",
       "      <td>0.126323</td>\n",
       "      <td>0.147658</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters:\n",
      "{'batch_size': 400.0, 'drop_out': 0.7000000000000001, 'epochs': 40.0, 'learning_rate': 0.001, 'n_neg_per_pos': 20.0, 'neurons': 200.0}\n"
     ]
    }
   ],
   "source": [
    "param_space = {\n",
    "    'n_neg_per_pos': hp.quniform('n_neg_per_pos', 15, 20, 2),\n",
    "    'epochs': hp.quniform('epochs', 20, 50, 20),\n",
    "    'batch_size': hp.quniform('batch_size', 200, 500, 200),\n",
    "    'drop_out' : hp.quniform('drop_out', 0.1, 1, 0.1),\n",
    "    'learning_rate': hp.quniform('learning_rate', 0.0005, 0.005, 0.0005),\n",
    "    'neurons': hp.quniform('neurons', 100, 300, 100)\n",
    "}\n",
    "\n",
    "best_param_set = tune_recommender(NNRecommender, interactions_df, items_df,\n",
    "                                  param_space, max_evals=400, show_progressbar=True, seed=seed)\n",
    "\n",
    "print(\"Best parameters:\")\n",
    "print(best_param_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "accredited-strap",
   "metadata": {},
   "source": [
    "# Final evaluation\n",
    "\n",
    "<span style=\"color:red\"><font size=\"4\">**Task:**</font></span><br> \n",
    "Run the final evaluation of your recommender and present its results against the Amazon and Netflix recommenders' results. You just need to give the class name of your recommender and its tuned parameters below.\n",
    "\n",
    "It's optional, but for better effect you can include here the results from all recommenders created during in this class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "given-homework",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NNRecommender</td>\n",
       "      <td>0.058384</td>\n",
       "      <td>0.143924</td>\n",
       "      <td>0.194162</td>\n",
       "      <td>0.269857</td>\n",
       "      <td>0.058384</td>\n",
       "      <td>0.106621</td>\n",
       "      <td>0.126814</td>\n",
       "      <td>0.151168</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn_recommender = NNRecommender(n_neg_per_pos=18, epochs=20, batch_size=400, drop_out=0.7000000000000001, learning_rate=0.0005, neurons=300)  # Initialize your recommender here\n",
    "\n",
    "# Give the name of your recommender in the line below\n",
    "nn_tts_results = [['NNRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    nn_recommender, interactions_df, items_df))]\n",
    "\n",
    "nn_tts_results = pd.DataFrame(\n",
    "    nn_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(nn_tts_results.to_html()))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "suited-nomination",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AmazonRecommender</td>\n",
       "      <td>0.044128</td>\n",
       "      <td>0.118805</td>\n",
       "      <td>0.160896</td>\n",
       "      <td>0.223693</td>\n",
       "      <td>0.044128</td>\n",
       "      <td>0.086755</td>\n",
       "      <td>0.104347</td>\n",
       "      <td>0.124505</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from recommenders.amazon_recommender import AmazonRecommender\n",
    "\n",
    "amazon_recommender = AmazonRecommender()\n",
    "\n",
    "amazon_tts_results = [['AmazonRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    amazon_recommender, interactions_df, items_df))]\n",
    "\n",
    "amazon_tts_results = pd.DataFrame(\n",
    "    amazon_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(amazon_tts_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "conservative-remedy",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAb8AAAI4CAYAAAD6TuePAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABR/ElEQVR4nO3dd3xUVf7/8ddnJo2E0AOEXqX3UBQLiLqAq9jFjq6i7rp+3fL96Vbd/X53v+6uuq5rQbCsa8Ou7C72ClIkSG8Seggl9J56fn/cCQ4YYAIhd5L7fj4e85iZe+9MPjPGvDnnnnuOOecQEREJkpDfBYiIiFQ1hZ+IiASOwk9ERAJH4SciIoGj8BMRkcBR+ImISOAo/EREJHAUfiJxyMxWm9k5ftchUlMp/EREJHAUfiLVhJklm9nDZpYXuT1sZsmRfY3M7N9mtsPMtpnZFDMLRfbdbWbrzWy3mS0zs2H+fhIR/yX4XYCIxOxXwCCgN+CAd4BfA78BfgbkAhmRYwcBzsw6AXcA/Z1zeWbWBghXbdki8UctP5Hq4xrg9865zc65fOB3wHWRfUVAJtDaOVfknJvivIl7S4BkoKuZJTrnVjvnVvhSvUgcUfiJVB/NgDVRz9dEtgH8BcgBPjCzlWZ2D4BzLge4C7gP2GxmE82sGSIBp/ATqT7ygNZRz1tFtuGc2+2c+5lzrh1wAfDTsnN7zrmXnHOnR17rgD9Vbdki8UfhJxK/Es0spewGvAz82swyzKwR8FvgBQAz+76ZdTAzA3bhdXeWmFknMzs7MjDmALA/sk8k0BR+IvFrMl5Yld1SgGxgPrAA+Br438ixHYGPgD3AdOBx59xneOf77ge2ABuBxsAvq+wTiMQp02K2IiISNGr5iYhI4Cj8REQkcBR+IiISODGFn5kNj0yLlFN2/dBh+68xs/mR2zQz6xXZ3snM5kbddpnZXZF990WmXCrbN7JSP5mIiMgRHHPAi5mFgW+Ac/GmT5oFXOWcWxx1zGnAEufcdjMbAdznnBtYzvusBwY659aY2X3AHufcA7EW26hRI9emTZtYDxcRkQCbPXv2FudcRnn7YpnbcwCQ45xbCWBmE4FRwMHwc85Nizp+BtCinPcZBqxwzq0pZ19M2rRpQ3Z29vG+XEREAsTMjpg3sXR7NgfWRT3PjWw7kh8A75azfTTeRbrR7oh0lT5jZvVjqEVEROSExRJ+Vs62cvtKzWwoXvjdfdj2JOBC4LWozU8A7fFmqN8APHiE9xxrZtlmlp2fnx9DuSIiIkcXS/jlAi2jnrcgMp9gNDPrCTwFjHLObT1s9wjga+fcprINzrlNzrkS51wpMAGve/U7nHPjnXNZzrmsjIxyu25FREQqJJbwmwV0NLO2kRbcaGBS9AFm1gp4E7jOOfdNOe9xFYd1eZpZZtTTi4GFFSlcRETkeB1zwItzrtjM7gDex1sE8xnn3CIzuy2yfxzeBLsNgce9eXUpds5lAZhZKt5I0VsPe+s/m1lvvC7U1eXsFxEROSmq1dyeWVlZTqM9RUQkFmY2u6whdjjN8CIiIoGj8BMRkcBR+ImISOAo/EREJHAUfiIiEjgKPxERCRyFn4iIBI7CT0REAkfhJyIigaPwExGRwFH4iYhI4Cj8REQkcBR+IiISOAo/EREJnOCFX242rPvK7ypERMRHx1zMtsb56D5wpXDjZL8rERERnwSv5RcKQ0mR31WIiIiPAhh+CVBa7HcVIiLiowCGX6LCT0Qk4AIYfmGFn4hIwAUv/MJq+YmIBF3wwi+UoAEvIiIBF8zwKy3xuwoREfFRQMNP3Z4iIkEW0PBTt6eISJAFL/w04EVEJPCCF36hBChR+ImIBFkAw0/X+YmIBF0Aw0/dniIiQRfA8IsMeHHO70pERMQnwQu/cKJ370r9rUNERHwTvPALhb17zfIiIhJYAQy/yPq9Ou8nIhJYAQy/SLenwk9EJLACGH5q+YmIBF3wwi+s8BMRCbrghV9Zy08DXkREAiu44aeWn4hIYAUw/MoGvGhNPxGRoApg+EWu89OyRiIigRW88AvrUgcRkaALXvhpwIuISOAFN/x0zk9EJLACHH7q9hQRCaoAh5+6PUVEgip44acBLyIigRe88Ds44EXhJyISVAEMv7Lr/BR+IiJBFcDwU7eniEjQBTD8NOBFRCToYgo/MxtuZsvMLMfM7iln/zVmNj9ym2ZmvaL2rTazBWY218yyo7Y3MLMPzWx55L5+5XykYwhrbk8RkaA7ZviZWRh4DBgBdAWuMrOuhx22CjjLOdcT+B9g/GH7hzrnejvnsqK23QN87JzrCHwceX7y6ZyfiEjgxdLyGwDkOOdWOucKgYnAqOgDnHPTnHPbI09nAC1ieN9RwHORx88BF8VU8YnS9GYiIoEXS/g1B9ZFPc+NbDuSHwDvRj13wAdmNtvMxkZtb+Kc2wAQuW9c3puZ2Vgzyzaz7Pz8/BjKPQYNeBERCbyEGI6xcra5cg80G4oXfqdHbR7snMszs8bAh2a21Dn3RawFOufGE+lGzcrKKvfnVoimNxMRCbxYWn65QMuo5y2AvMMPMrOewFPAKOfc1rLtzrm8yP1m4C28blSATWaWGXltJrD5eD5AhYUVfiIiQRdL+M0COppZWzNLAkYDk6IPMLNWwJvAdc65b6K2p5lZetlj4DxgYWT3JOCGyOMbgHdO5IPETC0/EZHAO2a3p3Ou2MzuAN4HwsAzzrlFZnZbZP844LdAQ+BxMwMojozsbAK8FdmWALzknHsv8tb3A6+a2Q+AtcDllfrJjkQDXkREAi+Wc3445yYDkw/bNi7q8c3AzeW8biXQ6/DtkX1bgWEVKbZShHSdn4hI0AVwhpcQYJrhRUQkwIIXfuDN8qJzfiIigRXM8AslKPxERAIsuOGn9fxERAIruOGnlp+ISGAFOPw04EVEJKgCHH5q+YmIBFUwwy+coOv8REQCLJjhF0rQDC8iIgEW0PDTdX4iIkEW0PDTOT8RkSALaPiFFX4iIgEWzPDT9GYiIoEWzPDTgBcRkUALaPgl6lIHEZEAC2j4hTXDi4hIgAU0/DTaU0QkyIIZfhrwIiISaMEMPy1pJCISaMENP7X8REQCK8DhpwEvIiJBFeDwU8tPRCSoghl+WtJIRCTQAhd+D3/0DfPz9mqGFxGRAAtc+M1cuY283cXq9hQRCbDAhV9C2ChyIYWfiEiABS78EsMhihV+IiKBFrjwC4eMQqf1/EREgixw4ZdY1u2pAS8iIoEVuPALh0IUEQYclJb6XY6IiPggcOGXGDIKSyMfW7O8iIgEUuDCLxyKdHuCzvuJiARU4MIvIRzyBryAwk9EJKACF34HB7yAljUSEQmowIVfOGQUlKrbU0QkyAIXfonhUNQ5Pw14EREJosCFX/iQ0Z5q+YmIBFHgwi8xZBw4GH5a1khEJIgCF37hUIgSIqM9NcuLiEggBS78EsIWmeEFdXuKiARU4MIvMWzftvw04EVEJJACF37hUIhidM5PRCTIAhd+iWGjmATvibo9RUQCKXDhFw4ZJWUfWwNeREQCKXDhlxgKUaS5PUVEAi1w4ee1/MrCT+f8RESCKHDhlxA2ijXaU0Qk0AIXfonhUFT4qdtTRCSIAhd+4VBUy08DXkREAilw4edd5K7r/EREgiym8DOz4Wa2zMxyzOyecvZfY2bzI7dpZtYrsr2lmX1qZkvMbJGZ/VfUa+4zs/VmNjdyG1l5H+vIwqGQpjcTEQm4hGMdYGZh4DHgXCAXmGVmk5xzi6MOWwWc5ZzbbmYjgPHAQKAY+Jlz7mszSwdmm9mHUa/9q3Pugcr8QMeSGDJKnAa8iIgEWSwtvwFAjnNupXOuEJgIjIo+wDk3zTm3PfJ0BtAisn2Dc+7ryOPdwBKgeWUVfzzCIU1sLSISdLGEX3NgXdTzXI4eYD8A3j18o5m1AfoAM6M23xHpKn3GzOqX92ZmNtbMss0sOz8/P4Zyjy4hHL2kkcJPRCSIYgk/K2ebK/dAs6F44Xf3YdtrA28AdznndkU2PwG0B3oDG4AHy3tP59x451yWcy4rIyMjhnKPzpvbUyu5i4gEWSzhlwu0jHreAsg7/CAz6wk8BYxyzm2N2p6IF3wvOufeLNvunNvknCtxzpUCE/C6V08671IHTWwtIhJksYTfLKCjmbU1syRgNDAp+gAzawW8CVznnPsmarsBTwNLnHMPHfaazKinFwMLj+8jVExiOBR1qYMGvIiIBNExR3s654rN7A7gfSAMPOOcW2Rmt0X2jwN+CzQEHvfyjmLnXBYwGLgOWGBmcyNv+Uvn3GTgz2bWG68LdTVwayV+riPyBryUtfx0nZ+ISBAdM/wAImE1+bBt46Ie3wzcXM7rplL+OUOcc9dVqNJKkhgKUVpWkmZ4EREJpMDN8BIOG2CUWoLO+YmIBFTgwi8x5LX6FH4iIsEVuPALHwy/sMJPRCSgAhd+CWHvI6vlJyISXIELv8RwVMtPA15ERAIpcOFX1u1ZHEqG4gKfqxERET8ELvwSQ95HLgqlQNE+n6sRERE/BC78QiHDDIpCyVC03+9yRETEB4ELP/Baf2r5iYgEVyDDLxwyCk0tPxGRoApk+CWEjUJLUfiJiARUTHN71jSJ4RCFoWR1e4pIzeKc93etYDcUH4CiA959cUHkPnJz5S7J6gmFIZwMCUmH3UduyemQlA7h6h0f1bv64xQOGQWWDIVq+YlIHHMOCnbBrg2wewPs3vjt/b6tcGAH7N8Rdb+z6pZqS0yFpNpeGJbdatWHtEaQ2hBSy+4bRG1rCIm1qqa+Ywhk+CWGjAJ0zk9E4kDRAdi+GrathO2rvPttK71tuzeW30OVXMcLklr1IKUe1GsJKXW9x7XqefsTa0VaaymH3ZLAwkeup7QYSgq91mJJARQXfntfvB8K9ngty4JdUFj2OHLLXwZrpsH+beBKy3//5DqQngnpTaFOM+9x2X16JtTJhLTGJ71lGcjwC4eNA6ZuTxGpQkUHIH8pbF4MmxZ5ty3LYdd6vGVNI1LqQoN2kNkLOo38NigO3jeFpDTfPkZMSku8VujeLV4LdV/kfu8W2LMZdud5rdlVU2DPxu9ONVm7Kfx82UktMZDhlxgKcYBkr3ugpAjCiX6XJCI1yYGdsP5rWD8bNi7wAm9rzretoYQUyOgMbU73gu7gra3XTVjdhcLe54jls5SWwt58LxB3b4RdeUduNVaiQIZfuKzbE7yuT4WfiByvkmIv3NZnQ+5s7z5/GQdbc/XbQONu0PUiaNIVmnT3gi50lK7HIAmFIL2Jd6tCgQy/hHCI/dHhl1LH34JEpPooKYK8ubD6C6/bbt1XULTX21erAbToD90vheb9vFuten5WK0cQyPBLDBsHXJL3ROf9RORoSktgw1wv6FZPgbUzvIEeAI27Qu+roeVAaNEP6rcFM1/LldgEMvzCIWN/SVn4acSniBxm7xbI+RiWfwA5H3mXEgA0OgV6Xgltz4DWp0PtDF/LlOMXyPBLDIXYXxzV7SkiwVZaChvnwfIPvcDLzQYcpGV4Iy47DPMGp6Q39btSqSSBDL9wyNh3sNtzr7/FiIg/Sopg9VRYMgmW/gf2bAIMmveFIb+AjudCZm9vQIbUOIEMv4RwdPip5ScSGMUFsOJTL/CWTYb9272ZSjqe67Xw2g9TV2ZABDP8Dmn5acCLSI1WXAg5H8LCN+Gb96FwNyTXhU7DocuFXpdmnEy5JVUnmOEXDrG3VC0/kRqrtBTWToP5r8Lid7wBK7UaQLeLoOsoaHuWN82XBFYgwy8xbOxUy0+kZnEONi30Am/hG960YYlp0Pl86HkFtBuiCS3koECGXzgUYq/O+YnUDHu3wPxXYM4L3kwroQTv3N25v4dOI+J/HkzxRSDDLzFk7C2NfHSFn0j1U1riXYc353lY9q43T2/zfjDyAeh2CaQ19LtCiXOBDL9wyCgoDUMoUd2eItXJtpUw50WY+5I3EXJqQxgwFvpc682bKRKjQIZfQjhEUYnzhjir5ScS34oOeINW5jzvTS9mIa9bc8T9cMoIDVyR4xLM8AsZJaWlkFJLLT+ReLVtJWQ/47X09m/z5s08+zfQ6yqo29zv6qSaC2b4hY3iEudd26OWn0j8KCmG5e/DrKdhxcfeiuOdz4f+P4A2Z2q2Fak0gQy/xHCI4lJ1e4rEjd2b4Ot/wuxnvUsU0pvBkF9C3+uhTqbf1UkNFMjwC4eM4tLSSMtP3Z4ivnAO1s2EmeNgyb+gtBjaDYURf/LO5YUD+edJqkggf7sSQxZp+anbU6TKlRR5A1hmPA7rZ0NKPRh4G2TdBA3b+12dBEQgwy8cCuEcuMRUbM9Gv8sRCYb9O+Dr52Dmk17XZoP23nV5va/WhehS5QIZfglhb6Xl0oQUwmr5iZxcW1d4XZtzXvSWEGtzBpz/EHQ8TwNYxDfBDL9QWfjVUviJnAzOwZppMP0xb+mgUAL0uAwG/RAye/pdnUhAwy/s/WuzNKwBLyKVqrgQFr0FMx6DDfO8lRTO/Dn0v1mroEtcCWT4JUZ1e2rAi0gl2LfNu0zhqwmwewM0OgW+/1foORqSUv2uTuQ7Ahl+4Ui3Z0lZy885MPO5KpFqaEuON2pz3sve/0vthsCFf/emH9P5PIljgQy/xMj/lCUJkdWbiw9oJWeRWDnnzbE5/TH45j0IJ3nr5Q36ITTp5nd1IjEJZPh92/JL9jYU7Vf4iRxLcYG3SOz0x2HTAkhtBGfd4009Vrux39WJVEggw6/sUofiUCTwivYBDfwrSCSe7d3qTTA9awLs2QQZnb2uzR5XQGKK39WJHJdghl+k27M4HPkfV4NeRL4rf1nkfN5E79RA+2Fw0RPQ/mydI5dqL5jhF2n5FYXKwk+XO4gA3vm8lZ96XZs5H0I4GXqN9s7nNe7sd3UilSaY4Rcq6/ZUy08E8BaMXfCa19LbvBjSGsPQX3nzbaY18rs6kUoXzPCLXORedHDAi1p+ElB78iH7aZj1FOzNhybdYdTj3mwsCcl+Vydy0gQy/BIjLb8iU8tPAmrTYm8WlvmvQUkBdPwenPpDaHuWzudJIAQy/MoudShSt6cESWkprPgEpj/qnddLqAV9roGBt0PGKX5XJ1KlYpqCwcyGm9kyM8sxs3vK2X+Nmc2P3KaZWa9jvdbMGpjZh2a2PHJfv3I+0rGVdXsWmLo9JQAK98Ksp+HxgfDipbB5CZz9G/jpYm8KMgWfBNAxW35mFgYeA84FcoFZZjbJObc46rBVwFnOue1mNgIYDww8xmvvAT52zt0fCcV7gLsr88MdScLBll8k/AoVflID7VwPX42H2f+AAzsgsxdc/CR0uwQSkvyuTsRXsXR7DgBynHMrAcxsIjAKOBh+zrlpUcfPAFrE8NpRwJDIcc8Bn1FV4Re51GF/qLa34cCOqvixIlUjN9sbtbnobcBB5+97lyq0GqTzeSIRsYRfc2Bd1PNcYOBRjv8B8G4Mr23inNsA4JzbYGblzo9kZmOBsQCtWrWKodxjO3iROwmQUs8b5SZSnZUUwZJJMOMJyJ0FyXVg0O0wYCzUb+13dSJxJ5bwK++fiq7cA82G4oXf6RV97ZE458bjdaOSlZVVodceycHpzUpLIS0D9m6pjLcVqXr7tsHXz3lLCe1aDw3awYi/QO+rIDnd7+pE4lYs4ZcLtIx63gLIO/wgM+sJPAWMcM5tjeG1m8wsM9LqywQ2V7T441W2qkNxifMu4FX4SXWzeYl3Pm/uy1C8H9qeCec/6F2yoKWERI4plvCbBXQ0s7bAemA0cHX0AWbWCngTuM45902Mr50E3ADcH7l/5wQ+R4WED2n5NYIty6vqR4scv5IiWPpv+OopWDPVm3qsx+Ve92bT7n5XJ1KtHDP8nHPFZnYH8D4QBp5xzi0ys9si+8cBvwUaAo+bd0K92DmXdaTXRt76fuBVM/sBsBa4vJI/2xEdvMi9xHnLsuyddoxXiPho90aY/Zy3UvruDVCvFZzzO+hzHaQ19Ls6kWoppovcnXOTgcmHbRsX9fhm4OZYXxvZvhUYVpFiK8vB9fxKnXfOb982KC2BUNiPckS+yzlYO8NbRmjxO1Ba7K2q8P2/Qsfz9LsqcoICOcPLwbk9SyLdnjgvAGtn+FuYSOFeb4LprybApoWQUhcG3OotGNuwvd/VidQYwQy/Q1p+kRnr921R+Il/Ni7wujbnvwoFO6FJD7jgEW+C6aQ0v6sTqXGCGX4HB7xEzvmBRnxK1SvYA4ve9GZgWT/bG8DSdZTXyms5UBeki5xEgQy/Qy91iLT2dKG7VJW8OV4rb8HrULgbMjrD8Puh55WQ2sDv6kQCIZDhFwoZZlGXOgDs23r0F4mciAO7YOHrXitvwzxISPHm2Ow3BloOUCtPpIoFMvzAa/0VlTio1QAwtfyk8pWN2Jz7Aix801s9pEl3GPmAd31erXp+VygSWIENv3DIKCkthXAC1Kqvc35SeXashXkTYe5LsH0VJKZB90uh343QvK9aeSJxILDhlxA2r+UHkfk91fKTE1C4FxZPgrkvwuop3rY2Z8BZ/w+6XAjJtf2tT0QOEdzwC5l3qQN45/10zk8qqrQU1k7zWniL34HCPVC/DQz9lTd4RaspiMStwIZfUkKIwuJS70laI2+iYJFjcQ42zvdGai58E3blQlI6dLsYel8NrU5Vt6ZINRDY8EtLSmBfUYn3JFUrO8gxbF0BC9/wZl/Z8g2EErzpxs65DzqfD0mpflcoIhUQ2PBLTQ6zr6DYe5KWAfu3Q0mxNwBGBLwJpRe+6QVe3teAQevB3ioKXS/SNXki1Vhg/9KnJiWwt7As/CLze+7fBrXLXVBegmJXHiz5lzd4Zc2XgIPMXnDe/3rX5dVt7neFIlIJAht+aUlhtu4tjDyJmuJM4Rc829fAkkle4OV+5W3L6AJn3e3Nrdmoo7/1iUilC2z4pSYlsHbbvsiTsvDbDHT1rSapQltyYMk7XuBtmOtta9oTzv6NN7+mAk+kRgtw+IXZVxgZ8NKgrXe/dQW0G+JbTXISlZZ4k0cvexe+eQ82L/a2N8+Cc3/vXYtX9nsgIjVeYMMvLTmBvWUDXuo0h+Q6utyhpinYAys/hWXvwfL3vYkMLAytT/Mmku5yAdRt4XeVIuKDwIZfWcvPOYeZQeMuCr+aYGeu17Jb9h6s+gJKCrwFYTucC51GQIdh3nR2IhJogQ2/tOQEiksdhSWlJCeEvfBbPMm7iFkXKVcfRfthzTRY8QnkfAT5S73tDdpB/5u9wGs1CMKJ/tYpInElsOGXmhQGYF9BSST8unrLzezZDOlN/C1Ojsw57yLznI+9sFvzJRQf8BaCbX0a9LkWOn7PG7Cif8SIyBEENvzSkryPvrewmPppSV7LD7yBEAq/+LJ7kzdZ9KrPIecTb0oxgEaneCsldBjmXXyuWVZEJEaBDb/U5EjLr2zEZ+PIJQ6bl0D7oT5VJQDs3Qprpnrn7FZNgS3LvO3JdaDdWXDmz73Aq9fK3zpFpNoKbPgdbPkVRM3ykpbx7RB4qTr7tnmLvq6e4gXepoXe9sQ0aH2qN2F02zO9mVZCYX9rFZEaIbDhd/CcX1nLDzTisyo4BzvWeGFXdsuPfOcJKdByIJz9a2hzprfwqwaqiMhJENjwS0s+rOUHXtfnnBe8ddpCIZ8qq2FKimHzokjQTffud2/w9iXX8cKux6XeUkAt+kNCsr/1ikggBDb8ym/5dfUWJN26HDI6+VRZNeYc7FjrzaSyfjas/9qbOqwoMo1c3ZbewJRWg7ywa9xF3Zgi4ovAht/Bll9hVMuvwzne/eJ34Kz/50NV1cy+bV7AHQy72bAvsi5iOBkye0LfG6BFltfCq9fS33pFRCICG37R1/kdVLe51yJZ+KbC73B7NnsrmG+YDxsXQN4c2L4qstO8lvIp3/PO0zXvB427QUKSryWLiBxJgMOvnJYfeGu2vfvf3sCXsmv/gqS01Au1DfO8kNu4wAu9PZu+PaZuK69V1+8GL+gye0NKHd9KFhGpqMCGXzhkpCSGDj3nB95yNu/d7bX+zv6VP8VVlb1bvJDPX+rdb1rkXWZQuMfbH0qAjM7Q/mxo2sNb8qdpd82NKSLVXmDDD7xr/Q4Z7Qne7C6tB8PC12HIPTVjQMa+bZGQWwKbl34bdmXn5wCS63ot3d5XR0Kuhxd8iSn+1S0icpIEOvxSk8PfbfmBNyHyazfAjMfhtB9XfWHHo6TIG2m5ZTlszfn2lr8sskhvRFI6NO7sTfjcuIsXcI27QHqm5sIUkcAIdPiV2/IDr+uz00j45H+9+4btq7648pSWetfIbVt5aMBtzYHtq6E06rOk1PMmd+547rcBl9HZW79OISciARfo8DtkNfdoZnD+Q/DYQK8FePWrUKdZ1RR1YKcXZNvXePc7IvfbV8OOdd76dGUSUqBBe+/6xK6joGGHb2+pDaqmXhGRaijQ4ZeWnMCe8lp+AHUy4dKn4LUx8ORZcMHDcMrwEzsHWFIEuzfCrvWRWx7sjDzesdYLuAM7Dn1NSj2o3xqadPNaofXbQIO20LCjtwK9ZqIREamwQIdfalKYzbsKjnzAKefBLR/DxGtg4tVQr7XXjZjZy2sJptQHnNfdWFrsLay6b+uht71bvK7KXXle8OEO/RmJad571WvlXTZQv7UXcPVae481slJEpNIFOvzSkhK+e53f4Rp3gR/OgKX/hjnPw7yJMOupY7+5hb2ux9SG3mCS9l28kKvb3Gux1WkeCdC6OgcnIlLFAh1+RxztebiEJOh+iXcrLYUdq2FPvtdFaSHvFkqAxFpe2KU28C4dUJekiEhcCnT4HXG059GEQtCgnXcTEZFqKdBNk9SkBAqKSykuKfW7FBERqUKBDr+05Mjk1kUxdH2KiEiNEejwK5vc+pCVHUREpMYLdPiVtfyOOeJTRERqlECHn1p+IiLBFOjwS0tSy09EJIgCHX6pyZGWn8JPRCRQAh1+B1t+6vYUEQmUQIefWn4iIsEU6PCrHRnwsvuAwk9EJEgCHX51aiWQlBAif/dRVnYQEZEaJ6bwM7PhZrbMzHLM7J5y9nc2s+lmVmBmP4/a3snM5kbddpnZXZF995nZ+qh9IyvtU8XIzMism0LezgNV/aNFRMRHx5zY2szCwGPAuUAuMMvMJjnnFkcdtg24E7go+rXOuWVA76j3WQ+8FXXIX51zD5xA/SesaZ0UNu7c72cJIiJSxWJp+Q0AcpxzK51zhcBEYFT0Ac65zc65WUDRUd5nGLDCObfmuKs9CZrVq0XeDrX8RESCJJYljZoD66Ke5wIDj+NnjQZePmzbHWZ2PZAN/Mw5t/3wF5nZWGAsQKtWrY7jxx5d07opbNp1gNJSRyikRWVF5OQrKioiNzeXAwf0D+/KkJKSQosWLUhMTIz5NbGEX3mJ4GL+CYCZJQEXAr+I2vwE8D+R9/of4EHgpu/8IOfGA+MBsrKyKvRzY9GsbgrFpY4tewpoXCelst9eROQ7cnNzSU9Pp02bNpjpH90nwjnH1q1byc3NpW3btjG/LpZuz1ygZdTzFkBeBesbAXztnNtUtsE5t8k5V+KcKwUm4HWvVrnMurUA2KBBLyJSRQ4cOEDDhg0VfJXAzGjYsGGFW9GxhN8soKOZtY204EYDkypY31Uc1uVpZplRTy8GFlbwPStF07pea2+DBr2ISBVS8FWe4/kujxl+zrli4A7gfWAJ8KpzbpGZ3WZmt0V+cFMzywV+CvzazHLNrE5kXyreSNE3D3vrP5vZAjObDwwFflLh6itBs3pey0+DXkQkKHbs2MHjjz9e4deNHDmSHTt2HPWY3/72t3z00UfHWVnVieWcH865ycDkw7aNi3q8Ea87tLzX7gMalrP9ugpVepLUT00kOSHExl0KPxEJhrLw++EPf3jI9pKSEsLh8BFfN3ny5CPuK/P73//+hOurCoGe4QWiLnTfoW5PEQmGe+65hxUrVtC7d2/69+/P0KFDufrqq+nRowcAF110Ef369aNbt26MHz/+4OvatGnDli1bWL16NV26dOGWW26hW7dunHfeeezf7/0NHTNmDK+//vrB4++991769u1Ljx49WLp0KQD5+fmce+659O3bl1tvvZXWrVuzZcuWKv0OYmr51XRN66awUQNeRMQHv/vXIhbn7arU9+zarA73XtDtiPvvv/9+Fi5cyNy5c/nss884//zzWbhw4cHRks888wwNGjRg//799O/fn0svvZSGDQ/twFu+fDkvv/wyEyZM4IorruCNN97g2muv/c7PatSoEV9//TWPP/44DzzwAE899RS/+93vOPvss/nFL37Be++9d0jAVpXAt/wAmtWtpdGeIhJYAwYMOOQygUceeYRevXoxaNAg1q1bx/Lly7/zmrZt29K7d28A+vXrx+rVq8t970suueQ7x0ydOpXRo0cDMHz4cOrXr195HyZGavkBmfW8C91LSh1hXeguIlXoaC20qpKWlnbw8WeffcZHH33E9OnTSU1NZciQIeVeRpCcnHzwcTgcPtjteaTjwuEwxcXeCjrOVfol2xWmlh/QtG6tgxe6i4jUdOnp6ezevbvcfTt37qR+/fqkpqaydOlSZsyYUek///TTT+fVV18F4IMPPmD79u9M7nXSqeWHN8sLeBe6N9EsLyJSwzVs2JDBgwfTvXt3atWqRZMmTQ7uGz58OOPGjaNnz5506tSJQYMGVfrPv/fee7nqqqt45ZVXOOuss8jMzCQ9Pb3Sf87RWDw0P2OVlZXlsrOzK/19czbv4ZyHPudPl/bgyv6VP3+oiEi0JUuW0KVLF7/L8E1BQQHhcJiEhASmT5/O7bffzty5c0/oPcv7Ts1stnMuq7zj1fID2mek0TAtiZmrtin8REROsrVr13LFFVdQWlpKUlISEyZMqPIaFH541/oNaNuAmSu3+V2KiEiN17FjR+bMmeNrDRrwEjGgbQPW79hP7vZ9fpciIiInmcIvYmBb7wLOr1ap9SciUtMp/CI6NU2nTkqCwk9EJAAUfhHhUOS8n8JPRKTGU/hFGdi2Iau27GXtVp33ExEpU7t2bQDy8vK47LLLyj1myJAhHOtStIcffph9+779+xrLEkkni8IvygW9mpEUDjHuixV+lyIiEneaNWt2cMWG43F4+E2ePJl69epVQmUVp/CL0rRuClf0b8Fr2eu0xJGI1Fh33333IYvZ3nffffzud79j2LBhB5cfeuedd77zutWrV9O9e3cA9u/fz+jRo+nZsydXXnnlIXN73n777WRlZdGtWzfuvfdewJssOy8vj6FDhzJ06FDg2yWSAB566CG6d+9O9+7defjhhw/+vCMtnXSidJ3fYW4f0oFXZq1j3Ocr+P2o7n6XIyI13bv3wMYFlfueTXvAiPuPuHv06NHcddddBxezffXVV3nvvff4yU9+Qp06ddiyZQuDBg3iwgsvxKz8yf6feOIJUlNTmT9/PvPnz6dv374H9/3hD3+gQYMGlJSUMGzYMObPn8+dd97JQw89xKeffkqjRo0Oea/Zs2fz7LPPMnPmTJxzDBw4kLPOOov69evHvHRSRanld5jm9WpxWb8WvDRzLZ9/k+93OSIila5Pnz5s3ryZvLw85s2bR/369cnMzOSXv/wlPXv25JxzzmH9+vVs2rTpiO/xxRdfHAyhnj170rNnz4P7Xn31Vfr27UufPn1YtGgRixcvPmo9U6dO5eKLLyYtLY3atWtzySWXMGXKFCD2pZMqSi2/cvxiZBfmrN3B7S/M5pWxp9KjRV2/SxKRmuooLbST6bLLLuP1119n48aNjB49mhdffJH8/Hxmz55NYmIibdq0KXcpo2jltQpXrVrFAw88wKxZs6hfvz5jxow55vscbY7pWJdOqii1/MpRJyWR524aQP3UJK4cP52XZq6Ni/WnREQqy+jRo5k4cSKvv/46l112GTt37qRx48YkJiby6aefsmbNmqO+/swzz+TFF18EYOHChcyfPx+AXbt2kZaWRt26ddm0aRPvvvvuwdccaSmlM888k7fffpt9+/axd+9e3nrrLc4444xK/LTfpfA7giZ1UnjttlPp3bIev3xrAVc8OZ3Pv8lXCIpIjdCtWzd2795N8+bNyczM5JprriE7O5usrCxefPFFOnfufNTX33777ezZs4eePXvy5z//mQEDBgDQq1cv+vTpQ7du3bjpppsYPHjwwdeMHTuWESNGHBzwUqZv376MGTOGAQMGMHDgQG6++Wb69OlT+R86ipY0OobSUsfLs9by6Cc5bNh5gDYNU7mgVzMu6NWMU5pU7fpTIlIzBH1Jo5NBSxpVslDIuGZgay7v15JJ8/J4e856Hvs0h79/kkOnJukM6ZzBWR0z6NemPskJYb/LFRGRGCj8YpSUEOKyfi24rF8LNu8+wLsLNjJ5wQaenrKKJz9fSa3EMIPaNeDMUzI4o2MG7TPSjjhEWERE/KXwOw6N01O44bQ23HBaG/YUFDNjxVa+WJ7PF9/k8+ky7/KI5vVqcUbHRpzesRGD2zeiflqSz1WLiEgZhd8Jqp2cwDldm3BO1yYArN2672AQ/mf+BibOWocZdG9Wl9M7NuKMDo3URSoiOOfUO1RJjmfsiga8nETFJaXMy93J1OVbmJqTz5y1OygudaQkhujfpoHXMuyQQZfMdP1PIBIgq1atIj09nYYNG+r//RPknGPr1q3s3r2btm3bHrLvaANeFH5VqKyLdGrOFqbmbCFn8x4AGtVOYnCHRpzeoRFndMygad0UnysVkZOpqKiI3NzcY178LbFJSUmhRYsWJCYmHrJd4RenNuzcH2kVbuHLnC1s2VMIQIfGtSNB2IiB7RpSO1m90yIiFaXwqwZKSx1LN+5mak4+U5Zv4atV2ygoLiUhZPRtVd87X9ixET1b1CMcUjeJiMixKPyqoQNFJcxes50pkfOFi/J24Rw0SEvizI6NGNKpMWeekkEDjSIVESmXLnKvhlISwwzu0IjBHRoBndm2t5Apy/P5bFk+n3+Tz9tz8zCDXi3qMaRTBkM7NaZH87qE1CoUETkmtfyqodJSx/z1O/ls2WY+W5bPvNwdOAcN05I485QMhnTK4MyOGbq2UEQCTd2eNdzWPQVMWb6FT5dt5otv8tm+r4iQQa+W9Ti7U2PO7daETk10OYWIBIvCL0BKSh3zc3fw6bJ8Plu2mfm5OwFoUb8W53RpwrldmzCgbQMSw1rQQ0RqNoVfgG3edYCPl27mo8WbmJqzhYLiUtJTEhjaqTHndG3CWadkULdW4rHfSESkmlH4CQD7CouZunwLHy3ZxMdLNrN1byEJIWNguwYHW4Ut6qf6XaaISKVQ+Ml3lJQ65q7bzoeLN/PRkk0HZ5vp2aIuI7pnMqJ7U9o0SvO5ShGR46fwk2NatWUv7y/ayLsLNzJv3Q4AumTWYUT3pozo3pSOWrhXRKoZhZ9UyPod+3lv4UbeXbCB7DXbAW/KNS8IMzURt4hUCwo/OW6bdh3g/UXewr1frdpGqYPWDVMZ2SOTUb2b0blpHb9LFBEpl8JPKsWWPQV8uHgTkxdsYNqKrZSUOk5pUptRvZtzQc9mtGqowTIiEj8UflLptu4pYPKCDUyal8es1V7XaJ9W9biwVzPO75lJ43QtyyQi/lL4yUmVu30f/56/gXfm5rFkwy5CBqe1b8SFvZsxskemlmQSEV8o/KTKLN+0m0nz8pg0L481W/dRKzHMiO5NuaxfCwa1a6iJt0Wkyij8pMo55/h67Q7e+DqXf83LY/eBYprXq8UlfZtzad8WuoZQRE46hZ/46kBRCR8s3sQbs3OZsjyfUgdZretzWb8WjOyZSZ0UTa8mIpVP4SdxY+POA7w1Zz1vfJ1LzuY9pCSGOL9HM64e2Iq+rerp+kERqTQKP4k7zjnm5e7k1ex1vDNnPXsLS+jcNJ2rB7bioj7N1RoUkROm8JO4tqegmElz83jpqzUsXL+LWolhLuiVydUDW9OrRV21BkXkuJxw+JnZcOBvQBh4yjl3/2H7OwPPAn2BXznnHojatxrYDZQAxWWFmFkD4BWgDbAauMI5t/1odSj8ar4FuTt56as1vDM3j32FJXTNrHOwNahLJkSkIk4o/MwsDHwDnAvkArOAq5xzi6OOaQy0Bi4CtpcTflnOuS2Hve+fgW3OufvN7B6gvnPu7qPVovALjt0Hinhnbh4vzVzL4g27SE9O4PKsllx/amuNFBWRmBwt/GL5p/QAIMc5tzLyZhOBUcDB8HPObQY2m9n5FahrFDAk8vg54DPgqOEnwZGeksi1g1pzzcBWzF23g+emreb5Gat5dtoqzu7UmDGD23B6h0bqEhWR4xJL+DUH1kU9zwUGVuBnOOADM3PAk8658ZHtTZxzGwCccxsircfvMLOxwFiAVq1aVeDHSk1gZvRpVZ8+rerzy5FdeGHmWl6auYbrnv6KDo1rc8NpbbikT3PS1CUqIhUQiuGY8v5pXZFRMoOdc32BEcCPzOzMCrwW59x451yWcy4rIyOjIi+VGqZxnRR+eu4pfHnP2Tx0RS9qJYb5zdsLOe3+T/jL+0vZvPuA3yWKSDURS/jlAi2jnrcA8mL9Ac65vMj9ZuAtvG5UgE1mlgkQud8c63tKsCUnhLmkbwsm3TGYN24/lVPbNeTxz1Zw+v2fcs8b81mRv8fvEkUkzsUSfrOAjmbW1sySgNHApFje3MzSzCy97DFwHrAwsnsScEPk8Q3AOxUpXMTM6Ne6AeOu68cnPxvC5VkteGvOeoY9+Dm3/DOb7NXb/C5RROJUrJc6jAQexrvU4Rnn3B/M7DYA59w4M2sKZAN1gFJgD9AVaITX2gPv/OJLzrk/RN6zIfAq0ApYC1zunDvqXyuN9pRj2bKngH9OX8M/p69mx74i+raqxx1nd2Bop8YaHCMSMLrIXQJnX2Exr2XnMv6LlazfsZ/uzetwx9COnNe1iVaWEAkIhZ8EVlFJKW/NWc/jn+aweus+OjVJ50dnd+D8HpmEFYIiNZrCTwKvuKSUf8/fwKOf5pCzeQ/tMtL40ZAOjOrdjIRwLKe+RaS6UfiJRJSWOt5btJG/f5LDkg27aNMwlZ+cewoX9Gym7lCRGuZo4ad/8kqghELGyB6ZTL7zdMZf14+UxDD/NXEuIx+ZwgeLNlKd/jEoIsdP4SeBZGac160pk+88g0eu6kNBcSljn5/NRY9PY8ryfIWgSA2n8JNAC4WMC3s148OfnMmfLu1B/q4DXPf0V1w1YQaz1+g6QZGaSuf8RKIUFJfw8sy1PPrpCrbsKWBE96bcPbyzVpIQqYY04EWkgvYVFvPUlFWM+3wFRSWlXDuoNXee3ZH6aUl+lyYiMdKAF5EKSk1K4M5hHfnsv4dwWb+WPDdtNWf+5VPGf7GCA0UlfpcnIidI4SdyFI3TU/i/S3rw3l1nktW6Pn+cvJRhD37OO3PXa1CMSDWm8BOJwSlN0nn2xgG8ePNA6tZK5L8mzuWycdNZuH6n36WJyHFQ+IlUwOAOjfj3j0/nz5f2ZPWWvVzw6FR++dYCtu8t9Ls0EakAhZ9IBYVCxhX9W/LJz4cw5rQ2vDJrHUMf/IznZ6yhpFRdoSLVgcJP5DjVrZXIvRd0Y/KdZ9ClaR1+8/ZCLvj7VGZpHUGRuKfwEzlBnZqm89ItA3n06j5s31fI5eOm89NX57J1T4HfpYnIESj8RCqBmfH9ns34+Gdn8aOh7fnXvDzOeehzXstep1GhInFI4SdSiVKTEvjv73XmP3eeQfuM2vz36/O5esJMVubv8bs0EYmi8BM5CU5pks6rt57KHy/uwcK8nQx/eAp/+2g5BcW6QF4kHij8RE6SUMi4emArPv7ZWZzXrQl//egbzn9kKl+t0oAYEb8p/EROssbpKTx6dV+evbE/+wtLuOLJ6dz7zkL2FRb7XZpIYCn8RKrI0E6N+fCnZzLmtDY8N30Nwx+ewoyVW/0uSySQFH4iVSg1KYH7LuzGK2MHATB6/Azum7RIrUCRKqbwE/HBwHYNee+uMxhzWhv+MW01wx+ewky1AkWqjMJPxCdlrcCJkVbglWoFilQZhZ+IzwZFWoE3nNqaf0xbzff/PpUFuVotQuRkUviJxIHUpAR+N6o7L908kH0FJVz8+Jc8/lmOJsoWOUkUfiJx5LQOjXjvrjP4Xrem/Pm9ZVw9YQbrd+z3uyyRGkfhJxJn6qUm8ejVfXjg8l4sXL+T4Q9/waR5eX6XJVKjKPxE4pCZcVm/Fkz+rzPo2Lg2d748h5+8MpddB4r8Lk2kRlD4icSx1g3TePXWU7nrnI5MmpfH+Y9MYX7uDr/LEqn2FH4icS4hHOKuc07h1VtPpbQULn1iGs9+uUpLJYmcAIWfSDXRr3V9/nPn6Zx1Sga/+9dibnthNjv3qxtU5Hgo/ESqkXqpSUy4Potfn9+Fj5ds5vxHpjB33Q6/yxKpdhR+ItWMmXHzGe147bZTcQ4uHzeNZ6aqG1SkIhR+ItVUn1b1mXznGQzp1Jjf/3sxtz6vblCRWCn8RKqxuqmJjL+uH7/5flc+WbqZUY9OZdnG3X6XJRL3FH4i1ZyZ8YPT2zJx7CD2FpZw0WNf8i9dFC9yVAo/kRoiq00D/vPj0+nWrA4/fnkOf/jPYopLSv0uSyQuKfxEapDGdVJ46ZZB3HBqayZMWcW1T89ky54Cv8sSiTsKP5EaJikhxO9GdefBy3sxZ+0OLvj7VOas3e53WSJxReEnUkNd2q8Fb9x+GuGQceWTM3hl1lq/SxKJGwo/kRqse/O6/OuO0xnYrgF3v7GA3/1rkc4DiqDwE6nx6qcl8eyY/tw4uA3PfrmaG/8xS9cDSuAp/EQCICEc4t4LuvF/l/Rg+oqtXPz4l6zM3+N3WSK+UfiJBMhVA1rx4s0D2bGviIse+5Ipy/P9LknEFwo/kYAZ2K4h7/xoMJl1azHm2Vn8Q8sjSQAp/EQCqGWDVN744WkM7ZTBff9azC/fWkiRBsJIgCj8RAKqdnIC46/L4vYh7Xn5q7Xc9I9Z7D6ggTASDAo/kQALhYy7h3fmT5f2YNqKrVw+bjp5O/b7XZbISafwExGu7N+Kf9zYn/Xb93PRY1+ycP1Ov0sSOaliCj8zG25my8wsx8zuKWd/ZzObbmYFZvbzqO0tzexTM1tiZovM7L+i9t1nZuvNbG7kNrJyPpKIHI8zOmbw+u2nkRAyrnhyOp8s3eR3SSInzTHDz8zCwGPACKArcJWZdT3ssG3AncADh20vBn7mnOsCDAJ+dNhr/+qc6x25TT7eDyEilaNT03Te/tFg2mWkcfNz2fxz+mq/SxI5KWJp+Q0AcpxzK51zhcBEYFT0Ac65zc65WUDRYds3OOe+jjzeDSwBmldK5SJyUjSuk8IrY0/l7M6N+e07i/iffy+mtFSXQkjNEkv4NQfWRT3P5TgCzMzaAH2AmVGb7zCz+Wb2jJnVP8LrxppZtpll5+frglyRqpCWnMCT12Ux5rQ2PD11FT+eOIeC4hK/yxKpNLGEn5WzrUL/DDSz2sAbwF3OuV2RzU8A7YHewAbgwfJe65wb75zLcs5lZWRkVOTHisgJCIeM+y7sxq9GduE/8zcw5plZ7NKlEFJDxBJ+uUDLqOctgLxYf4CZJeIF34vOuTfLtjvnNjnnSpxzpcAEvO5VEYkzt5zZjr9e2YtZq7dx5ZMz2LzrgN8liZywWMJvFtDRzNqaWRIwGpgUy5ubmQFPA0uccw8dti8z6unFwMLYShaRqnZxnxY8PaY/a7bu5ZInpmlSbKn2jhl+zrli4A7gfbwBK6865xaZ2W1mdhuAmTU1s1zgp8CvzSzXzOoAg4HrgLPLuaThz2a2wMzmA0OBn1T+xxORynLWKRm8fMsg9heWcNm46cxdt8PvkkSOm1WnCW2zsrJcdna232WIBNqqLXu5/pmZbNldyOPX9mVop8Z+lyRSLjOb7ZzLKm+fZngRkQpp2yiNN24/jXYZadzyXDZvzM71uySRClP4iUiFNU5PYeLYQQxs14CfvTaPcZ+v0LJIUq0o/ETkuKSnJPLMmP5c0KsZ97+7lD9OXqIAlGojwe8CRKT6Sk4I87cre9MwLYkJU1axa38xf7ykB+FQeZcHi8QPhZ+InJBQyLj3gq7USUngkU9y2FNQzF+v7E1SgjqWJH4p/ETkhJkZPz2vE3VqJfK//1nCnoJixl3bj1pJYb9LEymX/mkmIpXm5jPa8adLezBleT7XPzNT06FJ3FL4iUilurJ/K/5+VV/mrtvBVeNnsGVPgd8liXyHwk9EKt35PTOZcH0WK/L3cMWT08nbsd/vkkQOofATkZNiSKfGPP+DgeTvKuDycdNZtWWv3yWJHKTwE5GTpn+bBrw8dhD7i0q4fNx0lmzYdewXiVQBhZ+InFTdm9fl1VtPJTFsjB4/g3maEFvigMJPRE66Do1r8+qtp1KnVgLXPjWT7NXb/C5JAk7hJyJVomWDVF699VQapSdz/TNfMX3FVr9LkgBT+IlIlcmsW4tXxg6ieb1ajHn2K774Jt/vkiSgFH4iUqUa1/FWhGifUZubn8vmo8Wb/C5JAkjhJyJVrmHtZF6+ZRBdMtO57YXZTF6wwe+SJGAUfiLii7qpibxw80B6t6zHHS99zdtz1vtdkgSIwk9EfJOekshzNw1gYNuG/OTVubwya63fJUlAKPxExFdpyQk8e2N/zuiYwd1vLOCf01f7XZIEgMJPRHyXkhhmwvX9OKdLY377ziImfLHS75KkhlP4iUhcSE4I8/g1/RjZoyl/mLyERz9Z7ndJUoNpMVsRiRtJCSEeGd2H5IT5PPDBNxSWOH5yTkfMzO/SpIZR+IlIXEkIh3jg8l4khIxHPl5OcUkp//29TgpAqVQKPxGJO+GQ8adLe5IQDvH4ZysoKinllyO7KACl0ij8RCQuhULGHy/uTlLYmDBlFUUljnsv6KoAlEqh8BORuGVm3HdhNxLCIZ6euoqiklL+Z1R3QiEFoJwYhZ+IxDUz49fndyExHGLc514X6P9d0pOwAlBOgMJPROKemXH38E4khY1HPsmhuMTxl8t7KQDluCn8RKRaMDN+el4nEsIhHvrwG4pKHQ9d0YvEsC5XlopT+IlItXLnsI4khkP86b2lFJeU8rfRfUhKUABKxeg3RkSqnduHtOfX53fh3YUb+eGLX1NQXOJ3SVLNKPxEpFq6+Yx2/H5UNz5asolbn5/NgSIFoMRO4Sci1db1p7bhjxf34LNl+dzyz2z2FyoAJTYKPxGp1q4e2Io/X9aTqTlbuPEfX7G3oNjvkqQaUPiJSLV3RVZLHrqiF1+t2saYZ79ijwJQjkHhJyI1wsV9WvC30X34eu0Ornt6JrsOFPldksQxhZ+I1BgX9GrGY1f3YUHuTq59aiY79ykApXwKPxGpUYZ3z2Tctf1YumE3V02Ywba9hX6XJHFI4SciNc45XZvw5PX9yMnfw9UTZrBlT4HfJUmcUfiJSI00tFNjnrmhP6u37mX0+Bls3nXA75Ikjij8RKTGOr1jI54dM4C8HfsZPX4GG3cqAMWj8BORGu3U9g157qYBbNp1gCvHT2f9jv1+lyRxQOEnIjVe/zYNeP7mgWzbU8iVT05n3bZ9fpckPlP4iUgg9G1VnxdvGciu/UVc+eR0Vm/Z63dJ4iOFn4gERs8W9XjplkHsLyrhyvHTWZG/x++SxCcKPxEJlO7N6/Ly2EGUlDqufHIGyzft9rsk8YHCT0QCp3PTOkwcOwgzGD1+Bks27PK7JKliCj8RCaQOjdN5ZewgEsMhrpowg4Xrd/pdklShmMLPzIab2TIzyzGze8rZ39nMpptZgZn9PJbXmlkDM/vQzJZH7uuf+McREYldu4zavHLrINKSErh6wgzmrdvhd0lSRY4ZfmYWBh4DRgBdgavMrOthh20D7gQeqMBr7wE+ds51BD6OPBcRqVKtG6Yxcewg6qYmcu1TM5m9ZrvfJUkViKXlNwDIcc6tdM4VAhOBUdEHOOc2O+dmAYdPoX60144Cnos8fg646Pg+gojIiWnZIJVXxp5Kw9pJXP/0TL5atc3vkuQkiyX8mgProp7nRrbF4mivbeKc2wAQuW8c43uKiFS6ZvVq8cqtp9Kkbgo3PPMV01Zs8bskOYliCT8rZ5uL8f1P5LXeG5iNNbNsM8vOz8+vyEtFRCqkSZ0UXhl7Ki0b1OLGZ2fxxTf6m1NTxRJ+uUDLqOctgLwY3/9or91kZpkAkfvN5b2Bc268cy7LOZeVkZER448VETk+GenJvHzLINo2SuPmf2bz6dJy/zRJNRdL+M0COppZWzNLAkYDk2J8/6O9dhJwQ+TxDcA7sZctInLyNKztBeApTWoz9vlsPli00e+SpJIdM/ycc8XAHcD7wBLgVefcIjO7zcxuAzCzpmaWC/wU+LWZ5ZpZnSO9NvLW9wPnmtly4NzIcxGRuFA/LYkXbx5E12Z1+eGLX/Pugg1+lySVyJyr0Ck4X2VlZbns7Gy/yxCRANl1oIgbn53F3HU7+OuVvbmwVzO/S5IYmdls51xWefs0w4uIyFHUSUnkuZsG0K91fe6aOIc3v871uySpBAo/EZFjqJ2cwD9u7M+gdg352WvzeGnmWr9LkhOk8BMRiUFqUgLPjOnPWadk8Mu3FvD4ZzlUp9NGciiFn4hIjFISw0y4PotRvZvx5/eW8cfJSxSA1VSC3wWIiFQnieEQf72iN3VrJTJhyiq27yvi/kt6kBBWW6I6UfiJiFRQKGT87sJu1E9N4m8fL2fn/iL+flUfUhLDfpcmMdI/VUREjoOZ8ZNzT+HeC7ry4eJNjHn2K3YfOHxuf4lXCj8RkRNw4+C2PHxlb2at3s7VE2aydU+B3yVJDBR+IiIn6KI+zZlwfT++2bSby8dNZ/2O/X6XJMeg8BMRqQRnd27C8z8YSP7uAi57Yho5m3f7XZIchcJPRKSSDGjbgIm3DqKoxHHZuOlkr9aiuPFK4SciUom6NavLm7efRv3UJK55aibvLdSKEPFI4SciUslaNUzl9dtOpUtmHW5/cTb/nL7a75LkMAo/EZGToGxNwGGdm/DbdxZx/7tLKS3VbDDxQuEnInKS1EoKM+7avlw9sBXjPl/Bz16bR2Fxqd9lCZrhRUTkpEoIh/jDRd1pVjeFBz74hvzdBTxxbV/SUxL9Li3Q1PITETnJzIw7zu7IXy7ryYyVW7niyRls2KlrAf2k8BMRqSKXZ7Xk6TH9WbdtH6Me/ZIFuTv9LimwFH4iIlXorFMyeOP200gMh7jiyem8v0iXQvhB4SciUsU6NU3nrR+dxilN07nthdk8+fkKrQtYxRR+IiI+aJyewitjBzGyeyb/9+5SfvHmAopKNBK0qmi0p4iIT1ISw/z9qj60bZTGo5/msHbbPp64ph91UzUS9GRTy09ExEehkPHz73Xigct7MWv1Ni5+4ktWb9nrd1k1nsJPRCQOXNavBc//YCDb9hZy4aNT+fybfL9LqtEUfiIicWJQu4ZM+tHpNKtXixuf/YrxX2ggzMmi8BMRiSOtGqbyxu2n8b1uTfnj5KX85JW5HCgq8busGkfhJyISZ9KSE3j8mr787NxTeHtuHpePm06eVoevVAo/EZE4ZGb8eFhHJlyfxaote7nw0al8tUqL41YWhZ+ISBw7t2sT3v7RaaSnJHL1hBk8P2ONzgNWAoWfiEic69A4nbd/NJjTOzbiN28v5KevzmNfYbHfZVVrCj8RkWqgbq1EnrmhPz899xTenrueix77khX5e/wuq9pS+ImIVBOhkHHnsI7886YBbNlTyIV/n8q/5+f5XVa1pPATEalmzuiYwX/uPJ1OTdO546U53DdpkVaIryCFn4hINZRZtxav3HoqNw1uyz+mreaKJ3U5REUo/EREqqnEcIjfXtCVx6/pS87mPZz/yBQ+WrzJ77KqBYWfiEg1N7JHJpPuGExm3Vrc/M9s7n1noWaFOQaFn4hIDdAuozZv/eg0fnB6W56bvoaLHvuS5Zt2+11W3FL4iYjUEMkJYX7z/a48e2N/8ncX8P2/T+UFXRRfLoWfiEgNM7RTY9696wwGtG3Ar99eyG0vzGbHvkK/y4orCj8RkRqocXoKz904gF+O7MwnSzcz4m9TmLZii99lxQ2Fn4hIDRUKGWPPbM+btw8mJTHM1RNmct+kRewv1GAYhZ+ISA3Xo0Vd/nPn6Yw5rQ3/mLaakY9MYfaa7X6X5SuFn4hIAKQmJXDfhd146eaBFBaXcvm4adz/7lIKioPZClT4iYgEyGkdGvHeXWdwRVZLxn2+ggv//iUL1+/0u6wqp/ATEQmY9JRE7r+0J8+O6c/2fYVc9NiX/PXDbwLVClT4iYgE1NDOjfngJ2fy/Z6Z/O3j5Xz/kalkrw7GavEKPxGRAKuXmsTDo/vw7Jj+7Css4bJx0/n12wvYdaDI79JOKoWfiIgcbAXeNLgtL81cy7kPfc57Czf6XdZJo/ATEREA0pIT+O0FXXnrh4Opn5rEbS/M5tbns9m484DfpVU6hZ+IiByiV8t6/OvHp/P/hnfis2X5nPvQ5zwzdRXFJTVnwdyYws/MhpvZMjPLMbN7ytlvZvZIZP98M+sb2d7JzOZG3XaZ2V2RffeZ2fqofSMr9ZOJiMhxSwyH+OGQDrx/15n0blWP3/97Mec/MpXpK7b6XVqlOGb4mVkYeAwYAXQFrjKzrocdNgLoGLmNBZ4AcM4tc871ds71BvoB+4C3ol7317L9zrnJJ/phRESkcrVplMY/bxrAk9f1Y09BMVdNmMGPX57Dhp3Ve9X4WFp+A4Ac59xK51whMBEYddgxo4B/Os8MoJ6ZZR52zDBghXNuzQlXLSIiVcbM+F63pnz8s7P4r2EdeX/RRoY9+DlPfLai2l4bGEv4NQfWRT3PjWyr6DGjgZcP23ZHpJv0GTOrX94PN7OxZpZtZtn5+fkxlCsiIidDSmKYn5x7Ch/95CwGd2jEn95byvCHp/DJ0k3Vbs3AWMLPytl2+Kc86jFmlgRcCLwWtf8JoD3QG9gAPFjeD3fOjXfOZTnnsjIyMmIoV0RETqZWDVOZcH0W/7ixPwbc9I9srnlqZrWaJi2W8MsFWkY9bwHkVfCYEcDXzrlNZRucc5uccyXOuVJgAl73qoiIVBNDOjXmvbvO5L4LurJkwy6+//ep/PSVuazfEf/nA2MJv1lARzNrG2nBjQYmHXbMJOD6yKjPQcBO59yGqP1XcViX52HnBC8GFla4ehER8VVSQogxg9vy+f8byu1D2vPvBRsY+sBn/Om9pXE9S4zF0k8buQzhYSAMPOOc+4OZ3QbgnBtnZgY8CgzHG9F5o3MuO/LaVLzzge2cczuj3vN5vC5PB6wGbj0sML8jKyvLZWdnV/AjiohIVVm/Yz8Pvr+MN+esp0FaEj8+uwNXD2xFckK4ymsxs9nOuaxy91Wnk5QKPxGR6mHh+p38cfISpq3YSrO6Kfx4WEcu69eCxHDVza1ytPDTDC8iIlLpujevy4s3D+SFHwykcZ0UfvHmAoY9+DlvzM6lpNT/RpfCT0RETgoz4/SOjXjrh6fxzJgs0lMS+Nlr8zjvr5/zr3l5lPoYggo/ERE5qcyMszs34V93nM64a/sSDhk/fnkOIx+ZwuQFG3wJQZ3zExGRKlVS6vj3/Dz+9tFyVm7ZS/uMNH44pAMX9m5WqecENeBFRETiTkmpY/KCDTz2aQ5LN+6mZYNa3HZWey7r16JSRocq/EREJG455/h4yWb+/mkO89btoGWDWnz80yEkJZxYK/Bo4ZdwQu8sIiJygsyMc7o2YViXxnyZs5Xlm3efcPAdi8JPRETiQtno0NM7NjrpP0ujPUVEJHAUfiIiEjgKPxERCRyFn4iIBI7CT0REAkfhJyIigaPwExGRwFH4iYhI4Cj8REQkcBR+IiISOAo/EREJHIWfiIgEjsJPREQCR+EnIiKBo/ATEZHAUfiJiEjgKPxERCRwFH4iIhI4Cj8REQkchZ+IiASOwk9ERALHnHN+1xAzM8sH1lTCWzUCtlTC+1Q11V21VHfVUt1Vq7rWDbHX3to5l1HejmoVfpXFzLKdc1l+11FRqrtqqe6qpbqrVnWtGyqndnV7iohI4Cj8REQkcIIafuP9LuA4qe6qpbqrluquWtW1bqiE2gN5zk9ERIItqC0/EREJMIWfiIgETqDCz8yGm9kyM8sxs3v8rudIzKylmX1qZkvMbJGZ/Vdk+31mtt7M5kZuI/2utTxmttrMFkRqzI5sa2BmH5rZ8sh9fb/rjGZmnaK+17lmtsvM7orH79zMnjGzzWa2MGrbEb9fM/tF5Hd+mZl9z5+qj1j3X8xsqZnNN7O3zKxeZHsbM9sf9b2Pi7O6j/h7Eeff9ytRNa82s7mR7fH0fR/p71/l/o475wJxA8LACqAdkATMA7r6XdcRas0E+kYepwPfAF2B+4Cf+11fDPWvBhodtu3PwD2Rx/cAf/K7zmP8rmwEWsfjdw6cCfQFFh7r+4383swDkoG2kf8HwnFU93lAQuTxn6LqbhN9XBx+3+X+XsT7933Y/geB38bh932kv3+V+jsepJbfACDHObfSOVcITARG+VxTuZxzG5xzX0ce7waWAM39reqEjQKeizx+DrjIv1KOaRiwwjlXGbMJVTrn3BfAtsM2H+n7HQVMdM4VOOdWATl4/y9UufLqds594JwrjjydAbSo8sKO4Qjf95HE9fddxswMuAJ4uUqLisFR/v5V6u94kMKvObAu6nku1SBQzKwN0AeYGdl0R6SL6Jl46zqM4oAPzGy2mY2NbGvinNsA3i830Ni36o5tNIf+UagO3/mRvt/q9Ht/E/Bu1PO2ZjbHzD43szP8Kuooyvu9qC7f9xnAJufc8qhtcfd9H/b3r1J/x4MUflbOtri+zsPMagNvAHc553YBTwDtgd7ABrxui3g02DnXFxgB/MjMzvS7oFiZWRJwIfBaZFN1+c6PpFr83pvZr4Bi4MXIpg1AK+dcH+CnwEtmVsev+spxpN+LavF9A1dx6D/w4u77Lufv3xEPLWfbMb/zIIVfLtAy6nkLIM+nWo7JzBLx/sO/6Jx7E8A5t8k5V+KcKwUm4FN3yrE45/Ii95uBt/Dq3GRmmQCR+83+VXhUI4CvnXOboPp85xz5+43733szuwH4PnCNi5zEiXRhbY08no13HucU/6o81FF+L6rD950AXAK8UrYt3r7v8v7+Ucm/40EKv1lARzNrG/nX/Whgks81lSvSH/80sMQ591DU9syowy4GFh7+Wr+ZWZqZpZc9xhvQsBDvu74hctgNwDv+VHhMh/yLuDp85xFH+n4nAaPNLNnM2gIdga98qK9cZjYcuBu40Dm3L2p7hpmFI4/b4dW90p8qv+sovxdx/X1HnAMsdc7llm2Ip+/7SH//qOzfcb9H9lTlDRiJN3JoBfArv+s5Sp2n4zXb5wNzI7eRwPPAgsj2SUCm37WWU3s7vJFX84BFZd8z0BD4GFgeuW/gd63l1J4KbAXqRm2Lu+8cL5w3AEV4/+r9wdG+X+BXkd/5ZcCIOKs7B+98Tdnv+bjIsZdGfn/mAV8DF8RZ3Uf8vYjn7zuy/R/AbYcdG0/f95H+/lXq77imNxMRkcAJUreniIgIoPATEZEAUviJiEjgKPxERCRwFH4iIhI4Cj+RADCzIWb2b7/rEIkXCj8REQkchZ9IHDGza83sq8iaak+aWdjM9pjZg2b2tZl9bGYZkWN7m9kM+3YtvPqR7R3M7CMzmxd5TfvI29c2s9fNWz/vxchMGiKBpPATiRNm1gW4Em9i8N5ACXANkIY332hf4HPg3shL/gnc7ZzriTfbSNn2F4HHnHO9gNPwZvkAb3b8u/DWP2sHDD7JH0kkbiX4XYCIHDQM6AfMijTKauFN3lvKt5MQvwC8aWZ1gXrOuc8j258DXovMq9rcOfcWgHPuAEDk/b5ykfkcIyt4twGmnvRPJRKHFH4i8cOA55xzvzhko9lvDjvuaHMSHq0rsyDqcQn6/18CTN2eIvHjY+AyM2sMYGYNzKw13v+nl0WOuRqY6pzbCWyPWnT0OuBz5617lmtmF0XeI9nMUqvyQ4hUB/qXn0iccM4tNrNfAx+YWQhvNv4fAXuBbmY2G9iJd14QvGVdxkXCbSVwY2T7dcCTZvb7yHtcXoUfQ6Ra0KoOInHOzPY452r7XYdITaJuTxERCRy1/EREJHDU8hMRkcBR+ImISOAo/EREJHAUfiIiEjgKPxERCZz/D+r1fKhuzDBGAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss\n",
      "\ttraining         \t (min:    0.070, max:    0.237, cur:    0.070)\n",
      "\tvalidation       \t (min:    0.182, max:    0.266, cur:    0.216)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NetflixRecommender</td>\n",
       "      <td>0.044807</td>\n",
       "      <td>0.12186</td>\n",
       "      <td>0.165648</td>\n",
       "      <td>0.229803</td>\n",
       "      <td>0.044807</td>\n",
       "      <td>0.088844</td>\n",
       "      <td>0.107167</td>\n",
       "      <td>0.127658</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from recommenders.netflix_recommender import NetflixRecommender\n",
    "\n",
    "netflix_recommender = NetflixRecommender(embedding_dim=8, n_epochs=200, print_type='live')\n",
    "\n",
    "netflix_tts_results = [['NetflixRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    netflix_recommender, interactions_df, items_df))]\n",
    "\n",
    "netflix_tts_results = pd.DataFrame(\n",
    "    netflix_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(HTML(netflix_tts_results.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "moderate-printing",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NNRecommender</td>\n",
       "      <td>0.058384</td>\n",
       "      <td>0.143924</td>\n",
       "      <td>0.194162</td>\n",
       "      <td>0.269857</td>\n",
       "      <td>0.058384</td>\n",
       "      <td>0.106621</td>\n",
       "      <td>0.126814</td>\n",
       "      <td>0.151168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AmazonRecommender</td>\n",
       "      <td>0.044128</td>\n",
       "      <td>0.118805</td>\n",
       "      <td>0.160896</td>\n",
       "      <td>0.223693</td>\n",
       "      <td>0.044128</td>\n",
       "      <td>0.086755</td>\n",
       "      <td>0.104347</td>\n",
       "      <td>0.124505</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tts_results = pd.concat([nn_tts_results, amazon_tts_results]).reset_index(drop=True)\n",
    "display(HTML(tts_results.to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "uniform-vegetable",
   "metadata": {},
   "source": [
    "# Summary\n",
    "\n",
    "<span style=\"color:red\"><font size=\"4\">**Task:**</font></span><br> \n",
    "Write a summary of your experiments. What worked well and what did not? What are your thoughts how could you possibly further improve the model?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "declared-howard",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nAbout project:\\n\\n\\n'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "About project:\n",
    "I assess the project to be interesting, I have encountered neural networks earlier but in a completely different case (recognizing a photo), so I was really impressed when I notice \n",
    "that training only 3 hours gave better results than training with the same parameters all day.\n",
    "Unfortunately, recently owing to a lot of duties I couldn't devote as much time as I wanted to the project.\n",
    "Dealing with it has required a lot of patience because even small changes in parameters or codes gave quite big differences \n",
    "in outcomes. In the end, sadly I haven’t exceeded or even been close to 30% but stood at 26,6%. \n",
    "In my opinion, it is quite satisfying, bearing in mind how much time I have spent working on this project. \n",
    "The same result was recorded by Amazon and Netflix, which is for me a considerable success because last time \n",
    "I haven't even been close to Amazon recommender. I only remind myself scoring 4,6% vs 22,3%. \n",
    "All things considered, I feel good about the achieved score, but I want to develop this project in the summer \n",
    "to reach or even pass the upcome of 30%.\n",
    "\n",
    "\n",
    "O Przedmiocie:\n",
    "Według mnie jeden z lepiej prowadzonych przedmiotów na Uniwerystecie, widać że ma się do czynienia z osobą kompetentą \n",
    "zafascynowaną swoją profesją, która chce przekazać jak najwięcej informacji w krótkim czasie. \n",
    "Ilość materiałów oraz wiedzy bardzo duża, myślę że osoba, która przyswoiłaby ze zrozumieniem cały program zajęć, \n",
    "byłaby wystarczająco obeznana, aby otrzymać stanowisko juniora w pracy związanej z rekomenderami (lub szeroko pojętym \n",
    "data scientist). \n",
    "Sam przyznam, że materiał pojmuje dosyć ogólnikowo i mam pewne rozpoznanie, ale do całkowitego zrozumienia dużo brakuje.\n",
    "Jest to spowodowane pewnie tym, że w całym semestrze jest kilka dużych przedmiotów, którym również potrzeba poświęcić czas, \n",
    "jak i praca, która zabiera sporo cennego czasu. Najbardziej interesującą częścią przedmiotu dla mnie było przygotowywanie danych\n",
    "i ich obróbka. Utwierdziło mnie to w przekonaniu, że praca z danymi to coś co mnie interesuje na tyle, aby rozwijać się dalej w \n",
    "tym kierunku. Podsumowując przedmiot bardzo interesujący, bardzo się cieszę, że udało mi się na niego dostać. Chciałbym,\n",
    "aby było wiecej takich przedmiotów, które są ukierunkowane w jeden temat i poruszają go w obszerny oraz praktyczny sposóby.\n",
    "\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "222237f6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e257c25",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}