lecture_06_neighborhood_methods.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "emotional-exhibit",
   "metadata": {},
   "source": [
    "# Collaborative filtering - neighborhood methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "verified-accommodation",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from IPython.display import Markdown, display, HTML\n",
    "from collections import defaultdict\n",
    "\n",
    "np.set_printoptions(edgeitems=10, linewidth=500)\n",
    "\n",
    "# Fix the dying kernel problem (only a problem in some installations - you can remove it, if it works without it)\n",
    "import os\n",
    "os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "educated-tourist",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "looking-feeling",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>277</th>\n",
       "      <td>318</td>\n",
       "      <td>Shawshank Redemption, The (1994)</td>\n",
       "      <td>Crime|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>896</th>\n",
       "      <td>1193</td>\n",
       "      <td>One Flew Over the Cuckoo's Nest (1975)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>909</th>\n",
       "      <td>1208</td>\n",
       "      <td>Apocalypse Now (1979)</td>\n",
       "      <td>Action|Drama|War</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>915</th>\n",
       "      <td>1214</td>\n",
       "      <td>Alien (1979)</td>\n",
       "      <td>Horror|Sci-Fi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1291</th>\n",
       "      <td>1721</td>\n",
       "      <td>Titanic (1997)</td>\n",
       "      <td>Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2226</th>\n",
       "      <td>2959</td>\n",
       "      <td>Fight Club (1999)</td>\n",
       "      <td>Action|Crime|Drama|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2674</th>\n",
       "      <td>3578</td>\n",
       "      <td>Gladiator (2000)</td>\n",
       "      <td>Action|Adventure|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3194</th>\n",
       "      <td>4306</td>\n",
       "      <td>Shrek (2001)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8376</th>\n",
       "      <td>109487</td>\n",
       "      <td>Interstellar (2014)</td>\n",
       "      <td>Sci-Fi|IMAX</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of interactions left: 1689\n"
     ]
    }
   ],
   "source": [
    "ml_ratings_df = pd.read_csv(os.path.join(\"data\", \"movielens_small\", \"ratings.csv\")).rename(columns={'userId': 'user_id', 'movieId': 'item_id'})\n",
    "ml_movies_df = pd.read_csv(os.path.join(\"data\", \"movielens_small\", \"movies.csv\")).rename(columns={'movieId': 'item_id'})\n",
    "ml_df = pd.merge(ml_ratings_df, ml_movies_df, on='item_id')\n",
    "\n",
    "# Filter the data to reduce the number of movies\n",
    "left_ids = [1, 318, 1193, 1208, 1214, 1721, 2959, 3578, 4306, 109487]\n",
    "\n",
    "ml_ratings_df = ml_ratings_df.loc[ml_ratings_df['item_id'].isin(left_ids)]\n",
    "ml_movies_df = ml_movies_df.loc[ml_movies_df['item_id'].isin(left_ids)]\n",
    "ml_df = ml_df.loc[ml_df['item_id'].isin(left_ids)]\n",
    "\n",
    "display(ml_movies_df.head(10))\n",
    "\n",
    "print(\"Number of interactions left: {}\".format(len(ml_ratings_df)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "plastic-brooklyn",
   "metadata": {},
   "source": [
    "# Shift item ids and user ids so that they are consecutive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "valuable-modem",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>964982703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>964983250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>4.0</td>\n",
       "      <td>964981855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>5.0</td>\n",
       "      <td>964983282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>964980668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1445714835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1445714885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1445715145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>458</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>945078528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>516</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>847434962</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "interactions_df = ml_ratings_df.copy()\n",
    "\n",
    "unique_item_ids = interactions_df['item_id'].unique()\n",
    "item_id_mapping = dict(zip(unique_item_ids, list(range(len(unique_item_ids)))))\n",
    "item_id_reverse_mapping = dict(zip(list(range(len(unique_item_ids))), unique_item_ids))\n",
    "unique_user_ids = interactions_df['user_id'].unique()\n",
    "user_id_mapping = dict(zip(unique_user_ids, list(range(len(unique_user_ids)))))\n",
    "user_id_reverse_mapping = dict(zip(list(range(len(unique_user_ids))), unique_user_ids))\n",
    "\n",
    "interactions_df.replace({'item_id': item_id_mapping, 'user_id': user_id_mapping}, inplace=True)\n",
    "\n",
    "display(interactions_df.head(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "basic-meeting",
   "metadata": {},
   "source": [
    "# Get the number of items and users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "close-massachusetts",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_items=10\n",
      "n_users=521\n"
     ]
    }
   ],
   "source": [
    "n_items = np.max(interactions_df['item_id']) + 1\n",
    "n_users = np.max(interactions_df['user_id']) + 1\n",
    "\n",
    "print(\"n_items={}\\nn_users={}\".format(n_items, n_users))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acknowledged-threshold",
   "metadata": {},
   "source": [
    "# Get the user-item interaction matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "extraordinary-mexico",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 1. 1. 1. 0. 0. 0.]\n",
      " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n",
      " [1. 1. 0. 0. 1. 0. 0. 1. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 1. 1. 0. 1. 1. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 1. 0. 0. 1. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 1. 0.]\n",
      " ...\n",
      " [1. 0. 0. 1. 1. 1. 1. 1. 1. 1.]\n",
      " [0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 0. 0. 0. 1. 1.]\n",
      " [1. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 0. 0. 0. 0. 0. 0. 1. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 1. 0. 1. 1. 1.]\n",
      " [1. 0. 1. 0. 0. 1. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 1. 0. 1. 1. 1.]\n",
      " [1. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 1. 1. 1. 1. 0.]]\n"
     ]
    }
   ],
   "source": [
    "# mapping to int is necessary because of how iterrows works\n",
    "r = np.zeros(shape=(n_users, n_items))\n",
    "for idx, interaction in interactions_df.iterrows():\n",
    "    r[int(interaction['user_id'])][int(interaction['item_id'])] = 1\n",
    "    \n",
    "print(r)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "lovely-password",
   "metadata": {},
   "source": [
    "# Calculate cosine similarities of users\n",
    "\n",
    "<center>\n",
    "$$\n",
    "    \\text{Sim}(\\vec{u}, \\vec{v}) = \\text{Cos}(\\vec{u}, \\vec{v}) = \\frac{\\vec{u} \\cdot \\vec{v}}{\\lVert u \\rVert \\lVert v \\rVert} = \\frac{\\sum_{i = 1}^n u_i v_i}{\\sqrt{\\sum_{i = 1}^n u_i^2} \\sqrt{\\sum_{i = 1}^n v_i^2}}\n",
    "$$\n",
    "</center>\n",
    "\n",
    "For interaction vectors cosine similarity changes from 0 to 1. 1 means that both vectors are identical. 0 means that they have no 1's in common."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "rubber-detector",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7071067811865475\n",
      "1.0\n",
      "0.0\n"
     ]
    }
   ],
   "source": [
    "def cosine(u, v):\n",
    "    return np.sum(u * v) / np.sqrt(np.sum(u * u) * np.sum(v * v))\n",
    "\n",
    "print(cosine(np.array([1, 0, 1, 0]), np.array([1, 0, 0, 0])))\n",
    "print(cosine(np.array([1, 0, 1, 0]), np.array([1, 0, 1, 0])))\n",
    "print(cosine(np.array([1, 0, 1, 0]), np.array([0, 1, 0, 1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "wound-biotechnology",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cosine similarity between user 0 and 1\n",
      "[1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]\n",
      "[0. 0. 0. 0. 1. 1. 1. 0. 0. 0.]\n",
      "0.2581988897471611\n"
     ]
    }
   ],
   "source": [
    "print(\"Cosine similarity between user 0 and 1\")\n",
    "user_id_1 = 0\n",
    "user_id_2 = 1\n",
    "print(r[user_id_1])\n",
    "print(r[user_id_2])\n",
    "print(cosine(r[user_id_1], r[user_id_2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "signal-orchestra",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cosine similarity between user 0 and 5\n",
      "[1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]\n",
      "[1. 1. 0. 0. 1. 0. 0. 1. 0. 0.]\n",
      "0.6708203932499369\n"
     ]
    }
   ],
   "source": [
    "print(\"Cosine similarity between user 0 and 5\")\n",
    "user_id_1 = 0\n",
    "user_id_2 = 5\n",
    "print(r[user_id_1])\n",
    "print(r[user_id_2])\n",
    "print(cosine(r[user_id_1], r[user_id_2]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "distinguished-consequence",
   "metadata": {},
   "source": [
    "# Calculate Pearson similarities of users\n",
    "\n",
    "<center>\n",
    "$$\n",
    "    \\text{Sim}(\\vec{u}, \\vec{v}) = \\text{Pearson}(\\vec{u}, \\vec{v}) = \\frac{\\sum_{i = 1}^n (u_i - \\bar{u}) (v_i - \\bar{v})}{\\sqrt{\\sum_{i = 1}^n (u_i - \\bar{u})^2} \\sqrt{\\sum_{i = 1}^n (v_i - \\bar{v})^2}}\n",
    "$$\n",
    "</center>\n",
    "\n",
    "Correlation changes from -1 to 1. Correlation of 1 means that vectors are identical, -1 means they are opposites."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "pointed-deputy",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5773502691896258\n",
      "1.0\n",
      "-1.0\n"
     ]
    }
   ],
   "source": [
    "def pearson(u, v):\n",
    "    return np.sum((u - np.mean(u)) * (v - np.mean(v))) / (len(u) * np.std(u) * np.std(v))\n",
    "\n",
    "print(pearson(np.array([1, 0, 1, 0]), np.array([1, 0, 0, 0])))\n",
    "print(pearson(np.array([1, 0, 1, 0]), np.array([1, 0, 1, 0])))\n",
    "print(pearson(np.array([1, 0, 1, 0]), np.array([0, 1, 0, 1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "numerous-assignment",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pearson similarity between user 0 and 1\n",
      "[1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]\n",
      "[0. 0. 0. 0. 1. 1. 1. 0. 0. 0.]\n",
      "-0.2182178902359924\n"
     ]
    }
   ],
   "source": [
    "print(\"Pearson similarity between user 0 and 1\")\n",
    "user_id_1 = 0\n",
    "user_id_2 = 1\n",
    "print(r[user_id_1])\n",
    "print(r[user_id_2])\n",
    "print(pearson(r[user_id_1], r[user_id_2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "returning-porter",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pearson similarity between user 0 and 5\n",
      "[1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]\n",
      "[1. 1. 0. 0. 1. 0. 0. 1. 0. 0.]\n",
      "0.40824829046386296\n"
     ]
    }
   ],
   "source": [
    "print(\"Pearson similarity between user 0 and 5\")\n",
    "user_id_1 = 0\n",
    "user_id_2 = 5\n",
    "print(r[user_id_1])\n",
    "print(r[user_id_2])\n",
    "print(pearson(r[user_id_1], r[user_id_2]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "social-cancer",
   "metadata": {},
   "source": [
    "# All cosine similarities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "conscious-symbol",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scalar products\n",
      "[[5. 1. 1. 1. 0. 3. 0. 2. 0. 0. 1. 0. 4. 3. 2.]\n",
      " [1. 3. 0. 1. 1. 1. 1. 2. 1. 0. 1. 1. 3. 1. 1.]\n",
      " [1. 0. 1. 0. 0. 0. 0. 1. 0. 0. 0. 0. 1. 1. 1.]\n",
      " [1. 1. 0. 2. 1. 1. 1. 0. 1. 0. 0. 1. 2. 1. 2.]\n",
      " [0. 1. 0. 1. 1. 0. 1. 0. 1. 0. 0. 1. 1. 1. 1.]\n",
      " [3. 1. 0. 1. 0. 4. 0. 2. 0. 0. 1. 0. 3. 1. 1.]\n",
      " [0. 1. 0. 1. 1. 0. 1. 0. 1. 0. 0. 1. 1. 1. 1.]\n",
      " [2. 2. 1. 0. 0. 2. 0. 4. 0. 0. 1. 0. 4. 1. 1.]\n",
      " [0. 1. 0. 1. 1. 0. 1. 0. 2. 1. 1. 1. 1. 1. 1.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 1. 0. 0. 0. 0.]\n",
      " [1. 1. 0. 0. 0. 1. 0. 1. 1. 1. 2. 0. 1. 0. 0.]\n",
      " [0. 1. 0. 1. 1. 0. 1. 0. 1. 0. 0. 1. 1. 1. 1.]\n",
      " [4. 3. 1. 2. 1. 3. 1. 4. 1. 0. 1. 1. 7. 3. 3.]\n",
      " [3. 1. 1. 1. 1. 1. 1. 1. 1. 0. 0. 1. 3. 5. 3.]\n",
      " [2. 1. 1. 2. 1. 1. 1. 1. 1. 0. 0. 1. 3. 3. 4.]]\n",
      "\n",
      "Norms\n",
      "[2.236 1.732 1.    1.414 1.    2.    1.    2.    1.414 1.    1.414 1.    2.646 2.236 2.   ]\n",
      "\n",
      "Cosine similarities\n",
      "[[1.    0.258 0.447 0.316 0.    0.671 0.    0.447 0.    0.    0.316 0.    0.676 0.6   0.447]\n",
      " [0.258 1.    0.    0.408 0.577 0.289 0.577 0.577 0.408 0.    0.408 0.577 0.655 0.258 0.289]\n",
      " [0.447 0.    1.    0.    0.    0.    0.    0.5   0.    0.    0.    0.    0.378 0.447 0.5  ]\n",
      " [0.316 0.408 0.    1.    0.707 0.354 0.707 0.    0.5   0.    0.    0.707 0.535 0.316 0.707]\n",
      " [0.    0.577 0.    0.707 1.    0.    1.    0.    0.707 0.    0.    1.    0.378 0.447 0.5  ]\n",
      " [0.671 0.289 0.    0.354 0.    1.    0.    0.5   0.    0.    0.354 0.    0.567 0.224 0.25 ]\n",
      " [0.    0.577 0.    0.707 1.    0.    1.    0.    0.707 0.    0.    1.    0.378 0.447 0.5  ]\n",
      " [0.447 0.577 0.5   0.    0.    0.5   0.    1.    0.    0.    0.354 0.    0.756 0.224 0.25 ]\n",
      " [0.    0.408 0.    0.5   0.707 0.    0.707 0.    1.    0.707 0.5   0.707 0.267 0.316 0.354]\n",
      " [0.    0.    0.    0.    0.    0.    0.    0.    0.707 1.    0.707 0.    0.    0.    0.   ]\n",
      " [0.316 0.408 0.    0.    0.    0.354 0.    0.354 0.5   0.707 1.    0.    0.267 0.    0.   ]\n",
      " [0.    0.577 0.    0.707 1.    0.    1.    0.    0.707 0.    0.    1.    0.378 0.447 0.5  ]\n",
      " [0.676 0.655 0.378 0.535 0.378 0.567 0.378 0.756 0.267 0.    0.267 0.378 1.    0.507 0.567]\n",
      " [0.6   0.258 0.447 0.316 0.447 0.224 0.447 0.224 0.316 0.    0.    0.447 0.507 1.    0.671]\n",
      " [0.447 0.289 0.5   0.707 0.5   0.25  0.5   0.25  0.354 0.    0.    0.5   0.567 0.671 1.   ]]\n"
     ]
    }
   ],
   "source": [
    "n_uv = np.matmul(r, r.T)\n",
    "\n",
    "norms = np.sqrt(np.diag(n_uv))\n",
    "\n",
    "cos_sim = n_uv / norms.reshape(1, -1) / norms.reshape(-1, 1)\n",
    "# The above line is equivalent to the following:\n",
    "# cos_sim = dot_uv / norms[:, np.newaxis] / norms[np.newaxis, :]\n",
    "\n",
    "print(\"Scalar products\")\n",
    "print(n_uv[:15, :15])\n",
    "print()\n",
    "\n",
    "print(\"Norms\")\n",
    "print(np.around(norms[:15], 3))\n",
    "print()\n",
    "\n",
    "print(\"Cosine similarities\")\n",
    "print(np.around(cos_sim[:15, :15], 3))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "silver-collect",
   "metadata": {},
   "source": [
    "# All Pearson similarities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "yellow-franklin",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scalar products\n",
      "[[ 2.5 -0.5  0.5 -0.  -0.5  1.  -0.5 -0.  -1.  -0.5  0.  -0.5  0.5  0.5  0. ]\n",
      " [-0.5  2.1 -0.3  0.4  0.7 -0.2  0.7  0.8  0.4 -0.3  0.4  0.7  0.9 -0.5 -0.2]\n",
      " [ 0.5 -0.3  0.9 -0.2 -0.1 -0.4 -0.1  0.6 -0.2 -0.1 -0.2 -0.1  0.3  0.5  0.6]\n",
      " [-0.   0.4 -0.2  1.6  0.8  0.2  0.8 -0.8  0.6 -0.2 -0.4  0.8  0.6  0.   1.2]\n",
      " [-0.5  0.7 -0.1  0.8  0.9 -0.4  0.9 -0.4  0.8 -0.1 -0.2  0.9  0.3  0.5  0.6]\n",
      " [ 1.  -0.2 -0.4  0.2 -0.4  2.4 -0.4  0.4 -0.8 -0.4  0.2 -0.4  0.2 -1.  -0.6]\n",
      " [-0.5  0.7 -0.1  0.8  0.9 -0.4  0.9 -0.4  0.8 -0.1 -0.2  0.9  0.3  0.5  0.6]\n",
      " [-0.   0.8  0.6 -0.8 -0.4  0.4 -0.4  2.4 -0.8 -0.4  0.2 -0.4  1.2 -1.  -0.6]\n",
      " [-1.   0.4 -0.2  0.6  0.8 -0.8  0.8 -0.8  1.6  0.8  0.6  0.8 -0.4 -0.   0.2]\n",
      " [-0.5 -0.3 -0.1 -0.2 -0.1 -0.4 -0.1 -0.4  0.8  0.9  0.8 -0.1 -0.7 -0.5 -0.4]\n",
      " [ 0.   0.4 -0.2 -0.4 -0.2  0.2 -0.2  0.2  0.6  0.8  1.6 -0.2 -0.4 -1.  -0.8]\n",
      " [-0.5  0.7 -0.1  0.8  0.9 -0.4  0.9 -0.4  0.8 -0.1 -0.2  0.9  0.3  0.5  0.6]\n",
      " [ 0.5  0.9  0.3  0.6  0.3  0.2  0.3  1.2 -0.4 -0.7 -0.4  0.3  2.1 -0.5  0.2]\n",
      " [ 0.5 -0.5  0.5  0.   0.5 -1.   0.5 -1.  -0.  -0.5 -1.   0.5 -0.5  2.5  1. ]\n",
      " [ 0.  -0.2  0.6  1.2  0.6 -0.6  0.6 -0.6  0.2 -0.4 -0.8  0.6  0.2  1.   2.4]]\n",
      "\n",
      "Norms\n",
      "[1.581 1.449 0.949 1.265 0.949 1.549 0.949 1.549 1.265 0.949 1.265 0.949 1.449 1.581 1.549]\n",
      "\n",
      "Pearson similarities\n",
      "[[ 1.    -0.218  0.333 -0.    -0.333  0.408 -0.333 -0.    -0.5   -0.333  0.    -0.333  0.218  0.2    0.   ]\n",
      " [-0.218  1.    -0.218  0.218  0.509 -0.089  0.509  0.356  0.218 -0.218  0.218  0.509  0.429 -0.218 -0.089]\n",
      " [ 0.333 -0.218  1.    -0.167 -0.111 -0.272 -0.111  0.408 -0.167 -0.111 -0.167 -0.111  0.218  0.333  0.408]\n",
      " [-0.     0.218 -0.167  1.     0.667  0.102  0.667 -0.408  0.375 -0.167 -0.25   0.667  0.327  0.     0.612]\n",
      " [-0.333  0.509 -0.111  0.667  1.    -0.272  1.    -0.272  0.667 -0.111 -0.167  1.     0.218  0.333  0.408]\n",
      " [ 0.408 -0.089 -0.272  0.102 -0.272  1.    -0.272  0.167 -0.408 -0.272  0.102 -0.272  0.089 -0.408 -0.25 ]\n",
      " [-0.333  0.509 -0.111  0.667  1.    -0.272  1.    -0.272  0.667 -0.111 -0.167  1.     0.218  0.333  0.408]\n",
      " [-0.     0.356  0.408 -0.408 -0.272  0.167 -0.272  1.    -0.408 -0.272  0.102 -0.272  0.535 -0.408 -0.25 ]\n",
      " [-0.5    0.218 -0.167  0.375  0.667 -0.408  0.667 -0.408  1.     0.667  0.375  0.667 -0.218 -0.     0.102]\n",
      " [-0.333 -0.218 -0.111 -0.167 -0.111 -0.272 -0.111 -0.272  0.667  1.     0.667 -0.111 -0.509 -0.333 -0.272]\n",
      " [ 0.     0.218 -0.167 -0.25  -0.167  0.102 -0.167  0.102  0.375  0.667  1.    -0.167 -0.218 -0.5   -0.408]\n",
      " [-0.333  0.509 -0.111  0.667  1.    -0.272  1.    -0.272  0.667 -0.111 -0.167  1.     0.218  0.333  0.408]\n",
      " [ 0.218  0.429  0.218  0.327  0.218  0.089  0.218  0.535 -0.218 -0.509 -0.218  0.218  1.    -0.218  0.089]\n",
      " [ 0.2   -0.218  0.333  0.     0.333 -0.408  0.333 -0.408 -0.    -0.333 -0.5    0.333 -0.218  1.     0.408]\n",
      " [ 0.    -0.089  0.408  0.612  0.408 -0.25   0.408 -0.25   0.102 -0.272 -0.408  0.408  0.089  0.408  1.   ]]\n"
     ]
    }
   ],
   "source": [
    "r_shifted = r - np.mean(r, axis=1).reshape(-1, 1)\n",
    "\n",
    "n_uv = np.matmul(r_shifted, r_shifted.T)\n",
    "\n",
    "norms = np.sqrt(np.diag(n_uv))\n",
    "\n",
    "norms[norms == 0] = 0.000001\n",
    "\n",
    "person_sim = n_uv / norms.reshape(1, -1) / norms.reshape(-1, 1)\n",
    "\n",
    "print(\"Scalar products\")\n",
    "print(np.around(n_uv[:15, :15], 3))\n",
    "print()\n",
    "\n",
    "print(\"Norms\")\n",
    "print(np.around(norms[:15], 3))\n",
    "print()\n",
    "\n",
    "print(\"Pearson similarities\")\n",
    "print(np.around(person_sim[:15, :15], 3))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "lasting-porcelain",
   "metadata": {},
   "source": [
    "# Calculate scores of all items for user 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fifty-breeding",
   "metadata": {},
   "source": [
    "## Find n closest neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "advance-oriental",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nearest neighbors\n",
      "[138 387 240 399 513 285 473 172  24 270]\n",
      "\n",
      "User 0\n",
      "[1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]\n",
      "\n",
      "User 138\n",
      "[1. 1. 1. 1. 1. 0. 0. 1. 0. 0.]\n",
      "\n",
      "User 387\n",
      "[1. 1. 1. 1. 1. 0. 1. 0. 0. 1.]\n",
      "\n",
      "User 240\n",
      "[1. 1. 1. 1. 1. 1. 1. 0. 0. 0.]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "np.fill_diagonal(cos_sim, -1)\n",
    "\n",
    "user_id = 0\n",
    "n_neighbors = 10\n",
    "\n",
    "neighbor_ids = np.argsort(-cos_sim[user_id])[:n_neighbors]\n",
    "\n",
    "print(\"Nearest neighbors\")\n",
    "print(neighbor_ids)\n",
    "print()\n",
    "\n",
    "print(\"User {}\".format(user_id))\n",
    "print(r[user_id])\n",
    "print()\n",
    "for i in range(3):\n",
    "    print(\"User {}\".format(neighbor_ids[i]))\n",
    "    print(r[neighbor_ids[i]])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "described-marketplace",
   "metadata": {},
   "source": [
    "## Score all items\n",
    "\n",
    "<center>\n",
    "$$\n",
    "    \\text{score(i)} = \\frac{\\sum_{v \\in N(u)} \\text{Sim}(u, v) \\cdot v(i)}{\\sum_{v \\in N(u)} |\\text{Sim}(u, v)|}\n",
    "$$\n",
    "</center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "complex-lawsuit",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Interactions for nearest neighbors\n",
      "[[1. 1. 1. 1. 1. 0. 0. 1. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 0. 1. 0. 0. 1.]\n",
      " [1. 1. 1. 1. 1. 1. 1. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 1. 0. 0. 1. 0.]\n",
      " [1. 1. 1. 1. 1. 0. 0. 0. 1. 1.]\n",
      " [1. 1. 1. 1. 1. 1. 0. 1. 0. 0.]\n",
      " [1. 1. 1. 1. 0. 1. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 0. 1. 0. 0. 0. 0.]\n",
      " [0. 1. 1. 1. 1. 1. 0. 0. 0. 0.]\n",
      " [0. 1. 1. 1. 1. 1. 0. 0. 0. 0.]]\n",
      "\n",
      "similarities\n",
      "[0.91287093 0.84515425 0.84515425 0.84515425 0.84515425 0.84515425 0.8        0.8        0.8        0.8       ]\n",
      "\n",
      "v_i\n",
      "[1. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n",
      "\n",
      "score for user_id=0 and item_id=7\n",
      "0.21082871061546543\n"
     ]
    }
   ],
   "source": [
    "def score(similarities, v_i):\n",
    "    return np.sum(similarities * v_i) / np.sum(similarities)\n",
    "\n",
    "item_id = 7\n",
    "\n",
    "print(\"Interactions for nearest neighbors\")\n",
    "print(r[neighbor_ids])\n",
    "print()\n",
    "\n",
    "similarities = cos_sim[user_id, neighbor_ids]\n",
    "print(\"similarities\")\n",
    "print(similarities)\n",
    "print()\n",
    "\n",
    "v_i = r[neighbor_ids, item_id]\n",
    "print(\"v_i\")\n",
    "print(v_i)\n",
    "print()\n",
    "\n",
    "print(\"score for user_id={} and item_id={}\".format(user_id, item_id))\n",
    "print(score(similarities, v_i))\n",
    "\n",
    "\n",
    "# [1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "expanded-organization",
   "metadata": {},
   "source": [
    "### Calculate and print scores for the first 10 items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "governing-loading",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]\n",
      "score for user_id=0 and item_id=0\n",
      "0.81\n",
      "score for user_id=0 and item_id=1\n",
      "1.0\n",
      "score for user_id=0 and item_id=2\n",
      "1.0\n",
      "score for user_id=0 and item_id=3\n",
      "1.0\n",
      "score for user_id=0 and item_id=4\n",
      "0.81\n",
      "score for user_id=0 and item_id=5\n",
      "0.69\n",
      "score for user_id=0 and item_id=6\n",
      "0.2\n",
      "score for user_id=0 and item_id=7\n",
      "0.21\n",
      "score for user_id=0 and item_id=8\n",
      "0.2\n",
      "score for user_id=0 and item_id=9\n",
      "0.2\n"
     ]
    }
   ],
   "source": [
    "print(r[user_id])\n",
    "\n",
    "for i in range(10):\n",
    "    similarities = cos_sim[user_id, neighbor_ids]\n",
    "    v_i = r[neighbor_ids, i]\n",
    "    print(\"score for user_id={} and item_id={}\".format(user_id, i))\n",
    "    print(round(score(similarities, v_i), 2))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "initial-wednesday",
   "metadata": {},
   "source": [
    "### The same scoring with a single operation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "sorted-horizontal",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Neighbor ids\n",
      "[138 387 240 399 513 285 473 172  24 270]\n",
      "\n",
      "Item ids\n",
      "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n",
      "\n",
      "similarities\n",
      "[0.91287093 0.84515425 0.84515425 0.84515425 0.84515425 0.84515425 0.8        0.8        0.8        0.8       ]\n",
      "\n",
      "v_i\n",
      "[[1. 1. 1. 1. 1. 0. 0. 1. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 0. 1. 0. 0. 1.]\n",
      " [1. 1. 1. 1. 1. 1. 1. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 1. 0. 0. 1. 0.]\n",
      " [1. 1. 1. 1. 1. 0. 0. 0. 1. 1.]\n",
      " [1. 1. 1. 1. 1. 1. 0. 1. 0. 0.]\n",
      " [1. 1. 1. 1. 0. 1. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 0. 1. 0. 0. 0. 0.]\n",
      " [0. 1. 1. 1. 1. 1. 0. 0. 0. 0.]\n",
      " [0. 1. 1. 1. 1. 1. 0. 0. 0. 0.]]\n",
      "\n",
      "scores\n",
      "[0.81 1.   1.   1.   0.81 0.69 0.2  0.21 0.2  0.2 ]\n"
     ]
    }
   ],
   "source": [
    "item_ids = list(range(10))\n",
    "print(\"Neighbor ids\")\n",
    "print(neighbor_ids)\n",
    "print()\n",
    "print(\"Item ids\")\n",
    "print(item_ids)\n",
    "print()\n",
    "print(\"similarities\")\n",
    "print(similarities)\n",
    "print()\n",
    "\n",
    "v_i = r[neighbor_ids][:, item_ids]\n",
    "# Unfortunately the natural looking\n",
    "# v_i = r[neighbor_ids, item_ids]\n",
    "# does not work\n",
    "print(\"v_i\")\n",
    "print(v_i)\n",
    "print()\n",
    "\n",
    "scores = np.matmul(similarities, v_i) / np.sum(similarities)\n",
    "\n",
    "print(\"scores\")\n",
    "print(np.around(scores, 2))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "endangered-stomach",
   "metadata": {},
   "source": [
    "# Load a bigger dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "prepared-fraction",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>145</td>\n",
       "      <td>Bad Boys (1995)</td>\n",
       "      <td>Action|Comedy|Crime|Drama|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>171</td>\n",
       "      <td>Jeffrey (1995)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>228</td>\n",
       "      <td>Destiny Turns on the Radio (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>233</td>\n",
       "      <td>Exotica (1994)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>267</td>\n",
       "      <td>Major Payne (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>313</th>\n",
       "      <td>355</td>\n",
       "      <td>Flintstones, The (1994)</td>\n",
       "      <td>Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379</th>\n",
       "      <td>435</td>\n",
       "      <td>Coneheads (1993)</td>\n",
       "      <td>Comedy|Sci-Fi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>419</th>\n",
       "      <td>481</td>\n",
       "      <td>Kalifornia (1993)</td>\n",
       "      <td>Drama|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>615</th>\n",
       "      <td>780</td>\n",
       "      <td>Independence Day (a.k.a. ID4) (1996)</td>\n",
       "      <td>Action|Adventure|Sci-Fi|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>737</th>\n",
       "      <td>959</td>\n",
       "      <td>Of Human Bondage (1934)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of interactions left: 1170\n"
     ]
    }
   ],
   "source": [
    "ml_ratings_df = pd.read_csv(os.path.join(\"data\", \"movielens_small\", \"ratings.csv\")).rename(columns={'userId': 'user_id', 'movieId': 'item_id'})\n",
    "ml_movies_df = pd.read_csv(os.path.join(\"data\", \"movielens_small\", \"movies.csv\")).rename(columns={'movieId': 'item_id'})\n",
    "ml_df = pd.merge(ml_ratings_df, ml_movies_df, on='item_id')\n",
    "\n",
    "# Filter the data to reduce the number of movies\n",
    "seed = 6789\n",
    "rng = np.random.RandomState(seed=seed)\n",
    "left_ids = rng.choice(ml_movies_df['item_id'], size=100, replace=False)\n",
    "\n",
    "ml_ratings_df = ml_ratings_df.loc[ml_ratings_df['item_id'].isin(left_ids)]\n",
    "ml_movies_df = ml_movies_df.loc[ml_movies_df['item_id'].isin(left_ids)]\n",
    "ml_df = ml_df.loc[ml_df['item_id'].isin(left_ids)]\n",
    "\n",
    "display(ml_movies_df.head(10))\n",
    "\n",
    "print(\"Number of interactions left: {}\".format(len(ml_ratings_df)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "opponent-prediction",
   "metadata": {},
   "source": [
    "# User-based neighborhood recommender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "fancy-return",
   "metadata": {},
   "outputs": [],
   "source": [
    "from recommenders.recommender import Recommender\n",
    "\n",
    "class NearestNeighborsRecommender(Recommender):\n",
    "    \"\"\"\n",
    "    Nearest neighbors recommender allowing to do user-based or item-based collaborative filtering.\n",
    "\n",
    "    Possible similarity measures:\n",
    "        - 'cosine',\n",
    "        - 'pearson'.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.recommender_df = pd.DataFrame(columns=['user_id', 'item_id', 'score'])\n",
    "        self.interactions_df = None\n",
    "        self.item_id_mapping = None\n",
    "        self.user_id_mapping = None\n",
    "        self.item_id_reverse_mapping = None\n",
    "        self.user_id_reverse_mapping = None\n",
    "        self.r = None\n",
    "        self.similarities = None\n",
    "        self.most_popular_items = None\n",
    "\n",
    "        self.collaboration_type = 'user'\n",
    "        self.similarity_measure = 'cosine'\n",
    "        self.n_neighbors = 10\n",
    "        self.should_recommend_already_bought = False\n",
    "\n",
    "    def initialize(self, **params):\n",
    "        if 'n_neighbors' in params:\n",
    "            self.n_neighbors = params['n_neighbors']\n",
    "        if 'should_recommend_already_bought' in params:\n",
    "            self.should_recommend_already_bought = params['should_recommend_already_bought']\n",
    "\n",
    "    def fit(self, interactions_df, users_df, items_df):\n",
    "        \"\"\"\n",
    "        Training of the recommender.\n",
    "\n",
    "        :param pd.DataFrame interactions_df: DataFrame with recorded interactions between users and items\n",
    "            defined by user_id, item_id and features of the interaction.\n",
    "        :param pd.DataFrame users_df: DataFrame with users and their features defined by\n",
    "            user_id and the user feature columns.\n",
    "        :param pd.DataFrame items_df: DataFrame with items and their features defined\n",
    "            by item_id and the item feature columns.\n",
    "        \"\"\"\n",
    "\n",
    "        del users_df, items_df\n",
    "\n",
    "        # Shift item ids and user ids so that they are consecutive\n",
    "\n",
    "        unique_item_ids = interactions_df['item_id'].unique()\n",
    "        self.item_id_mapping = dict(zip(unique_item_ids, list(range(len(unique_item_ids)))))\n",
    "        self.item_id_reverse_mapping = dict(zip(list(range(len(unique_item_ids))), unique_item_ids))\n",
    "        unique_user_ids = interactions_df['user_id'].unique()\n",
    "        self.user_id_mapping = dict(zip(unique_user_ids, list(range(len(unique_user_ids)))))\n",
    "        self.user_id_reverse_mapping = dict(zip(list(range(len(unique_user_ids))), unique_user_ids))\n",
    "\n",
    "        interactions_df = interactions_df.copy()\n",
    "        interactions_df.replace({'item_id': self.item_id_mapping, 'user_id': self.user_id_mapping}, inplace=True)\n",
    "\n",
    "        # Get the number of items and users\n",
    "\n",
    "        self.interactions_df = interactions_df\n",
    "        n_items = np.max(interactions_df['item_id']) + 1\n",
    "        n_users = np.max(interactions_df['user_id']) + 1\n",
    "\n",
    "        # Get the user-item interaction matrix (mapping to int is necessary because of how iterrows works)\n",
    "        r = np.zeros(shape=(n_users, n_items))\n",
    "        for idx, interaction in interactions_df.iterrows():\n",
    "            r[int(interaction['user_id'])][int(interaction['item_id'])] = 1\n",
    "\n",
    "        if self.collaboration_type == 'item':\n",
    "            r = r.T\n",
    "\n",
    "        self.r = r\n",
    "\n",
    "        # Calculate all similarities\n",
    "\n",
    "        similarities = None\n",
    "        if self.similarity_measure == 'cosine':\n",
    "            n_uv = np.matmul(r, r.T)\n",
    "            norms = np.sqrt(np.diag(n_uv))\n",
    "            similarities = n_uv / norms[:, np.newaxis] / norms[np.newaxis, :]\n",
    "        elif self.similarity_measure == 'pearson':\n",
    "            r_shifted = r - np.mean(r, axis=1).reshape(-1, 1)\n",
    "            n_uv = np.matmul(r_shifted, r_shifted.T)\n",
    "            norms = np.sqrt(np.diag(n_uv))\n",
    "            norms[norms == 0] = 0.000001\n",
    "            similarities = n_uv / norms[:, np.newaxis] / norms[np.newaxis, :]\n",
    "\n",
    "        np.fill_diagonal(similarities, -1000)\n",
    "\n",
    "        self.similarities = similarities\n",
    "\n",
    "        # Find the most popular items for the cold start problem\n",
    "\n",
    "        offers_count = interactions_df.loc[:, ['item_id', 'user_id']].groupby(by='item_id').count()\n",
    "        offers_count = offers_count.sort_values('user_id', ascending=False)\n",
    "        self.most_popular_items = offers_count.index\n",
    "\n",
    "    def recommend(self, users_df, items_df, n_recommendations=1):\n",
    "        \"\"\"\n",
    "        Serving of recommendations. Scores items in items_df for each user in users_df and returns\n",
    "        top n_recommendations for each user.\n",
    "\n",
    "        :param pd.DataFrame users_df: DataFrame with users and their features for which\n",
    "            recommendations should be generated.\n",
    "        :param pd.DataFrame items_df: DataFrame with items and their features which should be scored.\n",
    "        :param int n_recommendations: Number of recommendations to be returned for each user.\n",
    "        :return: DataFrame with user_id, item_id and score as columns returning n_recommendations top recommendations\n",
    "            for each user.\n",
    "        :rtype: pd.DataFrame\n",
    "        \"\"\"\n",
    "\n",
    "        # Clean previous recommendations (iloc could be used alternatively)\n",
    "        self.recommender_df = self.recommender_df[:0]\n",
    "\n",
    "        # Handle users not in the training data\n",
    "\n",
    "        # Map item ids\n",
    "\n",
    "        items_df = items_df.copy()\n",
    "        items_df = items_df.loc[items_df['item_id'].isin(self.item_id_mapping)]\n",
    "        items_df.replace({'item_id': self.item_id_mapping}, inplace=True)\n",
    "\n",
    "        # Generate recommendations\n",
    "\n",
    "        for idx, user in users_df.iterrows():\n",
    "            recommendations = []\n",
    "\n",
    "            user_id = user['user_id']\n",
    "\n",
    "            if user_id in self.user_id_mapping:\n",
    "                chosen_ids = []\n",
    "                scores = []\n",
    "                mapped_user_id = self.user_id_mapping[user_id]\n",
    "\n",
    "                if self.collaboration_type == 'user':\n",
    "                    neighbor_ids = np.argsort(-self.similarities[mapped_user_id])[:self.n_neighbors]\n",
    "                    user_similarities = self.similarities[mapped_user_id][neighbor_ids]\n",
    "\n",
    "                    item_ids = items_df['item_id'].tolist()\n",
    "\n",
    "                    v_i = self.r[neighbor_ids][:, item_ids]\n",
    "\n",
    "                    scores = np.matmul(user_similarities, v_i) / np.maximum(np.sum(user_similarities), 0.0001)\n",
    "\n",
    "                    # Choose n recommendations based on highest scores\n",
    "                    if not self.should_recommend_already_bought:\n",
    "                        x_list = self.interactions_df.loc[\n",
    "                            self.interactions_df['user_id'] == mapped_user_id]['item_id'].tolist()\n",
    "                        scores[x_list] = -1e100\n",
    "\n",
    "                    chosen_ids = np.argsort(-scores)[:n_recommendations]\n",
    "\n",
    "                elif self.collaboration_type == 'item':\n",
    "                    x_list = self.interactions_df.loc[\n",
    "                        self.interactions_df['user_id'] == mapped_user_id]['item_id'].tolist()\n",
    "                    scores = np.sum(self.similarities[x_list], axis=0)\n",
    "\n",
    "                    # Choose n recommendations based on highest scores\n",
    "                    if not self.should_recommend_already_bought:\n",
    "                        scores[x_list] = -1e100\n",
    "\n",
    "                    chosen_ids = np.argsort(-scores)[:n_recommendations]\n",
    "\n",
    "                for item_id in chosen_ids:\n",
    "                    recommendations.append(\n",
    "                        {\n",
    "                            'user_id': self.user_id_reverse_mapping[mapped_user_id],\n",
    "                            'item_id': self.item_id_reverse_mapping[item_id],\n",
    "                            'score': scores[item_id]\n",
    "                        }\n",
    "                    )\n",
    "            else:  # For new users recommend most popular items\n",
    "                for i in range(n_recommendations):\n",
    "                    recommendations.append(\n",
    "                        {\n",
    "                            'user_id': user['user_id'],\n",
    "                            'item_id': self.item_id_reverse_mapping[self.most_popular_items[i]],\n",
    "                            'score': 1.0\n",
    "                        }\n",
    "                    )\n",
    "\n",
    "            user_recommendations = pd.DataFrame(recommendations)\n",
    "\n",
    "            self.recommender_df = pd.concat([self.recommender_df, user_recommendations])\n",
    "\n",
    "        return self.recommender_df\n",
    "    \n",
    "\n",
    "class UserBasedCosineNearestNeighborsRecommender(NearestNeighborsRecommender):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.collaboration_type = 'user'\n",
    "        self.similarity_measure = 'cosine'\n",
    "        \n",
    "        \n",
    "class UserBasedPearsonNearestNeighborsRecommender(NearestNeighborsRecommender):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.collaboration_type = 'user'\n",
    "        self.similarity_measure = 'pearson'\n",
    "        \n",
    "        \n",
    "class ItemBasedCosineNearestNeighborsRecommender(NearestNeighborsRecommender):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.collaboration_type = 'item'\n",
    "        self.similarity_measure = 'cosine'\n",
    "        \n",
    "\n",
    "class ItemBasedPearsonNearestNeighborsRecommender(NearestNeighborsRecommender):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.collaboration_type = 'item'\n",
    "        self.similarity_measure = 'pearson'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "nonprofit-roads",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recommendations\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>score</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>355</td>\n",
       "      <td>0.955688</td>\n",
       "      <td>Flintstones, The (1994)</td>\n",
       "      <td>Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>73323</td>\n",
       "      <td>0.291761</td>\n",
       "      <td>Girl Who Kicked the Hornet's Nest, The (Luftslottet som sprängdes) (2009)</td>\n",
       "      <td>Action|Crime|Mystery</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>8605</td>\n",
       "      <td>0.280261</td>\n",
       "      <td>Taxi 3 (2003)</td>\n",
       "      <td>Action|Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3628</td>\n",
       "      <td>0.059450</td>\n",
       "      <td>Flying Tigers (1942)</td>\n",
       "      <td>Action|Drama|Romance|War</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>6755</td>\n",
       "      <td>0.059450</td>\n",
       "      <td>Bubba Ho-tep (2002)</td>\n",
       "      <td>Comedy|Horror</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>3165</td>\n",
       "      <td>0.051167</td>\n",
       "      <td>Boiling Point (1993)</td>\n",
       "      <td>Action|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>4031</td>\n",
       "      <td>0.044312</td>\n",
       "      <td>All the Pretty Horses (2000)</td>\n",
       "      <td>Drama|Romance|Western</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>1914</td>\n",
       "      <td>0.044312</td>\n",
       "      <td>Smoke Signals (1998)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>7282</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Hip Hop Witch, Da (2000)</td>\n",
       "      <td>Comedy|Horror|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>2190</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Why Do Fools Fall In Love? (1998)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>4</td>\n",
       "      <td>4031</td>\n",
       "      <td>0.556855</td>\n",
       "      <td>All the Pretty Horses (2000)</td>\n",
       "      <td>Drama|Romance|Western</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>4</td>\n",
       "      <td>73323</td>\n",
       "      <td>0.556855</td>\n",
       "      <td>Girl Who Kicked the Hornet's Nest, The (Luftslottet som sprängdes) (2009)</td>\n",
       "      <td>Action|Crime|Mystery</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>4</td>\n",
       "      <td>355</td>\n",
       "      <td>0.098477</td>\n",
       "      <td>Flintstones, The (1994)</td>\n",
       "      <td>Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>4</td>\n",
       "      <td>2806</td>\n",
       "      <td>0.098477</td>\n",
       "      <td>Teaching Mrs. Tingle (1999)</td>\n",
       "      <td>Comedy|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>4</td>\n",
       "      <td>5673</td>\n",
       "      <td>0.056855</td>\n",
       "      <td>Punch-Drunk Love (2002)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>4</td>\n",
       "      <td>3567</td>\n",
       "      <td>0.056855</td>\n",
       "      <td>Bossa Nova (2000)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>4</td>\n",
       "      <td>145</td>\n",
       "      <td>0.049238</td>\n",
       "      <td>Bad Boys (1995)</td>\n",
       "      <td>Action|Comedy|Crime|Drama|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>4</td>\n",
       "      <td>3165</td>\n",
       "      <td>0.049238</td>\n",
       "      <td>Boiling Point (1993)</td>\n",
       "      <td>Action|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>4</td>\n",
       "      <td>2275</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Six-String Samurai (1998)</td>\n",
       "      <td>Action|Adventure|Sci-Fi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>4</td>\n",
       "      <td>4483</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Caddyshack II (1988)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>6</td>\n",
       "      <td>4896</td>\n",
       "      <td>0.653889</td>\n",
       "      <td>Harry Potter and the Sorcerer's Stone (a.k.a. Harry Potter and the Philosopher's Stone) (2001)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>6</td>\n",
       "      <td>1914</td>\n",
       "      <td>0.242896</td>\n",
       "      <td>Smoke Signals (1998)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>6</td>\n",
       "      <td>4031</td>\n",
       "      <td>0.045903</td>\n",
       "      <td>All the Pretty Horses (2000)</td>\n",
       "      <td>Drama|Romance|Western</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>6</td>\n",
       "      <td>2275</td>\n",
       "      <td>0.045903</td>\n",
       "      <td>Six-String Samurai (1998)</td>\n",
       "      <td>Action|Adventure|Sci-Fi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>6</td>\n",
       "      <td>3567</td>\n",
       "      <td>0.045903</td>\n",
       "      <td>Bossa Nova (2000)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>6</td>\n",
       "      <td>73323</td>\n",
       "      <td>0.045543</td>\n",
       "      <td>Girl Who Kicked the Hornet's Nest, The (Luftslottet som sprängdes) (2009)</td>\n",
       "      <td>Action|Crime|Mystery</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>6</td>\n",
       "      <td>1500</td>\n",
       "      <td>0.042938</td>\n",
       "      <td>Grosse Pointe Blank (1997)</td>\n",
       "      <td>Comedy|Crime|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>6</td>\n",
       "      <td>4483</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Caddyshack II (1988)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>6</td>\n",
       "      <td>7282</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Hip Hop Witch, Da (2000)</td>\n",
       "      <td>Comedy|Horror|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>6</td>\n",
       "      <td>2190</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Why Do Fools Fall In Love? (1998)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Quick test of the recommender\n",
    "\n",
    "nearest_neighbors_recommender = NearestNeighborsRecommender()\n",
    "nearest_neighbors_recommender.initialize(n_neighbors=20)\n",
    "nearest_neighbors_recommender.fit(ml_ratings_df, None, ml_movies_df)\n",
    "recommendations = nearest_neighbors_recommender.recommend(pd.DataFrame([[1], [4], [6]], columns=['user_id']), ml_movies_df, 10)\n",
    "\n",
    "recommendations = pd.merge(recommendations, ml_movies_df, on='item_id', how='left')\n",
    "print(\"Recommendations\")\n",
    "display(recommendations)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "framed-negative",
   "metadata": {},
   "source": [
    "# Training-test split evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "amended-future",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation_and_testing.testing import evaluate_train_test_split_implicit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "romantic-music",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UserBasedCosineNearestNeighborsRecommender</td>\n",
       "      <td>0.103896</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>0.409091</td>\n",
       "      <td>0.103896</td>\n",
       "      <td>0.126777</td>\n",
       "      <td>0.161141</td>\n",
       "      <td>0.219215</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ub_cos_nn_recommender = UserBasedCosineNearestNeighborsRecommender()\n",
    "ub_cos_nn_recommender.initialize(n_neighbors=30)\n",
    "\n",
    "ub_cos_nn_tts_results = [['UserBasedCosineNearestNeighborsRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    ub_cos_nn_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n",
    "\n",
    "ub_cos_nn_tts_results = pd.DataFrame(\n",
    "    ub_cos_nn_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(ub_cos_nn_tts_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "sharp-hearts",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UserBasedPearsonNearestNeighborsRecommender</td>\n",
       "      <td>0.103896</td>\n",
       "      <td>0.149351</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>0.409091</td>\n",
       "      <td>0.103896</td>\n",
       "      <td>0.130024</td>\n",
       "      <td>0.162161</td>\n",
       "      <td>0.219704</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ub_pearson_nn_recommender = UserBasedPearsonNearestNeighborsRecommender()\n",
    "ub_pearson_nn_recommender.initialize(n_neighbors=30)\n",
    "\n",
    "ub_pearson_nn_tts_results = [['UserBasedPearsonNearestNeighborsRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    ub_pearson_nn_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n",
    "\n",
    "ub_pearson_nn_tts_results = pd.DataFrame(\n",
    "    ub_pearson_nn_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(ub_pearson_nn_tts_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "mature-newton",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ItemBasedCosineNearestNeighborsRecommender</td>\n",
       "      <td>0.25974</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.694805</td>\n",
       "      <td>0.954545</td>\n",
       "      <td>0.25974</td>\n",
       "      <td>0.421302</td>\n",
       "      <td>0.482493</td>\n",
       "      <td>0.566174</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ib_cos_nn_recommender = ItemBasedCosineNearestNeighborsRecommender()\n",
    "ib_cos_nn_recommender.initialize(n_neighbors=30)\n",
    "\n",
    "ib_cos_nn_tts_results = [['ItemBasedCosineNearestNeighborsRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    ib_cos_nn_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n",
    "\n",
    "ib_cos_nn_tts_results = pd.DataFrame(\n",
    "    ib_cos_nn_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(ib_cos_nn_tts_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "adjacent-crazy",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ItemBasedPearsonNearestNeighborsRecommender</td>\n",
       "      <td>0.175325</td>\n",
       "      <td>0.350649</td>\n",
       "      <td>0.448052</td>\n",
       "      <td>0.558442</td>\n",
       "      <td>0.175325</td>\n",
       "      <td>0.27744</td>\n",
       "      <td>0.317397</td>\n",
       "      <td>0.352948</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ib_pearson_nn_recommender = ItemBasedPearsonNearestNeighborsRecommender()\n",
    "ib_pearson_nn_recommender.initialize(n_neighbors=30)\n",
    "\n",
    "ib_pearson_nn_tts_results = [['ItemBasedPearsonNearestNeighborsRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    ib_pearson_nn_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n",
    "\n",
    "ib_pearson_nn_tts_results = pd.DataFrame(\n",
    "    ib_pearson_nn_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(ib_pearson_nn_tts_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "standing-tiffany",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AmazonRecommender</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.311688</td>\n",
       "      <td>0.402597</td>\n",
       "      <td>0.551948</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.257806</td>\n",
       "      <td>0.294682</td>\n",
       "      <td>0.34147</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from recommenders.amazon_recommender import AmazonRecommender\n",
    "\n",
    "amazon_recommender = AmazonRecommender()\n",
    "\n",
    "amazon_tts_results = [['AmazonRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    amazon_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n",
    "\n",
    "amazon_tts_results = pd.DataFrame(\n",
    "    amazon_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(amazon_tts_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "saving-harrison",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TFIDFRecommender</td>\n",
       "      <td>0.025974</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.318182</td>\n",
       "      <td>0.025974</td>\n",
       "      <td>0.064393</td>\n",
       "      <td>0.083685</td>\n",
       "      <td>0.140799</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from recommenders.tfidf_recommender import TFIDFRecommender\n",
    "\n",
    "tfidf_recommender = TFIDFRecommender()\n",
    "\n",
    "tfidf_tts_results = [['TFIDFRecommender'] + list(evaluate_train_test_split_implicit(\n",
    "    tfidf_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df))]\n",
    "\n",
    "tfidf_tts_results = pd.DataFrame(\n",
    "    tfidf_tts_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(tfidf_tts_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "random-source",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UserBasedCosineNearestNeighborsRecommender</td>\n",
       "      <td>0.103896</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>0.409091</td>\n",
       "      <td>0.103896</td>\n",
       "      <td>0.126777</td>\n",
       "      <td>0.161141</td>\n",
       "      <td>0.219215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>UserBasedPearsonNearestNeighborsRecommender</td>\n",
       "      <td>0.103896</td>\n",
       "      <td>0.149351</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>0.409091</td>\n",
       "      <td>0.103896</td>\n",
       "      <td>0.130024</td>\n",
       "      <td>0.162161</td>\n",
       "      <td>0.219704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ItemBasedCosineNearestNeighborsRecommender</td>\n",
       "      <td>0.259740</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.694805</td>\n",
       "      <td>0.954545</td>\n",
       "      <td>0.259740</td>\n",
       "      <td>0.421302</td>\n",
       "      <td>0.482493</td>\n",
       "      <td>0.566174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ItemBasedPearsonNearestNeighborsRecommender</td>\n",
       "      <td>0.175325</td>\n",
       "      <td>0.350649</td>\n",
       "      <td>0.448052</td>\n",
       "      <td>0.558442</td>\n",
       "      <td>0.175325</td>\n",
       "      <td>0.277440</td>\n",
       "      <td>0.317397</td>\n",
       "      <td>0.352948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AmazonRecommender</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.311688</td>\n",
       "      <td>0.402597</td>\n",
       "      <td>0.551948</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.257806</td>\n",
       "      <td>0.294682</td>\n",
       "      <td>0.341470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>TFIDFRecommender</td>\n",
       "      <td>0.025974</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.318182</td>\n",
       "      <td>0.025974</td>\n",
       "      <td>0.064393</td>\n",
       "      <td>0.083685</td>\n",
       "      <td>0.140799</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tts_results = pd.concat([ub_cos_nn_tts_results, ub_pearson_nn_tts_results, ib_cos_nn_tts_results, \n",
    "                         ib_pearson_nn_tts_results, amazon_tts_results, tfidf_tts_results]).reset_index(drop=True)\n",
    "display(tts_results)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "continued-harassment",
   "metadata": {},
   "source": [
    "# Leave-one-out evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "exact-stuff",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation_and_testing.testing import evaluate_leave_one_out_implicit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "prerequisite-lounge",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UserBasedCosineNearestNeighborsRecommender</td>\n",
       "      <td>0.096667</td>\n",
       "      <td>0.146667</td>\n",
       "      <td>0.186667</td>\n",
       "      <td>0.306667</td>\n",
       "      <td>0.096667</td>\n",
       "      <td>0.124285</td>\n",
       "      <td>0.140782</td>\n",
       "      <td>0.178962</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ub_cos_nn_recommender = UserBasedCosineNearestNeighborsRecommender()\n",
    "ub_cos_nn_recommender.initialize(n_neighbors=30)\n",
    "\n",
    "ub_cos_nn_loo_results = [['UserBasedCosineNearestNeighborsRecommender'] + list(evaluate_leave_one_out_implicit(\n",
    "    ub_cos_nn_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df, max_evals=300, seed=6789))]\n",
    "\n",
    "ub_cos_nn_loo_results = pd.DataFrame(\n",
    "    ub_cos_nn_loo_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(ub_cos_nn_loo_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "olive-insurance",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UserBasedPearsonNearestNeighborsRecommender</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.18</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.127182</td>\n",
       "      <td>0.139518</td>\n",
       "      <td>0.180784</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ub_pearson_nn_recommender = UserBasedPearsonNearestNeighborsRecommender()\n",
    "ub_pearson_nn_recommender.initialize(n_neighbors=30)\n",
    "\n",
    "ub_pearson_nn_loo_results = [['UserBasedPearsonNearestNeighborsRecommender'] + list(evaluate_leave_one_out_implicit(\n",
    "    ub_pearson_nn_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df, max_evals=300, seed=6789))]\n",
    "\n",
    "ub_pearson_nn_loo_results = pd.DataFrame(\n",
    "    ub_pearson_nn_loo_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(ub_pearson_nn_loo_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "determined-orleans",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ItemBasedCosineNearestNeighborsRecommender</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.513333</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.357736</td>\n",
       "      <td>0.396033</td>\n",
       "      <td>0.440599</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ib_cos_nn_recommender = ItemBasedCosineNearestNeighborsRecommender()\n",
    "ib_cos_nn_recommender.initialize(n_neighbors=30)\n",
    "\n",
    "ib_cos_nn_loo_results = [['ItemBasedCosineNearestNeighborsRecommender'] + list(evaluate_leave_one_out_implicit(\n",
    "    ib_cos_nn_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df, max_evals=300, seed=6789))]\n",
    "\n",
    "ib_cos_nn_loo_results = pd.DataFrame(\n",
    "    ib_cos_nn_loo_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(ib_cos_nn_loo_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "worse-management",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ItemBasedPearsonNearestNeighborsRecommender</td>\n",
       "      <td>0.173333</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.336667</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.173333</td>\n",
       "      <td>0.234522</td>\n",
       "      <td>0.257759</td>\n",
       "      <td>0.284723</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ib_pearson_nn_recommender = ItemBasedPearsonNearestNeighborsRecommender()\n",
    "ib_pearson_nn_recommender.initialize(n_neighbors=30)\n",
    "\n",
    "ib_pearson_nn_loo_results = [['ItemBasedPearsonNearestNeighborsRecommender'] + list(evaluate_leave_one_out_implicit(\n",
    "    ib_pearson_nn_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df, max_evals=300, seed=6789))]\n",
    "\n",
    "ib_pearson_nn_loo_results = pd.DataFrame(\n",
    "    ib_pearson_nn_loo_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(ib_pearson_nn_loo_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "social-escape",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AmazonRecommender</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.256667</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.426667</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.219086</td>\n",
       "      <td>0.245486</td>\n",
       "      <td>0.279978</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from recommenders.amazon_recommender import AmazonRecommender\n",
    "\n",
    "amazon_recommender = AmazonRecommender()\n",
    "\n",
    "amazon_loo_results = [['AmazonRecommender'] + list(evaluate_leave_one_out_implicit(\n",
    "    amazon_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df, max_evals=300, seed=6789))]\n",
    "\n",
    "amazon_loo_results = pd.DataFrame(\n",
    "    amazon_loo_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(amazon_loo_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "behind-cambodia",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TFIDFRecommender</td>\n",
       "      <td>0.006667</td>\n",
       "      <td>0.053333</td>\n",
       "      <td>0.123333</td>\n",
       "      <td>0.233333</td>\n",
       "      <td>0.006667</td>\n",
       "      <td>0.033491</td>\n",
       "      <td>0.062178</td>\n",
       "      <td>0.096151</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tfidf_recommender = TFIDFRecommender()\n",
    "\n",
    "tfidf_loo_results = [['TFIDFRecommender'] + list(evaluate_leave_one_out_implicit(\n",
    "    tfidf_recommender, ml_ratings_df.loc[:, ['user_id', 'item_id']], ml_movies_df, max_evals=300, seed=6789))]\n",
    "\n",
    "tfidf_loo_results = pd.DataFrame(\n",
    "    tfidf_loo_results, columns=['Recommender', 'HR@1', 'HR@3', 'HR@5', 'HR@10', 'NDCG@1', 'NDCG@3', 'NDCG@5', 'NDCG@10'])\n",
    "\n",
    "display(tfidf_loo_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "lightweight-password",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recommender</th>\n",
       "      <th>HR@1</th>\n",
       "      <th>HR@3</th>\n",
       "      <th>HR@5</th>\n",
       "      <th>HR@10</th>\n",
       "      <th>NDCG@1</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UserBasedCosineNearestNeighborsRecommender</td>\n",
       "      <td>0.096667</td>\n",
       "      <td>0.146667</td>\n",
       "      <td>0.186667</td>\n",
       "      <td>0.306667</td>\n",
       "      <td>0.096667</td>\n",
       "      <td>0.124285</td>\n",
       "      <td>0.140782</td>\n",
       "      <td>0.178962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>UserBasedPearsonNearestNeighborsRecommender</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.150000</td>\n",
       "      <td>0.180000</td>\n",
       "      <td>0.310000</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.127182</td>\n",
       "      <td>0.139518</td>\n",
       "      <td>0.180784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ItemBasedCosineNearestNeighborsRecommender</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.420000</td>\n",
       "      <td>0.513333</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.357736</td>\n",
       "      <td>0.396033</td>\n",
       "      <td>0.440599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ItemBasedPearsonNearestNeighborsRecommender</td>\n",
       "      <td>0.173333</td>\n",
       "      <td>0.280000</td>\n",
       "      <td>0.336667</td>\n",
       "      <td>0.420000</td>\n",
       "      <td>0.173333</td>\n",
       "      <td>0.234522</td>\n",
       "      <td>0.257759</td>\n",
       "      <td>0.284723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AmazonRecommender</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.256667</td>\n",
       "      <td>0.320000</td>\n",
       "      <td>0.426667</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.219086</td>\n",
       "      <td>0.245486</td>\n",
       "      <td>0.279978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>TFIDFRecommender</td>\n",
       "      <td>0.006667</td>\n",
       "      <td>0.053333</td>\n",
       "      <td>0.123333</td>\n",
       "      <td>0.233333</td>\n",
       "      <td>0.006667</td>\n",
       "      <td>0.033491</td>\n",
       "      <td>0.062178</td>\n",
       "      <td>0.096151</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "loo_results = pd.concat([ub_cos_nn_loo_results, ub_pearson_nn_loo_results, ib_cos_nn_loo_results, \n",
    "                         ib_pearson_nn_loo_results, amazon_loo_results, tfidf_loo_results]).reset_index(drop=True)\n",
    "display(loo_results)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}