collaborative-filtering-model-using-surprise.ipynb

{"cells":[{"source":"<a href=\"https://www.kaggle.com/code/gpreda/collaborative-filtering-model-using-surprise?scriptVersionId=128758448\" target=\"_blank\"><img align=\"left\" alt=\"Kaggle\" title=\"Open in Kaggle\" src=\"https://kaggle.com/static/images/open-in-kaggle.svg\"></a>","metadata":{},"cell_type":"markdown"},{"cell_type":"markdown","id":"4d3bf8a3","metadata":{"papermill":{"duration":0.007819,"end_time":"2023-05-08T12:12:44.372806","exception":false,"start_time":"2023-05-08T12:12:44.364987","status":"completed"},"tags":[]},"source":["# Introduction\n","\n","This Notebook is experimenting with Surprise to build a simple collaborative filtering model using Surprise library."]},{"cell_type":"markdown","id":"d29d1671","metadata":{"papermill":{"duration":0.006345,"end_time":"2023-05-08T12:12:44.385949","exception":false,"start_time":"2023-05-08T12:12:44.379604","status":"completed"},"tags":[]},"source":["# Analysis preparation"]},{"cell_type":"code","execution_count":1,"id":"a27159f3","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:44.402295Z","iopub.status.busy":"2023-05-08T12:12:44.401243Z","iopub.status.idle":"2023-05-08T12:12:44.521706Z","shell.execute_reply":"2023-05-08T12:12:44.520641Z"},"papermill":{"duration":0.132141,"end_time":"2023-05-08T12:12:44.524956","exception":false,"start_time":"2023-05-08T12:12:44.392815","status":"completed"},"tags":[]},"outputs":[],"source":["import numpy as np\n","import pandas as pd\n","import re\n","\n","from surprise import accuracy\n","from surprise import Dataset\n","from surprise import Reader\n","from surprise.model_selection import cross_validate\n","from surprise import NormalPredictor, SVD\n"]},{"cell_type":"markdown","id":"de62e59f","metadata":{"papermill":{"duration":0.006234,"end_time":"2023-05-08T12:12:44.538054","exception":false,"start_time":"2023-05-08T12:12:44.53182","status":"completed"},"tags":[]},"source":["## Read users data"]},{"cell_type":"code","execution_count":2,"id":"debf5b8c","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:44.553646Z","iopub.status.busy":"2023-05-08T12:12:44.552924Z","iopub.status.idle":"2023-05-08T12:12:44.602292Z","shell.execute_reply":"2023-05-08T12:12:44.601117Z"},"papermill":{"duration":0.061697,"end_time":"2023-05-08T12:12:44.606237","exception":false,"start_time":"2023-05-08T12:12:44.54454","status":"completed"},"tags":[]},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>user_id</th>\n","      <th>age</th>\n","      <th>sex</th>\n","      <th>occupation</th>\n","      <th>zip_code</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>24</td>\n","      <td>M</td>\n","      <td>technician</td>\n","      <td>85711</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>53</td>\n","      <td>F</td>\n","      <td>other</td>\n","      <td>94043</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   user_id  age sex  occupation zip_code\n","0        1   24   M  technician    85711\n","1        2   53   F       other    94043"]},"execution_count":2,"metadata":{},"output_type":"execute_result"}],"source":["user_columns = ['user_id', 'age', 'sex', 'occupation', 'zip_code']\n","users_df = pd.read_csv('../input/movielens-100k-dataset/ml-100k/u.user', sep='|', names=user_columns) \n","users_df.head(2)"]},{"cell_type":"markdown","id":"ac740be3","metadata":{"papermill":{"duration":0.006912,"end_time":"2023-05-08T12:12:44.620392","exception":false,"start_time":"2023-05-08T12:12:44.61348","status":"completed"},"tags":[]},"source":["## Read ratings data"]},{"cell_type":"code","execution_count":3,"id":"53bd2fc2","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:44.636234Z","iopub.status.busy":"2023-05-08T12:12:44.635788Z","iopub.status.idle":"2023-05-08T12:12:44.734433Z","shell.execute_reply":"2023-05-08T12:12:44.733112Z"},"papermill":{"duration":0.110054,"end_time":"2023-05-08T12:12:44.737427","exception":false,"start_time":"2023-05-08T12:12:44.627373","status":"completed"},"tags":[]},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>user_id</th>\n","      <th>movie_id</th>\n","      <th>rating</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>196</td>\n","      <td>242</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>186</td>\n","      <td>302</td>\n","      <td>3</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   user_id  movie_id  rating\n","0      196       242       3\n","1      186       302       3"]},"execution_count":3,"metadata":{},"output_type":"execute_result"}],"source":["ratings_columns = ['user_id', 'movie_id', 'rating', 'unix_timestamp']\n","ratings_df = pd.read_csv('../input/movielens-100k-dataset/ml-100k/u.data', sep='\\t', names=ratings_columns)\n","ratings_df.drop( \"unix_timestamp\", inplace = True, axis = 1 ) \n","ratings_df.head(2)"]},{"cell_type":"markdown","id":"28cd599f","metadata":{"papermill":{"duration":0.00662,"end_time":"2023-05-08T12:12:44.751187","exception":false,"start_time":"2023-05-08T12:12:44.744567","status":"completed"},"tags":[]},"source":["## Read and clean movie data"]},{"cell_type":"code","execution_count":4,"id":"6aab6681","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:44.767175Z","iopub.status.busy":"2023-05-08T12:12:44.766751Z","iopub.status.idle":"2023-05-08T12:12:44.773429Z","shell.execute_reply":"2023-05-08T12:12:44.772312Z"},"papermill":{"duration":0.017344,"end_time":"2023-05-08T12:12:44.77557","exception":false,"start_time":"2023-05-08T12:12:44.758226","status":"completed"},"tags":[]},"outputs":[],"source":["def clean_title(title):\n","    return re.sub(\"[\\(\\[].*?[\\)\\]]\", \"\",title)\n","\n","def process_genre(series):\n","    genres = series.index[6:-2]\n","    \n","    text = []\n","    for i in genres:\n","        if series[i] == 1:\n","            text.append(i)\n","            break\n","    return \", \".join(text)"]},{"cell_type":"markdown","id":"9fa6f544","metadata":{"papermill":{"duration":0.007709,"end_time":"2023-05-08T12:12:44.790233","exception":false,"start_time":"2023-05-08T12:12:44.782524","status":"completed"},"tags":[]},"source":["### Read genres\n","\n","We get the list of names for movie genres from **u.genre** file."]},{"cell_type":"code","execution_count":5,"id":"100f7649","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:44.805753Z","iopub.status.busy":"2023-05-08T12:12:44.805355Z","iopub.status.idle":"2023-05-08T12:12:44.824459Z","shell.execute_reply":"2023-05-08T12:12:44.822791Z"},"papermill":{"duration":0.029938,"end_time":"2023-05-08T12:12:44.827088","exception":false,"start_time":"2023-05-08T12:12:44.79715","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["['unknown', 'Action', 'Adventure', 'Animation', \"Children's\", 'Comedy', 'Crime', 'Documentary', 'Drama', 'Fantasy', 'Film-Noir', 'Horror', 'Musical', 'Mystery', 'Romance', 'Sci-Fi', 'Thriller', 'War', 'Western']\n"]}],"source":["genre_df = pd.read_csv('../input/movielens-100k-dataset/ml-100k/u.genre', sep='|', encoding='latin-1')\n","genre_columns = [\"unknown\"] + list(genre_df[genre_df.columns[0]].values)\n","print(genre_columns)"]},{"cell_type":"code","execution_count":6,"id":"ba042931","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:44.843911Z","iopub.status.busy":"2023-05-08T12:12:44.843066Z","iopub.status.idle":"2023-05-08T12:12:44.871139Z","shell.execute_reply":"2023-05-08T12:12:44.869572Z"},"papermill":{"duration":0.039935,"end_time":"2023-05-08T12:12:44.874081","exception":false,"start_time":"2023-05-08T12:12:44.834146","status":"completed"},"tags":[]},"outputs":[],"source":["movie_columns = ['movie_id', 'title', 'release_date', 'video_release_date', 'imdb_url']\n","movies_df = pd.read_csv('../input/movielens-100k-dataset/ml-100k/u.item', sep='|', names=movie_columns+genre_columns,\n","                     encoding='latin-1')"]},{"cell_type":"code","execution_count":7,"id":"fb574b2f","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:44.890814Z","iopub.status.busy":"2023-05-08T12:12:44.889703Z","iopub.status.idle":"2023-05-08T12:12:44.916508Z","shell.execute_reply":"2023-05-08T12:12:44.915054Z"},"papermill":{"duration":0.038036,"end_time":"2023-05-08T12:12:44.919275","exception":false,"start_time":"2023-05-08T12:12:44.881239","status":"completed"},"tags":[]},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>movie_id</th>\n","      <th>title</th>\n","      <th>release_date</th>\n","      <th>video_release_date</th>\n","      <th>imdb_url</th>\n","      <th>unknown</th>\n","      <th>Action</th>\n","      <th>Adventure</th>\n","      <th>Animation</th>\n","      <th>Children's</th>\n","      <th>...</th>\n","      <th>Fantasy</th>\n","      <th>Film-Noir</th>\n","      <th>Horror</th>\n","      <th>Musical</th>\n","      <th>Mystery</th>\n","      <th>Romance</th>\n","      <th>Sci-Fi</th>\n","      <th>Thriller</th>\n","      <th>War</th>\n","      <th>Western</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>Toy Story (1995)</td>\n","      <td>01-Jan-1995</td>\n","      <td>NaN</td>\n","      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>GoldenEye (1995)</td>\n","      <td>01-Jan-1995</td>\n","      <td>NaN</td>\n","      <td>http://us.imdb.com/M/title-exact?GoldenEye%20(...</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>3</td>\n","      <td>Four Rooms (1995)</td>\n","      <td>01-Jan-1995</td>\n","      <td>NaN</td>\n","      <td>http://us.imdb.com/M/title-exact?Four%20Rooms%...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>4</td>\n","      <td>Get Shorty (1995)</td>\n","      <td>01-Jan-1995</td>\n","      <td>NaN</td>\n","      <td>http://us.imdb.com/M/title-exact?Get%20Shorty%...</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>5</td>\n","      <td>Copycat (1995)</td>\n","      <td>01-Jan-1995</td>\n","      <td>NaN</td>\n","      <td>http://us.imdb.com/M/title-exact?Copycat%20(1995)</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>5 rows × 24 columns</p>\n","</div>"],"text/plain":["   movie_id              title release_date  video_release_date  \\\n","0         1   Toy Story (1995)  01-Jan-1995                 NaN   \n","1         2   GoldenEye (1995)  01-Jan-1995                 NaN   \n","2         3  Four Rooms (1995)  01-Jan-1995                 NaN   \n","3         4  Get Shorty (1995)  01-Jan-1995                 NaN   \n","4         5     Copycat (1995)  01-Jan-1995                 NaN   \n","\n","                                            imdb_url  unknown  Action  \\\n","0  http://us.imdb.com/M/title-exact?Toy%20Story%2...        0       0   \n","1  http://us.imdb.com/M/title-exact?GoldenEye%20(...        0       1   \n","2  http://us.imdb.com/M/title-exact?Four%20Rooms%...        0       0   \n","3  http://us.imdb.com/M/title-exact?Get%20Shorty%...        0       1   \n","4  http://us.imdb.com/M/title-exact?Copycat%20(1995)        0       0   \n","\n","   Adventure  Animation  Children's  ...  Fantasy  Film-Noir  Horror  Musical  \\\n","0          0          1           1  ...        0          0       0        0   \n","1          1          0           0  ...        0          0       0        0   \n","2          0          0           0  ...        0          0       0        0   \n","3          0          0           0  ...        0          0       0        0   \n","4          0          0           0  ...        0          0       0        0   \n","\n","   Mystery  Romance  Sci-Fi  Thriller  War  Western  \n","0        0        0       0         0    0        0  \n","1        0        0       0         1    0        0  \n","2        0        0       0         1    0        0  \n","3        0        0       0         0    0        0  \n","4        0        0       0         1    0        0  \n","\n","[5 rows x 24 columns]"]},"execution_count":7,"metadata":{},"output_type":"execute_result"}],"source":["movies_df.head(5)"]},{"cell_type":"markdown","id":"3c329f6c","metadata":{"papermill":{"duration":0.007237,"end_time":"2023-05-08T12:12:44.934004","exception":false,"start_time":"2023-05-08T12:12:44.926767","status":"completed"},"tags":[]},"source":["Let's remove the issuing date from the movie title. Also, we create a copy of the movie dataset where we concatenate the genres, so that we have a list with the genres, and we eliminate the separate genre columns."]},{"cell_type":"code","execution_count":8,"id":"b8eea0db","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:44.950322Z","iopub.status.busy":"2023-05-08T12:12:44.949893Z","iopub.status.idle":"2023-05-08T12:12:45.027751Z","shell.execute_reply":"2023-05-08T12:12:45.026855Z"},"papermill":{"duration":0.088924,"end_time":"2023-05-08T12:12:45.030302","exception":false,"start_time":"2023-05-08T12:12:44.941378","status":"completed"},"tags":[]},"outputs":[],"source":["movies_df['title'] = movies_df['title'].apply(clean_title)\n","movies_df['title'] = movies_df['title'].str.strip()\n","\n","movies_df['genre'] = movies_df.apply(process_genre,axis=1)"]},{"cell_type":"code","execution_count":9,"id":"87d3d587","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:45.047106Z","iopub.status.busy":"2023-05-08T12:12:45.046576Z","iopub.status.idle":"2023-05-08T12:12:45.069769Z","shell.execute_reply":"2023-05-08T12:12:45.06833Z"},"papermill":{"duration":0.034477,"end_time":"2023-05-08T12:12:45.072161","exception":false,"start_time":"2023-05-08T12:12:45.037684","status":"completed"},"tags":[]},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>movie_id</th>\n","      <th>title</th>\n","      <th>release_date</th>\n","      <th>video_release_date</th>\n","      <th>imdb_url</th>\n","      <th>unknown</th>\n","      <th>Action</th>\n","      <th>Adventure</th>\n","      <th>Animation</th>\n","      <th>Children's</th>\n","      <th>...</th>\n","      <th>Film-Noir</th>\n","      <th>Horror</th>\n","      <th>Musical</th>\n","      <th>Mystery</th>\n","      <th>Romance</th>\n","      <th>Sci-Fi</th>\n","      <th>Thriller</th>\n","      <th>War</th>\n","      <th>Western</th>\n","      <th>genre</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>Toy Story</td>\n","      <td>01-Jan-1995</td>\n","      <td>NaN</td>\n","      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>Animation</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>GoldenEye</td>\n","      <td>01-Jan-1995</td>\n","      <td>NaN</td>\n","      <td>http://us.imdb.com/M/title-exact?GoldenEye%20(...</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>Action</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>2 rows × 25 columns</p>\n","</div>"],"text/plain":["   movie_id      title release_date  video_release_date  \\\n","0         1  Toy Story  01-Jan-1995                 NaN   \n","1         2  GoldenEye  01-Jan-1995                 NaN   \n","\n","                                            imdb_url  unknown  Action  \\\n","0  http://us.imdb.com/M/title-exact?Toy%20Story%2...        0       0   \n","1  http://us.imdb.com/M/title-exact?GoldenEye%20(...        0       1   \n","\n","   Adventure  Animation  Children's  ...  Film-Noir  Horror  Musical  Mystery  \\\n","0          0          1           1  ...          0       0        0        0   \n","1          1          0           0  ...          0       0        0        0   \n","\n","   Romance  Sci-Fi  Thriller  War  Western      genre  \n","0        0       0         0    0        0  Animation  \n","1        0       0         1    0        0     Action  \n","\n","[2 rows x 25 columns]"]},"execution_count":9,"metadata":{},"output_type":"execute_result"}],"source":["movies_df.head(2)"]},{"cell_type":"code","execution_count":10,"id":"dec0570e","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:45.089286Z","iopub.status.busy":"2023-05-08T12:12:45.088838Z","iopub.status.idle":"2023-05-08T12:12:45.104594Z","shell.execute_reply":"2023-05-08T12:12:45.103336Z"},"papermill":{"duration":0.027704,"end_time":"2023-05-08T12:12:45.107575","exception":false,"start_time":"2023-05-08T12:12:45.079871","status":"completed"},"tags":[]},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>movie_id</th>\n","      <th>title</th>\n","      <th>genre</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>Toy Story</td>\n","      <td>Animation</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>GoldenEye</td>\n","      <td>Action</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>3</td>\n","      <td>Four Rooms</td>\n","      <td>Thriller</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>4</td>\n","      <td>Get Shorty</td>\n","      <td>Action</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>5</td>\n","      <td>Copycat</td>\n","      <td>Crime</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   movie_id       title      genre\n","0         1   Toy Story  Animation\n","1         2   GoldenEye     Action\n","2         3  Four Rooms   Thriller\n","3         4  Get Shorty     Action\n","4         5     Copycat      Crime"]},"execution_count":10,"metadata":{},"output_type":"execute_result"}],"source":["movies_red_df = movies_df.copy()\n","movies_red_df.drop(movies_red_df.columns[[2,3,4]], inplace = True, axis = 1)\n","movies_red_df.drop(genre_columns,axis=1,inplace=True)\n","\n","movies_red_df.head(5)"]},{"cell_type":"markdown","id":"23518605","metadata":{"papermill":{"duration":0.008062,"end_time":"2023-05-08T12:12:45.124279","exception":false,"start_time":"2023-05-08T12:12:45.116217","status":"completed"},"tags":[]},"source":["# Prepare data in expected format for Surprise"]},{"cell_type":"code","execution_count":11,"id":"1bfdd476","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:45.141913Z","iopub.status.busy":"2023-05-08T12:12:45.141471Z","iopub.status.idle":"2023-05-08T12:12:45.152576Z","shell.execute_reply":"2023-05-08T12:12:45.151227Z"},"papermill":{"duration":0.022551,"end_time":"2023-05-08T12:12:45.154831","exception":false,"start_time":"2023-05-08T12:12:45.13228","status":"completed"},"tags":[]},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>user_id</th>\n","      <th>movie_id</th>\n","      <th>rating</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>196</td>\n","      <td>242</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>186</td>\n","      <td>302</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>22</td>\n","      <td>377</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>244</td>\n","      <td>51</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>166</td>\n","      <td>346</td>\n","      <td>1</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   user_id  movie_id  rating\n","0      196       242       3\n","1      186       302       3\n","2       22       377       1\n","3      244        51       2\n","4      166       346       1"]},"execution_count":11,"metadata":{},"output_type":"execute_result"}],"source":["ratings_df.head()"]},{"cell_type":"code","execution_count":12,"id":"dde712cd","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:45.172985Z","iopub.status.busy":"2023-05-08T12:12:45.172529Z","iopub.status.idle":"2023-05-08T12:12:45.182349Z","shell.execute_reply":"2023-05-08T12:12:45.1808Z"},"papermill":{"duration":0.021587,"end_time":"2023-05-08T12:12:45.184704","exception":false,"start_time":"2023-05-08T12:12:45.163117","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Users: 943\n"]}],"source":["print(f\"Users: {ratings_df.user_id.nunique()}\")"]},{"cell_type":"code","execution_count":13,"id":"26ca0037","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:45.203707Z","iopub.status.busy":"2023-05-08T12:12:45.20285Z","iopub.status.idle":"2023-05-08T12:12:45.208282Z","shell.execute_reply":"2023-05-08T12:12:45.20711Z"},"papermill":{"duration":0.017804,"end_time":"2023-05-08T12:12:45.211086","exception":false,"start_time":"2023-05-08T12:12:45.193282","status":"completed"},"tags":[]},"outputs":[],"source":["reader = Reader(rating_scale=(1, 5))"]},{"cell_type":"code","execution_count":14,"id":"38df188f","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:45.230227Z","iopub.status.busy":"2023-05-08T12:12:45.229167Z","iopub.status.idle":"2023-05-08T12:12:45.361914Z","shell.execute_reply":"2023-05-08T12:12:45.360697Z"},"papermill":{"duration":0.145493,"end_time":"2023-05-08T12:12:45.364686","exception":false,"start_time":"2023-05-08T12:12:45.219193","status":"completed"},"tags":[]},"outputs":[],"source":["data = Dataset.load_from_df(ratings_df, reader=reader)"]},{"cell_type":"markdown","id":"c72c6ada","metadata":{"papermill":{"duration":0.007893,"end_time":"2023-05-08T12:12:45.380832","exception":false,"start_time":"2023-05-08T12:12:45.372939","status":"completed"},"tags":[]},"source":["## Baseline using NormalPredictor\n","\n","We start by creating a baseline with a random predictor. This way we will have a basis to measure the performance of applying SVD model to our data."]},{"cell_type":"code","execution_count":15,"id":"aae3b930","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:45.398667Z","iopub.status.busy":"2023-05-08T12:12:45.397862Z","iopub.status.idle":"2023-05-08T12:12:45.403236Z","shell.execute_reply":"2023-05-08T12:12:45.401991Z"},"papermill":{"duration":0.016874,"end_time":"2023-05-08T12:12:45.405663","exception":false,"start_time":"2023-05-08T12:12:45.388789","status":"completed"},"tags":[]},"outputs":[],"source":["algo_np = NormalPredictor()"]},{"cell_type":"markdown","id":"e693e444","metadata":{"papermill":{"duration":0.007686,"end_time":"2023-05-08T12:12:45.421548","exception":false,"start_time":"2023-05-08T12:12:45.413862","status":"completed"},"tags":[]},"source":["We do cross-validation with 5 folds and we measure three metrics:\n","* RMSE - Root Mean Squared Error - smaller is better\n","* MAE - Mean Absolute Error - smaller is better\n","* FCP - Fraction of Concordant Pairs - higher is better (see: https://www.ijcai.org/Proceedings/13/Papers/449.pdf for reference)"]},{"cell_type":"code","execution_count":16,"id":"cfbf10d2","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:45.439482Z","iopub.status.busy":"2023-05-08T12:12:45.438928Z","iopub.status.idle":"2023-05-08T12:12:48.953164Z","shell.execute_reply":"2023-05-08T12:12:48.952262Z"},"papermill":{"duration":3.526639,"end_time":"2023-05-08T12:12:48.956249","exception":false,"start_time":"2023-05-08T12:12:45.42961","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Evaluating RMSE, MAE, FCP of algorithm NormalPredictor on 5 split(s).\n","\n","                  Fold 1  Fold 2  Fold 3  Fold 4  Fold 5  Mean    Std     \n","RMSE (testset)    1.5124  1.5193  1.5173  1.5241  1.5238  1.5194  0.0044  \n","MAE (testset)     1.2135  1.2225  1.2224  1.2271  1.2248  1.2221  0.0046  \n","FCP (testset)     0.4969  0.4925  0.4929  0.4989  0.4858  0.4934  0.0045  \n","Fit time          0.14    0.16    0.16    0.17    0.19    0.16    0.02    \n","Test time         0.28    0.15    0.15    0.27    0.15    0.20    0.06    \n"]},{"data":{"text/plain":["{'test_rmse': array([1.51236874, 1.51925514, 1.51730089, 1.52413816, 1.52382212]),\n"," 'test_mae': array([1.21346976, 1.22251733, 1.22237508, 1.22714034, 1.22475444]),\n"," 'test_fcp': array([0.49693499, 0.4925232 , 0.49290548, 0.49885622, 0.4858258 ]),\n"," 'fit_time': (0.137587308883667,\n","  0.1588761806488037,\n","  0.16019272804260254,\n","  0.1650855541229248,\n","  0.18841075897216797),\n"," 'test_time': (0.27613210678100586,\n","  0.1474018096923828,\n","  0.1464388370513916,\n","  0.27301025390625,\n","  0.14977288246154785)}"]},"execution_count":16,"metadata":{},"output_type":"execute_result"}],"source":["cross_validate(algo_np, data, measures = [\"RMSE\", \"MAE\", \"FCP\"], cv=5, verbose=True)"]},{"cell_type":"markdown","id":"adcb49b5","metadata":{"papermill":{"duration":0.01176,"end_time":"2023-05-08T12:12:48.980108","exception":false,"start_time":"2023-05-08T12:12:48.968348","status":"completed"},"tags":[]},"source":["## Model with SVD\n","\n","We train now the SVD (Singular Value Decomposition) model with cross-validation.  \n","We need to set at least the number of factors."]},{"cell_type":"code","execution_count":17,"id":"439847a2","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:49.00689Z","iopub.status.busy":"2023-05-08T12:12:49.005666Z","iopub.status.idle":"2023-05-08T12:12:49.011783Z","shell.execute_reply":"2023-05-08T12:12:49.010631Z"},"papermill":{"duration":0.021921,"end_time":"2023-05-08T12:12:49.014261","exception":false,"start_time":"2023-05-08T12:12:48.99234","status":"completed"},"tags":[]},"outputs":[],"source":["algo_svd = SVD(n_factors=40)"]},{"cell_type":"code","execution_count":18,"id":"8544ef17","metadata":{"execution":{"iopub.execute_input":"2023-05-08T12:12:49.033246Z","iopub.status.busy":"2023-05-08T12:12:49.032747Z","iopub.status.idle":"2023-05-08T12:13:07.209003Z","shell.execute_reply":"2023-05-08T12:13:07.207707Z"},"papermill":{"duration":18.189401,"end_time":"2023-05-08T12:13:07.212044","exception":false,"start_time":"2023-05-08T12:12:49.022643","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Evaluating RMSE, MAE, FCP of algorithm SVD on 5 split(s).\n","\n","                  Fold 1  Fold 2  Fold 3  Fold 4  Fold 5  Mean    Std     \n","RMSE (testset)    0.9347  0.9301  0.9382  0.9309  0.9329  0.9334  0.0029  \n","MAE (testset)     0.7360  0.7352  0.7382  0.7370  0.7349  0.7363  0.0012  \n","FCP (testset)     0.6999  0.7042  0.6939  0.7033  0.7055  0.7014  0.0042  \n","Fit time          3.05    3.10    3.09    3.07    3.13    3.09    0.03    \n","Test time         0.15    0.28    0.15    0.14    0.15    0.17    0.05    \n"]},{"data":{"text/plain":["{'test_rmse': array([0.93473246, 0.93014697, 0.93815803, 0.93090972, 0.93286119]),\n"," 'test_mae': array([0.73595842, 0.7352312 , 0.73819545, 0.73699584, 0.73494796]),\n"," 'test_fcp': array([0.69992859, 0.70421099, 0.69391508, 0.70333346, 0.70553694]),\n"," 'fit_time': (3.050109624862671,\n","  3.1021015644073486,\n","  3.0880095958709717,\n","  3.070892333984375,\n","  3.1302261352539062),\n"," 'test_time': (0.14779257774353027,\n","  0.27581119537353516,\n","  0.14616942405700684,\n","  0.14300203323364258,\n","  0.1465301513671875)}"]},"execution_count":18,"metadata":{},"output_type":"execute_result"}],"source":["cross_validate(algo_svd, data, measures = [\"RMSE\", \"MAE\", \"FCP\"], cv=5, verbose=True)"]},{"cell_type":"markdown","id":"7387421f","metadata":{"papermill":{"duration":0.008313,"end_time":"2023-05-08T12:13:07.228964","exception":false,"start_time":"2023-05-08T12:13:07.220651","status":"completed"},"tags":[]},"source":["# Final remarks\n","\n","With SVD we improved RMSE and MAE, but also FCP, from the values obtained with normal predictor (random values)."]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.12"},"papermill":{"default_parameters":{},"duration":33.846132,"end_time":"2023-05-08T12:13:08.061372","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-05-08T12:12:34.21524","version":"2.3.4"}},"nbformat":4,"nbformat_minor":5}