collaborative-filtering-svd-tuning-evaluate.ipynb

{"cells":[{"source":"<a href=\"https://www.kaggle.com/code/gpreda/collaborative-filtering-svd-tuning-evaluate?scriptVersionId=128777383\" target=\"_blank\"><img align=\"left\" alt=\"Kaggle\" title=\"Open in Kaggle\" src=\"https://kaggle.com/static/images/open-in-kaggle.svg\"></a>","metadata":{},"cell_type":"markdown"},{"cell_type":"markdown","id":"2133b2d2","metadata":{"papermill":{"duration":0.006329,"end_time":"2023-05-08T14:36:08.662767","exception":false,"start_time":"2023-05-08T14:36:08.656438","status":"completed"},"tags":[]},"source":["# Introduction\n","\n","We tune Singular Value Decomposition method and then compare tunned and not-tuned version.\n"]},{"cell_type":"markdown","id":"46275344","metadata":{"papermill":{"duration":0.004813,"end_time":"2023-05-08T14:36:08.673005","exception":false,"start_time":"2023-05-08T14:36:08.668192","status":"completed"},"tags":[]},"source":["# Analysis preparation"]},{"cell_type":"code","execution_count":1,"id":"340b00cd","metadata":{"execution":{"iopub.execute_input":"2023-05-08T14:36:08.68596Z","iopub.status.busy":"2023-05-08T14:36:08.685307Z","iopub.status.idle":"2023-05-08T14:36:08.814585Z","shell.execute_reply":"2023-05-08T14:36:08.813071Z"},"papermill":{"duration":0.139415,"end_time":"2023-05-08T14:36:08.817672","exception":false,"start_time":"2023-05-08T14:36:08.678257","status":"completed"},"tags":[]},"outputs":[],"source":["import numpy as np\n","import pandas as pd\n","import re\n","import os\n","from surprise.model_selection import cross_validate\n","from surprise.model_selection import GridSearchCV\n","from surprise import NormalPredictor, SVD"]},{"cell_type":"code","execution_count":2,"id":"e38dd737","metadata":{"execution":{"iopub.execute_input":"2023-05-08T14:36:08.831534Z","iopub.status.busy":"2023-05-08T14:36:08.830262Z","iopub.status.idle":"2023-05-08T14:36:08.881508Z","shell.execute_reply":"2023-05-08T14:36:08.879901Z"},"papermill":{"duration":0.061157,"end_time":"2023-05-08T14:36:08.884486","exception":false,"start_time":"2023-05-08T14:36:08.823329","status":"completed"},"tags":[]},"outputs":[],"source":["from recommender_metrics import RecommenderMetrics\n","from movie_lens_data import MovieLensData\n","from evaluator import Evaluator"]},{"cell_type":"markdown","id":"25cef376","metadata":{"papermill":{"duration":0.004924,"end_time":"2023-05-08T14:36:08.894749","exception":false,"start_time":"2023-05-08T14:36:08.889825","status":"completed"},"tags":[]},"source":["# Read the data"]},{"cell_type":"code","execution_count":3,"id":"9caf9eb9","metadata":{"execution":{"iopub.execute_input":"2023-05-08T14:36:08.907816Z","iopub.status.busy":"2023-05-08T14:36:08.907392Z","iopub.status.idle":"2023-05-08T14:36:13.969745Z","shell.execute_reply":"2023-05-08T14:36:13.968479Z"},"papermill":{"duration":5.072682,"end_time":"2023-05-08T14:36:13.97262","exception":false,"start_time":"2023-05-08T14:36:08.899938","status":"completed"},"tags":[]},"outputs":[],"source":["path = \"/kaggle/input/movielens-100k-dataset/ml-100k\"\n","movie_lens_data = MovieLensData(\n","    users_path = os.path.join(path, \"u.user\"),\n","    ratings_path = os.path.join(path, \"u.data\"), \n","    movies_path = os.path.join(path, \"u.item\"), \n","    genre_path = os.path.join(path, \"u.genre\") \n","    )\n","\n","evaluation_data = movie_lens_data.read_ratings_data()\n","movie_data = movie_lens_data.read_movies_data()\n","popularity_rankings = movie_lens_data.get_popularity_ranks()\n","ratings = movie_lens_data.get_ratings()"]},{"cell_type":"markdown","id":"f97725c0","metadata":{"papermill":{"duration":0.004806,"end_time":"2023-05-08T14:36:13.983195","exception":false,"start_time":"2023-05-08T14:36:13.978389","status":"completed"},"tags":[]},"source":["# Prepare evaluator"]},{"cell_type":"code","execution_count":4,"id":"37b0f687","metadata":{"execution":{"iopub.execute_input":"2023-05-08T14:36:13.996877Z","iopub.status.busy":"2023-05-08T14:36:13.995491Z","iopub.status.idle":"2023-05-08T14:36:19.984742Z","shell.execute_reply":"2023-05-08T14:36:19.983807Z"},"papermill":{"duration":5.999073,"end_time":"2023-05-08T14:36:19.987481","exception":false,"start_time":"2023-05-08T14:36:13.988408","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Number of full trainset users: 943\n","Number of full trainset items: 1682\n","Number of trainset users: 943\n","Number of trainset items: 1641\n","Size of testset: 25000\n","Estimating biases using als...\n","Computing the cosine similarity matrix...\n","Done computing similarity matrix.\n"]}],"source":["evaluator = Evaluator(evaluation_data, popularity_rankings)"]},{"cell_type":"markdown","id":"7d86f51b","metadata":{"papermill":{"duration":0.005334,"end_time":"2023-05-08T14:36:19.998738","exception":false,"start_time":"2023-05-08T14:36:19.993404","status":"completed"},"tags":[]},"source":["# Add random recommender to evaluator"]},{"cell_type":"code","execution_count":5,"id":"4cc453cb","metadata":{"execution":{"iopub.execute_input":"2023-05-08T14:36:20.011541Z","iopub.status.busy":"2023-05-08T14:36:20.011091Z","iopub.status.idle":"2023-05-08T14:36:20.016108Z","shell.execute_reply":"2023-05-08T14:36:20.015177Z"},"papermill":{"duration":0.014323,"end_time":"2023-05-08T14:36:20.018584","exception":false,"start_time":"2023-05-08T14:36:20.004261","status":"completed"},"tags":[]},"outputs":[],"source":["algo_np = NormalPredictor()\n","evaluator.add_algorithm(algo_np, \"Random\")"]},{"cell_type":"markdown","id":"28234ca5","metadata":{"papermill":{"duration":0.005619,"end_time":"2023-05-08T14:36:20.031185","exception":false,"start_time":"2023-05-08T14:36:20.025566","status":"completed"},"tags":[]},"source":["# Add SVD"]},{"cell_type":"code","execution_count":6,"id":"d37bda25","metadata":{"execution":{"iopub.execute_input":"2023-05-08T14:36:20.045079Z","iopub.status.busy":"2023-05-08T14:36:20.044549Z","iopub.status.idle":"2023-05-08T14:36:20.050535Z","shell.execute_reply":"2023-05-08T14:36:20.049155Z"},"papermill":{"duration":0.016054,"end_time":"2023-05-08T14:36:20.053382","exception":false,"start_time":"2023-05-08T14:36:20.037328","status":"completed"},"tags":[]},"outputs":[],"source":["algo_svd = SVD()\n","evaluator.add_algorithm(algo_svd, \"SVD\")"]},{"cell_type":"markdown","id":"0489cde4","metadata":{"papermill":{"duration":0.005414,"end_time":"2023-05-08T14:36:20.064484","exception":false,"start_time":"2023-05-08T14:36:20.05907","status":"completed"},"tags":[]},"source":["# Tune and add SVD tuned"]},{"cell_type":"code","execution_count":7,"id":"bf47e3b4","metadata":{"execution":{"iopub.execute_input":"2023-05-08T14:36:20.077823Z","iopub.status.busy":"2023-05-08T14:36:20.077421Z","iopub.status.idle":"2023-05-08T14:36:20.084452Z","shell.execute_reply":"2023-05-08T14:36:20.08249Z"},"papermill":{"duration":0.017036,"end_time":"2023-05-08T14:36:20.087066","exception":false,"start_time":"2023-05-08T14:36:20.07003","status":"completed"},"tags":[]},"outputs":[],"source":["param_grid = {'n_epochs': [20, 25, 30], 'lr_all': [0.005, 0.0075, 0.010],\n","              'n_factors': [50, 75, 100]}\n","gs = GridSearchCV(SVD, param_grid, measures=['rmse', 'mae'], cv=3)"]},{"cell_type":"code","execution_count":8,"id":"3d417218","metadata":{"execution":{"iopub.execute_input":"2023-05-08T14:36:20.100042Z","iopub.status.busy":"2023-05-08T14:36:20.099637Z","iopub.status.idle":"2023-05-08T14:43:13.432546Z","shell.execute_reply":"2023-05-08T14:43:13.43079Z"},"papermill":{"duration":413.343325,"end_time":"2023-05-08T14:43:13.435975","exception":false,"start_time":"2023-05-08T14:36:20.09265","status":"completed"},"tags":[]},"outputs":[],"source":["gs.fit(evaluation_data)"]},{"cell_type":"code","execution_count":9,"id":"f6fe8ee8","metadata":{"execution":{"iopub.execute_input":"2023-05-08T14:43:13.449619Z","iopub.status.busy":"2023-05-08T14:43:13.449211Z","iopub.status.idle":"2023-05-08T14:43:13.455086Z","shell.execute_reply":"2023-05-08T14:43:13.453792Z"},"papermill":{"duration":0.016792,"end_time":"2023-05-08T14:43:13.458805","exception":false,"start_time":"2023-05-08T14:43:13.442013","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["{'n_epochs': 25, 'lr_all': 0.005, 'n_factors': 50}\n"]}],"source":["print(gs.best_params['rmse'])"]},{"cell_type":"code","execution_count":10,"id":"5072d416","metadata":{"execution":{"iopub.execute_input":"2023-05-08T14:43:13.471954Z","iopub.status.busy":"2023-05-08T14:43:13.47156Z","iopub.status.idle":"2023-05-08T14:43:13.47772Z","shell.execute_reply":"2023-05-08T14:43:13.476253Z"},"papermill":{"duration":0.015972,"end_time":"2023-05-08T14:43:13.480416","exception":false,"start_time":"2023-05-08T14:43:13.464444","status":"completed"},"tags":[]},"outputs":[],"source":["params = gs.best_params['rmse']\n","algo_SVD_tuned = SVD(n_epochs = params['n_epochs'], lr_all = params['lr_all'], n_factors = params['n_factors'])\n","evaluator.add_algorithm(algo_SVD_tuned, \"SVD-Tuned\")"]},{"cell_type":"markdown","id":"98d1873b","metadata":{"papermill":{"duration":0.005208,"end_time":"2023-05-08T14:43:13.49138","exception":false,"start_time":"2023-05-08T14:43:13.486172","status":"completed"},"tags":[]},"source":["# Evaluate algorithms"]},{"cell_type":"code","execution_count":11,"id":"b913e100","metadata":{"execution":{"iopub.execute_input":"2023-05-08T14:43:13.50433Z","iopub.status.busy":"2023-05-08T14:43:13.503892Z","iopub.status.idle":"2023-05-08T14:43:24.74858Z","shell.execute_reply":"2023-05-08T14:43:24.747054Z"},"papermill":{"duration":11.254363,"end_time":"2023-05-08T14:43:24.751357","exception":false,"start_time":"2023-05-08T14:43:13.496994","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Evaluating  Random ...\n","Evaluating accuracy...\n","Analysis complete.\n","Evaluating  SVD ...\n","Evaluating accuracy...\n","Analysis complete.\n","Evaluating  SVD-Tuned ...\n","Evaluating accuracy...\n","Analysis complete.\n","\n","\n","Algorithm  RMSE       MAE        FCP       \n","Random     1.5205     1.2238     0.4964    \n","SVD        0.9437     0.7431     0.6960    \n","SVD-Tuned  0.9372     0.7369     0.7031    \n","\n","Legend:\n","\n","RMSE:      Root Mean Squared Error. Lower values mean better accuracy.\n","MAE:       Mean Absolute Error. Lower values mean better accuracy.\n","FCP:       Fraction of Concordant Pairs. Higher values mean better accuracy.\n"]}],"source":["evaluator.evaluate(do_top_n=False)"]},{"cell_type":"markdown","id":"6c65b838","metadata":{"papermill":{"duration":0.005557,"end_time":"2023-05-08T14:43:24.762826","exception":false,"start_time":"2023-05-08T14:43:24.757269","status":"completed"},"tags":[]},"source":["# Evaluate topN recommendations"]},{"cell_type":"code","execution_count":12,"id":"171f94ac","metadata":{"execution":{"iopub.execute_input":"2023-05-08T14:43:24.776181Z","iopub.status.busy":"2023-05-08T14:43:24.775757Z","iopub.status.idle":"2023-05-08T14:43:37.347623Z","shell.execute_reply":"2023-05-08T14:43:37.346087Z"},"papermill":{"duration":12.581979,"end_time":"2023-05-08T14:43:37.350596","exception":false,"start_time":"2023-05-08T14:43:24.768617","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","Using recommender  Random\n","\n","Building recommendation model...\n","Computing recommendations...\n","\n","We recommend:\n","Heavyweights 5\n","Romy and Michele's High School Reunion 5\n","Batman Forever 5\n","Crumb 5\n","French Twist 5\n","Dangerous Minds 5\n","Miller's Crossing 5\n","Amistad 5\n","Mimic 5\n","My Family 5\n","\n","Using recommender  SVD\n","\n","Building recommendation model...\n","Computing recommendations...\n","\n","We recommend:\n","12 Angry Men 4.3180115646420205\n","Glory 4.249907669569958\n","Wrong Trousers, The 4.2434423761891225\n","Wallace & Gromit: The Best of Aardman Animation 4.204108514558328\n","Close Shave, A 4.1851979248975795\n","L.A. Confidential 4.184611136400802\n","Rear Window 4.151459829873411\n","Hoop Dreams 4.14258805558699\n","M 4.142421504742167\n","Shall We Dance? 4.125151483020396\n","\n","Using recommender  SVD-Tuned\n","\n","Building recommendation model...\n","Computing recommendations...\n","\n","We recommend:\n","Close Shave, A 4.509246291043993\n","Wrong Trousers, The 4.3976631387361\n","12 Angry Men 4.377773485623745\n","Wallace & Gromit: The Best of Aardman Animation 4.335151730649278\n","Usual Suspects, The 4.289338524089054\n","Big Night 4.276886035607429\n","Rear Window 4.238888393924331\n","L.A. Confidential 4.213645754190902\n","Secrets & Lies 4.1556341335415325\n","Persuasion 4.150671708445124\n"]}],"source":["evaluator.sample_top_n_recs(movie_lens_data, test_subject=85, k=10)"]},{"cell_type":"code","execution_count":13,"id":"3e0b8433","metadata":{"execution":{"iopub.execute_input":"2023-05-08T14:43:37.365375Z","iopub.status.busy":"2023-05-08T14:43:37.364894Z","iopub.status.idle":"2023-05-08T14:43:50.011149Z","shell.execute_reply":"2023-05-08T14:43:50.009302Z"},"papermill":{"duration":12.657337,"end_time":"2023-05-08T14:43:50.01422","exception":false,"start_time":"2023-05-08T14:43:37.356883","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","Using recommender  Random\n","\n","Building recommendation model...\n","Computing recommendations...\n","\n","We recommend:\n","Jungle Book, The 5\n","Sabrina 5\n","Get Shorty 5\n","Conan the Barbarian 5\n","Angels and Insects 5\n","Brothers McMullen, The 5\n","To Kill a Mockingbird 5\n","Taxi Driver 5\n","Raiders of the Lost Ark 5\n","Annie Hall 5\n","\n","Using recommender  SVD\n","\n","Building recommendation model...\n","Computing recommendations...\n","\n","We recommend:\n","Jurassic Park 5\n","Game, The 4.975242863052553\n","Air Force One 4.937496600449121\n","Close Shave, A 4.9241345769940565\n","Raiders of the Lost Ark 4.897317898910013\n","Terminator 2: Judgment Day 4.833521405462666\n","Star Wars 4.777479988615321\n","Apartment, The 4.760170512473256\n","Singin' in the Rain 4.708449411519849\n","Good, The Bad and The Ugly, The 4.684318141382523\n","\n","Using recommender  SVD-Tuned\n","\n","Building recommendation model...\n","Computing recommendations...\n","\n","We recommend:\n","Return of the Jedi 5\n","Gone with the Wind 5\n","Game, The 5\n","Titanic 5\n","True Lies 4.988729825921464\n","Indiana Jones and the Last Crusade 4.969762428649412\n","Good Will Hunting 4.966071438803906\n","Patton 4.925783589339742\n","Star Wars 4.89490206154912\n","Dave 4.856392519481404\n"]}],"source":["evaluator.sample_top_n_recs(movie_lens_data, test_subject=314, k=10)"]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.12"},"papermill":{"default_parameters":{},"duration":474.896699,"end_time":"2023-05-08T14:43:51.047578","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-05-08T14:35:56.150879","version":"2.4.0"}},"nbformat":4,"nbformat_minor":5}