collaborative-filtering-knn-user-item-evaluation.ipynb

{"cells":[{"source":"<a href=\"https://www.kaggle.com/code/gpreda/collaborative-filtering-knn-user-item-evaluation?scriptVersionId=128769100\" target=\"_blank\"><img align=\"left\" alt=\"Kaggle\" title=\"Open in Kaggle\" src=\"https://kaggle.com/static/images/open-in-kaggle.svg\"></a>","metadata":{},"cell_type":"markdown"},{"cell_type":"markdown","id":"de62d9d0","metadata":{"papermill":{"duration":0.005948,"end_time":"2023-05-08T13:34:08.781213","exception":false,"start_time":"2023-05-08T13:34:08.775265","status":"completed"},"tags":[]},"source":["# Introduction\n","\n","\n","Compare collaborative filtering using KNN:   \n","* User-based collaborative filtering  \n","* Item-based collaborative filtering  \n","* Random generated recommendations (baseline)  \n"]},{"cell_type":"markdown","id":"cdca3041","metadata":{"papermill":{"duration":0.004514,"end_time":"2023-05-08T13:34:08.790813","exception":false,"start_time":"2023-05-08T13:34:08.786299","status":"completed"},"tags":[]},"source":["# Prepare analysis"]},{"cell_type":"code","execution_count":1,"id":"ae2895b2","metadata":{"execution":{"iopub.execute_input":"2023-05-08T13:34:08.802826Z","iopub.status.busy":"2023-05-08T13:34:08.802054Z","iopub.status.idle":"2023-05-08T13:34:09.881792Z","shell.execute_reply":"2023-05-08T13:34:09.880586Z"},"papermill":{"duration":1.088932,"end_time":"2023-05-08T13:34:09.884581","exception":false,"start_time":"2023-05-08T13:34:08.795649","status":"completed"},"tags":[]},"outputs":[],"source":["import numpy as np\n","import pandas as pd\n","import re\n","import os\n","import heapq\n","from surprise import accuracy\n","from surprise import Dataset\n","from surprise import Reader\n","from sklearn.metrics.pairwise import linear_kernel, cosine_similarity\n","from surprise.model_selection import cross_validate\n","from surprise.model_selection import train_test_split\n","from surprise.model_selection import LeaveOneOut\n","from surprise import NormalPredictor, KNNBasic"]},{"cell_type":"code","execution_count":2,"id":"726bcbdb","metadata":{"execution":{"iopub.execute_input":"2023-05-08T13:34:09.896408Z","iopub.status.busy":"2023-05-08T13:34:09.896034Z","iopub.status.idle":"2023-05-08T13:34:09.948475Z","shell.execute_reply":"2023-05-08T13:34:09.947586Z"},"papermill":{"duration":0.061587,"end_time":"2023-05-08T13:34:09.951226","exception":false,"start_time":"2023-05-08T13:34:09.889639","status":"completed"},"tags":[]},"outputs":[],"source":["from recommender_metrics import RecommenderMetrics\n","from movie_lens_data import MovieLensData\n","from evaluator import Evaluator"]},{"cell_type":"markdown","id":"c8e8c9da","metadata":{"papermill":{"duration":0.004714,"end_time":"2023-05-08T13:34:09.961179","exception":false,"start_time":"2023-05-08T13:34:09.956465","status":"completed"},"tags":[]},"source":["# Read the data"]},{"cell_type":"code","execution_count":3,"id":"2e9b3530","metadata":{"execution":{"iopub.execute_input":"2023-05-08T13:34:09.973244Z","iopub.status.busy":"2023-05-08T13:34:09.972563Z","iopub.status.idle":"2023-05-08T13:34:15.014459Z","shell.execute_reply":"2023-05-08T13:34:15.01337Z"},"papermill":{"duration":5.051371,"end_time":"2023-05-08T13:34:15.017375","exception":false,"start_time":"2023-05-08T13:34:09.966004","status":"completed"},"tags":[]},"outputs":[],"source":["path = \"/kaggle/input/movielens-100k-dataset/ml-100k\"\n","movie_lens_data = MovieLensData(\n","    users_path = os.path.join(path, \"u.user\"),\n","    ratings_path = os.path.join(path, \"u.data\"), \n","    movies_path = os.path.join(path, \"u.item\"), \n","    genre_path = os.path.join(path, \"u.genre\") \n","    )\n","\n","evaluation_data = movie_lens_data.read_ratings_data()\n","movie_data = movie_lens_data.read_movies_data()\n","popularity_rankings = movie_lens_data.get_popularity_ranks()\n","ratings = movie_lens_data.get_ratings()"]},{"cell_type":"markdown","id":"8320e09e","metadata":{"papermill":{"duration":0.00487,"end_time":"2023-05-08T13:34:15.027615","exception":false,"start_time":"2023-05-08T13:34:15.022745","status":"completed"},"tags":[]},"source":["# Prepare evaluator"]},{"cell_type":"code","execution_count":4,"id":"ca1c7fd0","metadata":{"execution":{"iopub.execute_input":"2023-05-08T13:34:15.039642Z","iopub.status.busy":"2023-05-08T13:34:15.038786Z","iopub.status.idle":"2023-05-08T13:34:20.612276Z","shell.execute_reply":"2023-05-08T13:34:20.611039Z"},"papermill":{"duration":5.582284,"end_time":"2023-05-08T13:34:20.614744","exception":false,"start_time":"2023-05-08T13:34:15.03246","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Number of full trainset users: 943\n","Number of full trainset items: 1682\n","Number of trainset users: 943\n","Number of trainset items: 1641\n","Size of testset: 25000\n","Estimating biases using als...\n","Computing the cosine similarity matrix...\n","Done computing similarity matrix.\n"]}],"source":["evaluator = Evaluator(evaluation_data, popularity_rankings)"]},{"cell_type":"markdown","id":"05d0177d","metadata":{"papermill":{"duration":0.005444,"end_time":"2023-05-08T13:34:20.625457","exception":false,"start_time":"2023-05-08T13:34:20.620013","status":"completed"},"tags":[]},"source":["# Add random recommender to evaluator"]},{"cell_type":"code","execution_count":5,"id":"235ac7d1","metadata":{"execution":{"iopub.execute_input":"2023-05-08T13:34:20.637811Z","iopub.status.busy":"2023-05-08T13:34:20.636845Z","iopub.status.idle":"2023-05-08T13:34:20.642522Z","shell.execute_reply":"2023-05-08T13:34:20.641209Z"},"papermill":{"duration":0.014369,"end_time":"2023-05-08T13:34:20.644859","exception":false,"start_time":"2023-05-08T13:34:20.63049","status":"completed"},"tags":[]},"outputs":[],"source":["algo_np = NormalPredictor()\n","evaluator.add_algorithm(algo_np, \"Random\")"]},{"cell_type":"markdown","id":"9d15fab9","metadata":{"papermill":{"duration":0.004779,"end_time":"2023-05-08T13:34:20.654996","exception":false,"start_time":"2023-05-08T13:34:20.650217","status":"completed"},"tags":[]},"source":["# Add item-based collaborative filtering RecSys to evaluator\n","\n","\n","Using the sim_options, we specify the type of similarity calculation and if the collaborative filtering is user based (in this case, No)"]},{"cell_type":"code","execution_count":6,"id":"d537396c","metadata":{"execution":{"iopub.execute_input":"2023-05-08T13:34:20.669092Z","iopub.status.busy":"2023-05-08T13:34:20.668036Z","iopub.status.idle":"2023-05-08T13:34:20.67389Z","shell.execute_reply":"2023-05-08T13:34:20.672716Z"},"papermill":{"duration":0.014564,"end_time":"2023-05-08T13:34:20.676205","exception":false,"start_time":"2023-05-08T13:34:20.661641","status":"completed"},"tags":[]},"outputs":[],"source":["item_KNN = KNNBasic(sim_options = {'name': 'pearson', 'user_based': False})\n","evaluator.add_algorithm(item_KNN, \"Item KNN\")"]},{"cell_type":"markdown","id":"eaaae01f","metadata":{"papermill":{"duration":0.005064,"end_time":"2023-05-08T13:34:20.686327","exception":false,"start_time":"2023-05-08T13:34:20.681263","status":"completed"},"tags":[]},"source":["# Add user-based collaborative filtering RecSys to evaluator\n","\n","Using the sim_options, we specify the type of similarity calculation and if the collaborative filtering is user based (in this case, Yes)"]},{"cell_type":"code","execution_count":7,"id":"b573a387","metadata":{"execution":{"iopub.execute_input":"2023-05-08T13:34:20.698322Z","iopub.status.busy":"2023-05-08T13:34:20.697908Z","iopub.status.idle":"2023-05-08T13:34:20.702981Z","shell.execute_reply":"2023-05-08T13:34:20.701934Z"},"papermill":{"duration":0.01373,"end_time":"2023-05-08T13:34:20.705094","exception":false,"start_time":"2023-05-08T13:34:20.691364","status":"completed"},"tags":[]},"outputs":[],"source":["user_KNN = KNNBasic(sim_options = {'name': 'pearson', 'user_based': True})\n","evaluator.add_algorithm(user_KNN, \"User KNN\")"]},{"cell_type":"markdown","id":"928a25b4","metadata":{"papermill":{"duration":0.004796,"end_time":"2023-05-08T13:34:20.714946","exception":false,"start_time":"2023-05-08T13:34:20.71015","status":"completed"},"tags":[]},"source":["# Evaluate algorithms"]},{"cell_type":"code","execution_count":8,"id":"2c1248d3","metadata":{"execution":{"iopub.execute_input":"2023-05-08T13:34:20.726578Z","iopub.status.busy":"2023-05-08T13:34:20.726155Z","iopub.status.idle":"2023-05-08T13:34:34.693574Z","shell.execute_reply":"2023-05-08T13:34:34.69181Z"},"papermill":{"duration":13.976423,"end_time":"2023-05-08T13:34:34.696237","exception":false,"start_time":"2023-05-08T13:34:20.719814","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Evaluating  Random ...\n","Evaluating accuracy...\n","Analysis complete.\n","Evaluating  Item KNN ...\n","Evaluating accuracy...\n","Computing the pearson similarity matrix...\n","Done computing similarity matrix.\n","Analysis complete.\n","Evaluating  User KNN ...\n","Evaluating accuracy...\n","Computing the pearson similarity matrix...\n","Done computing similarity matrix.\n","Analysis complete.\n","\n","\n","Algorithm  RMSE       MAE        FCP       \n","Random     1.5251     1.2224     0.4945    \n","Item KNN   1.0442     0.8338     0.5456    \n","User KNN   1.0166     0.8059     0.7080    \n","\n","Legend:\n","\n","RMSE:      Root Mean Squared Error. Lower values mean better accuracy.\n","MAE:       Mean Absolute Error. Lower values mean better accuracy.\n","FCP:       Fraction of Concordant Pairs. Higher values mean better accuracy.\n"]}],"source":["evaluator.evaluate(do_top_n=False)"]},{"cell_type":"code","execution_count":9,"id":"a2316442","metadata":{"execution":{"iopub.execute_input":"2023-05-08T13:34:34.71018Z","iopub.status.busy":"2023-05-08T13:34:34.709733Z","iopub.status.idle":"2023-05-08T13:44:32.306945Z","shell.execute_reply":"2023-05-08T13:44:32.305503Z"},"papermill":{"duration":597.607854,"end_time":"2023-05-08T13:44:32.309954","exception":false,"start_time":"2023-05-08T13:34:34.7021","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Evaluating  Random ...\n","Evaluating accuracy...\n","Evaluating top-N with leave-one-out...\n","Computing hit-rate and rank metrics...\n","Computing recommendations with full data set...\n","Analyzing coverage, diversity, and novelty...\n","Computing the cosine similarity matrix...\n","Done computing similarity matrix.\n","Analysis complete.\n","Evaluating  Item KNN ...\n","Evaluating accuracy...\n","Computing the pearson similarity matrix...\n","Done computing similarity matrix.\n","Evaluating top-N with leave-one-out...\n","Computing the pearson similarity matrix...\n","Done computing similarity matrix.\n","Computing hit-rate and rank metrics...\n","Computing recommendations with full data set...\n","Computing the pearson similarity matrix...\n","Done computing similarity matrix.\n","Analyzing coverage, diversity, and novelty...\n","Computing the cosine similarity matrix...\n","Done computing similarity matrix.\n","Analysis complete.\n","Evaluating  User KNN ...\n","Evaluating accuracy...\n","Computing the pearson similarity matrix...\n","Done computing similarity matrix.\n","Evaluating top-N with leave-one-out...\n","Computing the pearson similarity matrix...\n","Done computing similarity matrix.\n","Computing hit-rate and rank metrics...\n","Computing recommendations with full data set...\n","Computing the pearson similarity matrix...\n","Done computing similarity matrix.\n","Analyzing coverage, diversity, and novelty...\n","Computing the cosine similarity matrix...\n","Done computing similarity matrix.\n","Analysis complete.\n","\n","\n","Algorithm  RMSE       MAE        FCP        HR         cHR        ARHR       Coverage   Diversity  Novelty   \n","Random     1.5163     1.2192     0.4963     0.0276     0.0276     0.0097     1.0000     0.0726     355.2065  \n","Item KNN   1.0442     0.8338     0.5456     0.0000     0.0000     0.0000     0.9958     0.6690     1393.3134 \n","User KNN   1.0166     0.8059     0.7080     0.0011     0.0011     0.0011     1.0000     0.9199     1463.8406 \n","\n","Legend:\n","\n","RMSE:      Root Mean Squared Error. Lower values mean better accuracy.\n","MAE:       Mean Absolute Error. Lower values mean better accuracy.\n","FCP:       Fraction of Concordant Pairs. Higher values mean better accuracy.\n","HR:        Hit Rate; how often we are able to recommend a left-out rating. Higher is better.\n","cHR:       Cumulative Hit Rate; hit rate, confined to ratings above a certain threshold. Higher is better.\n","ARHR:      Average Reciprocal Hit Rank - Hit rate that takes the ranking into account. Higher is better.\n","Coverage:  Ratio of users for whom recommendations above a certain threshold exist. Higher is better.\n","Diversity: 1-S, where S is the average similarity score between every possible pair of recommendations\n","           for a given user. Higher means more diverse.\n","Novelty:   Average popularity rank of recommended items. Higher means more novel.\n"]}],"source":["# Time consuming, uncomment optionally\n","evaluator.evaluate(do_top_n=True)"]},{"cell_type":"markdown","id":"3cfe5118","metadata":{"papermill":{"duration":0.007948,"end_time":"2023-05-08T13:44:32.328305","exception":false,"start_time":"2023-05-08T13:44:32.320357","status":"completed"},"tags":[]},"source":["# Evaluate topN recommendations"]},{"cell_type":"code","execution_count":10,"id":"3c48f3f5","metadata":{"execution":{"iopub.execute_input":"2023-05-08T13:44:32.344975Z","iopub.status.busy":"2023-05-08T13:44:32.343736Z","iopub.status.idle":"2023-05-08T13:44:37.81484Z","shell.execute_reply":"2023-05-08T13:44:37.813424Z"},"papermill":{"duration":5.482215,"end_time":"2023-05-08T13:44:37.817621","exception":false,"start_time":"2023-05-08T13:44:32.335406","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","Using recommender  Random\n","\n","Building recommendation model...\n","Computing recommendations...\n","\n","We recommend:\n","Die Hard 5\n","Naked Gun 33 1/3: The Final Insult 5\n","Executive Decision 5\n","Swimming with Sharks 5\n","Devil's Own, The 5\n","Courage Under Fire 5\n","Homeward Bound: The Incredible Journey 5\n","It Could Happen to You 5\n","GoldenEye 5\n","Kazaam 5\n","\n","Using recommender  Item KNN\n","\n","Building recommendation model...\n","Computing the pearson similarity matrix...\n","Done computing similarity matrix.\n","Computing recommendations...\n","\n","We recommend:\n","Salut cousin! 4.5\n","To Have, or Not 4.5\n","Good Man in Africa, A 4.4\n","Amityville Curse, The 4.148591942911796\n","Amityville 3-D 4.137656281621579\n","Inkwell, The 4.0\n","Castle Freak 4.0\n","Turning, The 4.0\n","Commandments 4.0\n","Letter From Death Row, A 4.0\n","\n","Using recommender  User KNN\n","\n","Building recommendation model...\n","Computing the pearson similarity matrix...\n","Done computing similarity matrix.\n","Computing recommendations...\n","\n","We recommend:\n","Prefontaine 5\n","Santa with Muscles 5\n","Boys, Les 5\n","Great Day in Harlem, A 5\n","Aiqing wansui 5\n","For the Moment 5\n","Star Kid 5\n","The Deadly Cure 5\n","Leading Man, The 5\n","Someone Else's America 5\n"]}],"source":["evaluator.sample_top_n_recs(movie_lens_data, test_subject=85, k=10)"]},{"cell_type":"code","execution_count":11,"id":"f26f18bd","metadata":{"execution":{"iopub.execute_input":"2023-05-08T13:44:37.834817Z","iopub.status.busy":"2023-05-08T13:44:37.833724Z","iopub.status.idle":"2023-05-08T13:44:43.510051Z","shell.execute_reply":"2023-05-08T13:44:43.508575Z"},"papermill":{"duration":5.687957,"end_time":"2023-05-08T13:44:43.512814","exception":false,"start_time":"2023-05-08T13:44:37.824857","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","Using recommender  Random\n","\n","Building recommendation model...\n","Computing recommendations...\n","\n","We recommend:\n","Hunt for Red October, The 5\n","Age of Innocence, The 5\n","Casper 5\n","Mr. Holland's Opus 5\n","Jean de Florette 5\n","Dangerous Minds 5\n","Liar Liar 5\n","Terminator 2: Judgment Day 5\n","Clockers 5\n","Chain Reaction 5\n","\n","Using recommender  Item KNN\n","\n","Building recommendation model...\n","Computing the pearson similarity matrix...\n","Done computing similarity matrix.\n","Computing recommendations...\n","\n","We recommend:\n","Salut cousin! 4.5\n","To Have, or Not 4.5\n","Good Man in Africa, A 4.4\n","Amityville Curse, The 4.148591942911796\n","Amityville 3-D 4.137656281621579\n","Inkwell, The 4.0\n","Castle Freak 4.0\n","Turning, The 4.0\n","Commandments 4.0\n","Letter From Death Row, A 4.0\n","\n","Using recommender  User KNN\n","\n","Building recommendation model...\n","Computing the pearson similarity matrix...\n","Done computing similarity matrix.\n","Computing recommendations...\n","\n","We recommend:\n","Prefontaine 5\n","Santa with Muscles 5\n","Boys, Les 5\n","Great Day in Harlem, A 5\n","Aiqing wansui 5\n","For the Moment 5\n","Star Kid 5\n","The Deadly Cure 5\n","Leading Man, The 5\n","Someone Else's America 5\n"]}],"source":["evaluator.sample_top_n_recs(movie_lens_data, test_subject=85, k=5)"]},{"cell_type":"code","execution_count":12,"id":"e2e3aecb","metadata":{"execution":{"iopub.execute_input":"2023-05-08T13:44:43.531208Z","iopub.status.busy":"2023-05-08T13:44:43.530406Z","iopub.status.idle":"2023-05-08T13:44:49.087141Z","shell.execute_reply":"2023-05-08T13:44:49.085433Z"},"papermill":{"duration":5.569138,"end_time":"2023-05-08T13:44:49.089773","exception":false,"start_time":"2023-05-08T13:44:43.520635","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","Using recommender  Random\n","\n","Building recommendation model...\n","Computing recommendations...\n","\n","We recommend:\n","To Wong Foo, Thanks for Everything! Julie Newmar 5\n","Sabrina 5\n","Silence of the Lambs, The 5\n","Rear Window 5\n","Crumb 5\n","Tales from the Hood 5\n","Return of the Jedi 5\n","Angels and Insects 5\n","Ben-Hur 5\n","Batman 5\n","\n","Using recommender  Item KNN\n","\n","Building recommendation model...\n","Computing the pearson similarity matrix...\n","Done computing similarity matrix.\n","Computing recommendations...\n","\n","We recommend:\n","Hugo Pool 5\n","Ill Gotten Gains 5\n","Hugo Pool 5\n","Rough Magic 5\n","Temptress Moon 5\n","Men of Means 5\n","Truman Show, The 4.894106578227169\n","Leading Man, The 4.843018994729905\n","Relative Fear 4.818263625777179\n","Simple Wish, A 4.666666666666667\n","\n","Using recommender  User KNN\n","\n","Building recommendation model...\n","Computing the pearson similarity matrix...\n","Done computing similarity matrix.\n","Computing recommendations...\n","\n","We recommend:\n","Prefontaine 5\n","Great Day in Harlem, A 5\n","Innocents, The 5\n","Visitors, The 5\n","Star Kid 5\n","Saint of Fort Washington, The 5\n","Alphaville 4.695949837335183\n","Maya Lin: A Strong Clear Vision 4.683183332658394\n","Affair to Remember, An 4.578658759987711\n","Golden Earrings 4.52495893412099\n"]}],"source":["evaluator.sample_top_n_recs(movie_lens_data, test_subject=314, k=10)"]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.12"},"papermill":{"default_parameters":{},"duration":652.611605,"end_time":"2023-05-08T13:44:50.325795","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-05-08T13:33:57.71419","version":"2.4.0"}},"nbformat":4,"nbformat_minor":5}