DATA 602 Recommendations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>adult</th>\n",
       "      <th>belongs_to_collection</th>\n",
       "      <th>budget</th>\n",
       "      <th>genres</th>\n",
       "      <th>homepage</th>\n",
       "      <th>id</th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>original_language</th>\n",
       "      <th>original_title</th>\n",
       "      <th>overview</th>\n",
       "      <th>...</th>\n",
       "      <th>release_date</th>\n",
       "      <th>revenue</th>\n",
       "      <th>runtime</th>\n",
       "      <th>spoken_languages</th>\n",
       "      <th>status</th>\n",
       "      <th>tagline</th>\n",
       "      <th>title</th>\n",
       "      <th>video</th>\n",
       "      <th>vote_average</th>\n",
       "      <th>vote_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 10194, 'name': 'Toy Story Collection', ...</td>\n",
       "      <td>30000000</td>\n",
       "      <td>[{'id': 16, 'name': 'Animation'}, {'id': 35, '...</td>\n",
       "      <td>http://toystory.disney.com/toy-story</td>\n",
       "      <td>862</td>\n",
       "      <td>tt0114709</td>\n",
       "      <td>en</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>Led by Woody, Andy's toys live happily in his ...</td>\n",
       "      <td>...</td>\n",
       "      <td>1995-10-30</td>\n",
       "      <td>373554033.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>False</td>\n",
       "      <td>7.7</td>\n",
       "      <td>5415.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>65000000</td>\n",
       "      <td>[{'id': 12, 'name': 'Adventure'}, {'id': 14, '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8844</td>\n",
       "      <td>tt0113497</td>\n",
       "      <td>en</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>When siblings Judy and Peter discover an encha...</td>\n",
       "      <td>...</td>\n",
       "      <td>1995-12-15</td>\n",
       "      <td>262797249.0</td>\n",
       "      <td>104.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}, {'iso...</td>\n",
       "      <td>Released</td>\n",
       "      <td>Roll the dice and unleash the excitement!</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>False</td>\n",
       "      <td>6.9</td>\n",
       "      <td>2413.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 119050, 'name': 'Grumpy Old Men Collect...</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'id': 10749, 'name': 'Romance'}, {'id': 35, ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15602</td>\n",
       "      <td>tt0113228</td>\n",
       "      <td>en</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>A family wedding reignites the ancient feud be...</td>\n",
       "      <td>...</td>\n",
       "      <td>1995-12-22</td>\n",
       "      <td>0.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>Still Yelling. Still Fighting. Still Ready for...</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>False</td>\n",
       "      <td>6.5</td>\n",
       "      <td>92.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   adult                              belongs_to_collection    budget  \\\n",
       "0  False  {'id': 10194, 'name': 'Toy Story Collection', ...  30000000   \n",
       "1  False                                                NaN  65000000   \n",
       "2  False  {'id': 119050, 'name': 'Grumpy Old Men Collect...         0   \n",
       "\n",
       "                                              genres  \\\n",
       "0  [{'id': 16, 'name': 'Animation'}, {'id': 35, '...   \n",
       "1  [{'id': 12, 'name': 'Adventure'}, {'id': 14, '...   \n",
       "2  [{'id': 10749, 'name': 'Romance'}, {'id': 35, ...   \n",
       "\n",
       "                               homepage     id    imdb_id original_language  \\\n",
       "0  http://toystory.disney.com/toy-story    862  tt0114709                en   \n",
       "1                                   NaN   8844  tt0113497                en   \n",
       "2                                   NaN  15602  tt0113228                en   \n",
       "\n",
       "     original_title                                           overview  ...  \\\n",
       "0         Toy Story  Led by Woody, Andy's toys live happily in his ...  ...   \n",
       "1           Jumanji  When siblings Judy and Peter discover an encha...  ...   \n",
       "2  Grumpier Old Men  A family wedding reignites the ancient feud be...  ...   \n",
       "\n",
       "  release_date      revenue runtime  \\\n",
       "0   1995-10-30  373554033.0    81.0   \n",
       "1   1995-12-15  262797249.0   104.0   \n",
       "2   1995-12-22          0.0   101.0   \n",
       "\n",
       "                                    spoken_languages    status  \\\n",
       "0           [{'iso_639_1': 'en', 'name': 'English'}]  Released   \n",
       "1  [{'iso_639_1': 'en', 'name': 'English'}, {'iso...  Released   \n",
       "2           [{'iso_639_1': 'en', 'name': 'English'}]  Released   \n",
       "\n",
       "                                             tagline             title  video  \\\n",
       "0                                                NaN         Toy Story  False   \n",
       "1          Roll the dice and unleash the excitement!           Jumanji  False   \n",
       "2  Still Yelling. Still Fighting. Still Ready for...  Grumpier Old Men  False   \n",
       "\n",
       "  vote_average vote_count  \n",
       "0          7.7     5415.0  \n",
       "1          6.9     2413.0  \n",
       "2          6.5       92.0  \n",
       "\n",
       "[3 rows x 24 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import Pandas\n",
    "import pandas as pd\n",
    "\n",
    "# Load Movies Metadata\n",
    "metadata = pd.read_csv('d:/movies_metadata.csv', low_memory=False)\n",
    "\n",
    "# Print the first three rows\n",
    "metadata.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"A weighted rating that takes into account the average rating and the number of votes it has garnered. Such a system will make sure that a movie with a 9 rating from 100,000 voters gets a (far) higher score than a YouTube Web Series with the same rating but a few hundred voters.\n",
    "\n",
    "Since you are trying to build a clone of IMDB's Top 250, you will use its weighted rating formula as your metric/score. Mathematically, it is represented as follows:\n",
    "\n",
    "Weighted Rating (WR) = (vv+m.R)+(mv+m.C)\n",
    "where,\n",
    "\n",
    "v is the number of votes for the movie;\n",
    "m is the minimum votes required to be listed in the chart;\n",
    "R is the average rating of the movie; And\n",
    "C is the mean vote across the whole report\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The mean vote across the whole report 5.618207215133889\n"
     ]
    }
   ],
   "source": [
    "# Calculate C\n",
    "C = metadata['vote_average'].mean()\n",
    "print(\"The mean vote across the whole report\", C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's calculate the number of votes, m, received by a movie in the 90th percentile. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The minimum votes required to be listed in the chart is 160.0\n"
     ]
    }
   ],
   "source": [
    "# Calculate the minimum number of votes required to be in the chart, m\n",
    "m = metadata['vote_count'].quantile(0.90)\n",
    "print(\"The minimum votes required to be listed in the chart is\",m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4555, 24)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter out all qualified movies into a new DataFrame\n",
    "q_movies = metadata.copy().loc[metadata['vote_count'] >= m]\n",
    "q_movies.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function that computes the weighted rating of each movie\n",
    "def weighted_rating(x, m=m, C=C):\n",
    "    v = x['vote_count']\n",
    "    R = x['vote_average']\n",
    "    # Calculation based on the IMDB formula\n",
    "    return (v/(v+m) * R) + (m/(m+v) * C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a new feature 'score' and calculate its value with `weighted_rating()`\n",
    "q_movies['score'] = q_movies.apply(weighted_rating, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>vote_count</th>\n",
       "      <th>vote_average</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>314</th>\n",
       "      <td>The Shawshank Redemption</td>\n",
       "      <td>8358.0</td>\n",
       "      <td>8.5</td>\n",
       "      <td>8.445869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>834</th>\n",
       "      <td>The Godfather</td>\n",
       "      <td>6024.0</td>\n",
       "      <td>8.5</td>\n",
       "      <td>8.425439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10309</th>\n",
       "      <td>Dilwale Dulhania Le Jayenge</td>\n",
       "      <td>661.0</td>\n",
       "      <td>9.1</td>\n",
       "      <td>8.421453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12481</th>\n",
       "      <td>The Dark Knight</td>\n",
       "      <td>12269.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.265477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2843</th>\n",
       "      <td>Fight Club</td>\n",
       "      <td>9678.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.256385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>292</th>\n",
       "      <td>Pulp Fiction</td>\n",
       "      <td>8670.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.251406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>522</th>\n",
       "      <td>Schindler's List</td>\n",
       "      <td>4436.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.206639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23673</th>\n",
       "      <td>Whiplash</td>\n",
       "      <td>4376.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.205404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5481</th>\n",
       "      <td>Spirited Away</td>\n",
       "      <td>3968.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.196055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2211</th>\n",
       "      <td>Life Is Beautiful</td>\n",
       "      <td>3643.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.187171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1178</th>\n",
       "      <td>The Godfather: Part II</td>\n",
       "      <td>3418.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.180076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1152</th>\n",
       "      <td>One Flew Over the Cuckoo's Nest</td>\n",
       "      <td>3001.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.164256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>351</th>\n",
       "      <td>Forrest Gump</td>\n",
       "      <td>8147.0</td>\n",
       "      <td>8.2</td>\n",
       "      <td>8.150272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1154</th>\n",
       "      <td>The Empire Strikes Back</td>\n",
       "      <td>5998.0</td>\n",
       "      <td>8.2</td>\n",
       "      <td>8.132919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1176</th>\n",
       "      <td>Psycho</td>\n",
       "      <td>2405.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.132715</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 title  vote_count  vote_average     score\n",
       "314           The Shawshank Redemption      8358.0           8.5  8.445869\n",
       "834                      The Godfather      6024.0           8.5  8.425439\n",
       "10309      Dilwale Dulhania Le Jayenge       661.0           9.1  8.421453\n",
       "12481                  The Dark Knight     12269.0           8.3  8.265477\n",
       "2843                        Fight Club      9678.0           8.3  8.256385\n",
       "292                       Pulp Fiction      8670.0           8.3  8.251406\n",
       "522                   Schindler's List      4436.0           8.3  8.206639\n",
       "23673                         Whiplash      4376.0           8.3  8.205404\n",
       "5481                     Spirited Away      3968.0           8.3  8.196055\n",
       "2211                 Life Is Beautiful      3643.0           8.3  8.187171\n",
       "1178            The Godfather: Part II      3418.0           8.3  8.180076\n",
       "1152   One Flew Over the Cuckoo's Nest      3001.0           8.3  8.164256\n",
       "351                       Forrest Gump      8147.0           8.2  8.150272\n",
       "1154           The Empire Strikes Back      5998.0           8.2  8.132919\n",
       "1176                            Psycho      2405.0           8.3  8.132715"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Sort movies based on score calculated above\n",
    "q_movies = q_movies.sort_values('score', ascending=False)\n",
    "\n",
    "#Print the top 15 movies\n",
    "q_movies[['title', 'vote_count', 'vote_average', 'score']].head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Let's build a system that recommends movies that are similar to a particular movie. \n",
    "More specifically, you will compute pairwise similarity scores for all movies based on \n",
    "their plot descriptions and recommend movies based on that similarity score.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    Led by Woody, Andy's toys live happily in his ...\n",
       "1    When siblings Judy and Peter discover an encha...\n",
       "2    A family wedding reignites the ancient feud be...\n",
       "3    Cheated on, mistreated and stepped on, the wom...\n",
       "4    Just when George Banks has recovered from his ...\n",
       "Name: overview, dtype: object"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Print plot overviews of the first 5 movies.\n",
    "metadata['overview'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"It is not possible to compute the similarity between any two overviews. To do this, \n",
    "we need to compute the word vectors of each overview or document. We need to compute \n",
    "Term Frequency-Inverse Document Frequency (TF-IDF) vectors for each document. This will \n",
    "give us a matrix where each column represents a word in the overview vocabulary \n",
    "(all the words that appear in at least one document) and each column represents \n",
    "a movie, as before.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(45466, 75827)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Import TfIdfVectorizer from scikit-learn\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "#Define a TF-IDF Vectorizer Object. Remove all english stop words such as 'the', 'a'\n",
    "tfidf = TfidfVectorizer(stop_words='english')\n",
    "\n",
    "#Replace NaN with an empty string\n",
    "metadata['overview'] = metadata['overview'].fillna('')\n",
    "\n",
    "#Construct the required TF-IDF matrix by fitting and transforming the data\n",
    "tfidf_matrix = tfidf.fit_transform(metadata['overview'])\n",
    "\n",
    "#Output the shape of tfidf_matrix\n",
    "tfidf_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"With this matrix in hand, you can now compute a similarity score. There are several candidates \n",
    "for this; such as the euclidean, the Pearson and the cosine similarity scores. The cosine similarity \n",
    "to calculate a numeric quantity that denotes the similarity between two movies. You use the cosine \n",
    "similarity score since it is independent of magnitude and is relatively easy and fast to calculate \n",
    "(especially when used in conjunction with TF-IDF scores, which will be explained later). Mathematically, \n",
    "it is defined as follows:\n",
    "\n",
    "cosine(x,y)=x.y⊺/||x||.||y||\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import linear_kernel\n",
    "from sklearn.metrics.pairwise import linear_kernel\n",
    "\n",
    "# Compute the cosine similarity matrix\n",
    "cosine_sim = linear_kernel(tfidf_matrix, tfidf_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Construct a reverse map of indices and movie titles\n",
    "indices = pd.Series(metadata.index, index=metadata['title']).drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function that takes in movie title as input and outputs most similar movies\n",
    "def get_recommendations(title, cosine_sim=cosine_sim):\n",
    "    # Get the index of the movie that matches the title\n",
    "    idx = indices[title]\n",
    "\n",
    "    # Get the pairwsie similarity scores of all movies with that movie\n",
    "    sim_scores = list(enumerate(cosine_sim[idx]))\n",
    "\n",
    "    # Sort the movies based on the similarity scores\n",
    "    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)\n",
    "\n",
    "    # Get the scores of the 10 most similar movies\n",
    "    sim_scores = sim_scores[1:11]\n",
    "\n",
    "    # Get the movie indices\n",
    "    movie_indices = [i[0] for i in sim_scores]\n",
    "\n",
    "    # Return the top 10 most similar movies\n",
    "    return metadata['title'].iloc[movie_indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12481                                      The Dark Knight\n",
       "150                                         Batman Forever\n",
       "1328                                        Batman Returns\n",
       "15511                           Batman: Under the Red Hood\n",
       "585                                                 Batman\n",
       "21194    Batman Unmasked: The Psychology of the Dark Kn...\n",
       "9230                    Batman Beyond: Return of the Joker\n",
       "18035                                     Batman: Year One\n",
       "19792              Batman: The Dark Knight Returns, Part 1\n",
       "3095                          Batman: Mask of the Phantasm\n",
       "Name: title, dtype: object"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_recommendations('The Dark Knight Rises')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>adult</th>\n",
       "      <th>belongs_to_collection</th>\n",
       "      <th>budget</th>\n",
       "      <th>genres</th>\n",
       "      <th>homepage</th>\n",
       "      <th>id</th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>original_language</th>\n",
       "      <th>original_title</th>\n",
       "      <th>overview</th>\n",
       "      <th>...</th>\n",
       "      <th>spoken_languages</th>\n",
       "      <th>status</th>\n",
       "      <th>tagline</th>\n",
       "      <th>title</th>\n",
       "      <th>video</th>\n",
       "      <th>vote_average</th>\n",
       "      <th>vote_count</th>\n",
       "      <th>cast</th>\n",
       "      <th>crew</th>\n",
       "      <th>keywords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>{'id': 10194, 'name': 'Toy Story Collection', ...</td>\n",
       "      <td>30000000</td>\n",
       "      <td>[{'id': 16, 'name': 'Animation'}, {'id': 35, '...</td>\n",
       "      <td>http://toystory.disney.com/toy-story</td>\n",
       "      <td>862</td>\n",
       "      <td>tt0114709</td>\n",
       "      <td>en</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>Led by Woody, Andy's toys live happily in his ...</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>False</td>\n",
       "      <td>7.7</td>\n",
       "      <td>5415.0</td>\n",
       "      <td>[{'cast_id': 14, 'character': 'Woody (voice)',...</td>\n",
       "      <td>[{'credit_id': '52fe4284c3a36847f8024f49', 'de...</td>\n",
       "      <td>[{'id': 931, 'name': 'jealousy'}, {'id': 4290,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>65000000</td>\n",
       "      <td>[{'id': 12, 'name': 'Adventure'}, {'id': 14, '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8844</td>\n",
       "      <td>tt0113497</td>\n",
       "      <td>en</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>When siblings Judy and Peter discover an encha...</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}, {'iso...</td>\n",
       "      <td>Released</td>\n",
       "      <td>Roll the dice and unleash the excitement!</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>False</td>\n",
       "      <td>6.9</td>\n",
       "      <td>2413.0</td>\n",
       "      <td>[{'cast_id': 1, 'character': 'Alan Parrish', '...</td>\n",
       "      <td>[{'credit_id': '52fe44bfc3a36847f80a7cd1', 'de...</td>\n",
       "      <td>[{'id': 10090, 'name': 'board game'}, {'id': 1...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   adult                              belongs_to_collection    budget  \\\n",
       "0  False  {'id': 10194, 'name': 'Toy Story Collection', ...  30000000   \n",
       "1  False                                                NaN  65000000   \n",
       "\n",
       "                                              genres  \\\n",
       "0  [{'id': 16, 'name': 'Animation'}, {'id': 35, '...   \n",
       "1  [{'id': 12, 'name': 'Adventure'}, {'id': 14, '...   \n",
       "\n",
       "                               homepage    id    imdb_id original_language  \\\n",
       "0  http://toystory.disney.com/toy-story   862  tt0114709                en   \n",
       "1                                   NaN  8844  tt0113497                en   \n",
       "\n",
       "  original_title                                           overview  ...  \\\n",
       "0      Toy Story  Led by Woody, Andy's toys live happily in his ...  ...   \n",
       "1        Jumanji  When siblings Judy and Peter discover an encha...  ...   \n",
       "\n",
       "                                    spoken_languages    status  \\\n",
       "0           [{'iso_639_1': 'en', 'name': 'English'}]  Released   \n",
       "1  [{'iso_639_1': 'en', 'name': 'English'}, {'iso...  Released   \n",
       "\n",
       "                                     tagline      title  video  vote_average  \\\n",
       "0                                        NaN  Toy Story  False           7.7   \n",
       "1  Roll the dice and unleash the excitement!    Jumanji  False           6.9   \n",
       "\n",
       "   vote_count                                               cast  \\\n",
       "0      5415.0  [{'cast_id': 14, 'character': 'Woody (voice)',...   \n",
       "1      2413.0  [{'cast_id': 1, 'character': 'Alan Parrish', '...   \n",
       "\n",
       "                                                crew  \\\n",
       "0  [{'credit_id': '52fe4284c3a36847f8024f49', 'de...   \n",
       "1  [{'credit_id': '52fe44bfc3a36847f80a7cd1', 'de...   \n",
       "\n",
       "                                            keywords  \n",
       "0  [{'id': 931, 'name': 'jealousy'}, {'id': 4290,...  \n",
       "1  [{'id': 10090, 'name': 'board game'}, {'id': 1...  \n",
       "\n",
       "[2 rows x 27 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load keywords and credits\n",
    "credits = pd.read_csv('d:/credits.csv')\n",
    "keywords = pd.read_csv('d:/keywords.csv')\n",
    "\n",
    "# Remove rows with bad IDs.\n",
    "metadata = metadata.drop([19730, 29503, 35587])\n",
    "\n",
    "# Convert IDs to int. Required for merging\n",
    "keywords['id'] = keywords['id'].astype('int')\n",
    "credits['id'] = credits['id'].astype('int')\n",
    "metadata['id'] = metadata['id'].astype('int')\n",
    "\n",
    "# Merge keywords and credits into your main metadata dataframe\n",
    "metadata = metadata.merge(credits, on='id')\n",
    "metadata = metadata.merge(keywords, on='id')\n",
    "# Print the first two movies of your newly merged metadata\n",
    "metadata.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse the stringified features into their corresponding python objects\n",
    "from ast import literal_eval\n",
    "\n",
    "features = ['cast', 'crew', 'keywords', 'genres']\n",
    "for feature in features:\n",
    "    metadata[feature] = metadata[feature].apply(literal_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Numpy \n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the director's name from the crew feature. If director is not listed, return NaN\n",
    "def get_director(x):\n",
    "    for i in x:\n",
    "        if i['job'] == 'Director':\n",
    "            return i['name']\n",
    "    return np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Returns the list top 3 elements or entire list; whichever is more.\n",
    "def get_list(x):\n",
    "    if isinstance(x, list):\n",
    "        names = [i['name'] for i in x]\n",
    "        #Check if more than 3 elements exist. If yes, return only first three. If no, return entire list.\n",
    "        if len(names) > 3:\n",
    "            names = names[:3]\n",
    "        return names\n",
    "\n",
    "    #Return empty list in case of missing/malformed data\n",
    "    return []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define new director, cast, genres and keywords features that are in a suitable form.\n",
    "metadata['director'] = metadata['crew'].apply(get_director)\n",
    "\n",
    "features = ['cast', 'keywords', 'genres']\n",
    "for feature in features:\n",
    "    metadata[feature] = metadata[feature].apply(get_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>cast</th>\n",
       "      <th>director</th>\n",
       "      <th>keywords</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Toy Story</td>\n",
       "      <td>[Tom Hanks, Tim Allen, Don Rickles]</td>\n",
       "      <td>John Lasseter</td>\n",
       "      <td>[jealousy, toy, boy]</td>\n",
       "      <td>[Animation, Comedy, Family]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Jumanji</td>\n",
       "      <td>[Robin Williams, Jonathan Hyde, Kirsten Dunst]</td>\n",
       "      <td>Joe Johnston</td>\n",
       "      <td>[board game, disappearance, based on children'...</td>\n",
       "      <td>[Adventure, Fantasy, Family]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>[Walter Matthau, Jack Lemmon, Ann-Margret]</td>\n",
       "      <td>Howard Deutch</td>\n",
       "      <td>[fishing, best friend, duringcreditsstinger]</td>\n",
       "      <td>[Romance, Comedy]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              title                                            cast  \\\n",
       "0         Toy Story             [Tom Hanks, Tim Allen, Don Rickles]   \n",
       "1           Jumanji  [Robin Williams, Jonathan Hyde, Kirsten Dunst]   \n",
       "2  Grumpier Old Men      [Walter Matthau, Jack Lemmon, Ann-Margret]   \n",
       "\n",
       "        director                                           keywords  \\\n",
       "0  John Lasseter                               [jealousy, toy, boy]   \n",
       "1   Joe Johnston  [board game, disappearance, based on children'...   \n",
       "2  Howard Deutch       [fishing, best friend, duringcreditsstinger]   \n",
       "\n",
       "                         genres  \n",
       "0   [Animation, Comedy, Family]  \n",
       "1  [Adventure, Fantasy, Family]  \n",
       "2             [Romance, Comedy]  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print the new features of the first 3 films\n",
    "metadata[['title', 'cast', 'director', 'keywords', 'genres']].head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to convert all strings to lower case and strip names of spaces\n",
    "def clean_data(x):\n",
    "    if isinstance(x, list):\n",
    "        return [str.lower(i.replace(\" \", \"\")) for i in x]\n",
    "    else:\n",
    "        #Check if director exists. If not, return empty string\n",
    "        if isinstance(x, str):\n",
    "            return str.lower(x.replace(\" \", \"\"))\n",
    "        else:\n",
    "            return ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply clean_data function to your features.\n",
    "features = ['cast', 'keywords', 'director', 'genres']\n",
    "\n",
    "for feature in features:\n",
    "    metadata[feature] = metadata[feature].apply(clean_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_soup(x):\n",
    "    return ' '.join(x['keywords']) + ' ' + ' '.join(x['cast']) + ' ' + x['director'] + ' ' + ' '.join(x['genres'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a new soup feature\n",
    "metadata['soup'] = metadata.apply(create_soup, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import CountVectorizer and create the count matrix\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "count = CountVectorizer(stop_words='english')\n",
    "count_matrix = count.fit_transform(metadata['soup'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute the Cosine Similarity matrix based on the count_matrix\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "cosine_sim2 = cosine_similarity(count_matrix, count_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reset index of your main DataFrame and construct reverse mapping as before\n",
    "metadata = metadata.reset_index()\n",
    "indices = pd.Series(metadata.index, index=metadata['title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12589      The Dark Knight\n",
       "10210        Batman Begins\n",
       "9311                Shiner\n",
       "9874       Amongst Friends\n",
       "7772              Mitchell\n",
       "516      Romeo Is Bleeding\n",
       "11463         The Prestige\n",
       "24090            Quicksand\n",
       "25038             Deadfall\n",
       "41063                 Sara\n",
       "Name: title, dtype: object"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_recommendations('The Dark Knight Rises', cosine_sim2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1934            The Godfather: Part III\n",
       "1199             The Godfather: Part II\n",
       "15609                   The Rain People\n",
       "18940                         Last Exit\n",
       "34488                              Rege\n",
       "35802            Manuscripts Don't Burn\n",
       "35803            Manuscripts Don't Burn\n",
       "8001     The Night of the Following Day\n",
       "18261                 The Son of No One\n",
       "28683            In the Name of the Law\n",
       "Name: title, dtype: object"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_recommendations('The Godfather', cosine_sim2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}