streaming_options(utelly api).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collecting the streaming options for top 250 IMDb rated movies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Source Link: https://rapidapi.com/utelly/api/utelly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import dependencies\n",
    "\n",
    "import pandas as pd\n",
    "import requests\n",
    "import re\n",
    "from config_omdb import omdb_key\n",
    "\n",
    "import json\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 1: Collecting Top 250 IMDb from IBDb website\n",
    "\n",
    "Source Link: https://www.imdb.com/chart/top/?ref_=nv_mv_250"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# URL for top 250 IMdb movies\n",
    "\n",
    "url = 'https://www.imdb.com/chart/top/?ref_=nv_mv_250'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Rank &amp; Title</th>\n",
       "      <th>IMDb Rating</th>\n",
       "      <th>Your Rating</th>\n",
       "      <th>Unnamed: 4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.  The Shawshank Redemption  (1994)</td>\n",
       "      <td>9.2</td>\n",
       "      <td>12345678910 NOT YET RELEASED  Seen</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.  The Godfather  (1972)</td>\n",
       "      <td>9.1</td>\n",
       "      <td>12345678910 NOT YET RELEASED  Seen</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.  The Godfather: Part II  (1974)</td>\n",
       "      <td>9.0</td>\n",
       "      <td>12345678910 NOT YET RELEASED  Seen</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.  The Dark Knight  (2008)</td>\n",
       "      <td>9.0</td>\n",
       "      <td>12345678910 NOT YET RELEASED  Seen</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.  12 Angry Men  (1957)</td>\n",
       "      <td>8.9</td>\n",
       "      <td>12345678910 NOT YET RELEASED  Seen</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                          Rank & Title  IMDb Rating  \\\n",
       "0         NaN  1.  The Shawshank Redemption  (1994)          9.2   \n",
       "1         NaN             2.  The Godfather  (1972)          9.1   \n",
       "2         NaN    3.  The Godfather: Part II  (1974)          9.0   \n",
       "3         NaN           4.  The Dark Knight  (2008)          9.0   \n",
       "4         NaN              5.  12 Angry Men  (1957)          8.9   \n",
       "\n",
       "                          Your Rating  Unnamed: 4  \n",
       "0  12345678910 NOT YET RELEASED  Seen         NaN  \n",
       "1  12345678910 NOT YET RELEASED  Seen         NaN  \n",
       "2  12345678910 NOT YET RELEASED  Seen         NaN  \n",
       "3  12345678910 NOT YET RELEASED  Seen         NaN  \n",
       "4  12345678910 NOT YET RELEASED  Seen         NaN  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read all the tables \n",
    "\n",
    "tables = pd.read_html(url)\n",
    "\n",
    "# Printing the required table\n",
    "\n",
    "raw_imdb_df = tables[0]\n",
    "raw_imdb_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rank &amp; Title</th>\n",
       "      <th>IMDb Rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.  The Shawshank Redemption  (1994)</td>\n",
       "      <td>9.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.  The Godfather  (1972)</td>\n",
       "      <td>9.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.  The Godfather: Part II  (1974)</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.  The Dark Knight  (2008)</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5.  12 Angry Men  (1957)</td>\n",
       "      <td>8.9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           Rank & Title  IMDb Rating\n",
       "0  1.  The Shawshank Redemption  (1994)          9.2\n",
       "1             2.  The Godfather  (1972)          9.1\n",
       "2    3.  The Godfather: Part II  (1974)          9.0\n",
       "3           4.  The Dark Knight  (2008)          9.0\n",
       "4              5.  12 Angry Men  (1957)          8.9"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_imdb_df = raw_imdb_df.drop(['Unnamed: 0','Your Rating', 'Unnamed: 4'], axis = 1)\n",
    "raw_imdb_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "rank_list = []\n",
    "title_list = []\n",
    "year_list = []\n",
    "\n",
    "for movie in raw_imdb_df['Rank & Title']:\n",
    "    \n",
    "    rank = movie.split('.')[0]\n",
    "    rank_list.append(rank)\n",
    "    \n",
    "    title1 = re.split('[.(]', movie)[1]\n",
    "    title = re.sub(\"  \", \"\", title1)\n",
    "    title_list.append(title)\n",
    "        \n",
    "    year = re.split('[()]', movie)[1]\n",
    "    year_list.append(year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IMDb Rank</th>\n",
       "      <th>Movie Title</th>\n",
       "      <th>Year Released</th>\n",
       "      <th>IMDb Rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>The Shawshank Redemption</td>\n",
       "      <td>1994</td>\n",
       "      <td>9.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>The Godfather</td>\n",
       "      <td>1972</td>\n",
       "      <td>9.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>The Godfather: Part II</td>\n",
       "      <td>1974</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>The Dark Knight</td>\n",
       "      <td>2008</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>12 Angry Men</td>\n",
       "      <td>1957</td>\n",
       "      <td>8.9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  IMDb Rank               Movie Title Year Released  IMDb Rating\n",
       "0         1  The Shawshank Redemption          1994          9.2\n",
       "1         2             The Godfather          1972          9.1\n",
       "2         3    The Godfather: Part II          1974          9.0\n",
       "3         4           The Dark Knight          2008          9.0\n",
       "4         5              12 Angry Men          1957          8.9"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict_imdb = {'IMDb Rank': rank_list,\n",
    "             'Movie Title': title_list,\n",
    "             'Year Released': year_list,\n",
    "             'IMDb Rating' : raw_imdb_df['IMDb Rating']\n",
    "        }\n",
    "\n",
    "Top_250_IMDb_df = pd.DataFrame(dict_imdb)\n",
    "\n",
    "Top_250_IMDb_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "Top_250_IMDb_df.to_csv('Output/Top_250_IMDb.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 2: Collecting IMDb unique ID and other movie details from OMDb API\n",
    "\n",
    "Source Link: http://www.omdbapi.com/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"http://www.omdbapi.com/?apikey=\" + omdb_key + \"&t=\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_list = Top_250_IMDb_df['Movie Title']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_omdb = [];\n",
    "\n",
    "for movie in movie_list:\n",
    "    movie_data = requests.get(url + movie).json()\n",
    "    results_omdb.append(movie_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving into a JSON file\n",
    "# Source Link: https://stackabuse.com/reading-and-writing-json-files-in-python-with-pandas/\n",
    "\n",
    "with open ('Output/OMDb_250.json', 'w') as f:\n",
    "    json.dump(results_omdb, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Year</th>\n",
       "      <th>Rated</th>\n",
       "      <th>Released</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>Genre</th>\n",
       "      <th>Director</th>\n",
       "      <th>Writer</th>\n",
       "      <th>Actors</th>\n",
       "      <th>Plot</th>\n",
       "      <th>...</th>\n",
       "      <th>imdbRating</th>\n",
       "      <th>imdbVotes</th>\n",
       "      <th>imdbID</th>\n",
       "      <th>Type</th>\n",
       "      <th>DVD</th>\n",
       "      <th>BoxOffice</th>\n",
       "      <th>Production</th>\n",
       "      <th>Website</th>\n",
       "      <th>Response</th>\n",
       "      <th>Error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>The Shawshank Redemption</td>\n",
       "      <td>1994.0</td>\n",
       "      <td>R</td>\n",
       "      <td>14 Oct 1994</td>\n",
       "      <td>142 min</td>\n",
       "      <td>Drama</td>\n",
       "      <td>Frank Darabont</td>\n",
       "      <td>Stephen King (short story \"Rita Hayworth and S...</td>\n",
       "      <td>Tim Robbins, Morgan Freeman, Bob Gunton, Willi...</td>\n",
       "      <td>Two imprisoned men bond over a number of years...</td>\n",
       "      <td>...</td>\n",
       "      <td>9.3</td>\n",
       "      <td>2,217,195</td>\n",
       "      <td>tt0111161</td>\n",
       "      <td>movie</td>\n",
       "      <td>27 Jan 1998</td>\n",
       "      <td>N/A</td>\n",
       "      <td>Columbia Pictures</td>\n",
       "      <td>N/A</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>The Godfather</td>\n",
       "      <td>1972.0</td>\n",
       "      <td>R</td>\n",
       "      <td>24 Mar 1972</td>\n",
       "      <td>175 min</td>\n",
       "      <td>Crime, Drama</td>\n",
       "      <td>Francis Ford Coppola</td>\n",
       "      <td>Mario Puzo (screenplay by), Francis Ford Coppo...</td>\n",
       "      <td>Marlon Brando, Al Pacino, James Caan, Richard ...</td>\n",
       "      <td>The aging patriarch of an organized crime dyna...</td>\n",
       "      <td>...</td>\n",
       "      <td>9.2</td>\n",
       "      <td>1,532,092</td>\n",
       "      <td>tt0068646</td>\n",
       "      <td>movie</td>\n",
       "      <td>09 Oct 2001</td>\n",
       "      <td>N/A</td>\n",
       "      <td>Paramount Pictures</td>\n",
       "      <td>N/A</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>The Godfather: Part II</td>\n",
       "      <td>1974.0</td>\n",
       "      <td>R</td>\n",
       "      <td>18 Dec 1974</td>\n",
       "      <td>202 min</td>\n",
       "      <td>Crime, Drama</td>\n",
       "      <td>Francis Ford Coppola</td>\n",
       "      <td>Francis Ford Coppola (screenplay by), Mario Pu...</td>\n",
       "      <td>Al Pacino, Robert Duvall, Diane Keaton, Robert...</td>\n",
       "      <td>The early life and career of Vito Corleone in ...</td>\n",
       "      <td>...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1,072,495</td>\n",
       "      <td>tt0071562</td>\n",
       "      <td>movie</td>\n",
       "      <td>24 May 2005</td>\n",
       "      <td>N/A</td>\n",
       "      <td>Paramount Pictures</td>\n",
       "      <td>N/A</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>The Dark Knight</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>PG-13</td>\n",
       "      <td>18 Jul 2008</td>\n",
       "      <td>152 min</td>\n",
       "      <td>Action, Crime, Drama, Thriller</td>\n",
       "      <td>Christopher Nolan</td>\n",
       "      <td>Jonathan Nolan (screenplay), Christopher Nolan...</td>\n",
       "      <td>Christian Bale, Heath Ledger, Aaron Eckhart, M...</td>\n",
       "      <td>When the menace known as the Joker wreaks havo...</td>\n",
       "      <td>...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2,200,152</td>\n",
       "      <td>tt0468569</td>\n",
       "      <td>movie</td>\n",
       "      <td>09 Dec 2008</td>\n",
       "      <td>$533,316,061</td>\n",
       "      <td>Warner Bros. Pictures/Legendary</td>\n",
       "      <td>N/A</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>12 Angry Men</td>\n",
       "      <td>1957.0</td>\n",
       "      <td>Approved</td>\n",
       "      <td>10 Apr 1957</td>\n",
       "      <td>96 min</td>\n",
       "      <td>Drama</td>\n",
       "      <td>Sidney Lumet</td>\n",
       "      <td>Reginald Rose (story), Reginald Rose (screenplay)</td>\n",
       "      <td>Martin Balsam, John Fiedler, Lee J. Cobb, E.G....</td>\n",
       "      <td>A jury holdout attempts to prevent a miscarria...</td>\n",
       "      <td>...</td>\n",
       "      <td>8.9</td>\n",
       "      <td>650,851</td>\n",
       "      <td>tt0050083</td>\n",
       "      <td>movie</td>\n",
       "      <td>06 Mar 2001</td>\n",
       "      <td>N/A</td>\n",
       "      <td>Criterion Collection</td>\n",
       "      <td>N/A</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      Title    Year     Rated     Released  Runtime  \\\n",
       "0  The Shawshank Redemption  1994.0         R  14 Oct 1994  142 min   \n",
       "1             The Godfather  1972.0         R  24 Mar 1972  175 min   \n",
       "2    The Godfather: Part II  1974.0         R  18 Dec 1974  202 min   \n",
       "3           The Dark Knight  2008.0     PG-13  18 Jul 2008  152 min   \n",
       "4              12 Angry Men  1957.0  Approved  10 Apr 1957   96 min   \n",
       "\n",
       "                            Genre              Director  \\\n",
       "0                           Drama        Frank Darabont   \n",
       "1                    Crime, Drama  Francis Ford Coppola   \n",
       "2                    Crime, Drama  Francis Ford Coppola   \n",
       "3  Action, Crime, Drama, Thriller     Christopher Nolan   \n",
       "4                           Drama          Sidney Lumet   \n",
       "\n",
       "                                              Writer  \\\n",
       "0  Stephen King (short story \"Rita Hayworth and S...   \n",
       "1  Mario Puzo (screenplay by), Francis Ford Coppo...   \n",
       "2  Francis Ford Coppola (screenplay by), Mario Pu...   \n",
       "3  Jonathan Nolan (screenplay), Christopher Nolan...   \n",
       "4  Reginald Rose (story), Reginald Rose (screenplay)   \n",
       "\n",
       "                                              Actors  \\\n",
       "0  Tim Robbins, Morgan Freeman, Bob Gunton, Willi...   \n",
       "1  Marlon Brando, Al Pacino, James Caan, Richard ...   \n",
       "2  Al Pacino, Robert Duvall, Diane Keaton, Robert...   \n",
       "3  Christian Bale, Heath Ledger, Aaron Eckhart, M...   \n",
       "4  Martin Balsam, John Fiedler, Lee J. Cobb, E.G....   \n",
       "\n",
       "                                                Plot  ... imdbRating  \\\n",
       "0  Two imprisoned men bond over a number of years...  ...        9.3   \n",
       "1  The aging patriarch of an organized crime dyna...  ...        9.2   \n",
       "2  The early life and career of Vito Corleone in ...  ...        9.0   \n",
       "3  When the menace known as the Joker wreaks havo...  ...        9.0   \n",
       "4  A jury holdout attempts to prevent a miscarria...  ...        8.9   \n",
       "\n",
       "   imdbVotes     imdbID   Type          DVD     BoxOffice  \\\n",
       "0  2,217,195  tt0111161  movie  27 Jan 1998           N/A   \n",
       "1  1,532,092  tt0068646  movie  09 Oct 2001           N/A   \n",
       "2  1,072,495  tt0071562  movie  24 May 2005           N/A   \n",
       "3  2,200,152  tt0468569  movie  09 Dec 2008  $533,316,061   \n",
       "4    650,851  tt0050083  movie  06 Mar 2001           N/A   \n",
       "\n",
       "                        Production Website Response Error  \n",
       "0                Columbia Pictures     N/A     True   NaN  \n",
       "1               Paramount Pictures     N/A     True   NaN  \n",
       "2               Paramount Pictures     N/A     True   NaN  \n",
       "3  Warner Bros. Pictures/Legendary     N/A     True   NaN  \n",
       "4             Criterion Collection     N/A     True   NaN  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading JSON file through dataframe\n",
    "\n",
    "raw_omdb_df = pd.read_json('Output/OMDb_250.json')\n",
    "raw_omdb_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(250, 26)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_omdb_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Title',\n",
       " 'Year',\n",
       " 'Rated',\n",
       " 'Released',\n",
       " 'Runtime',\n",
       " 'Genre',\n",
       " 'Director',\n",
       " 'Writer',\n",
       " 'Actors',\n",
       " 'Plot',\n",
       " 'Language',\n",
       " 'Country',\n",
       " 'Awards',\n",
       " 'Poster',\n",
       " 'Ratings',\n",
       " 'Metascore',\n",
       " 'imdbRating',\n",
       " 'imdbVotes',\n",
       " 'imdbID',\n",
       " 'Type',\n",
       " 'DVD',\n",
       " 'BoxOffice',\n",
       " 'Production',\n",
       " 'Website',\n",
       " 'Response',\n",
       " 'Error']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Showing all column titles\n",
    "list(raw_omdb_df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# raw_omdb_df.to_csv('Output/OMDb_raw_df.csv', index=False)\n",
    "# raw_omdb_df.to_csv('Output/OMDb_raw_df_index.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_imdb_list = raw_omdb_df['imdbID']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "250"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(id_imdb_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 3: Collecting streaming options for Top 250 IMDb movies\n",
    "\n",
    "Source Link: Utelly API Documentation\n",
    "https://rapidapi.com/utelly/api/utelly?endpoint=apiendpoint_3cad787b-ca7b-449a-84b4-23b40d64fd73\n",
    "\n",
    "Endpoint: GET/idlookup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "from config import api_key\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# url = \"https://utelly-tv-shows-and-movies-availability-v1.p.rapidapi.com/idlookup\"\n",
    "# querystring = {\"country\":\"US\",\"source_id\":\"tt0468569\",\"source\":\"imdb\"}\n",
    "# headers = {\n",
    "#     'x-rapidapi-host': \"utelly-tv-shows-and-movies-availability-v1.p.rapidapi.com\",\n",
    "#     'x-rapidapi-key': api_key\n",
    "#     }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# response = requests.request(\"GET\", url, headers=headers, params=querystring)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'collection': {'id': '5d97dab59a76a40056de6bcb',\n",
       "  'picture': 'https://utellyassets9-1.imgix.net/api/Images/149c8a3dab9ffbd65c1cc655724f29aa/Redirect',\n",
       "  'name': 'Dark Knight - The Dark Knight',\n",
       "  'locations': [{'icon': 'https://utellyassets7.imgix.net/locations_icons/utelly/black_new/iTunesIVAUS.png?w=92&auto=compress&app_version=ae3576e2-0796-4eda-b953-80cadc8e2619_eww2020-05-09',\n",
       "    'country': ['us'],\n",
       "    'display_name': 'iTunes',\n",
       "    'name': 'iTunesIVAUS',\n",
       "    'id': '5d80a9a5d51bef861d3740d3',\n",
       "    'url': 'https://itunes.apple.com/us/movie/the-dark-knight/id764632601'},\n",
       "   {'icon': 'https://utellyassets7.imgix.net/locations_icons/utelly/black_new/GooglePlayIVAUS.png?w=92&auto=compress&app_version=ae3576e2-0796-4eda-b953-80cadc8e2619_eww2020-05-09',\n",
       "    'country': ['us'],\n",
       "    'display_name': 'Google Play',\n",
       "    'name': 'GooglePlayIVAUS',\n",
       "    'id': '5d8260b128fbcd0052aed197',\n",
       "    'url': 'https://play.google.com/store/movies/details/The_Dark_Knight?gl=US&hl=en&id=qY3UkAHufLY'},\n",
       "   {'icon': 'https://utellyassets7.imgix.net/locations_icons/utelly/black_new/AmazonInstantVideoIVAUS.png?w=92&auto=compress&app_version=ae3576e2-0796-4eda-b953-80cadc8e2619_eww2020-05-09',\n",
       "    'country': ['us'],\n",
       "    'display_name': 'Amazon Instant Video',\n",
       "    'name': 'AmazonInstantVideoIVAUS',\n",
       "    'id': '5d82609332ac2f0051962fe6',\n",
       "    'url': 'https://www.amazon.com/gp/product/B001I189MQ?creativeASIN=B001I189MQ&ie=UTF8&linkCode=xm2&tag=utellycom00-21'}],\n",
       "  'provider': 'iva',\n",
       "  'weight': 0,\n",
       "  'source_ids': {'imdb': {'url': 'https://www.imdb.com/title/tt0468569',\n",
       "    'id': 'tt0468569'}}},\n",
       " 'type': 'imdb',\n",
       " 'id': 'tt0468569',\n",
       " 'status_code': 200,\n",
       " 'variant': 'ivafull'}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# json_result = response.json()\n",
    "# json_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://utelly-tv-shows-and-movies-availability-v1.p.rapidapi.com/idlookup\"\n",
    "\n",
    "querystring = {\"country\":\"US\",\"source\":\"imdb\"}\n",
    "\n",
    "headers = {\n",
    "    'x-rapidapi-host': \"utelly-tv-shows-and-movies-availability-v1.p.rapidapi.com\",\n",
    "    'x-rapidapi-key': api_key\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# id_imdb_list_sample = ['tt0111161', 'tt0068646', 'tt0071562']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "imdb_id = []\n",
    "title = []\n",
    "streaming_service = []\n",
    "streaming_url = []\n",
    "all_requests_json = []\n",
    "\n",
    "for id_imdb in id_imdb_list:\n",
    "    time.sleep(2)\n",
    "    querystring[\"source_id\"] = id_imdb\n",
    "    \n",
    "    try:\n",
    "        response = requests.request(\"GET\", url, headers=headers, params=querystring)\n",
    "\n",
    "        json_result = response.json()\n",
    "        all_requests_json.append(json_result)\n",
    "\n",
    "        for location in json_result['collection']['locations']:\n",
    "            imdb_id.append(json_result['id'])\n",
    "            title.append(json_result['collection']['name'])\n",
    "            streaming_service.append(location['display_name'])\n",
    "            streaming_url.append(location['url'])\n",
    "        \n",
    "    except:\n",
    "        print('Data Not Available')\n",
    "        print(id_imdb)\n",
    "        print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open ('Output/json_files/utelly_all_requests_json.json', 'w') as f:\n",
    "    json.dump(all_requests_json, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "streaming_dict = {'IMDb ID': imdb_id,\n",
    "                 'Title': title,\n",
    "                 'Streaming Service': streaming_service,\n",
    "                 'Streaming URL' : streaming_url\n",
    "                }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IMDb ID</th>\n",
       "      <th>Title</th>\n",
       "      <th>Streaming Service</th>\n",
       "      <th>Streaming URL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>tt0111161</td>\n",
       "      <td>The Shawshank Redemption</td>\n",
       "      <td>Netflix</td>\n",
       "      <td>https://www.netflix.com/title/70005379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>tt0111161</td>\n",
       "      <td>The Shawshank Redemption</td>\n",
       "      <td>Google Play</td>\n",
       "      <td>https://play.google.com/store/movies/details/T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>tt0111161</td>\n",
       "      <td>The Shawshank Redemption</td>\n",
       "      <td>Amazon Instant Video</td>\n",
       "      <td>https://www.amazon.com/gp/product/B001EBV0P8?c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>tt0111161</td>\n",
       "      <td>The Shawshank Redemption</td>\n",
       "      <td>iTunes</td>\n",
       "      <td>https://itunes.apple.com/us/movie/the-shawshan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>tt0068646</td>\n",
       "      <td>The Godfather</td>\n",
       "      <td>Google Play</td>\n",
       "      <td>https://play.google.com/store/movies/details/T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>694</td>\n",
       "      <td>tt0103639</td>\n",
       "      <td>Aladdin</td>\n",
       "      <td>Google Play</td>\n",
       "      <td>https://play.google.com/store/movies/details/A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>695</td>\n",
       "      <td>tt0103639</td>\n",
       "      <td>Aladdin</td>\n",
       "      <td>DisneyPlusIVAUS</td>\n",
       "      <td>https://www.disneyplus.com/movies/aladdin-1992...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>696</td>\n",
       "      <td>tt2338151</td>\n",
       "      <td>PK</td>\n",
       "      <td>Netflix</td>\n",
       "      <td>https://www.netflix.com/title/70303496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>697</td>\n",
       "      <td>tt0094625</td>\n",
       "      <td>Akira</td>\n",
       "      <td>Hulu</td>\n",
       "      <td>https://www.hulu.com/watch/64a5a8d0-1406-4178-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>698</td>\n",
       "      <td>tt0050613</td>\n",
       "      <td>Throne of Blood</td>\n",
       "      <td>iTunes</td>\n",
       "      <td>https://itunes.apple.com/us/movie/throne-of-bl...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>699 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       IMDb ID                     Title     Streaming Service  \\\n",
       "0    tt0111161  The Shawshank Redemption               Netflix   \n",
       "1    tt0111161  The Shawshank Redemption           Google Play   \n",
       "2    tt0111161  The Shawshank Redemption  Amazon Instant Video   \n",
       "3    tt0111161  The Shawshank Redemption                iTunes   \n",
       "4    tt0068646             The Godfather           Google Play   \n",
       "..         ...                       ...                   ...   \n",
       "694  tt0103639                   Aladdin           Google Play   \n",
       "695  tt0103639                   Aladdin       DisneyPlusIVAUS   \n",
       "696  tt2338151                        PK               Netflix   \n",
       "697  tt0094625                     Akira                  Hulu   \n",
       "698  tt0050613           Throne of Blood                iTunes   \n",
       "\n",
       "                                         Streaming URL  \n",
       "0               https://www.netflix.com/title/70005379  \n",
       "1    https://play.google.com/store/movies/details/T...  \n",
       "2    https://www.amazon.com/gp/product/B001EBV0P8?c...  \n",
       "3    https://itunes.apple.com/us/movie/the-shawshan...  \n",
       "4    https://play.google.com/store/movies/details/T...  \n",
       "..                                                 ...  \n",
       "694  https://play.google.com/store/movies/details/A...  \n",
       "695  https://www.disneyplus.com/movies/aladdin-1992...  \n",
       "696             https://www.netflix.com/title/70303496  \n",
       "697  https://www.hulu.com/watch/64a5a8d0-1406-4178-...  \n",
       "698  https://itunes.apple.com/us/movie/throne-of-bl...  \n",
       "\n",
       "[699 rows x 4 columns]"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "streaming_df = pd.DataFrame(streaming_dict)\n",
    "streaming_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(699, 4)"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "streaming_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "# streaming_df.to_csv('Output/streaming_df_index.csv')\n",
    "streaming_df.to_csv('Output/streaming_df.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}