2. Get Bechdel test data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "import urllib\n",
    "import json\n",
    "import pandas as pd\n",
    "import html\n",
    "print('Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 7,485 movies in the Bechdel test corpus.\n",
      "7,485 of them have IMDb IDs.\n"
     ]
    }
   ],
   "source": [
    "bechdel_url = 'http://bechdeltest.com/api/v1/getAllMovieIds'\n",
    "bechdel_page = urllib.request.urlopen(bechdel_url).read().decode('utf-8')\n",
    "#print(bechdel_page)\n",
    "movie_ids = json.loads(bechdel_page)\n",
    "bechdel_ids = [movie[u'id'] for movie in movie_ids]\n",
    "imdb_ids = [movie[u'imdbid'] for movie in movie_ids]\n",
    "print(u\"There are {0:,.0f} movies in the Bechdel test corpus.\".format(len(bechdel_ids)))\n",
    "print(u\"{0:,.0f} of them have IMDb IDs.\".format(len(imdb_ids)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bechdel_url = 'http://bechdeltest.com/api/v1/getMovieByImdbId?imdbid=0235712'\n",
    "bechdel_page = urllib.request.urlopen(bechdel_url).read().decode('utf-8')\n",
    "bechdel_page = html.unescape(bechdel_page)\n",
    "x = json.loads(bechdel_page)\n",
    "x\n",
    "#imdb_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 7,485 movies in the Bechdel test corpus.\n",
      "7,485 of them have IMDb IDs.\n",
      "Processing movie number 500 of 7,485...\n",
      "Processing movie number 1,000 of 7,485...\n",
      "Processing movie number 1,500 of 7,485...\n",
      "Processing movie number 2,000 of 7,485...\n",
      "Processing movie number 2,500 of 7,485...\n",
      "Processing movie number 3,000 of 7,485...\n",
      "Processing movie number 3,500 of 7,485...\n",
      "Processing movie number 4,000 of 7,485...\n",
      "Processing movie number 4,500 of 7,485...\n",
      "Processing movie number 5,000 of 7,485...\n",
      "Processing movie number 5,500 of 7,485...\n",
      "Processing movie number 6,000 of 7,485...\n",
      "Processing movie number 6,500 of 7,485...\n",
      "Processing movie number 7,000 of 7,485...\n",
      "Movies processed!\n",
      "Creating dataframe...\n",
      "\n",
      "\n",
      "7,456 movies written. Exceptions below.\n",
      "0 0170817\n",
      "1 0643106\n",
      "2 0643110\n",
      "3 0757061\n",
      "4 0779496\n",
      "5 0782399\n",
      "6 0785532\n",
      "7 0785533\n",
      "8 0805420\n",
      "9 0817400\n",
      "10 0817401\n",
      "11 0817402\n",
      "12 0833098\n",
      "13 0854548\n",
      "14 0868394\n",
      "15 0871197\n",
      "16 1095573\n",
      "17 1340768\n",
      "18 2165859\n",
      "19 5540928\n",
      "Writing CSV...\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "bechdel_url = 'http://bechdeltest.com/api/v1/getAllMovieIds'\n",
    "bechdel_page = urllib.request.urlopen(bechdel_url).read().decode('utf-8')\n",
    "#print(bechdel_page)\n",
    "movie_ids = json.loads(bechdel_page)\n",
    "bechdel_ids = [movie[u'id'] for movie in movie_ids]\n",
    "imdb_ids = [movie[u'imdbid'] for movie in movie_ids]\n",
    "print(u\"There are {0:,.0f} movies in the Bechdel test corpus.\".format(len(bechdel_ids)))\n",
    "print(u\"{0:,.0f} of them have IMDb IDs.\".format(len(imdb_ids)))\n",
    "\n",
    "ratings = dict()\n",
    "exceptions = list()\n",
    "\n",
    "\n",
    "totalmovies = len(imdb_ids)\n",
    "#i = 0\n",
    "#for num,imdb_id in enumerate(imdb_ids):\n",
    "#for i in range(1,5):\n",
    "for i in range(1,totalmovies):\n",
    "    imdb_id = imdb_ids[i]\n",
    "    try:\n",
    "        bechdel_url = 'http://bechdeltest.com/api/v1/getMovieByImdbId?imdbid={0}'.format(imdb_id)\n",
    "#        print(bechdel_url)\n",
    "        bechdel_info = urllib.request.urlopen(bechdel_url).info()\n",
    "        bechdel_page = urllib.request.urlopen(bechdel_url).read().decode('ISO-8859-1')\n",
    "        bechdel_page = html.unescape(bechdel_page)\n",
    "#        print(bechdel_info)\n",
    "        ratings[imdb_id] = json.loads(bechdel_page)\n",
    "    except:\n",
    "        exceptions.append(imdb_id)\n",
    "        pass\n",
    "#    i = i + 1\n",
    "    if ((i % 500 == 0) | (i == totalmovies)):\n",
    "        print('Processing movie number {0:,.0f} of {1:,.0f}...'.format(i,totalmovies))\n",
    "print('Movies processed!')\n",
    "\n",
    "del ratings['']\n",
    "\n",
    "print('Creating dataframe...')\n",
    "ratings_df = pd.DataFrame.from_dict(ratings,orient='index')\n",
    "ratings_df.index.name = 'imdb_id'\n",
    "ratings_df = ratings_df.drop(['imdbid'],axis=1)\n",
    "ratings_df.columns = ['bechdel_title','bechdel_submitter_id','bechdel_submitted_date','bechdel_dubious','bechdel_rating','bechdel_year','bechdel_site_id','bechdel_visible']\n",
    "#ratings_df\n",
    "\n",
    "print('\\n')\n",
    "print('{0:,.0f} movies written. Exceptions below.'.format(len(ratings_df)))\n",
    "for i in range(0, len(exceptions)):\n",
    "    print(i, exceptions[i])\n",
    "\n",
    "print('Writing CSV...')\n",
    "ratings_df.to_csv('bechdel.csv',encoding='utf-8')\n",
    "print('Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16 1095573 http://www.imdb.com/title/tt1095573\n",
      "17 1340768 http://www.imdb.com/title/tt1340768\n",
      "18 2165859 http://www.imdb.com/title/tt2165859\n",
      "19 5540928 http://www.imdb.com/title/tt5540928\n"
     ]
    }
   ],
   "source": [
    "##ratings_bk = ratings\n",
    "#ratings = ratings_bk\n",
    "for i in range(16,len(exceptions)):\n",
    "    print(i, exceptions[i],'http://www.imdb.com/title/tt{0:}'.format(exceptions[i]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}