Tweet Topic Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T01:43:32.705163Z",
     "start_time": "2017-11-10T01:43:32.700927Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T01:03:19.483387Z",
     "start_time": "2017-11-10T01:03:19.329933Z"
    }
   },
   "outputs": [],
   "source": [
    "# Read Twitter JSON file and convert to Pandas Dataframe\n",
    "with open('twitter-data.json', 'r', encoding='utf8') as f:\n",
    "    line = f.readline()\n",
    "    df_from_twitter_data = pd.read_json(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T01:03:20.141272Z",
     "start_time": "2017-11-10T01:03:20.084232Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>homeTimeline</th>\n",
       "      <th>userTimeline</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>results</th>\n",
       "      <td>[{'__v': 0, 'highestTweetId': '928061258355658...</td>\n",
       "      <td>[{'__v': 0, 'highestTweetId': '926217842910904...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>serviceName</th>\n",
       "      <td>twitter</td>\n",
       "      <td>twitter</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>totalResultCount</th>\n",
       "      <td>642</td>\n",
       "      <td>1850</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                       homeTimeline  \\\n",
       "results           [{'__v': 0, 'highestTweetId': '928061258355658...   \n",
       "serviceName                                                 twitter   \n",
       "totalResultCount                                                642   \n",
       "\n",
       "                                                       userTimeline  \n",
       "results           [{'__v': 0, 'highestTweetId': '926217842910904...  \n",
       "serviceName                                                 twitter  \n",
       "totalResultCount                                               1850  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_from_twitter_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Home Timeline Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T01:03:21.158075Z",
     "start_time": "2017-11-10T01:03:21.152204Z"
    }
   },
   "outputs": [],
   "source": [
    "homeTimeline = df_from_twitter_data.loc['results','homeTimeline']\n",
    "homeTimeline = pd.DataFrame(homeTimeline[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T01:03:21.825815Z",
     "start_time": "2017-11-10T01:03:21.799812Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'contributors': None,\n",
      " 'coordinates': None,\n",
      " 'created_at': 'Wed Nov 08 00:43:38 +0000 2017',\n",
      " 'entities': {'hashtags': [],\n",
      "              'symbols': [],\n",
      "              'urls': [{'display_url': 'twitter.com/i/web/status/9…',\n",
      "                        'expanded_url': 'https://twitter.com/i/web/status/928060408618876929',\n",
      "                        'indices': [117, 140],\n",
      "                        'url': 'https://t.co/kOfU764Cuz'}],\n",
      "              'user_mentions': []},\n",
      " 'favorite_count': 1,\n",
      " 'favorited': False,\n",
      " 'geo': None,\n",
      " 'id': 928060408618876900,\n",
      " 'id_str': '928060408618876929',\n",
      " 'in_reply_to_screen_name': None,\n",
      " 'in_reply_to_status_id': None,\n",
      " 'in_reply_to_status_id_str': None,\n",
      " 'in_reply_to_user_id': None,\n",
      " 'in_reply_to_user_id_str': None,\n",
      " 'is_quote_status': True,\n",
      " 'lang': 'en',\n",
      " 'place': None,\n",
      " 'possibly_sensitive': False,\n",
      " 'possibly_sensitive_appealable': False,\n",
      " 'quoted_status': {'contributors': None,\n",
      "                   'coordinates': None,\n",
      "                   'created_at': 'Tue Nov 07 19:40:33 +0000 2017',\n",
      "                   'entities': {'hashtags': [],\n",
      "                                'symbols': [],\n",
      "                                'urls': [{'display_url': 'twitter.com/i/web/status/9…',\n",
      "                                          'expanded_url': 'https://twitter.com/i/web/status/927984135167029248',\n",
      "                                          'indices': [117, 140],\n",
      "                                          'url': 'https://t.co/G9hQig3sJ8'}],\n",
      "                                'user_mentions': []},\n",
      "                   'favorite_count': 393,\n",
      "                   'favorited': False,\n",
      "                   'geo': None,\n",
      "                   'id': 927984135167029200,\n",
      "                   'id_str': '927984135167029248',\n",
      "                   'in_reply_to_screen_name': None,\n",
      "                   'in_reply_to_status_id': None,\n",
      "                   'in_reply_to_status_id_str': None,\n",
      "                   'in_reply_to_user_id': None,\n",
      "                   'in_reply_to_user_id_str': None,\n",
      "                   'is_quote_status': False,\n",
      "                   'lang': 'en',\n",
      "                   'place': None,\n",
      "                   'possibly_sensitive': False,\n",
      "                   'possibly_sensitive_appealable': False,\n",
      "                   'retweet_count': 133,\n",
      "                   'retweeted': False,\n",
      "                   'source': '<a href=\"http://twitter.com/download/iphone\" '\n",
      "                             'rel=\"nofollow\">Twitter for iPhone</a>',\n",
      "                   'text': 'Apple Pay Cash is rolling out to iOS 11.2 beta. '\n",
      "                           'Nice design touches with opalescence moving with '\n",
      "                           'device orientation… https://t.co/G9hQig3sJ8',\n",
      "                   'truncated': True,\n",
      "                   'user': {'id': 6503412, 'id_str': '6503412'}},\n",
      " 'quoted_status_id': 927984135167029200,\n",
      " 'quoted_status_id_str': '927984135167029248',\n",
      " 'retweet_count': 1,\n",
      " 'retweeted': False,\n",
      " 'source': '<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>',\n",
      " 'text': 'Whichever team thought of using the accelerometer to create a foil '\n",
      "         'effect is brilliant. This is probably going to b… '\n",
      "         'https://t.co/kOfU764Cuz',\n",
      " 'truncated': True,\n",
      " 'user_id_str': '13767622'}\n"
     ]
    }
   ],
   "source": [
    "pprint(homeTimeline.loc[1,'tweets']) # sample tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T01:03:22.870161Z",
     "start_time": "2017-11-10T01:03:22.327192Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10069172</td>\n",
       "      <td>Customer Operations - An Idea for Maximizing E...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13767622</td>\n",
       "      <td>Whichever team thought of using the accelerome...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1344951</td>\n",
       "      <td>Here are the iOS 11 privacy settings you shoul...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13767622</td>\n",
       "      <td>RT @mathowie: https://t.co/pULhe78X71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15249166</td>\n",
       "      <td>Thanks to rapidly advancing biotechnology, hum...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>16870421</td>\n",
       "      <td>Guess all of our #xMed updates come in 280 cha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>97418412</td>\n",
       "      <td>RT @AsiaMatos: .@Hull just got WAY better. Get...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>15492359</td>\n",
       "      <td>This man has one goal: to make you love birds....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>16870421</td>\n",
       "      <td>RT @cristina_djsj: #StemCell trial given arm m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>16870421</td>\n",
       "      <td>At #xMed right now @drhanskeirstead is droppin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>7144422</td>\n",
       "      <td>You can still upgrade to Windows 10 if you use...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>16870421</td>\n",
       "      <td>This technology, dubbed tissue nanotransfectio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>16870421</td>\n",
       "      <td>We may be on our way to an entirely new regene...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>3018841323</td>\n",
       "      <td>Application of AI and Data Science in Medical ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>11518572</td>\n",
       "      <td>#TheCricle https://t.co/i9U17xMl4L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>809760</td>\n",
       "      <td>Fuck every single one of these additional 140 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1344951</td>\n",
       "      <td>Waymo just took a massive step toward bringing...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18156146</td>\n",
       "      <td>Ew!! 280 character tweets just look gross!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>159121042</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20167623</td>\n",
       "      <td>20 Questions to Detect Fake #DataScientists #K...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>20167623</td>\n",
       "      <td>Building #AI That Can Build AI via The New Yor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>37083551</td>\n",
       "      <td>RT @Mikel_Jollett: Guns are not allowed in the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>16870421</td>\n",
       "      <td>RT @ExponentialMed: Learn how @singularityu is...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>387332001</td>\n",
       "      <td>Keep your attention glued to the breath. Follo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1344951</td>\n",
       "      <td>RT @WIREDTransport: Yes, self-driving cars wil...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>16870421</td>\n",
       "      <td>Our own @PeterDiamandis and @HaririRobert (@hu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>7144422</td>\n",
       "      <td>How to move out and not screw over your roomma...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>3119988399</td>\n",
       "      <td>Search interest in @RalphNortham and @EdWGille...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1344951</td>\n",
       "      <td>Hackers seem to be exploiting last week's ISIS...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>9207632</td>\n",
       "      <td>Marie Curie was born 150 years ago today. Most...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>612</th>\n",
       "      <td>231723415</td>\n",
       "      <td>RT @coindesk: Opinion: Why Segwit2x Is Doomed ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>613</th>\n",
       "      <td>231723415</td>\n",
       "      <td>RT @Beautyon_: The \"Lex Luthor\" of Bitcoin. ht...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>614</th>\n",
       "      <td>231723415</td>\n",
       "      <td>RT @SatoshiLite: Email from Daniel Vogel at @b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>615</th>\n",
       "      <td>231723415</td>\n",
       "      <td>RT @JulianAssange: UK justice (for me): jail, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>616</th>\n",
       "      <td>20167623</td>\n",
       "      <td>Top Stories, Oct 30-Nov 5: 6 Books For Every #...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>617</th>\n",
       "      <td>20167623</td>\n",
       "      <td>7 Steps to Mastering #MachineLearning With #Py...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>618</th>\n",
       "      <td>11518572</td>\n",
       "      <td>A reasonable question. https://t.co/A0dE1l9eX5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>619</th>\n",
       "      <td>231723415</td>\n",
       "      <td>RT @SatoshiLite: While some are splitting the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>620</th>\n",
       "      <td>231723415</td>\n",
       "      <td>RT @Xentagz: CME Futures market will launch on...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>621</th>\n",
       "      <td>36125830</td>\n",
       "      <td>Common theme of my sidebar convos at #southsou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>622</th>\n",
       "      <td>231723415</td>\n",
       "      <td>RT @real_or_random: This paper makes Confident...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>623</th>\n",
       "      <td>231723415</td>\n",
       "      <td>RT @lopp: In 11 days we will discover who hold...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>624</th>\n",
       "      <td>36125830</td>\n",
       "      <td>Mass shootings forcibly remind us of the darke...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>625</th>\n",
       "      <td>231723415</td>\n",
       "      <td>RT @GLove39: Siri, show me a visual metaphor f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>626</th>\n",
       "      <td>11518572</td>\n",
       "      <td>RT @divbyzero: The trig functions, geometrical...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>627</th>\n",
       "      <td>19489341</td>\n",
       "      <td>Solenne was offered installments to pay her €1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>628</th>\n",
       "      <td>231723415</td>\n",
       "      <td>RT @JulianAssange: Spain seizes peaceful Catal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>629</th>\n",
       "      <td>36125830</td>\n",
       "      <td>My thoughts exactly. https://t.co/EPXyzRk1Wx</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>630</th>\n",
       "      <td>36125830</td>\n",
       "      <td>RT @NOSaturn: Tacoma is the coffee to rub on a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>631</th>\n",
       "      <td>36125830</td>\n",
       "      <td>RT @HannahWVR: Tacoma is the future liberals y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>632</th>\n",
       "      <td>36125830</td>\n",
       "      <td>RT @365womeninSTEM: #243 Faiza Al-Kharafi is c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>633</th>\n",
       "      <td>36125830</td>\n",
       "      <td>RT @MoveToTacoma: Everything you wanted to kno...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>634</th>\n",
       "      <td>36125830</td>\n",
       "      <td>Diverse stock photos! Search \"holiday\" on the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>635</th>\n",
       "      <td>20167623</td>\n",
       "      <td>The 10 Algorithms #MachineLearning Engineers N...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>636</th>\n",
       "      <td>1344951</td>\n",
       "      <td>Here are the most exciting rumors out of the '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>637</th>\n",
       "      <td>30901808</td>\n",
       "      <td>RT @barstoolsports: Me when I heard the alarm ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>638</th>\n",
       "      <td>15249166</td>\n",
       "      <td>Join Us Live at Singularity University’s Expon...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>639</th>\n",
       "      <td>7144422</td>\n",
       "      <td>Why you still shouldn’t download iOS 11 on an ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>640</th>\n",
       "      <td>3018841323</td>\n",
       "      <td>Using GRAKN.AI to detect patterns in credit fr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>641</th>\n",
       "      <td>2207665903</td>\n",
       "      <td>#DataScientist #Interviews &amp;gt;&amp;gt; 15 In-Dept...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>642 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id                                               text\n",
       "0      10069172  Customer Operations - An Idea for Maximizing E...\n",
       "1      13767622  Whichever team thought of using the accelerome...\n",
       "2       1344951  Here are the iOS 11 privacy settings you shoul...\n",
       "3      13767622              RT @mathowie: https://t.co/pULhe78X71\n",
       "4      15249166  Thanks to rapidly advancing biotechnology, hum...\n",
       "5      16870421  Guess all of our #xMed updates come in 280 cha...\n",
       "6      97418412  RT @AsiaMatos: .@Hull just got WAY better. Get...\n",
       "7      15492359  This man has one goal: to make you love birds....\n",
       "8      16870421  RT @cristina_djsj: #StemCell trial given arm m...\n",
       "9      16870421  At #xMed right now @drhanskeirstead is droppin...\n",
       "10      7144422  You can still upgrade to Windows 10 if you use...\n",
       "11     16870421  This technology, dubbed tissue nanotransfectio...\n",
       "12     16870421  We may be on our way to an entirely new regene...\n",
       "13   3018841323  Application of AI and Data Science in Medical ...\n",
       "14     11518572                 #TheCricle https://t.co/i9U17xMl4L\n",
       "15       809760  Fuck every single one of these additional 140 ...\n",
       "16      1344951  Waymo just took a massive step toward bringing...\n",
       "17     18156146         Ew!! 280 character tweets just look gross!\n",
       "18    159121042                                                  2\n",
       "19     20167623  20 Questions to Detect Fake #DataScientists #K...\n",
       "20     20167623  Building #AI That Can Build AI via The New Yor...\n",
       "21     37083551  RT @Mikel_Jollett: Guns are not allowed in the...\n",
       "22     16870421  RT @ExponentialMed: Learn how @singularityu is...\n",
       "23    387332001  Keep your attention glued to the breath. Follo...\n",
       "24      1344951  RT @WIREDTransport: Yes, self-driving cars wil...\n",
       "25     16870421  Our own @PeterDiamandis and @HaririRobert (@hu...\n",
       "26      7144422  How to move out and not screw over your roomma...\n",
       "27   3119988399  Search interest in @RalphNortham and @EdWGille...\n",
       "28      1344951  Hackers seem to be exploiting last week's ISIS...\n",
       "29      9207632  Marie Curie was born 150 years ago today. Most...\n",
       "..          ...                                                ...\n",
       "612   231723415  RT @coindesk: Opinion: Why Segwit2x Is Doomed ...\n",
       "613   231723415  RT @Beautyon_: The \"Lex Luthor\" of Bitcoin. ht...\n",
       "614   231723415  RT @SatoshiLite: Email from Daniel Vogel at @b...\n",
       "615   231723415  RT @JulianAssange: UK justice (for me): jail, ...\n",
       "616    20167623  Top Stories, Oct 30-Nov 5: 6 Books For Every #...\n",
       "617    20167623  7 Steps to Mastering #MachineLearning With #Py...\n",
       "618    11518572     A reasonable question. https://t.co/A0dE1l9eX5\n",
       "619   231723415  RT @SatoshiLite: While some are splitting the ...\n",
       "620   231723415  RT @Xentagz: CME Futures market will launch on...\n",
       "621    36125830  Common theme of my sidebar convos at #southsou...\n",
       "622   231723415  RT @real_or_random: This paper makes Confident...\n",
       "623   231723415  RT @lopp: In 11 days we will discover who hold...\n",
       "624    36125830  Mass shootings forcibly remind us of the darke...\n",
       "625   231723415  RT @GLove39: Siri, show me a visual metaphor f...\n",
       "626    11518572  RT @divbyzero: The trig functions, geometrical...\n",
       "627    19489341  Solenne was offered installments to pay her €1...\n",
       "628   231723415  RT @JulianAssange: Spain seizes peaceful Catal...\n",
       "629    36125830       My thoughts exactly. https://t.co/EPXyzRk1Wx\n",
       "630    36125830  RT @NOSaturn: Tacoma is the coffee to rub on a...\n",
       "631    36125830  RT @HannahWVR: Tacoma is the future liberals y...\n",
       "632    36125830  RT @365womeninSTEM: #243 Faiza Al-Kharafi is c...\n",
       "633    36125830  RT @MoveToTacoma: Everything you wanted to kno...\n",
       "634    36125830  Diverse stock photos! Search \"holiday\" on the ...\n",
       "635    20167623  The 10 Algorithms #MachineLearning Engineers N...\n",
       "636     1344951  Here are the most exciting rumors out of the '...\n",
       "637    30901808  RT @barstoolsports: Me when I heard the alarm ...\n",
       "638    15249166  Join Us Live at Singularity University’s Expon...\n",
       "639     7144422  Why you still shouldn’t download iOS 11 on an ...\n",
       "640  3018841323  Using GRAKN.AI to detect patterns in credit fr...\n",
       "641  2207665903  #DataScientist #Interviews &gt;&gt; 15 In-Dept...\n",
       "\n",
       "[642 rows x 2 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Build a dataframe with just the columns we're interested in\n",
    "homeTimelineTweets = pd.DataFrame(columns=['user_id', 'text'])\n",
    "\n",
    "for i, tweet_dict in enumerate(homeTimeline.loc[:,\"tweets\"]):\n",
    "    if tweet_dict['text']:\n",
    "        homeTimelineTweets.loc[i,'user_id'] = tweet_dict['user_id_str']\n",
    "        homeTimelineTweets.loc[i,'text'] = tweet_dict['text']\n",
    "        \n",
    "homeTimelineTweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T02:04:53.390713Z",
     "start_time": "2017-11-10T02:04:53.387118Z"
    }
   },
   "outputs": [],
   "source": [
    "homeTweetList = [tweet for tweet in homeTimelineTweets.loc[:,'text']] \n",
    "#pprint(homeTweetList)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Personal User Timeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T01:03:31.820120Z",
     "start_time": "2017-11-10T01:03:31.813360Z"
    }
   },
   "outputs": [],
   "source": [
    "userTimeline = df_from_twitter_data.loc['results','userTimeline']\n",
    "userTimeline = pd.DataFrame(userTimeline[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T01:05:37.881601Z",
     "start_time": "2017-11-10T01:05:37.863704Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'contributors': None,\n",
      " 'coordinates': None,\n",
      " 'created_at': 'Thu Nov 02 16:34:40 +0000 2017',\n",
      " 'entities': {'hashtags': [],\n",
      "              'symbols': [],\n",
      "              'urls': [],\n",
      "              'user_mentions': [{'id': 3018841323,\n",
      "                                 'id_str': '3018841323',\n",
      "                                 'indices': [21, 26],\n",
      "                                 'name': 'Open Data Science',\n",
      "                                 'screen_name': 'odsc'}]},\n",
      " 'favorite_count': 0,\n",
      " 'favorited': False,\n",
      " 'geo': None,\n",
      " 'id': 926125417580265500,\n",
      " 'id_str': '926125417580265472',\n",
      " 'in_reply_to_screen_name': None,\n",
      " 'in_reply_to_status_id': None,\n",
      " 'in_reply_to_status_id_str': None,\n",
      " 'in_reply_to_user_id': None,\n",
      " 'in_reply_to_user_id_str': None,\n",
      " 'is_quote_status': False,\n",
      " 'lang': 'en',\n",
      " 'place': {'bounding_box': {'coordinates': [[[-122.4039361, 37.5681114],\n",
      "                                             [-122.3308186, 37.5681114],\n",
      "                                             [-122.3308186, 37.604114],\n",
      "                                             [-122.4039361, 37.604114]]],\n",
      "                            'type': 'Polygon'},\n",
      "           'contained_within': [],\n",
      "           'country': 'United States',\n",
      "           'country_code': 'US',\n",
      "           'full_name': 'Burlingame, CA',\n",
      "           'id': '9cee92fec370baf9',\n",
      "           'name': 'Burlingame',\n",
      "           'place_type': 'city',\n",
      "           'url': 'https://api.twitter.com/1.1/geo/id/9cee92fec370baf9.json'},\n",
      " 'retweet_count': 0,\n",
      " 'retweeted': False,\n",
      " 'source': '<a href=\"http://twitter.com/download/iphone\" '\n",
      "           'rel=\"nofollow\">Twitter for iPhone</a>',\n",
      " 'text': 'Paid extra to attend @odsc trainings today and was turned away from '\n",
      "         'most classes at the door because room was at capacity. Poor '\n",
      "         'planning. 😡',\n",
      " 'truncated': False,\n",
      " 'user_id_str': '12924742'}\n"
     ]
    }
   ],
   "source": [
    "#pprint(userTimeline.loc[1,'tweets']) # sample tweet dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T01:06:50.961132Z",
     "start_time": "2017-11-10T01:06:49.082798Z"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Reminder: NLP enthusiast meetup tonight at 6pm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Paid extra to attend @odsc trainings today and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12924742</td>\n",
       "      <td>RT @r2d3us: Machine learning explained in inte...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12924742</td>\n",
       "      <td>RT @NextGent: Real time data from an app that ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12924742</td>\n",
       "      <td>RT @ruchowdh: Most of #teamlead!\\n#100women @B...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Sorry can't talk: mouth tired from savoring pe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>12924742</td>\n",
       "      <td>RT @wolfpupy: let's turn this sinking ship aro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>12924742</td>\n",
       "      <td>\"Good enough\" habits die hard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>12924742</td>\n",
       "      <td>.@SlackHQ I need a way to see a list of my his...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Companies are fair-weather communities. It's t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Refreshing my coding skills! Feels good, man. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12924742</td>\n",
       "      <td>RT @reinpk: Today we're announcing the Segment...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12924742</td>\n",
       "      <td>\"Everything that irritates us about others can...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>12924742</td>\n",
       "      <td>#thanksobama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Contemplation https://t.co/gtPdsBT1Ow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>12924742</td>\n",
       "      <td>TIL: Soylent alternating with Fresca tastes li...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>12924742</td>\n",
       "      <td>\"Think lightly of yourself and deeply of the w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>12924742</td>\n",
       "      <td>What's a great fiction book I should read?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Definitely need to be locked out of Slack.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Burrito eating contest at the first annual Seg...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>12924742</td>\n",
       "      <td>RT @kendrawcandraw: Is the person naming these...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Kids born today won't give a crap about gettin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>12924742</td>\n",
       "      <td>The Colors of Lunch @ Segment https://t.co/Uks...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Ultimate Frisbee overlooking the Pacific @ Wes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>12924742</td>\n",
       "      <td>RT @brainpicker: A wonderfully honest illustra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>12924742</td>\n",
       "      <td>We made a Guess Who game with Segment faces! @...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Segment Friday Lunch Shuffle group! @ Goat Hil...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>12924742</td>\n",
       "      <td>The cop isn't real @ SFMOMA San Francisco Muse...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>12924742</td>\n",
       "      <td>One of four piles of dirt in this artwork @ SF...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>12924742</td>\n",
       "      <td>RT @segment: Introducing the Native Mobile Spe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1820</th>\n",
       "      <td>12924742</td>\n",
       "      <td>I foresee my case of the Mondays extending int...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1821</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Preliminary estimates show dish washing up 200...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1822</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Overhead in the EPA: Mothafucka! I'm gon get s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1823</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Kyle Wild ladies and gentlemen: http://tinyurl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1824</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Pidgin's Coin Flip plugin makes me a virtual T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1825</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Take me to New York, I'd like to C.L.A.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1826</th>\n",
       "      <td>12924742</td>\n",
       "      <td>http://www.thesixtyone.com/ is a music MMORPG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1827</th>\n",
       "      <td>12924742</td>\n",
       "      <td>I can understand talking to yourself under you...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1828</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Fuck Balmoral Apartments. They put granite nea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1829</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Fighting a time-sensitive fire that I don't kn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1830</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Civilization in the bathroom. Civilization at ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1831</th>\n",
       "      <td>12924742</td>\n",
       "      <td>dealing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1832</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Trying out lunarr.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1833</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Saddened by the transience of weekends.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1834</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Playing Civilization on the DS! Never again wi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1835</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Thinking about getting a iPhone this weekend.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1836</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Contacted by a Google recruiter today. I assum...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1837</th>\n",
       "      <td>12924742</td>\n",
       "      <td>I want a nap. And I want it in a nap machine w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1838</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Ahhh, just love it when the bosses fight like ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1839</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Best delivery ever.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1840</th>\n",
       "      <td>12924742</td>\n",
       "      <td>I want more time.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1841</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Dark Knight meets expectations. Loved Ledger's...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1842</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Spent 6 hours talking with my boss today about...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1843</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Still unpacking...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1844</th>\n",
       "      <td>12924742</td>\n",
       "      <td>You know you're alignment is off when you can ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1845</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Appreciating the lyrics and message of Flobot'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1846</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Moving.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1847</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Not looking forward to packing and moving tomo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1848</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Note to self: bring more cash next time.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1849</th>\n",
       "      <td>12924742</td>\n",
       "      <td>Greg and Fabric are visiting! A brief stop on ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1850 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       user_id                                               text\n",
       "0     12924742  Reminder: NLP enthusiast meetup tonight at 6pm...\n",
       "1     12924742  Paid extra to attend @odsc trainings today and...\n",
       "2     12924742  RT @r2d3us: Machine learning explained in inte...\n",
       "3     12924742  RT @NextGent: Real time data from an app that ...\n",
       "4     12924742  RT @ruchowdh: Most of #teamlead!\\n#100women @B...\n",
       "5     12924742  Sorry can't talk: mouth tired from savoring pe...\n",
       "6     12924742  RT @wolfpupy: let's turn this sinking ship aro...\n",
       "7     12924742                      \"Good enough\" habits die hard\n",
       "8     12924742  .@SlackHQ I need a way to see a list of my his...\n",
       "9     12924742  Companies are fair-weather communities. It's t...\n",
       "10    12924742  Refreshing my coding skills! Feels good, man. ...\n",
       "11    12924742  RT @reinpk: Today we're announcing the Segment...\n",
       "12    12924742  \"Everything that irritates us about others can...\n",
       "13    12924742                                       #thanksobama\n",
       "14    12924742              Contemplation https://t.co/gtPdsBT1Ow\n",
       "15    12924742  TIL: Soylent alternating with Fresca tastes li...\n",
       "16    12924742  \"Think lightly of yourself and deeply of the w...\n",
       "17    12924742         What's a great fiction book I should read?\n",
       "18    12924742         Definitely need to be locked out of Slack.\n",
       "19    12924742  Burrito eating contest at the first annual Seg...\n",
       "20    12924742  RT @kendrawcandraw: Is the person naming these...\n",
       "21    12924742  Kids born today won't give a crap about gettin...\n",
       "22    12924742  The Colors of Lunch @ Segment https://t.co/Uks...\n",
       "23    12924742  Ultimate Frisbee overlooking the Pacific @ Wes...\n",
       "24    12924742  RT @brainpicker: A wonderfully honest illustra...\n",
       "25    12924742  We made a Guess Who game with Segment faces! @...\n",
       "26    12924742  Segment Friday Lunch Shuffle group! @ Goat Hil...\n",
       "27    12924742  The cop isn't real @ SFMOMA San Francisco Muse...\n",
       "28    12924742  One of four piles of dirt in this artwork @ SF...\n",
       "29    12924742  RT @segment: Introducing the Native Mobile Spe...\n",
       "...        ...                                                ...\n",
       "1820  12924742  I foresee my case of the Mondays extending int...\n",
       "1821  12924742  Preliminary estimates show dish washing up 200...\n",
       "1822  12924742  Overhead in the EPA: Mothafucka! I'm gon get s...\n",
       "1823  12924742  Kyle Wild ladies and gentlemen: http://tinyurl...\n",
       "1824  12924742  Pidgin's Coin Flip plugin makes me a virtual T...\n",
       "1825  12924742            Take me to New York, I'd like to C.L.A.\n",
       "1826  12924742      http://www.thesixtyone.com/ is a music MMORPG\n",
       "1827  12924742  I can understand talking to yourself under you...\n",
       "1828  12924742  Fuck Balmoral Apartments. They put granite nea...\n",
       "1829  12924742  Fighting a time-sensitive fire that I don't kn...\n",
       "1830  12924742  Civilization in the bathroom. Civilization at ...\n",
       "1831  12924742                                            dealing\n",
       "1832  12924742                              Trying out lunarr.com\n",
       "1833  12924742            Saddened by the transience of weekends.\n",
       "1834  12924742  Playing Civilization on the DS! Never again wi...\n",
       "1835  12924742      Thinking about getting a iPhone this weekend.\n",
       "1836  12924742  Contacted by a Google recruiter today. I assum...\n",
       "1837  12924742  I want a nap. And I want it in a nap machine w...\n",
       "1838  12924742  Ahhh, just love it when the bosses fight like ...\n",
       "1839  12924742                                Best delivery ever.\n",
       "1840  12924742                                  I want more time.\n",
       "1841  12924742  Dark Knight meets expectations. Loved Ledger's...\n",
       "1842  12924742  Spent 6 hours talking with my boss today about...\n",
       "1843  12924742                                 Still unpacking...\n",
       "1844  12924742  You know you're alignment is off when you can ...\n",
       "1845  12924742  Appreciating the lyrics and message of Flobot'...\n",
       "1846  12924742                                            Moving.\n",
       "1847  12924742  Not looking forward to packing and moving tomo...\n",
       "1848  12924742           Note to self: bring more cash next time.\n",
       "1849  12924742  Greg and Fabric are visiting! A brief stop on ...\n",
       "\n",
       "[1850 rows x 2 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Build a dataframe with just the columns we're interested in\n",
    "userTimelineTweets = pd.DataFrame(columns=['user_id', 'text'])\n",
    "\n",
    "for i, tweet_dict in enumerate(userTimeline.loc[:,\"tweets\"]):\n",
    "    if tweet_dict['text']:\n",
    "        userTimelineTweets.loc[i,'user_id'] = tweet_dict['user_id_str']\n",
    "        userTimelineTweets.loc[i,'text'] = tweet_dict['text']\n",
    "        \n",
    "userTimelineTweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T01:37:23.217623Z",
     "start_time": "2017-11-10T01:37:23.210423Z"
    }
   },
   "outputs": [],
   "source": [
    "userTextblob = str()\n",
    "for text in userTimelineTweets.loc[:,'text']:\n",
    "    userTextblob += text + \" \\n \"\n",
    "#userTextblob # one giant string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T02:31:39.901074Z",
     "start_time": "2017-11-10T02:31:39.884318Z"
    }
   },
   "outputs": [],
   "source": [
    "userTweetList = [tweet for tweet in userTimelineTweets.loc[:,'text']] \n",
    "#userTweetList"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Latent Dirichlet Allocation (LDA) with CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T02:00:24.495355Z",
     "start_time": "2017-11-10T02:00:24.443032Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer, ENGLISH_STOP_WORDS\n",
    "custom_stopwords = [\"http\", \"com\", \"bit\", \"ly\"]\n",
    "vect = CountVectorizer(stop_words=list(ENGLISH_STOP_WORDS) + custom_stopwords)\n",
    "X = vect.fit_transform(userTweetList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T02:00:30.049319Z",
     "start_time": "2017-11-10T02:00:24.621522Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "num_topics = 10\n",
    "lda = LatentDirichletAllocation(n_topics=num_topics, learning_method=\"batch\",\n",
    "                                max_iter=25, random_state=0)\n",
    "\n",
    "# build the model and transform the data in one step\n",
    "# computing transform takes some time,\n",
    "# and we can save time by doing both at once.\n",
    "document_topics = lda.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T02:00:30.056437Z",
     "start_time": "2017-11-10T02:00:30.051046Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lda.components_.shape: (10, 6927)\n"
     ]
    }
   ],
   "source": [
    "print(\"lda.components_.shape: {}\".format(lda.components_.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T02:00:30.092491Z",
     "start_time": "2017-11-10T02:00:30.059274Z"
    }
   },
   "outputs": [],
   "source": [
    "# for each topic (a row in the components_), sort the features (ascending).\n",
    "# Invert rows with [:, ::-1]\n",
    "sorting = np.argsort(lda.components_, axis=1)[:, ::-1]\n",
    "# get the feature names from the vectorizer:\n",
    "feature_names = np.array(vect.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T02:00:30.106066Z",
     "start_time": "2017-11-10T02:00:30.094654Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_topics(topics, feature_names, sorting, topics_per_chunk=6,\n",
    "                 n_words=20):\n",
    "    for i in range(0, len(topics), topics_per_chunk):\n",
    "        # for each chunk:\n",
    "        these_topics = topics[i: i + topics_per_chunk]\n",
    "        # maybe we have less than topics_per_chunk left\n",
    "        len_this_chunk = len(these_topics)\n",
    "        # print topic headers\n",
    "        print((\"topic {:<8}\" * len_this_chunk).format(*these_topics))\n",
    "        print((\"-------- {0:<5}\" * len_this_chunk).format(\"\"))\n",
    "        # print top n_words frequent words\n",
    "        for i in range(n_words):\n",
    "            try:\n",
    "                print((\"{:<14}\" * len_this_chunk).format(\n",
    "                    *feature_names[sorting[these_topics, i]]))\n",
    "            except:\n",
    "                pass\n",
    "        print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T02:00:30.140934Z",
     "start_time": "2017-11-10T02:00:30.107992Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "topic 0       topic 1       topic 2       topic 3       topic 4       \n",
      "--------      --------      --------      --------      --------      \n",
      "like          rt            just          rt            https         \n",
      "rt            https         like          going         segment       \n",
      "work          just          rt            new           going         \n",
      "ultimate      tinyurl       stanford      ted           time          \n",
      "right         got           ted           work          new           \n",
      "frisbee       segment       stars         maria         day           \n",
      "going         best          going         today         rt            \n",
      "new           party         world         good          ted           \n",
      "today         today         https         just          tonight       \n",
      "man           time          systems       time          twitpic       \n",
      "\n",
      "\n",
      "topic 5       topic 6       topic 7       topic 8       topic 9       \n",
      "--------      --------      --------      --------      --------      \n",
      "rt            rt            www           just          rt            \n",
      "time          today         reading       rt            man           \n",
      "maria         ve            review        like          factlets      \n",
      "new           just          goodreads     great         burning       \n",
      "tinyurl       ah            stars         today         new           \n",
      "just          like          just          read          today         \n",
      "playing       alto          mind          got           ted           \n",
      "wnycradiolab  palo          today         day           google        \n",
      "house         great         don           watching      radiolab      \n",
      "brain         new           like          good          know          \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print out the n topics:\n",
    "print_topics(topics=range(num_topics), feature_names=feature_names,\n",
    "                           sorting=sorting, topics_per_chunk=5, n_words=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "80px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}