02_clean_tweets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 02_clean_tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "pd.options.display.max_columns = 999\n",
    "pd.options.display.max_colwidth = 999\n",
    "pd.options.display.max_rows = 999\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_df = pd.read_pickle('01_Künast.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2992, 30)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                 object\n",
       "conversation_id    object\n",
       "created_at          int64\n",
       "date               object\n",
       "timezone           object\n",
       "place              object\n",
       "tweet              object\n",
       "hashtags           object\n",
       "cashtags           object\n",
       "user_id             int64\n",
       "user_id_str        object\n",
       "username           object\n",
       "name               object\n",
       "day                 int64\n",
       "hour               object\n",
       "link               object\n",
       "retweet              bool\n",
       "nlikes              int64\n",
       "nreplies            int64\n",
       "nretweets           int64\n",
       "quote_url          object\n",
       "search             object\n",
       "near               object\n",
       "geo                object\n",
       "source             object\n",
       "user_rt_id         object\n",
       "user_rt            object\n",
       "retweet_id         object\n",
       "reply_to           object\n",
       "retweet_date       object\n",
       "dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_df.date = pd.to_datetime(tweets_df.date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>conversation_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>timezone</th>\n",
       "      <th>place</th>\n",
       "      <th>tweet</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>cashtags</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_id_str</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "      <th>link</th>\n",
       "      <th>retweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "      <th>search</th>\n",
       "      <th>near</th>\n",
       "      <th>geo</th>\n",
       "      <th>source</th>\n",
       "      <th>user_rt_id</th>\n",
       "      <th>user_rt</th>\n",
       "      <th>retweet_id</th>\n",
       "      <th>reply_to</th>\n",
       "      <th>retweet_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>411</td>\n",
       "      <td>1138061455755022336</td>\n",
       "      <td>1138061455755022336</td>\n",
       "      <td>1560169969000</td>\n",
       "      <td>2019-06-10 14:32:49</td>\n",
       "      <td>CEST</td>\n",
       "      <td></td>\n",
       "      <td>Grüne wollen Folgen ihrer Politik verbieten.   Eine gewisse Frau Künast, ebenfalls grüne Theoretikerin, hat diesen Wahnsinn eingeführt.   Grüne Richter halten daran fest   http://tagesschau.de : Online-Handel: Grüne wollen Retouren-Vernichtung verbieten. https://www.tagesschau.de/wirtschaft/gruene-amazon-101.html …</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>17775954</td>\n",
       "      <td>17775954</td>\n",
       "      <td>bad_deluxe</td>\n",
       "      <td>Deichmann</td>\n",
       "      <td>4</td>\n",
       "      <td>19</td>\n",
       "      <td>https://twitter.com/bad_deluxe/status/1138061455755022336</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>Künast</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[{'user_id': '17775954', 'username': 'bad_deluxe'}]</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      id      conversation_id     created_at  \\\n",
       "411  1138061455755022336  1138061455755022336  1560169969000   \n",
       "\n",
       "                   date timezone place  \\\n",
       "411 2019-06-10 14:32:49     CEST         \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                            tweet  \\\n",
       "411  Grüne wollen Folgen ihrer Politik verbieten.   Eine gewisse Frau Künast, ebenfalls grüne Theoretikerin, hat diesen Wahnsinn eingeführt.   Grüne Richter halten daran fest   http://tagesschau.de : Online-Handel: Grüne wollen Retouren-Vernichtung verbieten. https://www.tagesschau.de/wirtschaft/gruene-amazon-101.html …   \n",
       "\n",
       "    hashtags cashtags   user_id user_id_str    username       name  day hour  \\\n",
       "411       []       []  17775954    17775954  bad_deluxe  Deichmann    4   19   \n",
       "\n",
       "                                                          link  retweet  \\\n",
       "411  https://twitter.com/bad_deluxe/status/1138061455755022336    False   \n",
       "\n",
       "     nlikes  nreplies  nretweets quote_url  search near geo source user_rt_id  \\\n",
       "411       0         0          0            Künast                              \n",
       "\n",
       "    user_rt retweet_id                                             reply_to  \\\n",
       "411                     [{'user_id': '17775954', 'username': 'bad_deluxe'}]   \n",
       "\n",
       "    retweet_date  \n",
       "411               "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "tweets_df.sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_df = tweets_df[['date', 'username', 'name', 'tweet', 'nlikes', 'nreplies', 'nretweets', 'quote_url']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>tweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2019-06-29 20:43:41</td>\n",
       "      <td>wolfgs2</td>\n",
       "      <td>Rev. Wolf Schawaller</td>\n",
       "      <td>Pädophile bei den Grünen: Renate Künast gerät in Erklärungsnot - WELT  https://www.welt.de/politik/deutschland/article141406874/Gruenen-Politikerin-Kuenast-geraet-in-Erklaerungsnot.html …</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2019-06-29 20:32:26</td>\n",
       "      <td>weigl_</td>\n",
       "      <td>Peter Weigl</td>\n",
       "      <td>ISRC’s Portier mit lukrativem Vertrag mit den US tort lawyers. Stinkt zum Himmel. Das BfR wurde 2003 von der grünen Ministerin Künast auf wissenschaftliche Unabhängigkeit hin eingerichtet. Gut, dass sich mit CRISPR die Anti GMO Hysterie sowieso gegessen hat.</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2019-06-29 20:16:34</td>\n",
       "      <td>LunaLara404</td>\n",
       "      <td>LunaLara 🇪🇺 ☮️ 🛡</td>\n",
       "      <td>Das manche ältere Herren immer wieder großen Mist von sich geben, quasi ohne Konsequenzen, Seehofer, Gabriel, habe ich kein Verständnis für. Aber auch in den älteren Generationen gibt es Positivbeispiele, Polenz, Künast…</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2019-06-29 15:57:58</td>\n",
       "      <td>exveganer</td>\n",
       "      <td>Ex-Veganer Klarname</td>\n",
       "      <td>Die (((#Grünen))) eben. Informiere Dich. Es geht nicht primär um Öko. Es geht um die Farbe und was damit verbunden ist. Wie bei Leuten mit Namen wie #Greenspan, #Greenhill, #Roth, #Künast/#Kynast/#Kienast etc.</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2019-06-29 13:12:41</td>\n",
       "      <td>FlaviusJosephu7</td>\n",
       "      <td>Flavius Josephus</td>\n",
       "      <td>Sie und die C.Roth oder die Künast? Ich bitte dich. Nichtmal ein Flüchtling würde sich an denen vergehen. Naja. Höchstens die Frau Hofreiter müsste vorsichtig sein. 😂😂😂</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 date         username                  name  \\\n",
       "0 2019-06-29 20:43:41          wolfgs2  Rev. Wolf Schawaller   \n",
       "1 2019-06-29 20:32:26           weigl_           Peter Weigl   \n",
       "2 2019-06-29 20:16:34      LunaLara404      LunaLara 🇪🇺 ☮️ 🛡   \n",
       "3 2019-06-29 15:57:58        exveganer   Ex-Veganer Klarname   \n",
       "4 2019-06-29 13:12:41  FlaviusJosephu7      Flavius Josephus   \n",
       "\n",
       "                                                                                                                                                                                                                                                                tweet  \\\n",
       "0                                                                         Pädophile bei den Grünen: Renate Künast gerät in Erklärungsnot - WELT  https://www.welt.de/politik/deutschland/article141406874/Gruenen-Politikerin-Kuenast-geraet-in-Erklaerungsnot.html …   \n",
       "1  ISRC’s Portier mit lukrativem Vertrag mit den US tort lawyers. Stinkt zum Himmel. Das BfR wurde 2003 von der grünen Ministerin Künast auf wissenschaftliche Unabhängigkeit hin eingerichtet. Gut, dass sich mit CRISPR die Anti GMO Hysterie sowieso gegessen hat.   \n",
       "2                                        Das manche ältere Herren immer wieder großen Mist von sich geben, quasi ohne Konsequenzen, Seehofer, Gabriel, habe ich kein Verständnis für. Aber auch in den älteren Generationen gibt es Positivbeispiele, Polenz, Künast…   \n",
       "3                                                   Die (((#Grünen))) eben. Informiere Dich. Es geht nicht primär um Öko. Es geht um die Farbe und was damit verbunden ist. Wie bei Leuten mit Namen wie #Greenspan, #Greenhill, #Roth, #Künast/#Kynast/#Kienast etc.   \n",
       "4                                                                                            Sie und die C.Roth oder die Künast? Ich bitte dich. Nichtmal ein Flüchtling würde sich an denen vergehen. Naja. Höchstens die Frau Hofreiter müsste vorsichtig sein. 😂😂😂   \n",
       "\n",
       "   nlikes  nreplies  nretweets quote_url  \n",
       "0       1         0          0            \n",
       "1       1         1          0            \n",
       "2       1         0          0            \n",
       "3       2         1          1            \n",
       "4       4         1          0            "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_df.to_pickle('clean_kuenast.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}