03_get_toxicity.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 03_get_toxicity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json \n",
    "import requests\n",
    "import time\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "pd.options.display.max_columns = 999\n",
    "pd.options.display.max_colwidth = 999\n",
    "pd.options.display.max_rows = 999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_df = pd.read_pickle('clean_kuenast.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2992, 8)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>tweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1074</td>\n",
       "      <td>2019-05-10 13:46:55</td>\n",
       "      <td>travellmax</td>\n",
       "      <td>travellmax</td>\n",
       "      <td>Appropos Zitate.  Zur Diskussion ob sexuelle Handlungen an Kindern straffrei sein sollen, sagte Renate Künast, Zitat:  „Komma, wenn keine Gewalt im Spiel ist!“ #zitate #pädophilie #grüne  https://www.google.de/amp/s/amp.welt.de/politik/deutschland/article141406874/Gruenen-Politikerin-Kuenast-geraet-in-Erklaerungsnot.html …</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    date    username        name  \\\n",
       "1074 2019-05-10 13:46:55  travellmax  travellmax   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                     tweet  \\\n",
       "1074  Appropos Zitate.  Zur Diskussion ob sexuelle Handlungen an Kindern straffrei sein sollen, sagte Renate Künast, Zitat:  „Komma, wenn keine Gewalt im Spiel ist!“ #zitate #pädophilie #grüne  https://www.google.de/amp/s/amp.welt.de/politik/deutschland/article141406874/Gruenen-Politikerin-Kuenast-geraet-in-Erklaerungsnot.html …   \n",
       "\n",
       "      nlikes  nreplies  nretweets quote_url  \n",
       "1074       8         1          1            "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df.sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_df.to_csv('kuenast2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "api_key = 'AIzaSyDmzXrYen1XtcQTGa5xxnygTuWkPwPlSRU'\n",
    "url = ('https://commentanalyzer.googleapis.com/v1alpha1/comments:analyze' +    \n",
    "    '?key=' + api_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_df = tweets_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                                                           Pädophile bei den Grünen: Renate Künast gerät in Erklärungsnot - WELT  https://www.welt.de/politik/deutschland/article141406874/Gruenen-Politikerin-Kuenast-geraet-in-Erklaerungsnot.html …\n",
       "1    ISRC’s Portier mit lukrativem Vertrag mit den US tort lawyers. Stinkt zum Himmel. Das BfR wurde 2003 von der grünen Ministerin Künast auf wissenschaftliche Unabhängigkeit hin eingerichtet. Gut, dass sich mit CRISPR die Anti GMO Hysterie sowieso gegessen hat.\n",
       "2                                          Das manche ältere Herren immer wieder großen Mist von sich geben, quasi ohne Konsequenzen, Seehofer, Gabriel, habe ich kein Verständnis für. Aber auch in den älteren Generationen gibt es Positivbeispiele, Polenz, Künast…\n",
       "3                                                     Die (((#Grünen))) eben. Informiere Dich. Es geht nicht primär um Öko. Es geht um die Farbe und was damit verbunden ist. Wie bei Leuten mit Namen wie #Greenspan, #Greenhill, #Roth, #Künast/#Kynast/#Kienast etc.\n",
       "4                                                                                              Sie und die C.Roth oder die Künast? Ich bitte dich. Nichtmal ein Flüchtling würde sich an denen vergehen. Naja. Höchstens die Frau Hofreiter müsste vorsichtig sein. 😂😂😂\n",
       "Name: tweet, dtype: object"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df.tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_toxity(column):\n",
    "    tox_list = []\n",
    "    for i in column:\n",
    "        time.sleep(2)\n",
    "        data_dict = {\n",
    "                    'comment': {'text': i},\n",
    "                    'languages': ['de'],\n",
    "                    'requestedAttributes': {'TOXICITY_EXPERIMENTAL': {}}\n",
    "                    }\n",
    "        response = requests.post(url=url, data=json.dumps(data_dict))\n",
    "        response_dict = json.loads(response.content)\n",
    "        tox_list.append(json.dumps(response_dict, indent=2))\n",
    "        print(json.dumps(response_dict, indent=2))\n",
    "        # df.at[i, 'Toxity'] = response_dict['attributeScores']['TOXICITY_EXPERIMENTAL']['summaryScore']['value']\n",
    "    return tox_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"attributeScores\": {\n",
      "    \"TOXICITY_EXPERIMENTAL\": {\n",
      "      \"spanScores\": [\n",
      "        {\n",
      "          \"begin\": 0,\n",
      "          \"end\": 187,\n",
      "          \"score\": {\n",
      "            \"value\": 0.15297964,\n",
      "            \"type\": \"PROBABILITY\"\n",
      "          }\n",
      "        }\n",
      "      ],\n",
      "      \"summaryScore\": {\n",
      "        \"value\": 0.15297964,\n",
      "        \"type\": \"PROBABILITY\"\n",
      "      }\n",
      "    }\n",
      "  },\n",
      "  \"languages\": [\n",
      "    \"de\"\n",
      "  ],\n",
      "  \"detectedLanguages\": [\n",
      "    \"de\"\n",
      "  ]\n",
      "}\n",
      "{\n",
      "  \"attributeScores\": {\n",
      "    \"TOXICITY_EXPERIMENTAL\": {\n",
      "      \"spanScores\": [\n",
      "        {\n",
      "          \"begin\": 0,\n",
      "          \"end\": 258,\n",
      "          \"score\": {\n",
      "            \"value\": 0.2629246,\n",
      "            \"type\": \"PROBABILITY\"\n",
      "          }\n",
      "        }\n",
      "      ],\n",
      "      \"summaryScore\": {\n",
      "        \"value\": 0.2629246,\n",
      "        \"type\": \"PROBABILITY\"\n",
      "      }\n",
      "    }\n",
      "  },\n",
      "  \"languages\": [\n",
      "    \"de\"\n",
      "  ],\n",
      "  \"detectedLanguages\": [\n",
      "    \"de\"\n",
      "  ]\n",
      "}\n",
      "{\n",
      "  \"attributeScores\": {\n",
      "    \"TOXICITY_EXPERIMENTAL\": {\n",
      "      \"spanScores\": [\n",
      "        {\n",
      "          \"begin\": 0,\n",
      "          \"end\": 220,\n",
      "          \"score\": {\n",
      "            \"value\": 0.2638486,\n",
      "            \"type\": \"PROBABILITY\"\n",
      "          }\n",
      "        }\n",
      "      ],\n",
      "      \"summaryScore\": {\n",
      "        \"value\": 0.2638486,\n",
      "        \"type\": \"PROBABILITY\"\n",
      "      }\n",
      "    }\n",
      "  },\n",
      "  \"languages\": [\n",
      "    \"de\"\n",
      "  ],\n",
      "  \"detectedLanguages\": [\n",
      "    \"de\"\n",
      "  ]\n",
      "}\n",
      "{\n",
      "  \"attributeScores\": {\n",
      "    \"TOXICITY_EXPERIMENTAL\": {\n",
      "      \"spanScores\": [\n",
      "        {\n",
      "          \"begin\": 0,\n",
      "          \"end\": 209,\n",
      "          \"score\": {\n",
      "            \"value\": 0.05818355,\n",
      "            \"type\": \"PROBABILITY\"\n",
      "          }\n",
      "        }\n",
      "      ],\n",
      "      \"summaryScore\": {\n",
      "        \"value\": 0.05818355,\n",
      "        \"type\": \"PROBABILITY\"\n",
      "      }\n",
      "    }\n",
      "  },\n",
      "  \"languages\": [\n",
      "    \"de\"\n",
      "  ],\n",
      "  \"detectedLanguages\": [\n",
      "    \"de\"\n",
      "  ]\n",
      "}\n",
      "{\n",
      "  \"attributeScores\": {\n",
      "    \"TOXICITY_EXPERIMENTAL\": {\n",
      "      \"spanScores\": [\n",
      "        {\n",
      "          \"begin\": 0,\n",
      "          \"end\": 171,\n",
      "          \"score\": {\n",
      "            \"value\": 0.17539881,\n",
      "            \"type\": \"PROBABILITY\"\n",
      "          }\n",
      "        }\n",
      "      ],\n",
      "      \"summaryScore\": {\n",
      "        \"value\": 0.17539881,\n",
      "        \"type\": \"PROBABILITY\"\n",
      "      }\n",
      "    }\n",
      "  },\n",
      "  \"languages\": [\n",
      "    \"de\"\n",
      "  ],\n",
      "  \"detectedLanguages\": [\n",
      "    \"de\"\n",
      "  ]\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "response_dict = get_toxity(tweets_df.tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "list indices must be integers or slices, not str",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-34-7489199fe030>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mresponse_dict\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'attributeScores'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'TOXICITY_EXPERIMENTAL'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'summaryScore'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'value'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m: list indices must be integers or slices, not str"
     ]
    }
   ],
   "source": [
    "for i in response_dict:\n",
    "    print(response_dict['attributeScores']['TOXICITY_EXPERIMENTAL']['summaryScore']['value'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'attributeScores': {'TOXICITY_EXPERIMENTAL': {'spanScores': [{'begin': 0,\n",
       "     'end': 171,\n",
       "     'score': {'value': 0.17539881, 'type': 'PROBABILITY'}}],\n",
       "   'summaryScore': {'value': 0.17539881, 'type': 'PROBABILITY'}}},\n",
       " 'languages': ['de'],\n",
       " 'detectedLanguages': ['de']}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in response_dict:\n",
    "    tox_list.append(response_dict['attributeScores']['TOXICITY_EXPERIMENTAL']['summaryScore']['value'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.17539881, 0.17539881, 0.17539881]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tox_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>tweet</th>\n",
       "      <th>nlikes</th>\n",
       "      <th>nreplies</th>\n",
       "      <th>nretweets</th>\n",
       "      <th>quote_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2019-06-29 20:43:41</td>\n",
       "      <td>wolfgs2</td>\n",
       "      <td>Rev. Wolf Schawaller</td>\n",
       "      <td>Pädophile bei den Grünen: Renate Künast gerät in Erklärungsnot - WELT  https://www.welt.de/politik/deutschland/article141406874/Gruenen-Politikerin-Kuenast-geraet-in-Erklaerungsnot.html …</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2019-06-29 20:32:26</td>\n",
       "      <td>weigl_</td>\n",
       "      <td>Peter Weigl</td>\n",
       "      <td>ISRC’s Portier mit lukrativem Vertrag mit den US tort lawyers. Stinkt zum Himmel. Das BfR wurde 2003 von der grünen Ministerin Künast auf wissenschaftliche Unabhängigkeit hin eingerichtet. Gut, dass sich mit CRISPR die Anti GMO Hysterie sowieso gegessen hat.</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2019-06-29 20:16:34</td>\n",
       "      <td>LunaLara404</td>\n",
       "      <td>LunaLara 🇪🇺 ☮️ 🛡</td>\n",
       "      <td>Das manche ältere Herren immer wieder großen Mist von sich geben, quasi ohne Konsequenzen, Seehofer, Gabriel, habe ich kein Verständnis für. Aber auch in den älteren Generationen gibt es Positivbeispiele, Polenz, Künast…</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2019-06-29 15:57:58</td>\n",
       "      <td>exveganer</td>\n",
       "      <td>Ex-Veganer Klarname</td>\n",
       "      <td>Die (((#Grünen))) eben. Informiere Dich. Es geht nicht primär um Öko. Es geht um die Farbe und was damit verbunden ist. Wie bei Leuten mit Namen wie #Greenspan, #Greenhill, #Roth, #Künast/#Kynast/#Kienast etc.</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2019-06-29 13:12:41</td>\n",
       "      <td>FlaviusJosephu7</td>\n",
       "      <td>Flavius Josephus</td>\n",
       "      <td>Sie und die C.Roth oder die Künast? Ich bitte dich. Nichtmal ein Flüchtling würde sich an denen vergehen. Naja. Höchstens die Frau Hofreiter müsste vorsichtig sein. 😂😂😂</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 date         username                  name  \\\n",
       "0 2019-06-29 20:43:41          wolfgs2  Rev. Wolf Schawaller   \n",
       "1 2019-06-29 20:32:26           weigl_           Peter Weigl   \n",
       "2 2019-06-29 20:16:34      LunaLara404      LunaLara 🇪🇺 ☮️ 🛡   \n",
       "3 2019-06-29 15:57:58        exveganer   Ex-Veganer Klarname   \n",
       "4 2019-06-29 13:12:41  FlaviusJosephu7      Flavius Josephus   \n",
       "\n",
       "                                                                                                                                                                                                                                                                tweet  \\\n",
       "0                                                                         Pädophile bei den Grünen: Renate Künast gerät in Erklärungsnot - WELT  https://www.welt.de/politik/deutschland/article141406874/Gruenen-Politikerin-Kuenast-geraet-in-Erklaerungsnot.html …   \n",
       "1  ISRC’s Portier mit lukrativem Vertrag mit den US tort lawyers. Stinkt zum Himmel. Das BfR wurde 2003 von der grünen Ministerin Künast auf wissenschaftliche Unabhängigkeit hin eingerichtet. Gut, dass sich mit CRISPR die Anti GMO Hysterie sowieso gegessen hat.   \n",
       "2                                        Das manche ältere Herren immer wieder großen Mist von sich geben, quasi ohne Konsequenzen, Seehofer, Gabriel, habe ich kein Verständnis für. Aber auch in den älteren Generationen gibt es Positivbeispiele, Polenz, Künast…   \n",
       "3                                                   Die (((#Grünen))) eben. Informiere Dich. Es geht nicht primär um Öko. Es geht um die Farbe und was damit verbunden ist. Wie bei Leuten mit Namen wie #Greenspan, #Greenhill, #Roth, #Künast/#Kynast/#Kienast etc.   \n",
       "4                                                                                            Sie und die C.Roth oder die Künast? Ich bitte dich. Nichtmal ein Flüchtling würde sich an denen vergehen. Naja. Höchstens die Frau Hofreiter müsste vorsichtig sein. 😂😂😂   \n",
       "\n",
       "   nlikes  nreplies  nretweets quote_url  \n",
       "0       1         0          0            \n",
       "1       1         1          0            \n",
       "2       1         0          0            \n",
       "3       2         1          1            \n",
       "4       4         1          0            "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}