SentimentDictionaries.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
    "import collections\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.util import ngrams\n",
    "from collections import Counter\n",
    "import pickle\n",
    "\n",
    "def saveInFile(data,fileName):\n",
    "    filehandler = open(fileName+\".p\", \"wb\")\n",
    "    pickle.dump(data, filehandler)\n",
    "    filehandler.close()\n",
    "\n",
    "def get_ngrams(sentence, n ):\n",
    "    n_grams = ngrams(word_tokenize(sentence), n)\n",
    "    return [ ' '.join(grams) for grams in n_grams]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id\n",
      "0000997932d777bf    explanation why the edit make under -PRON- use...\n",
      "000103f0d9cfb60f    d'aww ! -PRON- match this background colour -P...\n",
      "000113f07ec002fd    hey man , -PRON- be really not try to edit war...\n",
      "0001b41b1c6bb37e    \" more i can not make any real suggestion on i...\n",
      "0001d958c54c6e35    -PRON- , sir , be -PRON- hero . any chance -PR...\n",
      "00025465d4725e87    \" congratulation from -PRON- as well , use the...\n",
      "0002bcb3da6cb337    cocksucker before -PRON- pis around on -PRON- ...\n",
      "00031b1e95af7921    -PRON- vandalism to the matt shirvington artic...\n",
      "00037261f536c51d    sorry if the word ' nonsense ' be offensive to...\n",
      "00040093b2687caa    alignment on this subject and which be contrar...\n",
      "0005300084f90edc    \" fair use rationale for image : wonju.jpg tha...\n",
      "00054a5e18b50dd4    bbq be a man and let discus -PRON- - maybe ove...\n",
      "0005c987bdfc9d4b    hey ... what be -PRON- .. @ | talk . what be -...\n",
      "0006f16e4e9f292e    before -PRON- start throw accusation and warni...\n",
      "00070ef96486d6f9    oh , and the girl above start -PRON- argument ...\n",
      "00078f8ce7eb276d    \" juelz santana age in , juelz santana be year...\n",
      "0007e25b2121310b    bye ! do not look , come or think of comm back...\n",
      "000897889268bc93    redirect talk : voydan pop georgiev- chernodri...\n",
      "0009801bd85e5806    the mitsurugi point make no sense - why not ar...\n",
      "0009eaea3325de8c    do not mean to bother -PRON- i see that -PRON-...\n",
      "000b08c464718505    \" regard -PRON- recent edit once again , pleas...\n",
      "000bfd0867774845    \" good to know . about -PRON- , yeah , -PRON- ...\n",
      "000c0dfd995809fa    \" snowflake be not always symmetrical ! under ...\n",
      "000c6a3f0cd3ba8e    \" the signpost : september read this signpost ...\n",
      "000cfee90f50d471    \" re - consider st paragraph edit ? i do not u...\n",
      "000eefc67a2c930f    radial symmetry several now extinct lineage in...\n",
      "000f35deef84dc4a    there be no need to apologize . a wikipedia ar...\n",
      "000ffab30195c5e1    yes , because the mother of the child in the c...\n",
      "0010307a3a50a353    \" ok . but -PRON- will take a bit of work but ...\n",
      "0010833a96e1f886    \" = = a barnstar for -PRON- ! = = the real lif...\n",
      "                                          ...                        \n",
      "ffa33d3122b599d6    -PRON- absurd edit -PRON- absurd edit on great...\n",
      "ffa95244f261527f    maybe -PRON- be get good thing to do than spen...\n",
      "ffad104337fe9891    scrap that , -PRON- do meet criterion and -PRO...\n",
      "ffaed63c487a2b42                                -PRON- could do bad .\n",
      "ffb268f37788a011    , march ( utc ) be -PRON- also user : bmattson...\n",
      "ffb47123b2d82762    \" hey listen do not -PRON- ever ! ! ! ! delete...\n",
      "ffb7b4c3d3ae5842                   thank -PRON- very , very much . ·✆\n",
      "ffb93b0a0a1e78f9                                 talkback : september\n",
      "ffb998f9749bd83e                                      ( utc ) : , mar\n",
      "ffba5332d6b8fd14    i agree/ on another note lil wayne be a talent...\n",
      "ffbc2db4225258dd    while about half the reference be from byu - i...\n",
      "ffbcd64a71775e04    prague spring i think that prague spring deser...\n",
      "ffbd331a3aa269b9    i see this as have be merge ; undo one side of...\n",
      "ffbdbb0483ed0841    and -PRON- be go to keep post the stuff u dele...\n",
      "ffc2f409658571f1    \" how come when -PRON- download that mp -PRON-...\n",
      "ffc671f2acdd80e1    -PRON- will be on irc , too , if -PRON- have a...\n",
      "ffc7bbb177c3c966    -PRON- be -PRON- opinion that that happen to b...\n",
      "ffca1e81aefc48ac    please stop remove content from wikipedia ; -P...\n",
      "ffca8d71d71a3fae    image : barack - obama - mother.jpg list for d...\n",
      "ffcdcb71854f6d8a    \" editing of article without consensus & remov...\n",
      "ffd2e85b07b3c7e4    \" no -PRON- do not , read -PRON- again ( i wou...\n",
      "ffd72e9766c09c97    \" auto guide and the motoring press be not goo...\n",
      "ffe029a7c79dc7fe    \" please identify what part of blp apply becau...\n",
      "ffe897e7f7182c90    catalan independentism be the social movement ...\n",
      "ffe8b9316245be30    the number in parenthesis be the additional de...\n",
      "ffe987279560d7ff    \" : : : : : and for the second time of ask , w...\n",
      "ffea4adeee384e90    -PRON- should be ashamed of -PRON- that be a h...\n",
      "ffee36eab5c267c9    spitzer umm , there s no actual article for pr...\n",
      "fff125370e4aaaf3    and -PRON- look like -PRON- be actually -PRON-...\n",
      "fff46fc426af1f9a    \" and ... i really do not think -PRON- underst...\n",
      "Name: comment_text, Length: 159571, dtype: object\n",
      "(159571, 7)\n"
     ]
    }
   ],
   "source": [
    "sid = SentimentIntensityAnalyzer()\n",
    "\n",
    "train = pickle.load(open('train_TestwithPunc.p','rb'))\n",
    "x_train=train['comment_text']\n",
    "y_train=train.iloc[:, 1:]\n",
    "\n",
    "print(x_train)\n",
    "print(y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nigger {'neg': 1.0, 'neu': 0.0, 'pos': 0.0, 'compound': -0.6486}\n",
      "fuck {'neg': 1.0, 'neu': 0.0, 'pos': 0.0, 'compound': -0.5423}\n",
      "could {'neg': 0.0, 'neu': 1.0, 'pos': 0.0, 'compound': 0.0}\n",
      "jerk {'neg': 1.0, 'neu': 0.0, 'pos': 0.0, 'compound': -0.34}\n",
      "-PRON- hate your face express {'neg': 0.481, 'neu': 0.519, 'pos': 0.0, 'compound': -0.5719}\n",
      "I hate your facial expression {'neg': 0.552, 'neu': 0.448, 'pos': 0.0, 'compound': -0.5719}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ola.alnaameh\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:15: DeprecationWarning: generator 'ngrams' raised StopIteration\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    }
   ],
   "source": [
    "wordsInsult = Counter()\n",
    "for item in train[(train['insult'] == 1) & (train['toxic'] == 0) & (train['severe_toxic'] == 0) & (train['identity_hate'] == 0) & (\n",
    "            train['threat'] == 0) & (train['obscene']==0)]['comment_text'].tolist():\n",
    "    wordsInsult.update(get_ngrams(item,2))\n",
    "    wordsInsult.update(get_ngrams(item, 3))\n",
    "    wordsInsult.update(item.split(\" \"))\n",
    "\n",
    "for w in ['nigger','fuck','could','jerk','-PRON- hate your face express','I hate your facial expression']  :  \n",
    "    print(w,sid.polarity_scores(w))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ola.alnaameh\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\nltk\\twitter\\__init__.py:20: UserWarning: The twython library has not been installed. Some functionality from the twitter package will not be available.\n",
      "  warnings.warn(\"The twython library has not been installed. \"\n",
      "C:\\Users\\ola.alnaameh\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:16: DeprecationWarning: generator 'ngrams' raised StopIteration\n",
      "  app.launch_new_instance()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "insultDict Counter({'': 171, 'not': 154, '-PRON-': 148, 'be': 107, 'do not': 98, 'do': 98, '-PRON- be': 92, 'not do': 41, 'not do not': 41, 'be -PRON-': 31, '-PRON- be -PRON-': 31, 'do not do': 28, 'not -PRON-': 27, '-PRON- have': 25, 'have': 25, 'be -PRON- be': 22, 'will': 18, 'do not -PRON-': 18, 'be not': 18, 'can not': 18, 'can': 18, 'not -PRON- be': 17, 'be do': 16, '-PRON- be do': 16, 'be do not': 16, 'have -PRON-': 12, '-PRON- have -PRON-': 12, 'would': 11, 'that': 11, '-PRON- will': 10, 'will not': 8, 'because': 8, 'have -PRON- be': 7, 'will -PRON-': 7, '-PRON- will -PRON-': 7, 'would not': 6, 'be not do': 6, 'will -PRON- be': 5, 'be -PRON- have': 5, '-PRON- would': 5, '-PRON- -PRON-': 5, 'be be': 4, '-PRON- be be': 4, 'be be not': 4, 'not because': 4, 'not that': 4, 'not can': 4, 'be not -PRON-': 4, 'do not can': 4, 'not can not': 4, 'can not do': 4, 'not -PRON- have': 4, 'should not': 4, 'should': 4, 'be would': 3, '-PRON- be would': 3, 'be would not': 3, 'have -PRON- have': 3, 'because -PRON-': 3, 'because -PRON- be': 3, 'can not -PRON-': 3, 'not -PRON- will': 3, 'who': 3, 'not would': 3, 'not would not': 3, 'that do': 3, 'that do not': 3, 'have do': 3, '-PRON- have do': 3, 'have do not': 3, '-PRON- -PRON- be': 3, '-PRON- be not': 3, 'have will': 2, '-PRON- have will': 2, 'have will not': 2, 'not because -PRON-': 2, 'could not': 2, 'could': 2, 'that be': 2, 'that be not': 2, 'do not would': 2, 'would not do': 2, 'that -PRON-': 2, 'that -PRON- be': 2, 'do not because': 2, 'would -PRON-': 2, '-PRON- would -PRON-': 2, 'not -PRON- -PRON-': 2, 'not will': 1, 'do not will': 1, 'not will not': 1, 'would not -PRON-': 1, 'what be': 1, 'what be not': 1, 'what': 1, 'will not because': 1, 'because because': 1, 'because because -PRON-': 1, 'will will': 1, '-PRON- will will': 1, 'will will not': 1, 'go to': 1, 'go': 1, 'to': 1, 'would not that': 1, 'not that be': 1, 'will -PRON- will': 1, 'can not because': 1, 'will not do': 1, 'have that': 1, '-PRON- have that': 1, 'be can': 1, '-PRON- be can': 1, 'be can not': 1, 'not be': 1, 'do not be': 1, 'not be not': 1, 'who -PRON-': 1, 'who -PRON- be': 1, 'would not would': 1, '-PRON- do': 1, '-PRON- do not': 1, 'do not that': 1, 'not that -PRON-': 1, 'because can': 1, 'because can not': 1, 'have -PRON- will': 1, 'be will': 1, 'will -PRON- would': 1, 'would -PRON- will': 1, '-PRON- be will': 1, 'be will not': 1, 'will not -PRON-': 1, 'be who': 1, '-PRON- be who': 1, 'will not that': 1, 'be -PRON- would': 1, '-PRON- -PRON- -PRON-': 1, 'have -PRON- would': 1, 'would -PRON- be': 1, 'not -PRON- would': 1, 'not could': 1, 'do not could': 1, 'not could not': 1, 'could not that': 1, 'because do': 1, 'because do not': 1, 'not should': 1, 'be -PRON- -PRON-': 1, 'be not should': 1, 'not should not': 1, '--': 1, 'be that': 1, '-PRON- be that': 1, 'be that do': 1})\n",
      "toxicDict Counter({'not': 3218, '': 3051, '-PRON-': 2242, 'do': 2079, 'do not': 2077, 'be': 1865, '-PRON- be': 1518, 'not do': 836, 'not do not': 836, 'do not do': 684, 'can not': 491, 'can': 491, 'be -PRON-': 478, '-PRON- be -PRON-': 478, 'not -PRON-': 476, 'be -PRON- be': 366, 'have': 355, 'be not': 351, '-PRON- have': 346, 'not -PRON- be': 314, 'that': 291, 'do not -PRON-': 284, 'will': 235, 'be do': 184, '-PRON- be do': 184, 'be do not': 184, 'not can': 180, 'not can not': 180, 'would': 171, 'will not': 146, 'not be': 117, 'not be not': 117, '-PRON- would': 116, 'have -PRON-': 115, '-PRON- have -PRON-': 115, 'can not can': 104, 'to': 90, '-PRON- will': 83, 'because': 81, 'what': 79, 'go': 75, 'go to': 72, 'not that': 71, 'there': 71, 'not -PRON- have': 70, 'can not -PRON-': 64, 'be not -PRON-': 64, 'do not be': 60, 'be -PRON- have': 60, 'that -PRON-': 59, 'be not do': 59, 'have -PRON- be': 58, 'do not can': 57, '-PRON- -PRON-': 55, 'would not': 55, 'be can': 54, '-PRON- be can': 54, 'be can not': 54, 'be be': 52, '-PRON- be be': 52, 'be be not': 52, 'should not': 51, 'should': 51, 'can not do': 51, 'have do': 48, '-PRON- have do': 48, 'have do not': 48, 'do not that': 47, 'not will': 46, 'not will not': 44, '--': 43, 'have -PRON- have': 40, 'that do': 39, 'that do not': 39, 'could not': 39, 'could': 39, 'will not -PRON-': 38, 'be not be': 34, 'not -PRON- would': 31, 'who': 31, 'do not will': 30, 'that -PRON- be': 30, 'be go': 28, '-PRON- be go': 28, 'be go to': 28, 'would -PRON-': 25, '-PRON- would -PRON-': 25, 'will not do': 24, 'be that': 24, '-PRON- be that': 24, 'be will': 22, '-PRON- be will': 22, 'be will not': 22, '-PRON- do': 22, '-PRON- do not': 22, 'will -PRON-': 21, 'not -PRON- will': 21, '-PRON- will -PRON-': 20, 'not should': 19, 'not should not': 19, 'what -PRON-': 19, 'not there': 18, 'be -PRON- would': 18, 'that that': 18, 'not because': 18, 'not what': 18, '-PRON- -PRON- be': 16, 'can not be': 16, 'would -PRON- be': 16, 'not would': 16, 'not would not': 16, 'there -PRON-': 15, 'have be': 15, 'have be not': 15, '-PRON- have be': 14, 'be -PRON- will': 14, 'have that': 14, 'not that -PRON-': 14, 'because -PRON-': 14, 'do not because': 14, 'to -PRON-': 13, 'not that do': 13, 'be -PRON- -PRON-': 13, '-PRON- have that': 13, 'what -PRON- be': 13, 'to do': 12, 'go to do': 12, 'not could': 12, 'not could not': 12, 'be not can': 12, 'because do': 12, 'because do not': 12, 'that -PRON- have': 11, 'not go': 11, 'not go to': 11, 'to do not': 11, 'would do': 11, '-PRON- would do': 11, 'would do not': 11, 'not --': 11, 'would not -PRON-': 11, 'do not there': 11, 'do not what': 11, 'be because': 11, '-PRON- be because': 11, 'there -PRON- be': 10, 'can not that': 10, 'do not could': 10, 'not -PRON- -PRON-': 10, 'be there': 10, '-PRON- be there': 10, 'do not should': 10, 'go to -PRON-': 9, 'will -PRON- be': 9, 'that be': 9, 'that be not': 9, '-PRON- -PRON- have': 9, '-PRON- -PRON- -PRON-': 9, 'because -PRON- be': 9, 'that can': 9, 'that can not': 9, 'be what': 9, '-PRON- be what': 9, 'would can': 8, '-PRON- would can': 8, 'would can not': 8, 'have can': 8, '-PRON- have can': 8, 'have can not': 8, '-PRON- can': 8, '-PRON- can not': 8, 'could not do': 8, 'what do': 8, 'what do not': 8, 'to can': 8, 'go to can': 8, 'to can not': 8, 'to -PRON- be': 7, 'have not': 7, 'have -PRON- will': 7, 'that -PRON- would': 7, 'there do': 7, 'there do not': 7, 'that will': 7, 'not that that': 7, 'do not --': 7, 'be that -PRON-': 7, 'would that': 7, '-PRON- would that': 7, 'could not -PRON-': 7, 'be because -PRON-': 7, 'who -PRON-': 6, 'be not will': 6, 'have would': 6, '-PRON- have would': 6, 'have would not': 6, 'how': 6, 'be would': 6, '-PRON- be would': 6, 'be would not': 6, 'have -PRON- would': 6, 'be not that': 6, 'not -PRON- do': 6, 'do not would': 6, 'should not do': 6, 'to go': 6, 'be not would': 6, 'be going': 5, 'going to': 5, '-PRON- be going': 5, 'be going to': 5, 'going': 5, 'where': 5, 'not there -PRON-': 5, 'can not will': 5, '-PRON- go': 5, '-PRON- go to': 5, 'should not -PRON-': 5, 'that -PRON- will': 5, 'can not should': 5, 'who do': 5, 'who do not': 5, 'be who': 5, '-PRON- be who': 5, 'go to go': 5, 'to go to': 5, 'will do': 5, 'will do not': 5, 'be not go': 5, 'will that': 5, 'will not that': 5, 'that could': 5, 'that could not': 5, '-PRON- that': 4, 'be not there': 4, 'not have': 4, 'not have not': 4, 'not who': 4, 'because can': 4, 'because can not': 4, 'be there -PRON-': 4, 'will -PRON- have': 4, 'be that do': 4, 'because that': 4, 'nothing': 4, 'be --': 4, '-PRON- be --': 4, 'what can': 4, 'what can not': 4, 'who -PRON- have': 4, 'will not will': 4, '-PRON- be not': 4, 'old': 4, 'that that -PRON-': 4, '-PRON- will do': 4, 'not because do': 4, 'there -PRON- will': 4, '-PRON- will that': 4, 'would -PRON- would': 4, 'do not go': 4, 'be should': 4, '-PRON- be should': 4, 'be should not': 4, 'something': 4, 'what -PRON- have': 4, 'who who': 3, 'have who': 3, '-PRON- have who': 3, 'have not -PRON-': 3, 'there be': 3, 'there be not': 3, 'would not can': 3, 'not because -PRON-': 3, 'enough': 3, '-- --': 3, '-- do': 3, '-- do not': 3, 'to -PRON- have': 3, 'not to': 3, 'would -PRON- have': 3, 'not -PRON- can': 3, 'be not what': 3, 'will can': 3, 'will can not': 3, 'have will': 3, '-PRON- have will': 3, 'have will not': 3, 'not nothing': 3, 'would not do': 3, 'to be': 3, 'to be not': 3, 'do not have': 3, 'there that': 3, 'what that': 3, 'that what': 3, 'that what -PRON-': 3, 'because because': 3, 'will -PRON- do': 3, 'not that could': 3, 'have that -PRON-': 3, 'will -PRON- would': 3, 'have -PRON- -PRON-': 3, '-PRON- -PRON- would': 3, 'because -PRON- have': 3, 'will not be': 3, 'be not should': 3, 'should not can': 3, 'will that -PRON-': 3, 'be could': 3, '-PRON- be could': 3, 'be could not': 3, 'be what -PRON-': 3, 'will -PRON- will': 2, 'who -PRON- be': 2, 'that who': 2, 'do not who': 2, 'that there': 2, 'not -PRON- go': 2, '-PRON- what': 2, 'not that be': 2, 'do not to': 2, 'that because': 2, 'that would': 2, 'could not be': 2, 'that would not': 2, 'to because': 2, 'go to because': 2, 'not who -PRON-': 2, 'be who do': 2, 'have go': 2, '-PRON- have go': 2, 'be not nothing': 2, 'there there': 2, '-PRON- because': 2, 'go to be': 2, 'would not that': 2, 'can not there': 2, 'that will not': 2, 'not what that': 2, 'not because because': 2, 'can not would': 2, '-PRON- -PRON- can': 2, '-PRON- will not': 2, 'be there do': 2, 'be what do': 2, 'be something': 2, '-PRON- be something': 2, 'that that will': 2, 'would that do': 2, 'what be': 2, 'what be not': 2, 'because there': 2, 'there will': 2, 'have because': 2, '-PRON- have because': 2, 'that -PRON- do': 2, 'to that': 2, 'go to that': 2, 'not what -PRON-': 2, 'can not because': 2, '-- be': 2, '-- be not': 2, 'can not --': 2, 'have there': 2, '-PRON- have there': 2, 'there because': 2, 'not there because': 2, 'what that -PRON-': 2, 'would will': 2, '-PRON- would will': 2, 'would will not': 2, 'because -PRON- will': 2, '-PRON- will can': 2, '-PRON- there': 2, 'what what': 2, 'would be': 2, '-PRON- would be': 2, 'would be not': 2, 'would have': 2, '-PRON- would have': 2, 'would have not': 2, 'there can': 2, 'there can not': 2, 'have should': 2, '-PRON- have should': 2, 'have should not': 2, '-PRON- that -PRON-': 1, 'will who': 1, '-PRON- will who': 1, 'will who who': 1, 'to -PRON- would': 1, 'have who -PRON-': 1, 'not where': 1, 'do not where': 1, 'who can': 1, 'that who can': 1, 'who can not': 1, 'have to': 1, '-PRON- have to': 1, 'have to -PRON-': 1, 'how that': 1, 'can not have': 1, 'be -PRON- can': 1, 'would not go': 1, 'not that there': 1, 'that there do': 1, 'would would': 1, '-PRON- would would': 1, 'would would not': 1, 'will because': 1, 'that will because': 1, 'not -PRON- what': 1, 'could not would': 1, 'be enough': 1, '-PRON- be enough': 1, '-- -- do': 1, 'should not be': 1, 'should not should': 1, 'when do': 1, 'to what': 1, 'when do not': 1, 'not there do': 1, 'not to what': 1, 'to what do': 1, 'when': 1, 'that because that': 1, 'because that be': 1, 'have nothing': 1, '-PRON- have nothing': 1, 'not will can': 1, 'be that would': 1, 'to because -PRON-': 1, 'not what can': 1, 'be because that': 1, 'will not who': 1, 'be because do': 1, 'going to -PRON-': 1, 'go -PRON-': 1, 'be to': 1, 'go will': 1, 'nothing -PRON-': 1, 'have go -PRON-': 1, 'go -PRON- be': 1, '-PRON- be to': 1, 'be to -PRON-': 1, 'will not to': 1, 'not to go': 1, 'to go will': 1, 'go will not': 1, 'not nothing -PRON-': 1, 'nothing -PRON- be': 1, 'old can': 1, 'old can not': 1, 'not there there': 1, '-PRON- -PRON- because': 1, 'not that can': 1, 'there --': 1, 'be there --': 1, 'not there that': 1, 'there that can': 1, 'would that will': 1, 'can not what': 1, 'what that what': 1, 'will not because': 1, 'be -PRON- what': 1, 'because because do': 1, 'have could': 1, '-PRON- have could': 1, 'have could not': 1, 'that that do': 1, 'have that what': 1, 'not that who': 1, 'that who who': 1, 'would that that': 1, 'can not go': 1, 'would that can': 1, 'something go': 1, 'be something go': 1, 'something go to': 1, 'something -PRON-': 1, 'be something -PRON-': 1, 'something -PRON- be': 1, 'not will do': 1, 'because there will': 1, 'would not what': 1, 'not because can': 1, 'that that be': 1, 'that because can': 1, 'there will not': 1, 'will not what': 1, 'will could': 1, '-PRON- will could': 1, 'will could not': 1, 'have shall': 1, 'shall not': 1, '-PRON- have shall': 1, 'have shall not': 1, 'shall': 1, 'something be': 1, 'because old': 1, 'old -PRON-': 1, 'something be not': 1, 'to because because': 1, 'because because old': 1, 'because old -PRON-': 1, 'because be': 1, 'not because be': 1, 'because be not': 1, 'going to be': 1, 'there would': 1, 'there would not': 1, 'would who': 1, '-PRON- would who': 1, 'there go': 1, 'be there go': 1, 'there go to': 1, 'be that what': 1, 'because because can': 1, 'to --': 1, 'go to --': 1, 'to -- be': 1, 'would could': 1, '-PRON- would could': 1, 'would could not': 1, 'could not that': 1, 'not -- be': 1, 'would not --': 1, 'not -- do': 1, 'do not nothing': 1, 'there because do': 1, 'could not there': 1, 'will there': 1, '-PRON- will there': 1, 'will there that': 1, 'there that will': 1, 'that will that': 1, 'would go': 1, '-PRON- would go': 1, 'would go to': 1, '-PRON- -PRON- will': 1, 'where do': 1, 'where do not': 1, 'be what be': 1, 'have not do': 1, 'be -- --': 1, 'would not will': 1, 'that go': 1, 'would that go': 1, 'that go to': 1, 'would not could': 1, 'have that do': 1, 'not -PRON- there': 1, '-PRON- there be': 1, 'would what': 1, '-PRON- would what': 1, 'would what -PRON-': 1, 'old do': 1, 'old do not': 1, 'be old': 1, '-PRON- be old': 1, 'how -PRON-': 1, 'have what': 1, 'what because': 1, 'be how': 1, 'how -PRON- have': 1, '-PRON- have what': 1, 'have what because': 1, 'what because do': 1, '-PRON- be how': 1, 'be not could': 1, 'be what what': 1, 'what what do': 1, 'should not would': 1, 'will --': 1, '-PRON- will --': 1, 'be that that': 1, 'that will -PRON-': 1, 'have because -PRON-': 1, 'will what': 1, '-PRON- will what': 1, 'not what do': 1, 'have not who': 1, 'who have': 1, 'have who have': 1, 'who have be': 1, 'have --': 1, '-PRON- have --': 1, 'could not can': 1, '-PRON- -PRON- do': 1, '-- will': 1, '-- will not': 1, 'would --': 1, '-PRON- would --': 1, 'should not what': 1, 'to -PRON- go': 1, 'have there can': 1, 'be who -PRON-': 1, 'have go to': 1, '-PRON- should': 1, '-PRON- -PRON- should': 1, '-PRON- should not': 1, 'be that could': 1, 'would because': 1, '-PRON- would because': 1, '-PRON- -PRON- there': 1, 'not -PRON- because': 1, '-PRON- because there': 1, 'because there that': 1, 'there that do': 1, 'because that can': 1, 'there could': 1, 'there could not': 1, 'should not because': 1, 'not because that': 1, 'because that do': 1, 'there -PRON- would': 1, 'have -PRON- do': 1, 'how do': 1, 'how do not': 1, 'that to': 1, 'to will': 1, 'not that to': 1, 'that to will': 1, 'to will not': 1, 'be not --': 1, 'would not be': 1})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "severe_toxicDict Counter({'': 1120, 'not': 753, '-PRON-': 648, 'be': 533, '-PRON- be': 493, 'do not': 400, 'do': 400, 'be -PRON-': 255, '-PRON- be -PRON-': 255, 'be -PRON- be': 240, 'can not': 234, 'can': 234, 'not do': 209, 'not do not': 209, 'not can': 192, 'not can not': 192, 'do not do': 149, 'can not can': 135, 'will': 80, '-PRON- -PRON-': 61, 'to': 60, 'will not': 59, 'go to': 56, 'go': 56, 'not -PRON-': 55, 'do not can': 55, '-PRON- -PRON- -PRON-': 53, 'can not do': 49, 'not will': 47, 'not will not': 47, 'be go': 44, '-PRON- be go': 44, 'be go to': 44, 'will not will': 44, 'be not': 41, '-PRON- have': 34, 'have': 34, 'do not -PRON-': 33, 'not -PRON- be': 32, 'to -PRON-': 32, 'go to -PRON-': 30, 'to -PRON- be': 30, 'would': 26, 'that': 26, 'be do': 23, '-PRON- be do': 23, 'be do not': 23, '-PRON- will': 21, '-PRON- would': 17, 'not be': 15, 'not be not': 15, 'what': 12, 'have -PRON-': 11, '-PRON- have -PRON-': 11, 'be -PRON- have': 10, 'be not -PRON-': 10, 'because': 10, 'not -PRON- have': 9, 'would not': 9, 'have -PRON- be': 8, 'can not -PRON-': 8, 'do not be': 8, 'be can': 8, '-PRON- be can': 8, 'be can not': 8, 'will -PRON-': 7, '-PRON- will -PRON-': 7, 'that -PRON-': 7, 'have do': 6, '-PRON- have do': 6, 'have do not': 6, 'should not': 6, 'should': 6, 'be not do': 6, 'would -PRON-': 6, '-PRON- would -PRON-': 6, 'there': 6, 'be not be': 6, 'not -PRON- will': 5, 'be that': 5, '-PRON- be that': 5, 'that -PRON- be': 5, '-PRON- -PRON- be': 4, 'could not': 4, 'could': 4, 'that do': 4, 'that do not': 4, 'would -PRON- be': 4, 'not that': 4, 'to do': 4, 'go to do': 4, 'to do not': 4, 'not -PRON- -PRON-': 3, 'be be': 3, '-PRON- be be': 3, 'be be not': 3, 'will -PRON- will': 3, 'be -PRON- will': 3, 'will not -PRON-': 3, 'not go': 3, 'not go to': 3, 'because -PRON-': 3, 'not there': 3, 'how': 3, 'be going': 3, 'going to': 3, '-PRON- be going': 3, 'be going to': 3, 'going': 3, 'not would': 3, 'not would not': 3, 'who': 3, 'have -PRON- have': 2, 'should not do': 2, '-PRON- do': 2, '-PRON- do not': 2, 'to -PRON- will': 2, 'do not go': 2, 'enough': 2, 'because -PRON- -PRON-': 2, '-PRON- -PRON- would': 2, 'will -PRON- be': 2, 'not what': 2, 'not because': 2, 'do not what': 2, 'will not do': 2, 'do not because': 2, 'do not there': 2, 'have would': 2, '-PRON- have would': 2, 'have would not': 2, 'not that -PRON-': 2, 'be that -PRON-': 2, '--': 2, 'not -PRON- would': 2, 'would -PRON- would': 2, 'do not would': 2, 'do not that': 2, 'what -PRON-': 2, 'going to -PRON-': 2, 'will -PRON- would': 1, 'can not be': 1, 'that -PRON- do': 1, 'not something': 1, 'something -PRON-': 1, 'be not something': 1, 'not something -PRON-': 1, 'something -PRON- be': 1, 'something': 1, 'will could': 1, 'be will': 1, '-PRON- will could': 1, 'will could not': 1, 'could not will': 1, '-PRON- be will': 1, 'be will not': 1, 'can not will': 1, 'be because': 1, '-PRON- be because': 1, 'be because -PRON-': 1, 'because -PRON- be': 1, 'will would': 1, '-PRON- will would': 1, 'will would not': 1, 'will do': 1, '-PRON- will do': 1, 'will do not': 1, 'what can': 1, 'what can not': 1, '-PRON- should': 1, '-PRON- should not': 1, 'should not -PRON-': 1, 'what will': 1, 'because do': 1, 'not what will': 1, 'what will not': 1, 'not because do': 1, 'because do not': 1, 'be -PRON- would': 1, 'have -PRON- would': 1, 'what go': 1, 'what go to': 1, '-PRON- be not': 1, 'can not that': 1, 'not should': 1, 'do not should': 1, 'not should not': 1, 'be not can': 1, 'what would': 1, 'not what would': 1, 'what would not': 1, 'will not can': 1, 'have be': 1, '-PRON- have be': 1, 'have be not': 1, '-PRON- that': 1, 'what -PRON- that': 1, '-PRON- that do': 1, 'what do': 1, 'what do not': 1, 'that who': 1, 'who -PRON-': 1, 'that who -PRON-': 1, 'who -PRON- be': 1, 'not how': 1, 'how do': 1, 'do not how': 1, 'not how do': 1, 'how do not': 1, 'be not that': 1, 'what -PRON- have': 1, 'will -PRON- have': 1, 'have --': 1, '-PRON- have --': 1, 'be not go': 1, 'be what': 1, '-PRON- be what': 1, 'do not will': 1, 'that -PRON- would': 1, 'there can': 1, 'there should': 1, 'not who': 1, 'who do': 1, 'there can not': 1, 'can not would': 1, 'would not there': 1, 'not there should': 1, 'there should not': 1, 'should not who': 1, 'not who do': 1, 'who do not': 1, 'would not do': 1})\n",
      "identity_hateDict Counter({'': 32, 'not': 32, 'do not': 18, 'do': 18, '-PRON-': 15, 'be': 14, '-PRON- be': 9, 'not -PRON-': 7, 'not do': 7, 'not do not': 7, 'can not': 5, 'can': 5, 'be not': 5, '-PRON- have': 4, 'not -PRON- have': 4, 'have': 4, 'do not do': 4, 'do not -PRON-': 4, 'be do': 3, 'not -PRON- be': 3, '-PRON- be do': 3, 'be do not': 3, 'will': 3, 'not should': 2, 'should not': 2, 'not should not': 2, 'should': 2, 'be not -PRON-': 2, 'can not do': 2, 'not will': 2, 'will not': 2, 'not will not': 2, 'there': 2, '--': 1, 'can not should': 1, 'not be': 1, 'do not should': 1, 'should not be': 1, 'not be not': 1, 'be not do': 1, 'not can': 1, 'have do': 1, 'do not will': 1, 'will not can': 1, 'not can not': 1, '-PRON- have do': 1, 'have do not': 1, 'what do': 1, 'what do not': 1, 'what': 1, '-PRON- will': 1, 'will -PRON-': 1, '-PRON- will -PRON-': 1, 'be -PRON-': 1, '-PRON- be -PRON-': 1, 'be -PRON- be': 1, 'there who': 1, 'who': 1, 'not there': 1, 'there -PRON-': 1, 'be not there': 1, 'not there -PRON-': 1, 'there -PRON- be': 1, 'be that': 1, '-PRON- be that': 1, 'that': 1, 'because be': 1, 'because be not': 1, 'be not will': 1, 'will not -PRON-': 1, 'because': 1})\n",
      "threatDict Counter({'': 14, '-PRON-': 12, 'be': 11, '-PRON- be': 10, 'be -PRON-': 5, '-PRON- be -PRON-': 5, 'not': 4, 'do not': 3, 'do': 3, 'be -PRON- be': 3, 'have': 3, 'could have': 2, 'could': 2, 'that': 1, 'be not': 1, 'be going': 1, 'going to': 1, 'to -PRON-': 1, '-PRON- be going': 1, 'be going to': 1, 'going to -PRON-': 1, 'to -PRON- be': 1, 'going': 1, 'to': 1, '-PRON- will': 1, 'will -PRON-': 1, 'be -PRON- will': 1, '-PRON- will -PRON-': 1, 'will -PRON- be': 1, 'will': 1, 'have do': 1, 'not do': 1, 'not could': 1, 'have -PRON-': 1, '-PRON- have': 1, 'could have do': 1, 'have do not': 1, 'do not do': 1, 'not do not': 1, 'do not could': 1, 'not could have': 1, 'could have -PRON-': 1, 'have -PRON- be': 1, 'be -PRON- have': 1})\n",
      "obsceneDict Counter({'not': 257, '-PRON-': 218, 'be': 169, 'do': 150, 'do not': 149, '': 132, '-PRON- be': 123, 'not -PRON-': 59, 'not do': 51, 'not do not': 51, 'be not': 46, 'have': 39, '-PRON- have': 38, 'be -PRON-': 36, '-PRON- be -PRON-': 36, 'do not -PRON-': 34, 'can not': 33, 'can': 33, 'that': 33, 'be -PRON- be': 32, 'not -PRON- be': 28, 'do not do': 27, 'would': 24, '-PRON- would': 23, 'will': 19, 'be do': 18, '-PRON- be do': 18, 'not -PRON- have': 18, 'be do not': 17, 'there': 16, 'have -PRON-': 16, '-PRON- have -PRON-': 16, 'will not': 14, 'not be': 13, 'not be not': 13, 'can not -PRON-': 12, 'that -PRON-': 11, 'be not do': 10, 'not can': 9, 'not can not': 9, 'would -PRON-': 9, '-PRON- would -PRON-': 9, 'could not': 8, 'be not -PRON-': 8, 'could': 8, 'go': 8, 'to': 8, '-PRON- -PRON-': 8, 'what': 8, 'have do': 7, '-PRON- have do': 7, 'have do not': 7, 'not that': 7, 'have -PRON- be': 7, 'be that': 7, '-PRON- be that': 7, 'that -PRON- be': 7, 'because': 6, 'not could': 6, 'be be': 6, 'go to': 6, 'not could not': 6, 'could not do': 6, '-PRON- be be': 6, 'be be not': 6, 'do not be': 6, 'not -PRON- would': 6, '--': 6, 'that do': 6, 'that do not': 6, 'not will': 5, 'not will not': 5, 'should not': 5, 'should': 5, 'have -PRON- have': 5, 'can not do': 5, 'there -PRON-': 5, 'do not that': 5, '-PRON- will': 5, 'would -PRON- would': 5, 'do not will': 4, 'do not could': 4, 'be not can': 4, 'be can': 4, '-PRON- be can': 4, 'be can not': 4, 'there -PRON- be': 4, 'there do': 4, 'there do not': 4, 'be not be': 4, 'be that -PRON-': 4, 'will not -PRON-': 3, '-PRON- do': 3, 'not because': 3, '-PRON- do not': 3, 'do not because': 3, 'do not can': 3, 'be go': 3, '-PRON- be go': 3, 'not there': 3, 'be there': 3, 'will not do': 3, '-PRON- be there': 3, 'who': 3, 'that -PRON- have': 3, 'not that -PRON-': 3, '-PRON- that': 3, 'would -PRON- be': 3, 'be will': 2, '-PRON- be will': 2, 'be will not': 2, '-PRON- can': 2, '-PRON- can not': 2, 'should not -PRON-': 2, 'that that': 2, 'can not can': 2, 'that there': 2, 'there be': 2, 'there be not': 2, 'to -PRON-': 2, 'be go to': 2, 'go to -PRON-': 2, 'to -PRON- be': 2, 'would do': 2, 'would can': 2, '-PRON- would do': 2, 'would do not': 2, '-PRON- would can': 2, 'would can not': 2, 'go do': 2, 'go do not': 2, 'there that': 2, 'to do': 2, 'go to do': 2, 'to do not': 2, 'that be': 2, 'that be not': 2, 'what -PRON-': 2, 'would that': 2, '-PRON- would that': 2, 'not --': 2, 'do not --': 2, 'have be': 2, '-PRON- have be': 2, 'have be not': 2, '-PRON- -PRON- -PRON-': 2, 'because do': 2, 'not because do': 2, 'because do not': 2, 'be going': 2, 'going to': 2, '-PRON- be going': 2, 'be going to': 2, 'going': 2, 'not -PRON- do': 1, 'that should': 1, 'that should not': 1, 'be not that': 1, 'not that there': 1, 'that there be': 1, 'have not': 1, 'have not be': 1, 'be not could': 1, 'could not could': 1, 'have would': 1, 'would not': 1, '-PRON- have would': 1, 'have would not': 1, 'have can': 1, '-PRON- have can': 1, 'have can not': 1, '-PRON- there': 1, 'be -PRON- do': 1, 'not -PRON- there': 1, '-PRON- there -PRON-': 1, 'there -PRON- -PRON-': 1, '-PRON- -PRON- can': 1, 'that go': 1, 'do not there': 1, 'not there that': 1, 'there that go': 1, 'that go to': 1, 'be there -PRON-': 1, 'that what': 1, 'not that what': 1, 'have that': 1, '-PRON- have that': 1, 'have that do': 1, 'will not be': 1, 'be -PRON- would': 1, 'have there': 1, '-PRON- have there': 1, 'have there do': 1, 'not who': 1, 'who -PRON-': 1, 'do not who': 1, 'not who -PRON-': 1, 'who -PRON- be': 1, 'be there do': 1, 'be not there': 1, 'can not that': 1, 'not that be': 1, 'what -PRON- be': 1, 'who because': 1, 'because -PRON-': 1, 'who because -PRON-': 1, 'would that -PRON-': 1, 'be that there': 1, 'that there -PRON-': 1, '-PRON- -PRON- be': 1, 'be who': 1, 'who can': 1, '-PRON- be who': 1, 'be who can': 1, 'who can not': 1, 'what -PRON- have': 1, 'have -PRON- that': 1, '-PRON- that do': 1, 'will not will': 1, '-- -PRON-': 1, '-- -PRON- be': 1, 'will will': 1, 'there that -PRON-': 1, 'that -PRON- will': 1, '-PRON- will will': 1, 'will will not': 1, 'something do': 1, 'something do not': 1, 'something': 1, '-- --': 1, 'not -- --': 1, 'would go': 1, 'have -PRON- would': 1, '-PRON- would go': 1, 'would go to': 1, 'be -PRON- have': 1, 'what do': 1, 'what do not': 1, 'have -PRON- do': 1, 'be what': 1, '-PRON- be what': 1, 'will -PRON-': 1, 'be -PRON- -PRON-': 1, '-PRON- -PRON- would': 1, 'would -PRON- will': 1, '-PRON- will -PRON-': 1, 'will -PRON- would': 1, 'will should': 1, 'have -PRON- will': 1, '-PRON- will should': 1, 'will should not': 1, 'would be': 1, '-PRON- would be': 1, 'would be not': 1, '-PRON- should': 1, 'that will': 1, 'not -PRON- should': 1, '-PRON- should not': 1, 'not -PRON- -PRON-': 1, '-PRON- -PRON- that': 1, '-PRON- that will': 1, 'that will not': 1, 'be that that': 1, 'can not there': 1, 'not -PRON- that': 1, 'should not be': 1, '-PRON- --': 1, '-PRON- -PRON- --': 1, 'would there': 1, '-PRON- would there': 1, 'would there be': 1, 'would that be': 1, 'be go do': 1, 'to can': 1, 'do -PRON-': 1, 'going to can': 1, 'to can not': 1, 'be do -PRON-': 1, 'do -PRON- be': 1, 'have what': 1, 'what there': 1, '-PRON- have what': 1, 'have what there': 1, 'what there -PRON-': 1, 'will do': 1, '-PRON- will do': 1, 'will do not': 1})\n",
      "Ordered_ToxicDict OrderedDict()\n",
      "Ordered_Severe_toxicDict OrderedDict()\n",
      "Ordered_Identity_hateDict OrderedDict()\n",
      "Ordered_ThreatDict OrderedDict()\n",
      "Ordered_InsultDict OrderedDict()\n",
      "Ordered_ObsceneDict OrderedDict()\n",
      "All {'-PRON- have': 0.0, 'have -PRON-': 0.0, '-PRON- be': 0.0, 'be do': 0.0, 'do not': 0.0, 'not will': 0.0, 'will not': 0.0, '-PRON- have -PRON-': 0.0, 'have -PRON- be': 0.0, '-PRON- be do': 0.0, 'be do not': 0.0, 'do not will': 0.0, 'not will not': 0.0, '-PRON-': 0.0, 'have': 0.0, 'be': 0.0, 'do': 0.0, 'not': 0.0, 'will': 0.0, '': 0.0, 'not do': 0.0, 'do not do': 0.0, 'not do not': 0.0, 'be would': 0.0, 'would not': 0.0, 'not -PRON-': 0.0, '-PRON- be would': 0.0, 'be would not': 0.0, 'would not -PRON-': 0.0, 'not -PRON- be': 0.0, 'would': 0.0, 'do not -PRON-': 0.0, 'what be': 0.0, 'be not': 0.0, 'what be not': 0.0, 'what': 0.0, 'have -PRON- have': 0.0, 'be be': 0.0, '-PRON- be be': 0.0, 'be be not': 0.0, 'have will': 0.0, 'not because': 0.0, 'because -PRON-': 0.0, '-PRON- have will': 0.0, 'have will not': 0.0, 'will not because': 0.0, 'not because -PRON-': 0.0, 'because -PRON- be': 0.0, 'because': 0.0, 'because because': 0.0, 'because because -PRON-': 0.0, 'can not': 0.0, '-PRON- will': 0.0, 'will will': 0.0, 'can not -PRON-': 0.0, 'not -PRON- will': 0.0, '-PRON- will will': 0.0, 'will will not': 0.0, 'can': 0.0, 'go to': 0.0, 'go': 0.0, 'to': 0.0, 'be -PRON-': 0.0, '-PRON- be -PRON-': 0.0, 'be -PRON- be': 0.0, 'could not': 0.0, 'could': 0.0, 'not that': 0.0, 'that be': 0.0, 'not can': 0.0, 'will -PRON-': 0.0, 'would not that': 0.0, 'not that be': 0.0, 'that be not': 0.0, 'be not -PRON-': 0.0, 'do not can': 0.0, 'not can not': 0.0, '-PRON- will -PRON-': 0.0, 'will -PRON- will': 0.0, 'will -PRON- be': 0.0, 'that': 0.0, 'can not because': 0.0, 'can not do': 0.0, 'will not do': 0.0, 'have that': 0.0, 'be -PRON- have': 0.0, '-PRON- have that': 0.0, 'be can': 0.0, '-PRON- be can': 0.0, 'be can not': 0.0, 'not be': 0.0, 'do not be': 0.0, 'not be not': 0.0, 'be not do': 0.0, 'who -PRON-': 0.0, 'who -PRON- be': 0.0, 'who': 0.0, 'not would': 0.0, 'would not would': 0.0, 'not would not': 0.0, '-PRON- do': 0.0, '-PRON- do not': 0.0, 'not -PRON- have': 0.0, 'do not would': 0.0, 'would not do': 0.0, 'that do': 0.0, 'that do not': 0.0, 'that -PRON-': 0.0, 'do not that': 0.0, 'not that -PRON-': 0.0, 'that -PRON- be': 0.0, 'do not because': 0.0, 'should not': 0.0, 'should': 0.0, 'because can': 0.0, 'because can not': 0.0, 'have -PRON- will': 0.0, '-PRON- would': 0.0, 'would -PRON-': 0.0, 'be will': 0.0, 'will -PRON- would': 0.0, '-PRON- would -PRON-': 0.0, 'would -PRON- will': 0.0, '-PRON- be will': 0.0, 'be will not': 0.0, 'will not -PRON-': 0.0, 'be who': 0.0, '-PRON- be who': 0.0, 'will not that': 0.0, 'have do': 0.0, '-PRON- -PRON-': 0.0, '-PRON- have do': 0.0, 'have do not': 0.0, 'not -PRON- -PRON-': 0.0, '-PRON- -PRON- be': 0.0, '-PRON- be not': 0.0, 'be -PRON- would': 0.0, '-PRON- -PRON- -PRON-': 0.0, 'have -PRON- would': 0.0, 'would -PRON- be': 0.0, 'not -PRON- would': 0.0, 'not could': 0.0, 'do not could': 0.0, 'not could not': 0.0, 'could not that': 0.0, 'because do': 0.0, 'because do not': 0.0, 'not should': 0.0, 'be -PRON- -PRON-': 0.0, 'be not should': 0.0, 'not should not': 0.0, '--': 0.0, 'be that': 0.0, '-PRON- be that': 0.0, 'be that do': 0.0, 'be going': 0.0, 'going to': 0.0, 'to -PRON-': 0.0, '-PRON- be going': 0.0, 'be going to': 0.0, 'going to -PRON-': 0.0, 'to -PRON- be': 0.0, 'going': 0.0, 'be -PRON- will': 0.0, 'could have': 0.0, 'could have do': 0.0, 'not could have': 0.0, 'could have -PRON-': 0.0, 'can not should': 0.0, 'do not should': 0.0, 'should not be': 0.0, 'will not can': 0.0, 'what do': 0.0, 'what do not': 0.0, 'there who': 0.0, 'there': 0.0, 'not there': 0.0, 'there -PRON-': 0.0, 'be not there': 0.0, 'not there -PRON-': 0.0, 'there -PRON- be': 0.0, 'because be': 0.0, 'because be not': 0.0, 'be not will': 0.0, 'should not do': 0.0, 'can not be': 0.0, 'that -PRON- do': 0.0, 'be go': 0.0, '-PRON- be go': 0.0, 'be go to': 0.0, 'go to -PRON-': 0.0, 'to -PRON- will': 0.0, 'not something': 0.0, 'something -PRON-': 0.0, 'be not something': 0.0, 'not something -PRON-': 0.0, 'something -PRON- be': 0.0, 'something': 0.0, 'will could': 0.0, '-PRON- will could': 0.0, 'will could not': 0.0, 'could not will': 0.0, 'can not will': 0.0, 'not go': 0.0, 'do not go': 0.0, 'not go to': 0.0, 'be because': 0.0, '-PRON- be because': 0.0, 'be because -PRON-': 0.0, 'will not will': 0.0, 'will would': 0.0, '-PRON- will would': 0.0, 'will would not': 0.0, 'will do': 0.0, '-PRON- will do': 0.0, 'will do not': 0.0, 'enough': 0.0, 'what can': 0.0, 'what can not': 0.0, 'because -PRON- -PRON-': 0.0, '-PRON- -PRON- would': 0.0, '-PRON- should': 0.0, '-PRON- should not': 0.0, 'should not -PRON-': 0.0, 'not what': 0.0, 'what will': 0.0, 'do not what': 0.0, 'not what will': 0.0, 'what will not': 0.0, 'not because do': 0.0, 'do not there': 0.0, 'be not be': 0.0, 'have would': 0.0, '-PRON- have would': 0.0, 'have would not': 0.0, 'what go': 0.0, 'what go to': 0.0, 'can not that': 0.0, 'be that -PRON-': 0.0, 'to do': 0.0, 'go to do': 0.0, 'to do not': 0.0, 'be not can': 0.0, 'what would': 0.0, 'not what would': 0.0, 'what would not': 0.0, 'would -PRON- would': 0.0, 'have be': 0.0, '-PRON- have be': 0.0, 'have be not': 0.0, 'how': 0.0, 'what -PRON-': 0.0, '-PRON- that': 0.0, 'what -PRON- that': 0.0, '-PRON- that do': 0.0, 'that who': 0.0, 'that who -PRON-': 0.0, 'not how': 0.0, 'how do': 0.0, 'do not how': 0.0, 'not how do': 0.0, 'how do not': 0.0, 'be not that': 0.0, 'what -PRON- have': 0.0, 'will -PRON- have': 0.0, 'have --': 0.0, '-PRON- have --': 0.0, 'be not go': 0.0, 'be what': 0.0, '-PRON- be what': 0.0, 'that -PRON- would': 0.0, 'there can': 0.0, 'there should': 0.0, 'not who': 0.0, 'who do': 0.0, 'there can not': 0.0, 'can not would': 0.0, 'would not there': 0.0, 'not there should': 0.0, 'there should not': 0.0, 'should not who': 0.0, 'not who do': 0.0, 'who do not': 0.0, 'can not can': 0.0, 'would can': 0.0, '-PRON- would can': 0.0, 'would can not': 0.0, '-PRON- that -PRON-': 0.0, 'will who': 0.0, 'who who': 0.0, '-PRON- will who': 0.0, 'will who who': 0.0, 'to -PRON- would': 0.0, 'not that do': 0.0, 'have who': 0.0, '-PRON- have who': 0.0, 'have who -PRON-': 0.0, 'not where': 0.0, 'do not where': 0.0, 'where': 0.0, 'who can': 0.0, 'that who can': 0.0, 'who can not': 0.0, 'have can': 0.0, '-PRON- have can': 0.0, 'have can not': 0.0, 'have to': 0.0, '-PRON- have to': 0.0, 'have to -PRON-': 0.0, 'that -PRON- have': 0.0, 'how that': 0.0, 'not have': 0.0, 'have not': 0.0, 'can not have': 0.0, 'not have not': 0.0, 'have not -PRON-': 0.0, '-PRON- can': 0.0, 'be -PRON- can': 0.0, '-PRON- can not': 0.0, 'do not who': 0.0, 'there be': 0.0, 'there be not': 0.0, 'could not do': 0.0, 'would not go': 0.0, 'that there': 0.0, 'there do': 0.0, 'not that there': 0.0, 'that there do': 0.0, 'there do not': 0.0, 'would would': 0.0, '-PRON- would would': 0.0, 'would would not': 0.0, 'that will': 0.0, 'will because': 0.0, 'that will because': 0.0, 'that that': 0.0, 'not that that': 0.0, '-PRON- go': 0.0, 'not -PRON- go': 0.0, '-PRON- go to': 0.0, 'would do': 0.0, '-PRON- would do': 0.0, 'would do not': 0.0, '-PRON- what': 0.0, 'not -PRON- what': 0.0, 'could not would': 0.0, '-PRON- -PRON- have': 0.0, 'not -PRON- do': 0.0, 'would not can': 0.0, 'be there': 0.0, '-PRON- be there': 0.0, 'be there -PRON-': 0.0, 'that -PRON- will': 0.0, 'be enough': 0.0, '-PRON- be enough': 0.0, 'not --': 0.0, 'do not --': 0.0, '-- --': 0.0, '-- do': 0.0, '-- -- do': 0.0, '-- do not': 0.0, 'should not should': 0.0, 'to -PRON- have': 0.0, 'when do': 0.0, 'not to': 0.0, 'to what': 0.0, 'when do not': 0.0, 'not there do': 0.0, 'do not to': 0.0, 'not to what': 0.0, 'to what do': 0.0, 'when': 0.0, 'that because': 0.0, 'because that': 0.0, 'that because that': 0.0, 'because that be': 0.0, 'have nothing': 0.0, '-PRON- have nothing': 0.0, 'nothing': 0.0, 'would -PRON- have': 0.0, 'be --': 0.0, '-PRON- be --': 0.0, 'not -PRON- can': 0.0, 'be not what': 0.0, 'will can': 0.0, 'not will can': 0.0, 'will can not': 0.0, 'that would': 0.0, 'could not be': 0.0, 'be that would': 0.0, 'that would not': 0.0, 'to because': 0.0, 'go to because': 0.0, 'to because -PRON-': 0.0, 'not what can': 0.0, 'be because that': 0.0, 'will not who': 0.0, 'not who -PRON-': 0.0, 'who -PRON- have': 0.0, 'be who do': 0.0, 'be because do': 0.0, 'have go': 0.0, 'go -PRON-': 0.0, 'be to': 0.0, 'to go': 0.0, 'go will': 0.0, 'not nothing': 0.0, 'nothing -PRON-': 0.0, '-PRON- have go': 0.0, 'have go -PRON-': 0.0, 'go -PRON- be': 0.0, '-PRON- be to': 0.0, 'be to -PRON-': 0.0, 'will not to': 0.0, 'not to go': 0.0, 'to go will': 0.0, 'go will not': 0.0, 'be not nothing': 0.0, 'not nothing -PRON-': 0.0, 'nothing -PRON- be': 0.0, 'old can': 0.0, 'old can not': 0.0, 'old': 0.0, 'go to go': 0.0, 'to go to': 0.0, 'there there': 0.0, 'not there there': 0.0, '-PRON- because': 0.0, '-PRON- -PRON- because': 0.0, 'to be': 0.0, 'go to be': 0.0, 'to be not': 0.0, 'be not would': 0.0, 'that that -PRON-': 0.0, 'that can': 0.0, 'not that can': 0.0, 'that can not': 0.0, 'there --': 0.0, 'do not have': 0.0, 'be there --': 0.0, 'there that': 0.0, 'can not there': 0.0, 'not there that': 0.0, 'there that can': 0.0, 'would that': 0.0, '-PRON- would that': 0.0, 'would that will': 0.0, 'that will not': 0.0, 'what that': 0.0, 'that what': 0.0, 'can not what': 0.0, 'not what that': 0.0, 'what that what': 0.0, 'that what -PRON-': 0.0, 'what -PRON- be': 0.0, 'be -PRON- what': 0.0, 'not because because': 0.0, 'because because do': 0.0, 'will -PRON- do': 0.0, 'have could': 0.0, '-PRON- have could': 0.0, 'have could not': 0.0, '-PRON- -PRON- can': 0.0, 'will that': 0.0, 'there -PRON- will': 0.0, '-PRON- will that': 0.0, 'that that do': 0.0, 'have that what': 0.0, 'to can': 0.0, 'go to can': 0.0, 'to can not': 0.0, 'not that who': 0.0, 'that who who': 0.0, '-PRON- will not': 0.0, 'would that that': 0.0, 'that could': 0.0, 'not that could': 0.0, 'that could not': 0.0, 'could not -PRON-': 0.0, 'be there do': 0.0, 'can not go': 0.0, 'would that can': 0.0, 'have that -PRON-': 0.0, 'be should': 0.0, '-PRON- be should': 0.0, 'be should not': 0.0, 'have -PRON- -PRON-': 0.0, 'because -PRON- have': 0.0, 'be what do': 0.0, 'be something': 0.0, 'something go': 0.0, '-PRON- be something': 0.0, 'be something go': 0.0, 'something go to': 0.0, 'that that will': 0.0, 'be something -PRON-': 0.0, 'would that do': 0.0, 'not will do': 0.0, 'because there': 0.0, 'there will': 0.0, 'because there will': 0.0, 'would not what': 0.0, 'not because can': 0.0, 'have because': 0.0, '-PRON- have because': 0.0, 'that that be': 0.0, 'that because can': 0.0, 'there will not': 0.0, 'will not what': 0.0, 'will not be': 0.0, 'have shall': 0.0, 'shall not': 0.0, '-PRON- have shall': 0.0, 'have shall not': 0.0, 'shall': 0.0, 'something be': 0.0, 'because old': 0.0, 'old -PRON-': 0.0, 'something be not': 0.0, 'to because because': 0.0, 'because because old': 0.0, 'because old -PRON-': 0.0, 'to that': 0.0, 'go to that': 0.0, 'not because be': 0.0, 'going to be': 0.0, 'there would': 0.0, 'there would not': 0.0, 'would who': 0.0, '-PRON- would who': 0.0, 'there go': 0.0, 'be there go': 0.0, 'there go to': 0.0, 'not what -PRON-': 0.0, 'be that what': 0.0, 'because because can': 0.0, 'to --': 0.0, '-- be': 0.0, 'go to --': 0.0, 'to -- be': 0.0, '-- be not': 0.0, 'should not can': 0.0, 'would could': 0.0, '-PRON- would could': 0.0, 'would could not': 0.0, 'can not --': 0.0, 'not -- be': 0.0, 'would not --': 0.0, 'not -- do': 0.0, 'do not nothing': 0.0, 'have there': 0.0, '-PRON- have there': 0.0, 'there because': 0.0, 'not there because': 0.0, 'there because do': 0.0, 'will that -PRON-': 0.0, 'be could': 0.0, '-PRON- be could': 0.0, 'be could not': 0.0, 'could not there': 0.0, 'will there': 0.0, '-PRON- will there': 0.0, 'will there that': 0.0, 'there that will': 0.0, 'that will that': 0.0, 'what that -PRON-': 0.0, 'would go': 0.0, '-PRON- would go': 0.0, 'would go to': 0.0, 'would will': 0.0, '-PRON- -PRON- will': 0.0, '-PRON- would will': 0.0, 'would will not': 0.0, 'where do': 0.0, 'where do not': 0.0, 'be what be': 0.0, 'have not do': 0.0, 'be -- --': 0.0, 'would not will': 0.0, 'that go': 0.0, 'would that go': 0.0, 'that go to': 0.0, 'because -PRON- will': 0.0, 'would not could': 0.0, 'have that do': 0.0, '-PRON- will can': 0.0, '-PRON- there': 0.0, 'not -PRON- there': 0.0, '-PRON- there be': 0.0, 'would what': 0.0, '-PRON- would what': 0.0, 'would what -PRON-': 0.0, 'old do': 0.0, 'old do not': 0.0, 'be old': 0.0, '-PRON- be old': 0.0, 'how -PRON-': 0.0, 'have what': 0.0, 'what because': 0.0, 'be how': 0.0, 'how -PRON- have': 0.0, '-PRON- have what': 0.0, 'have what because': 0.0, 'what because do': 0.0, '-PRON- be how': 0.0, 'be not could': 0.0, 'what what': 0.0, 'be what what': 0.0, 'what what do': 0.0, 'be what -PRON-': 0.0, 'should not would': 0.0, 'will --': 0.0, '-PRON- will --': 0.0, 'be that that': 0.0, 'that will -PRON-': 0.0, 'would be': 0.0, '-PRON- would be': 0.0, 'would be not': 0.0, 'have because -PRON-': 0.0, 'will what': 0.0, '-PRON- will what': 0.0, 'not what do': 0.0, 'would have': 0.0, '-PRON- would have': 0.0, 'would have not': 0.0, 'have not who': 0.0, 'who have': 0.0, 'have who have': 0.0, 'who have be': 0.0, 'could not can': 0.0, '-PRON- -PRON- do': 0.0, '-- will': 0.0, '-- will not': 0.0, 'would --': 0.0, '-PRON- would --': 0.0, 'have should': 0.0, '-PRON- have should': 0.0, 'have should not': 0.0, 'should not what': 0.0, 'to -PRON- go': 0.0, 'have there can': 0.0, 'be who -PRON-': 0.0, 'have go to': 0.0, '-PRON- -PRON- should': 0.0, 'be that could': 0.0, 'would because': 0.0, '-PRON- would because': 0.0, '-PRON- -PRON- there': 0.0, 'not -PRON- because': 0.0, '-PRON- because there': 0.0, 'because there that': 0.0, 'there that do': 0.0, 'because that can': 0.0, 'there could': 0.0, 'there could not': 0.0, 'should not because': 0.0, 'not because that': 0.0, 'because that do': 0.0, 'there -PRON- would': 0.0, 'have -PRON- do': 0.0, 'that to': 0.0, 'to will': 0.0, 'not that to': 0.0, 'that to will': 0.0, 'to will not': 0.0, 'be not --': 0.0, 'would not be': 0.0, 'that should': 0.0, 'that should not': 0.0, 'that there be': 0.0, 'have not be': 0.0, 'could not could': 0.0, 'be -PRON- do': 0.0, '-PRON- there -PRON-': 0.0, 'there -PRON- -PRON-': 0.0, 'go do': 0.0, 'go do not': 0.0, 'there that go': 0.0, 'not that what': 0.0, 'have there do': 0.0, 'who because': 0.0, 'who because -PRON-': 0.0, 'would that -PRON-': 0.0, 'be that there': 0.0, 'that there -PRON-': 0.0, 'be who can': 0.0, 'have -PRON- that': 0.0, '-- -PRON-': 0.0, '-- -PRON- be': 0.0, 'there that -PRON-': 0.0, 'something do': 0.0, 'something do not': 0.0, 'not -- --': 0.0, 'will should': 0.0, '-PRON- will should': 0.0, 'will should not': 0.0, 'not -PRON- should': 0.0, '-PRON- -PRON- that': 0.0, '-PRON- that will': 0.0, 'not -PRON- that': 0.0, '-PRON- --': 0.0, '-PRON- -PRON- --': 0.0, 'would there': 0.0, '-PRON- would there': 0.0, 'would there be': 0.0, 'would that be': 0.0, 'be go do': 0.0, 'do -PRON-': 0.0, 'going to can': 0.0, 'be do -PRON-': 0.0, 'do -PRON- be': 0.0, 'what there': 0.0, 'have what there': 0.0, 'what there -PRON-': 0.0}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "wordsToxic = Counter()\n",
    "for item in train[(train['insult'] == 0) & (train['toxic'] == 1) & (train['severe_toxic'] == 0) & (train['identity_hate'] == 0) & (\n",
    "            train['threat'] == 0) & (train['obscene']==0)]['comment_text'].tolist():\n",
    "    wordsToxic.update(get_ngrams(item,2))\n",
    "    wordsToxic.update(get_ngrams(item, 3))\n",
    "    wordsToxic.update(item.split(\" \"))\n",
    "print('toxicDict',wordsToxic)\n",
    "\n",
    "wordsSevere_toxic = Counter()\n",
    "for item in train[train['severe_toxic'] == 1]['comment_text'].tolist():\n",
    "    wordsSevere_toxic.update(get_ngrams(item,2))\n",
    "    wordsSevere_toxic.update(get_ngrams(item, 3))\n",
    "    wordsSevere_toxic.update(item.split(\" \"))\n",
    "print('severe_toxicDict',wordsSevere_toxic)\n",
    "\n",
    "wordsIdentity_hate = Counter()\n",
    "for item in train[(train['insult'] == 0) & (train['toxic'] == 0) & (train['severe_toxic'] == 0) & (train['identity_hate'] == 1) & (\n",
    "            train['threat'] == 0) & (train['obscene']==0)]['comment_text'].tolist():\n",
    "    wordsIdentity_hate.update(get_ngrams(item,2))\n",
    "    wordsIdentity_hate.update(get_ngrams(item, 3))\n",
    "    wordsIdentity_hate.update(item.split(\" \"))\n",
    "print('identity_hateDict',wordsIdentity_hate)\n",
    "\n",
    "wordsThreat = Counter()\n",
    "for item in train[(train['insult'] == 0) & (train['toxic'] == 0) & (train['severe_toxic'] == 0) & (train['identity_hate'] == 0) & (\n",
    "            train['threat'] == 1) & (train['obscene']==0)]['comment_text'].tolist():\n",
    "    wordsThreat.update(get_ngrams(item,2))\n",
    "    wordsThreat.update(get_ngrams(item, 3))\n",
    "    wordsThreat.update(item.split(\" \"))\n",
    "print('threatDict',wordsThreat)\n",
    "\n",
    "wordsObscene = Counter()\n",
    "for item in train[(train['insult'] == 0) & (train['toxic'] == 0) & (train['severe_toxic'] == 0) & (train['identity_hate'] == 0) & (\n",
    "            train['threat'] == 0) & (train['obscene']==1)]['comment_text'].tolist():\n",
    "    wordsObscene.update(get_ngrams(item,2))\n",
    "    wordsObscene.update(get_ngrams(item, 3))\n",
    "    wordsObscene.update(item.split(\" \"))\n",
    "print('obsceneDict',wordsObscene)\n",
    "\n",
    "FullDict=dict()\n",
    "InsultDict = dict()\n",
    "for ele in wordsInsult:\n",
    "    if ele not in FullDict.keys():\n",
    "        polarity = sid.polarity_scores(ele).get('neg')\n",
    "        FullDict.update({ele: polarity})\n",
    "#print('InsultDict',InsultDict)\n",
    "#saveInFile(InsultDict,'SentimentInsultDict')\n",
    "\n",
    "ThreatDict = dict()\n",
    "for ele in wordsThreat:\n",
    "    if ele not in FullDict.keys():\n",
    "        polarity = sid.polarity_scores(ele).get('neg')\n",
    "        FullDict.update({ele: polarity})\n",
    "#print('ThreatDict',ThreatDict)\n",
    "#saveInFile(ThreatDict,'SentimentThreatDict')\n",
    "\n",
    "Identity_hateDict = dict()\n",
    "for ele in wordsIdentity_hate:\n",
    "    if ele not in FullDict.keys():\n",
    "        polarity = sid.polarity_scores(ele).get('neg')\n",
    "        FullDict.update({ele: polarity})\n",
    "#print('Identity_hateDict',Identity_hateDict)\n",
    "#saveInFile(Identity_hateDict,'SentimentIdentity_hateDict')\n",
    "\n",
    "Severe_toxicDict = dict()\n",
    "for ele in wordsSevere_toxic:\n",
    "    if ele not in FullDict.keys():\n",
    "        polarity = sid.polarity_scores(ele).get('neg')\n",
    "        FullDict.update({ele: polarity})\n",
    "#print('Severe_toxicDict',Severe_toxicDict)\n",
    "#saveInFile(Severe_toxicDict,'SentimentSevere_toxicDict')\n",
    "\n",
    "ToxicDict = dict()\n",
    "for ele in wordsToxic:\n",
    "    if ele not in FullDict.keys():\n",
    "        polarity = sid.polarity_scores(ele).get('neg')\n",
    "        FullDict.update({ele: polarity})\n",
    "#print('ToxicDict',ToxicDict)\n",
    "#saveInFile(ToxicDict,'SentimentToxicDict')\n",
    "\n",
    "ObsceneDict = dict()\n",
    "for ele in wordsObscene:\n",
    "    if ele not in FullDict.keys():\n",
    "        polarity = sid.polarity_scores(ele).get('neg')\n",
    "        FullDict.update({ele: polarity})\n",
    "#print('ObsceneDict',ObsceneDict)\n",
    "saveInFile(FullDict,'SentimentFullDict')\n",
    "\n",
    "ToxicDict = collections.OrderedDict(sorted(ToxicDict.items(), key=lambda t: t[1],reverse=True))\n",
    "print('Ordered_ToxicDict',ToxicDict)\n",
    "Severe_toxicDict = collections.OrderedDict(sorted(Severe_toxicDict.items(), key=lambda t: t[1],reverse=True))\n",
    "print('Ordered_Severe_toxicDict',Severe_toxicDict)\n",
    "Identity_hateDict = collections.OrderedDict(sorted(Identity_hateDict.items(), key=lambda t: t[1],reverse=True))\n",
    "print('Ordered_Identity_hateDict',Identity_hateDict)\n",
    "ThreatDict = collections.OrderedDict(sorted(ThreatDict.items(), key=lambda t: t[1],reverse=True))\n",
    "print('Ordered_ThreatDict',ThreatDict)\n",
    "InsultDict = collections.OrderedDict(sorted(InsultDict.items(), key=lambda t: t[1],reverse=True))\n",
    "print('Ordered_InsultDict',InsultDict)\n",
    "ObsceneDict = collections.OrderedDict(sorted(ObsceneDict.items(), key=lambda t: t[1],reverse=True))\n",
    "print('Ordered_ObsceneDict',ObsceneDict)\n",
    "\n",
    "print('All',FullDict)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(159571,)\n",
      "158966\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import sent_tokenize\n",
    "print(x_train.shape)\n",
    "FullDict=dict()\n",
    "i=0\n",
    "for comm in x_train:\n",
    "    #print(comm)\n",
    "    sumV=0\n",
    "    i+=1\n",
    "    negCounter=0\n",
    "    sent_tokenize_list = sent_tokenize(comm)\n",
    "    for sentence in sent_tokenize_list:\n",
    "        polarity = sid.polarity_scores(sentence).get('compound')\n",
    "        if polarity<0:\n",
    "            sumV+=polarity\n",
    "            negCounter+=1\n",
    "    if negCounter!=0:\n",
    "        FullDict.update({comm: (sumV/negCounter)})\n",
    "        #print(i)\n",
    "    else:\n",
    "        FullDict.update({comm: (polarity)})\n",
    "        #print(i)\n",
    "#FullDict={k: v for k, v in FullDict.items() if v > 0}\n",
    "#FullDict1={k: v for k, v in FullDict.items() if v > 0.8}\n",
    "saveInFile(FullDict,'SentimentFullDict')\n",
    "print(len(FullDict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "423\n"
     ]
    }
   ],
   "source": [
    "import collections\n",
    "print (len([item for item, count in collections.Counter(x_train).items() if count > 1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}