09-Scraping-Create-Debate-Data-Analysis.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"9-Scraping-Create-Debate-Data-Analysis.ipynb","private_outputs":true,"provenance":[],"collapsed_sections":["-Az4DDXEz01-"],"authorship_tag":"ABX9TyPMepM3IRUI7tWRN/cwOe/D"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","metadata":{"id":"LXZshlL0ibeI"},"source":["from google.colab import drive\n","drive.mount('/content/gdrive', force_remount=True)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ayHqEJHjUujI"},"source":["%%shell\n","git clone https://github.com/utkarsh512/CreateDebate-Scraper.git"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"CgKQl76ZVbES"},"source":["%%shell\n","pip install bs4\n","pip install glom"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"jWD0YxArU7_t"},"source":["# Uncomment to start scraping\n","#\n","# %%shell\n","# cd CreateDebate-Scraper/src/nested/\n","# python scrape.py --page_count 104 --tag Politics --data_dir /content/gdrive/MyDrive/DL/CreateDebate/Politics"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"invXkA5j49Ar"},"source":["%cd CreateDebate-Scraper/src/nested/"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"WplxT7dHA0Yi"},"source":["reader_addr = '/content/gdrive/MyDrive/DL/CreateDebate/Politics/threads.log'\n","\n","from thread import Thread, Comment\n","import pickle\n","from collections import OrderedDict\n","from wordcloud import WordCloud, STOPWORDS\n","from graphviz import Digraph"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"lZ1ZCuc9zq1-"},"source":["# Analysis of Dataset"]},{"cell_type":"code","metadata":{"id":"6TTJwK4BGZcc"},"source":["threadCount = 0\n","commentCount = 0\n","\n","reader = open(reader_addr, 'rb')\n","\n","threads = []\n","\n","e = Thread()\n","\n","try:\n","    while True:\n","        e = pickle.load(reader)\n","        threads.append(e)\n","except:\n","    reader.close()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"hTT038G2Ho2i"},"source":["len(threads)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"zgICEziQHsuH"},"source":["maxCommentCount = 0\n","\n","for x in threads:\n","    commentCount += len(x.comments)\n","    maxCommentCount = max(maxCommentCount, len(x.comments))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"-Sy-jNSnKAPi"},"source":["print(commentCount, maxCommentCount)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"TxPMIvMhKGdH"},"source":["dis = [0] * (maxCommentCount + 1)\n","for x in threads:\n","    dis[len(x.comments)] += 1"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"W1aUO-ZBKiy_"},"source":["X = []\n","for i in range(maxCommentCount + 1):\n","    X.append(i)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"vWrBft-dK6nk"},"source":["import matplotlib.pyplot as plt\n","plt.figure(dpi=100)\n","plt.plot(X, dis)\n","plt.xlabel('# arguments')\n","plt.ylabel('# debates')\n","plt.title('Distribution of arguments in threads')\n","plt.show()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"fmMyvyFS25LI"},"source":["v = OrderedDict()\n","v['0'] = dis[0]\n","v['1'] = dis[1]\n","v['2'] = dis[2]\n","v['3-5'] = dis[3] + dis[4] + dis[5]\n","v['6-10'] = 0\n","for i in range(6, 11):\n","    v['6-10'] += dis[i]\n","v['11-20'] = 0\n","for i in range(11, 21):\n","    v['11-20'] += dis[i]\n","v['>20'] = 0\n","for i in range(21, len(dis)):\n","    v['>20'] += dis[i]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"cZAN-jnx7yIB"},"source":["x = ['0', '1', '2', '3-5', '6-10', '11-20', '>20']\n","y = []\n","for xx in x:\n","    y.append(v[xx])\n","plt.bar(x, y)\n","plt.xlabel('Number of arguments per post')\n","plt.ylabel('Number of posts')\n","plt.savefig('distribution_of_number_of_arguments_per_post_create_debate.eps', format='eps', dpi=1200)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"f2AD1_kdz1OX"},"source":["# Printing any random thread\n","\n","import random\n","\n","idx = random.randint(0, len(threads) - 1)\n","print(threads[idx])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"tnvQjfGL6Zzi"},"source":["# Extracting distribution of comments by their level\n","\n","maxDepth = 0\n","\n","def dfs(Map, cid='root', depth=0):\n","    global maxDepth\n","    maxDepth = max(maxDepth, depth)\n","    if cid == 'root':\n","        for key in Map[cid].keys():\n","            dfs(Map[cid], key, depth + 1)\n","        return\n","\n","    for key in Map[cid].keys():\n","        dfs(Map[cid], key, depth + 1)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1altju1O6e7g"},"source":["for e in threads:\n","    if 'root' in e.metaL.keys():\n","        dfs(e.metaL)\n","    if 'root' in e.metaR.keys():\n","        dfs(e.metaR)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"-whFyB4M64Wn"},"source":["print(maxDepth)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"7P7dk0yU7DiP"},"source":["cnt = [0] * (maxDepth + 1)\n","\n","def dfs(Map, cid='root', depth=0, cnt=None):\n","    if depth != 0:\n","        cnt[depth] += 1\n","    if cid == 'root':\n","        for key in Map[cid].keys():\n","            dfs(Map[cid], key, depth + 1, cnt)\n","        return\n","    for key in Map[cid].keys():\n","        dfs(Map[cid], key, depth + 1, cnt)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"qEmnn7Z97ZK3"},"source":["for e in threads:\n","    if 'root' in e.metaL.keys():\n","        dfs(e.metaL, cnt=cnt)\n","    if 'root' in e.metaR.keys():\n","        dfs(e.metaR, cnt=cnt)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"z8sNeC9o7mEf"},"source":["x = []\n","for i in range(maxDepth + 1):\n","    x.append(i)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"2OZYs6M47zD_"},"source":["plt.figure(dpi = 100)\n","plt.plot(x[1:], cnt[1:])\n","plt.xlabel('Level')\n","plt.ylabel('Number of comments')\n","plt.title('Distribution of Comments (Level-wise)')\n","plt.show()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"VYKoJSPN-d7E"},"source":["v = dict()\n","v['1'] = cnt[1]\n","v['2'] = cnt[2]\n","v['3-5'] = cnt[3] + cnt[4] + cnt[5]\n","v['6-10'] = 0\n","for i in range(6, 11):\n","    v['6-10'] += cnt[i]\n","v['>10'] = 0\n","for i in range(11, len(cnt)):\n","    v['>10'] += cnt[i]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"PgXBCUxz_RpR"},"source":["\n","x = ['1', '2', '3-5', '6-10', '>10']\n","y = []\n","for xx in x:\n","    y.append(v[xx])\n","plt.bar(x, y)\n","plt.xlabel('Level')\n","plt.ylabel('Number of arguments')\n","plt.savefig('distribution_of_number_of_arguments_on_each_level_create_debate.eps', format='eps', dpi=1200)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"U6qBZjPAq6FE"},"source":["# Extracting the length of comments in CreateDebate (both word-wise and character-wise)\n","\n","maxLength = 0\n","\n","for e in threads:\n","    for x in e.comments.keys():\n","        cmnt = e.comments[x]\n","        cur = len(cmnt.body.split())\n","        maxLength = max(maxLength, cur)\n","\n","print(maxLength)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"9OUnJwjgsPTm"},"source":["cnt = [0 for x in range(maxLength + 1)]\n","for e in threads:\n","    for x in e.comments.keys():\n","        cmnt = e.comments[x]\n","        cnt[len(cmnt.body.split())] += 1\n","\n","x = [i for i in range(maxLength + 1)]\n","\n","plt.figure(dpi = 100)\n","plt.plot(x[1:], cnt[1:])\n","plt.xlabel('Level')\n","plt.ylabel('Number of comments')\n","plt.title('Distribution of Comments (Level-wise)')\n","plt.show()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ukNKVD_AtIxP"},"source":["v = dict()\n","v['0-50'] = 0\n","for i in range(51):\n","    v['0-50'] += cnt[i]\n","v['51-100'] = 0\n","for i in range(51, 101):\n","    v['51-100'] += cnt[i]\n","v['101-200'] = 0\n","for i in range(101, 201):\n","    v['101-200'] += cnt[i]\n","v['201-500'] = 0\n","for i in range(201, 501):\n","    v['201-500'] += cnt[i]\n","v['501-1000'] = 0\n","for i in range(501, 1001):\n","    v['501-1000'] += cnt[i]\n","v['>1000'] = 0\n","for i in range(1001, maxLength + 1):\n","    v['>1000'] += cnt[i]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"RC5nIHvduSlm"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Q0WvTW3p8BjO"},"source":["# Extracting how often the authors comment\n","\n","athr = dict()\n","\n","text = \"\"\n","\n","for e in threads:\n","    for x in e.comments.keys():\n","        cmnt = e.comments[x]\n","        cur_athr = cmnt.author\n","        text = text + \" \" + cur_athr\n","        try:\n","            athr[cur_athr] += 1\n","        except:\n","            athr[cur_athr] = 1"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"D-Ow1-if_qn4"},"source":["len(athr)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"0vFSyXpGEyEp"},"source":["def genWordCloud(text, max_words=100):\n","    wordcloud = WordCloud(background_color='white', max_words=max_words, width=800, height=400, relative_scaling=1.0, stopwords=STOPWORDS).generate(text)\n","    fig = plt.figure(1, figsize=(12, 12))\n","    plt.axis('off')\n","    plt.imshow(wordcloud)\n","    plt.show()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"wwBGpJKZFjmL"},"source":["genWordCloud(text)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"awlDOiM1_sek"},"source":["s = []\n","for x in athr.keys():\n","    s.append((athr[x], x))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ha26cbAWBTAA"},"source":["s = sorted(s, reverse=True)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"OcH5xfDrKwTD"},"source":["print(s[0])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"UMtrUxKNBXWn"},"source":["cnt = [0] * (9000)\n","for x in athr.keys():\n","    cnt[athr[x]] += 1"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ZdE6g2BIBdlO"},"source":["x = []\n","for i in range(9000):\n","    x.append(i)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"aW28E8MwCmIs"},"source":["plt.figure(dpi = 100)\n","plt.plot(x[1:50], cnt[1:50])\n","plt.xlabel('Comment count')\n","plt.ylabel('Number of authors')\n","plt.title('Distribution of Authors')\n","plt.show()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"IxCfBVCuHksS"},"source":["v = dict()\n","v['1'] = cnt[1]\n","v['2'] = cnt[2]\n","v['3-5'] = cnt[3] + cnt[4] + cnt[5]\n","v['6-10'] = 0\n","for i in range(6, 11):\n","    v['6-10'] += cnt[i]\n","v['11-20'] = 0\n","for i in range(11, 21):\n","    v['11-20'] += cnt[i]\n","v['>20'] = 0\n","for i in range(21, len(cnt)):\n","    v['>20'] += cnt[i]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"a7EdfLH4I1t_"},"source":["\n","x = ['1', '2', '3-5', '6-10', '11-20', '>20']\n","y = []\n","for xx in x:\n","    y.append(v[xx])\n","plt.bar(x, y)\n","plt.xlabel('Number of arguments')\n","plt.ylabel('Number of authors')\n","plt.savefig('distribution_of_authors_wrt_comment_frequency_create_debate.eps', format='eps', dpi=1200)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"iaOXG3LcMA2T"},"source":["athr = dict()\n","for e in threads:\n","    if 'root' in e.metaL.keys():\n","        for key in e.metaL['root'].keys():\n","            cmnt = e.comments[key]\n","            cur_athr = cmnt.author\n","            try:\n","                athr[cur_athr] += 1\n","            except:\n","                athr[cur_athr] = 1\n","    if 'root' in e.metaR.keys():\n","        for key in e.metaR['root'].keys():\n","            cmnt = e.comments[key]\n","            cur_athr = cmnt.author\n","            try:\n","                athr[cur_athr] += 1\n","            except:\n","                athr[cur_athr] = 1"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"XsEsQ_NtNBUX"},"source":["len(athr)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"9Kdbd6lRNuuj"},"source":["s = []\n","for x in athr.keys():\n","    s.append((athr[x], x))\n","s = sorted(s, reverse=True)\n","for i in range(10):\n","    x, y = s[i]\n","    print(f'# L1: {x} \\t Author: {y}')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"jy9XsMT6j9dL"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"bAc15X7yO5vB"},"source":["cnt = [0] * 1500\n","for x in athr:\n","    cnt[athr[x]] += 1\n","x = []\n","for i in range(1500):\n","    x.append(i)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"pfmtG5hAPMzH"},"source":["plt.figure(dpi = 100)\n","plt.plot(x[1:50], cnt[1:50])\n","plt.xlabel('Comment count')\n","plt.ylabel('Number of authors')\n","plt.title('Distribution of Authors')\n","plt.show()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"a4nvgkzoPc3z"},"source":["v = dict()\n","v['1'] = cnt[1]\n","v['2'] = cnt[2]\n","v['3-5'] = cnt[3] + cnt[4] + cnt[5]\n","v['6-10'] = 0\n","for i in range(6, 11):\n","    v['6-10'] += cnt[i]\n","v['11-20'] = 0\n","for i in range(11, 21):\n","    v['11-20'] += cnt[i]\n","v['>20'] = 0\n","for i in range(21, len(cnt)):\n","    v['>20'] += cnt[i]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"CwrnWrYyPkSC"},"source":["\n","x = ['1', '2', '3-5', '6-10', '11-20', '>20']\n","y = []\n","for xx in x:\n","    y.append(v[xx])\n","plt.bar(x, y)\n","plt.xlabel('Number of level-1 arguments')\n","plt.ylabel('Number of authors')\n","plt.savefig('distribution_of_authors_wrt_level_1_comment_frequency_create_debate.eps', format='eps', dpi=1200)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"qVxaTGA3Ql2h"},"source":["# Storing the authors who posted more than 200 L1 comments\n","\n","L1_athr = dict()\n","for x in athr:\n","    if athr[x] > 200:\n","        L1_athr[x] = True"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"lW5VPMH_C3K2"},"source":["# Popularity of author (by number of direct replies received)\n","\n","athr = dict()\n","\n","def dfs(Map, cmntMap, athr, cid='root'):\n","    if cid == 'root':\n","        for key in Map[cid].keys():\n","            dfs(Map[cid], cmntMap, athr, key)\n","        return\n","    cur_author = cmntMap[cid].author\n","\n","    try:\n","        athr[cur_author] += len(Map[cid].keys())\n","    except:\n","        athr[cur_author] = len(Map[cid].keys())\n","\n","    for key in Map[cid].keys():\n","        dfs(Map[cid], cmntMap, athr, key)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"wB8SkjSkFJ7D"},"source":["for e in threads:\n","    if 'root' in e.metaL.keys():\n","        dfs(e.metaL, e.comments, athr)\n","    if 'root' in e.metaR.keys():\n","        dfs(e.metaR, e.comments, athr) "],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Io0OX6GRFgw-"},"source":["len(athr)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"14ASoqCSFjLa"},"source":["s = []\n","for x in athr:\n","    s.append((athr[x], x))\n","\n","s = sorted(s, reverse=True)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"erS462GQFxCc"},"source":["for i in range(10):\n","    x, y = s[i]\n","    print(f'# Direct Replies: {x} \\t Author: {y}')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"yeKS8htqF0ME"},"source":["cnt = [0] * (6000)\n","for x in athr:\n","    cnt[athr[x]] += 1"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"b0wRnasYGSWX"},"source":["x = []\n","for i in range(6000):\n","    x.append(i)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"HJM2UKkEGYC8"},"source":["plt.figure(dpi = 100)\n","plt.plot(x[1:50], cnt[1:50])\n","plt.xlabel('# direct replies received')\n","plt.ylabel('# authors')\n","plt.title('Distribution of direct replies')\n","plt.show()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"2JUu3C94GcPE"},"source":["v = dict()\n","v['1'] = cnt[1]\n","v['2'] = cnt[2]\n","v['3-5'] = cnt[3] + cnt[4] + cnt[5]\n","v['6-10'] = 0\n","for i in range(6, 11):\n","    v['6-10'] += cnt[i]\n","v['11-20'] = 0\n","for i in range(11, 21):\n","    v['11-20'] += cnt[i]\n","v['>20'] = 0\n","for i in range(21, len(cnt)):\n","    v['>20'] += cnt[i]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"3cma8v4GTP7H"},"source":["\n","x = ['1', '2', '3-5', '6-10', '11-20', '>20']\n","y = []\n","for xx in x:\n","    y.append(v[xx])\n","plt.bar(x, y)\n","plt.xlabel('Number of direct replies')\n","plt.ylabel('Number of authors')\n","plt.savefig('distribution_of_authors_wrt_direct_reply_frequency_create_debate.eps', format='eps', dpi=1200)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"dZnQVEiiTZVq"},"source":["# Storing the authors who posted more than 20 L1 comments\n","\n","A = []\n","for x in athr:\n","    if athr[x] > 100:\n","        try:\n","            z = L1_athr[x]\n","            A.append(x)\n","        except KeyError:\n","            pass"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"i2oKqSvYUa98"},"source":["len(A)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"jYLsCSbVUcRo"},"source":["athr_id = dict()\n","which = [\"\"] * len(A)\n","for i in range(len(A)):\n","    athr_id[A[i]] = i\n","    which[i] = A[i]\n","print(which)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"RnxUShh8aTzX"},"source":["G = [[0 for i in range(len(A))] for j in range(len(A))]\n","H = [[0 for i in range(len(A))] for j in range(len(A))]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"vXzAV6Hfcdpw"},"source":["def dfs(Map, cmntMap, cid='root'):\n","    if cid == 'root':\n","        for key in Map[cid].keys():\n","            dfs(Map[cid], cmntMap, key)\n","        return\n","\n","    cur_author = cmntMap[cid].author\n","    cur_pol = cmntMap[cid].polarity\n","    \n","    if cur_author in athr_id and cur_pol != 'Not Available':\n","        cur_author_id = athr_id[cur_author]\n","        for key in Map[cid].keys():\n","            nxt_author = cmntMap[key].author\n","            nxt_pol = cmntMap[key].polarity\n","            if nxt_author in athr_id and nxt_pol != 'Not Available':\n","                nxt_author_id = athr_id[nxt_author]\n","                if cur_pol == nxt_pol:\n","                    G[nxt_author_id][cur_author_id] += 1\n","                else:\n","                    H[nxt_author_id][cur_author_id] += 1\n","\n","    for key in Map[cid].keys():\n","        dfs(Map[cid], cmntMap, key)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"7xx8mdU-cfR9"},"source":["for e in threads:\n","    if 'root' in e.metaL:\n","        dfs(e.metaL, e.comments)\n","    if 'root' in e.metaR:\n","        dfs(e.metaR, e.comments)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"PYZccvCngfaU"},"source":["G_max = 0\n","H_max = 0\n","\n","for i in range(len(A)):\n","    for j in range(len(A)):\n","        G_max = max(G_max, G[i][j])\n","        H_max = max(H_max, H[i][j])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"4sEa49RXgxht"},"source":["print(G_max, H_max)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"hw2uz8wThhAc"},"source":["# Visualizing Support Graph\n","\n","u = Digraph('unix', filename='unix.gv', node_attr={'color': 'lightblue2', 'style': 'filled'}, format='png')\n","\n","for i in range(len(A)):\n","    for j in range(len(A)):\n","        node1 = which[i]\n","        node2 = which[j]\n","        if H[i][j] != 0:\n","            u.edge(node1, node2, label=str(H[i][j]))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"qiY85tQ_hxFk"},"source":["u.render(view=True)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"GSni5P6oltck"},"source":["import numpy as np \n","\n","fig = plt.figure()\n","fig.dpi= 100\n","ax = fig.add_subplot(111)\n","cax = ax.matshow(H, interpolation='nearest')\n","fig.colorbar(cax)\n","ax.set_xticks(np.arange(len(A)))\n","ax.set_yticks(np.arange(len(A)))\n","ax.set_xticklabels(which)\n","ax.set_yticklabels(which)\n","plt.setp(ax.get_xticklabels(), rotation=-90)\n","plt.show()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"BocReDCfr3wd"},"source":["def Peek(author, max_words=200):\n","    text = \"\"\n","    for e in threads:\n","        for x in e.comments:\n","            cmnt = e.comments[x]\n","            if cmnt.author == author:\n","                text = text + ' ' + cmnt.body\n","    genWordCloud(text, max_words)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"bCxvAhKJ2fBs"},"source":["Peek('outlaw60', 400)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"IxYU_TOX2i_5"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"-Az4DDXEz01-"},"source":["# Community Detection"]},{"cell_type":"code","metadata":{"id":"SUlsFxwVz4JP"},"source":["authorMapL1 = {} # Contains author name who wrote atleast one L1 comment and thier index (in building the graphs)\n","authorCountL1 = 0"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"aZ55SlTP0XQ5"},"source":["for e in threads:\n","    if 'root' in e.metaL.keys():\n","        for key in e.metaL['root'].keys():\n","            cmnt = e.comments[key]\n","            curAuthor = cmnt.author\n","            if curAuthor not in authorMapL1:\n","                authorMapL1[curAuthor] = authorCountL1\n","                authorCountL1 += 1\n","    if 'root' in e.metaR.keys():\n","        for key in e.metaR['root'].keys():\n","            cmnt = e.comments[key]\n","            curAuthor = cmnt.author\n","            if curAuthor not in authorMapL1:\n","                authorMapL1[curAuthor] = authorCountL1\n","                authorCountL1 += 1"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"d5aqMB6O3ULR"},"source":["G = [[0 for j in range(authorCountL1)] for i in range(authorCountL1)] # Support matrix\n","H = [[0 for j in range(authorCountL1)] for i in range(authorCountL1)] # Dispute matrix"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Ph3ika0O3aFL"},"source":["# Calculating support matrix and dispute matrix\n","\n","athr_id = authorMapL1\n","\n","def dfs(Map, cmntMap, cid='root'):\n","    if cid == 'root':\n","        for key in Map[cid].keys():\n","            dfs(Map[cid], cmntMap, key)\n","        return\n","\n","    cur_author = cmntMap[cid].author\n","    cur_pol = cmntMap[cid].polarity\n","    \n","    if cur_author in athr_id and cur_pol != 'Not Available':\n","        cur_author_id = athr_id[cur_author]\n","        for key in Map[cid].keys():\n","            nxt_author = cmntMap[key].author\n","            nxt_pol = cmntMap[key].polarity\n","            if nxt_author in athr_id and nxt_pol != 'Not Available':\n","                nxt_author_id = athr_id[nxt_author]\n","                if cur_pol == nxt_pol:\n","                    G[nxt_author_id][cur_author_id] += 1\n","                else:\n","                    H[nxt_author_id][cur_author_id] += 1\n","\n","    for key in Map[cid].keys():\n","        dfs(Map[cid], cmntMap, key)\n","\n","for e in threads:\n","    if 'root' in e.metaL:\n","        dfs(e.metaL, e.comments)\n","    if 'root' in e.metaR:\n","        dfs(e.metaR, e.comments)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"2qdTS0zZ5RMz"},"source":["G_max = 0\n","H_max = 0\n","\n","for i in range(authorCountL1):\n","    for j in range(authorCountL1):\n","        G_max = max(G_max, G[i][j])\n","        H_max = max(H_max, H[i][j])\n","\n","print(G_max, H_max)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"BRAKnqNn5hn4"},"source":["import numpy as np \n","\n","fig = plt.figure()\n","fig.dpi= 100\n","ax = fig.add_subplot(111)\n","cax = ax.matshow(G, interpolation='nearest')\n","fig.colorbar(cax)\n","plt.show()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"z-XTXzk56tZF"},"source":["which = [\"lol\"] * (authorCountL1)\n","for key in authorMapL1.keys():\n","    which[authorMapL1[key]] = key"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"2cNi52pX7bTi"},"source":["import networkx as nx"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"7bx0gnNK8U9a"},"source":["G_ = nx.DiGraph()\n","for i in range(authorCountL1):\n","    for j in range(authorCountL1):\n","        if G[i][j] != 0:\n","            G_.add_weighted_edges_from([(i, j, G[i][j])])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"TvtpxSwt9NZa"},"source":["H_ = nx.DiGraph()\n","for i in range(authorCountL1):\n","    for j in range(authorCountL1):\n","        if H[i][j] != 0:\n","            H_.add_weighted_edges_from([(i, j, H[i][j])])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"_Z9ActVI9VmI"},"source":["support_graph_reciprocity = nx.algorithms.reciprocity(G_)\n","dispute_graph_reciprocity = nx.algorithms.reciprocity(H_)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"goryBN4H9qRz"},"source":["print(support_graph_reciprocity, dispute_graph_reciprocity)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"aWwrRzNl-LLL"},"source":[""],"execution_count":null,"outputs":[]}]}