49-Diversity-in-CreateDebate.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"private_outputs":true,"provenance":[],"authorship_tag":"ABX9TyM/YQ2+G5uxS95Fen93tQ4L"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":null,"metadata":{"id":"TKMrlRE0RDKm"},"outputs":[],"source":["from google.colab import drive\n","drive.mount('/content/gdrive', force_remount=True)"]},{"cell_type":"code","source":["!pip install scikit-bio"],"metadata":{"id":"jZjmO1HPRPyp"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Clone `CreateDebateScraper` library from github\n","!git clone https://github.com/utkarsh512/CreateDebateScraper.git\n","%cd CreateDebateScraper/src/nested/"],"metadata":{"id":"e0UaoNaBRbuZ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from   copy                     import deepcopy\n","from   itertools                import accumulate\n","import json\n","from   matplotlib               import pyplot as plt\n","import networkx as nx\n","import nltk\n","import numpy as np\n","import pandas as pd\n","import pickle\n","import re\n","from   scipy                    import stats\n","import textwrap\n","from   thread                   import Comment, Thread\n","from   tqdm                     import tqdm\n","nltk.download('punkt') # For tokenizers\n","import matplotlib\n","from   nltk.tokenize            import TweetTokenizer\n","from   pprint                   import pprint\n","import skbio\n","matplotlib.rcParams.update({'font.size': 18})\n","matplotlib.rcParams[\"figure.figsize\"] = (12, 5)"],"metadata":{"id":"0xhjN2TzRycs"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Helper function"],"metadata":{"id":"IrPN5V3pR97O"}},{"cell_type":"code","source":["tknz = TweetTokenizer()\n","\n","def clean_text(text):\n","    \"\"\"\n","    Preprocessing text\n","    \"\"\"\n","    text = text.lower()\n","    text = re.sub(r\"http\\S+\", \"\", text)\n","    text = re.sub(r\"www\\S+\", \"\", text)\n","    text = re.sub(\"-\", \" \", text)\n","    text = re.sub(\"\\s+\", \" \", text)\n","    text = re.sub(\"\\u2018\", \"X\", text) \n","    text = re.sub(\"\\u2019\", \"X\", text) \n","    text = re.sub(\"\\'\", \"X\", text) \n","    wordTokens_ = tknz.tokenize(text)\n","    wordTokens = list()\n","    for x in wordTokens_:\n","        x = ''.join([v for v in x if v.isalnum() or v == ' '])\n","        if len(x) > 0 and x != 'X':\n","            x = x.replace('X', '\\'')\n","            wordTokens.append(x)\n","    return wordTokens"],"metadata":{"id":"uvi-S0g_R190"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["SLUR_WORDS = {\n","  \"jews\": [\n","    \"jews\",\n","    \"oven dodger\",\n","    \"nazi\",\n","    \"dirty jew\",\n","    \"holocaust\",\n","    \"kikesucker\",\n","    \"hook nose\",\n","    \"kike\"\n","  ],\n","  \"homosexual\": [\n","    \"faggots usually\",\n","    \"fucking queer\",\n","    \"the biggest faggot\",\n","    \"dyke\",\n","    \"you fucking faggot\",\n","    \"hate faggots\",\n","    \"queer\",\n","    \"homosexual\",\n","    \"the faggots\",\n","    \"faggot\",\n","    \"faggots usually have\",\n","    \"gay\",\n","    \"faggots\",\n","    \"dykey\",\n","    \"ugly dyke\",\n","    \"faggots like you\",\n","    \"you a fag\",\n","    \"lesbian\",\n","    \"homo\",\n","    \"is a faggot\",\n","    \"like a faggot\",\n","    \"dykes\",\n","    \"faggots like\",\n","    \"faggot if you ever\"\n","  ],\n","  \"women\": [\n","    \"ugly dyke\",\n","    \"woman terrorist\",\n","    \"nigress\",\n","    \"bitch\",\n","    \"slut\",\n","    \"women\",\n","    \"sheeboon\",\n","    \"negress\",\n","    \"mud shark\",\n","    \"women threat\",\n","    \"you a lame bitch\",\n","    \"your a cunt\",\n","    \"white bitch\",\n","    \"niggeress\",\n","    \"hoe\",\n","    \"dykes\",\n","    \"niggress\",\n","    \"sheboon\",\n","    \"feminazi\"\n","  ],\n","  \"blacks\": [\n","    \"pavement ape\",\n","    \"the niggers\",\n","    \"negress\",\n","    \"porch monkey\",\n","    \"that nigger\",\n","    \"this nigger\",\n","    \"sheboon\",\n","    \"all niggers\",\n","    \"eurafrica\",\n","    \"shut up nigger\",\n","    \"picaninny\",\n","    \"african attack\",\n","    \"spearchucker\",\n","    \"how many niggers\",\n","    \"nigger\",\n","    \"africa\",\n","    \"niggers are in my\",\n","    \"dindu nuffin\",\n","    \"stupid nigger\",\n","    \"moolie\",\n","    \"niggers\",\n","    \"bluegum\",\n","    \"nigger ass\",\n","    \"you niggers\",\n","    \"fucking nigger\",\n","    \"nigger music\",\n","    \"niggress\",\n","    \"you a nigger\",\n","    \"many niggers are\",\n","    \"nigress\",\n","    \"blacks\",\n","    \"teenaper\",\n","    \"sheeboon\",\n","    \"dumb nigger\",\n","    \"niggeress\",\n","    \"pickaninny\",\n","    \"nigga\"\n","  ],\n","  \"muslim\": [\n","    \"muslim immigrant\",\n","    \"islam\",\n","    \"mudslime\",\n","    \"mooslem\",\n","    \"muslim refugee\",\n","    \"musslime\",\n","    \"shitlam\",\n","    \"muslim invasion\",\n","    \"moslime\",\n","    \"mooslamic\",\n","    \"muzzie\",\n","    \"allah akbar\",\n","    \"mooslime\",\n","    \"musloid\",\n","    \"mudslimes\",\n","    \"muslim\",\n","    \"muslimes\",\n","    \"moslum\",\n","    \"mussie\",\n","    \"muzrat\",\n","    \"muslim countries\",\n","    \"muzzy\",\n","    \"moslim\",\n","    \"jihadi\",\n","    \"muslim country\",\n","    \"moslem\",\n","    \"muzzrat\",\n","    \"mooslim\"\n","  ],\n","  \"arabs\": [\n","    \"towel head\",\n","    \"goatfucker\",\n","    \"arabs\",\n","    \"goathumper\",\n","    \"raghead\",\n","    \"rag head\",\n","    \"goathumping\",\n","    \"towelhead\",\n","    \"camel jockey\",\n","    \"sandnigger\",\n","    \"camel fucker\",\n","    \"sand nigger\"\n","  ],\n","  \"generic\": [\n","    \"to rape\",\n","    \"raped and\",\n","    \"shithole country\",\n","    \"get raped\",\n","    \"raped\",\n","    \"is a fucking\",\n","    \"shit skin\",\n","    \"raped by\",\n","    \"hate you\",\n","    \"fake empowerment\",\n","    \"abusive women\",\n","    \"fuck you too\",\n","    \"violence\",\n","    \"wit a lame nigga\",\n","    \"they all look\",\n","    \"alllivesmatter\",\n","    \"shithole countries\",\n","    \"fucking hate\",\n","    \"trailer trash\",\n","    \"kill all\",\n","    \"terrorist threat\",\n","    \"harassment\",\n","    \"kill yourself\",\n","    \"shitskin\",\n","    \"okay to be white\",\n","    \"fucking hate you\"\n","  ],\n","  \"white\": [\n","    \"full of white\",\n","    \"white trash\",\n","    \"white devil\",\n","    \"white\",\n","    \"are all white\",\n","    \"white boy\",\n","    \"white ass\",\n","    \"white bitch\",\n","    \"hillbilly\",\n","    \"whigger\",\n","    \"white christian\",\n","    \"white person\",\n","    \"all white\",\n","    \"white nigger\",\n","    \"redneck\",\n","    \"white honky\",\n","    \"wigger\",\n","    \"them white\"\n","  ],\n","  \"economy\": [\n","    \"ghetto\"\n","  ],\n","  \"immigrant\": [\n","    \"illegal immigrants\",\n","    \"immigrant not welcome\",\n","    \"immigrant terror\",\n","    \"mexcrement\",\n","    \"go back to where you come from\",\n","    \"muslim refugee\",\n","    \"illegal aliens\",\n","    \"refugee\",\n","    \"protect from immigrants\",\n","    \"negro\",\n","    \"refugees\",\n","    \"immigrant\",\n","    \"refugee invasion\",\n","    \"go back to where they come from\",\n","    \"refugees impact\",\n","    \"bring ebola\",\n","    \"immigrants\",\n","    \"illegal alien\",\n","    \"immigrant invasion\",\n","    \"bring disease\"\n","  ],\n","  \"mental\": [\n","    \"retard\",\n","    \"mongoloid\",\n","    \"retarded\"\n","  ],\n","  \"asians\": [\n","    \"asians\",\n","    \"ching chong\",\n","    \"chinaman\"\n","  ]\n","}"],"metadata":{"id":"VWchDXdtSBQY"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Loading CreateDebate dataset"],"metadata":{"id":"y8onZGbGSIji"}},{"cell_type":"code","source":["comments = dict()\n","\n","# Topical forums on CreateDebate. We have scraped comments for all of the\n","# following forurm.\n","categories = ['business', 'comedy', 'entertainment', 'health', 'law', 'nsfw',\n","              'politics2', 'religion', 'science', 'shopping', 'sports',\n","              'technology', 'travel', 'world']\n","\n","# However, we will be analyzing comments from selected forum only!\n","# These forum have at least 10k comments each.\n","categories_selected = ['politics2', 'religion', 'world', \n","                       'science', 'law', 'technology']\n","\n","for x in categories_selected:\n","    comments[x] = list()"],"metadata":{"id":"ivFJT4EQSFlO"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Loading comments from select forums\n","\n","for cat in tqdm(categories_selected):\n","    fp = open('/content/gdrive/MyDrive/DL/CreateDebate/' + cat + '/threads.log', 'rb')\n","\n","    # Get all the `Thread` objects pickled while scraping.\n","    threads = list()\n","    try:\n","        while True:\n","            e = pickle.load(fp)\n","            threads.append(e)\n","    except EOFError:\n","        fp.close()\n","\n","    # While classifying CreateDebate comments, we used comments as per author mode.\n","    # Hence, using the same mode to attach classification score with the comments.\n","    # \n","    # score < 0.5 -> ad hominem comment\n","    #       > 0.5 -> non ad hominem comment\n","    authors = dict()\n","    for thread in threads:\n","        for k, v in thread.comments.items():\n","            try:\n","                authors[v.author].append(v)\n","            except:\n","                authors[v.author] = list()\n","                authors[v.author].append(v)\n","\n","    ctr = 0\n","    # Load the classification score of the comments.\n","    with open('/content/gdrive/MyDrive/DL/CreateDebate/' + cat + '/comments_with_score.log', 'rb') as fp:\n","        cws = pickle.load(fp)\n","    # Attach classification score with the comments.\n","    for author in authors.keys():\n","        for i in range(len(authors[author])):\n","            comment = authors[author][i]\n","            foo = deepcopy(comment.__dict__)\n","            foo['tag'] = cat\n","            foo['score'] = cws[ctr][0]\n","            foo['validation'] = cws[ctr][1][0]\n","            comments[cat].append(foo)\n","            ctr += 1"],"metadata":{"id":"uyl_1EB4SND6"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Loading slur count data for CreateDebate"],"metadata":{"id":"LjyndbFISXPT"}},{"cell_type":"code","source":["# Loading computation from cache\n","\n","with open('/content/gdrive/MyDrive/Temp/47-ah-score.pkl', 'rb') as fp:\n","    ah_score = pickle.load(fp)\n","\n","with open('/content/gdrive/MyDrive/Temp/47-slur-count.pkl', 'rb') as fp:\n","    slur_count = pickle.load(fp)"],"metadata":{"id":"ZPDpJGETSPzt"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Loading CreateDebate user profile data"],"metadata":{"id":"ljBJ8jPuYaWc"}},{"cell_type":"code","source":["df = pd.read_json('/content/gdrive/MyDrive/DL/CreateDebate/profile/results.json', lines=True)"],"metadata":{"id":"jqXKvF_AYZ9U"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df"],"metadata":{"id":"KEaigXe7YlcX"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["reward_points_map = {k : v for k, v in zip(df['username'].tolist(), df['reward_points'].tolist())}\n","efficiency_map    = {k : v for k, v in zip(df['username'].tolist(), df['efficiency'].tolist())}\n","allies_map        = {k : len(v) for k, v in zip(df['username'].tolist(), df['allies'].tolist())}\n","enemies_map       = {k : len(v) for k, v in zip(df['username'].tolist(), df['enemies'].tolist())}\n","hostiles_map      = {k : len(v) for k, v in zip(df['username'].tolist(), df['hostiles'].tolist())}"],"metadata":{"id":"bv5KXQFAYqfi"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def get_stats_from_profile_data(user_subset):\n","    rewards_ = list()\n","    efficiency_ = list()\n","    n_allies = list()\n","    n_enemies = list()\n","    n_hostiles = list()\n","\n","    for user in user_subset:\n","        rewards_.append(reward_points_map[user])\n","        efficiency_.append(efficiency_map[user])\n","        n_allies.append(allies_map[user])\n","        n_enemies.append(enemies_map[user])\n","        n_hostiles.append(hostiles_map[user])\n","    \n","    grpd_data = [rewards_, efficiency_, n_allies, n_enemies, n_hostiles]\n","    avgs = [np.average(x) for x in grpd_data]\n","    stds = [np.std(x) for x in grpd_data]\n","    \n","    return avgs, stds"],"metadata":{"id":"5f27wj4cZHqD"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Analysis"],"metadata":{"id":"l0xL-r0MStp8"}},{"cell_type":"code","source":["# Median ah score per category per author\n","#   key: category -> author\n","#   value: median ah score\n","ah_score_median = dict()\n","\n","for category, author_data in ah_score.items():\n","    ah_score_median[category] = dict()\n","    for author, ah_scores in author_data.items():\n","        ah_score_median[category][author] = np.median(ah_scores)"],"metadata":{"id":"ryDx07l0SeVW"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["user_list = set()\n","\n","for category in categories_selected:\n","    for comment in comments[category]:\n","        user_list.add(comment['author'])\n","\n","user_list = list(user_list)"],"metadata":{"id":"FseHNpNBSvl0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["len(user_list)"],"metadata":{"id":"HdOMtaevS0T_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["ah_activity_matrix = [[0 for j in range(4)] for i in range(len(user_list))]"],"metadata":{"id":"sGrUxx4QS22A"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["EXTREME_AH_UPPER = 1\n","EXTREME_AH_LOWER = 0.95\n","\n","MODERATE_AH_UPPER = 0.8\n","MODERATE_AH_LOWER = 0.7\n","\n","LOW_AH_UPPER = 0.6\n","LOW_AH_LOWER = 0.5"],"metadata":{"id":"fln3eBGmTEvZ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for i, user in enumerate(user_list):\n","    for j, category in enumerate(categories_selected):\n","        try:\n","            current_score = ah_score_median[category][user]\n","            if EXTREME_AH_LOWER <= current_score and current_score <= EXTREME_AH_UPPER:\n","                ah_activity_matrix[i][1] += 1\n","            elif MODERATE_AH_LOWER <= current_score and current_score <= MODERATE_AH_UPPER:\n","                ah_activity_matrix[i][2] += 1\n","            elif LOW_AH_LOWER <= current_score and current_score <= LOW_AH_UPPER:\n","                ah_activity_matrix[i][3] += 1\n","            else:\n","                ah_activity_matrix[i][0] += 1\n","        except KeyError:\n","            ah_activity_matrix[i][0] += 1\n","            pass"],"metadata":{"id":"oibMYJMFS5QE"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["diversity_map = dict()\n","# key: author\n","# value: diversity of the user\n","\n","# Note that we include only those users who are present in the following categories for at least one topical forum:\n","# * EXTREME_AH\n","# * MODERATE_AH\n","# * LOW_AH\n","\n","for i, user in enumerate(user_list):\n","\n","    # Check whether to include this user in the study or not\n","    flag = ah_activity_matrix[i][1] or ah_activity_matrix[i][2] or ah_activity_matrix[i][3]\n","    if not flag:\n","        continue\n","    \n","    # It's okay to include this user in the study\n","    div = skbio.diversity.alpha.shannon(ah_activity_matrix[i]) / 2 # divided by 2 to normalize `div` in [0, 1] range\n","    diversity_map[user] = div"],"metadata":{"id":"Ahdy3s6QS8-X"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["len(diversity_map)"],"metadata":{"id":"xx0K-bwbTMLl"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plt.hist(diversity_map.values(), bins=[0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1])\n","plt.xlabel('Normalized Shannon diversity index')\n","plt.ylabel('#Users (in log-scale)')\n","plt.yscale('log')\n","plt.title('Normalized Shannon diversity index for CreateDebate users')"],"metadata":{"id":"EQnMK7LkTdhc"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Thresholds to partition these users as \n","# - highly diverse\n","# - moderately diverse\n","\n","HIGH_DIVERSITY_LOWER = 0.6\n","\n","MODERATE_DIVERSITY_LOWER = 0.3\n","MODERATE_DIVERSITY_UPPER = 0.4\n","\n","high_diversity_users = set()\n","moderate_diversity_users = set()"],"metadata":{"id":"X-p7FDTLTfh4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for user, d in diversity_map.items():\n","    if d >= HIGH_DIVERSITY_LOWER:\n","        high_diversity_users.add(user)\n","    elif d >= MODERATE_DIVERSITY_LOWER and d <= MODERATE_DIVERSITY_UPPER:\n","        moderate_diversity_users.add(user)"],"metadata":{"id":"MuXsyXGnTmhX"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(\"Number of highly diverse users: \", len(high_diversity_users))\n","print(\"Number of moderately diverse users: \", len(moderate_diversity_users))"],"metadata":{"id":"ikK4BSJ_TpZ4"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Network study for highly diverse / moderately diverse groups\n","\n","**Ref**: Notebook #12"],"metadata":{"id":"sJol5w1MUTIp"}},{"cell_type":"code","source":["reader_addr = '/content/gdrive/MyDrive/DL/CreateDebate/Politics/threads.log'\n","reader = open(reader_addr, 'rb')\n","threads = []\n","try:\n","    while True:\n","        e = pickle.load(reader)\n","        threads.append(e)\n","except:\n","    reader.close()"],"metadata":{"id":"7AKT_hdATsFW"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def build_graph(user_subset, n1 = 0, n2 = 0):\n","    \"\"\"Builds support graph and dispute graph from hyper-parameters n1 and n2\n","    inputs\n","    :param n1: threshold on number of level-1 comments\n","    :param n2: threshold on number of direct replies\n","\n","    output\n","    (author_map : dict, reverse_map : list, author_count : int, support_graph : nx.DiGraph, support_matrix: list, dispute_graph : nxDiGraph, dispute_matrix : list)\n","    \"\"\"\n","\n","    athr = dict()\n","    for e in threads:\n","        if 'root' in e.metaL.keys():\n","            for key in e.metaL['root'].keys():\n","                cmnt = e.comments[key]\n","                cur_athr = cmnt.author\n","                try:\n","                    athr[cur_athr] += 1\n","                except:\n","                    athr[cur_athr] = 1\n","        if 'root' in e.metaR.keys():\n","            for key in e.metaR['root'].keys():\n","                cmnt = e.comments[key]\n","                cur_athr = cmnt.author\n","                try:\n","                    athr[cur_athr] += 1\n","                except:\n","                    athr[cur_athr] = 1\n","\n","    L1_athr = dict()\n","    for x in athr:\n","        if athr[x] >= n1:\n","            L1_athr[x] = True\n","\n","    athr = dict()\n","\n","    def dfs(Map, cmntMap, athr, cid='root'):\n","        if cid == 'root':\n","            for key in Map[cid].keys():\n","                dfs(Map[cid], cmntMap, athr, key)\n","            return\n","        cur_author = cmntMap[cid].author\n","\n","        try:\n","            athr[cur_author] += len(Map[cid].keys())\n","        except:\n","            athr[cur_author] = len(Map[cid].keys())\n","\n","        for key in Map[cid].keys():\n","            dfs(Map[cid], cmntMap, athr, key)\n","\n","    for e in threads:\n","        if 'root' in e.metaL.keys():\n","            dfs(e.metaL, e.comments, athr)\n","        if 'root' in e.metaR.keys():\n","            dfs(e.metaR, e.comments, athr) \n","\n","    A = []\n","    for x in athr:\n","        if x not in user_subset:\n","            continue\n","        if athr[x] >= n2:\n","            try:\n","                z = L1_athr[x]\n","                A.append(x)\n","            except KeyError:\n","                pass\n","\n","    author_map = dict()\n","    reverse_map = [\"\"] * len(A)\n","    author_count = len(A)\n","\n","    for i in range(author_count):\n","        author_map[A[i]] = i\n","        reverse_map[i] = A[i]\n","\n","    support_matrix = [[0 for j in range(author_count)] for i in range(author_count)]\n","    dispute_matrix = [[0 for j in range(author_count)] for i in range(author_count)]\n","\n","    def dfs1(Map, cmntMap, cid='root'):\n","        if cid == 'root':\n","            for key in Map[cid].keys():\n","                dfs1(Map[cid], cmntMap, key)\n","            return\n","\n","        cur_author = cmntMap[cid].author\n","        cur_pol = cmntMap[cid].polarity\n","        \n","        if cur_author in author_map and cur_pol != 'Not Available':\n","            cur_author_id = author_map[cur_author]\n","            for key in Map[cid].keys():\n","                nxt_author = cmntMap[key].author\n","                nxt_pol = cmntMap[key].polarity\n","                if nxt_author in author_map and nxt_pol != 'Not Available':\n","                    nxt_author_id = author_map[nxt_author]\n","                    if cur_pol == nxt_pol:\n","                        support_matrix[nxt_author_id][cur_author_id] += 1\n","                    else:\n","                        dispute_matrix[nxt_author_id][cur_author_id] += 1\n","\n","        for key in Map[cid].keys():\n","            dfs1(Map[cid], cmntMap, key)\n","\n","    for e in threads:\n","        if 'root' in e.metaL:\n","            dfs1(e.metaL, e.comments)\n","        if 'root' in e.metaR:\n","            dfs1(e.metaR, e.comments)\n","\n","    support_graph = nx.DiGraph()\n","    for i in range(author_count):\n","        for j in range(author_count):\n","            if support_matrix[i][j] != 0:\n","                support_graph.add_weighted_edges_from([(i, j, support_matrix[i][j])])\n","\n","    dispute_graph = nx.DiGraph()\n","    for i in range(author_count):\n","        for j in range(author_count):\n","            if dispute_matrix[i][j] != 0:\n","                dispute_graph.add_weighted_edges_from([(i, j, dispute_matrix[i][j])])\n","    \n","    return (author_map, reverse_map, author_count, support_graph, support_matrix, dispute_graph, dispute_matrix)"],"metadata":{"id":"p5V58Hm8UhKM"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Highly diverse users"],"metadata":{"id":"ydHsPiPVVOIF"}},{"cell_type":"code","source":["# Observing variation in properties of group G with variations in n1 and n2\n","\n","thresholds = [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000]\n","thresholds_str = []\n","for x in thresholds:\n","    thresholds_str.append(str(x))\n","n = len(thresholds)\n","count = [[0 for j in range(n)] for i in range(n)]\n","support_graph_r = [[0 for j in range(n)] for i in range(n)]\n","dispute_graph_r = [[0 for j in range(n)] for i in range(n)]\n","s_scc = [[0 for j in range(n)] for i in range(n)]\n","d_scc = [[0 for j in range(n)] for i in range(n)]\n","for i in range(n):\n","    for j in range(n):\n","        try:\n","            _1, _2, cnt, support_graph, _4, dispute_graph, _6 = build_graph(high_diversity_users, thresholds[i], thresholds[j])\n","            count[i][j] = cnt\n","            support_graph_r[i][j] = nx.algorithms.reciprocity(support_graph)\n","            dispute_graph_r[i][j] = nx.algorithms.reciprocity(dispute_graph)\n","            s_scc[i][j] = nx.number_strongly_connected_components(support_graph)\n","            d_scc[i][j] = nx.number_strongly_connected_components(dispute_graph)\n","        except:\n","            pass"],"metadata":{"id":"xpOApNF0VMlK"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["fig = plt.figure()\n","ax = fig.add_subplot(111)\n","cax = ax.matshow(dispute_graph_r, interpolation='nearest')\n","fig.colorbar(cax)\n","ax.set_xticks(np.arange(n))\n","ax.set_yticks(np.arange(n))\n","ax.set_xticklabels(thresholds_str)\n","ax.set_yticklabels(thresholds_str)\n","ax.set_ylabel(\"$\\lambda$\", rotation='horizontal')\n","ax.set_xlabel(\"$\\\\rho$\")\n","plt.setp(ax.get_xticklabels(), rotation=90)"],"metadata":{"id":"Pha1_c2LV2gy"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["avgs, stds = get_stats_from_profile_data(high_diversity_users)"],"metadata":{"id":"SrVzqWGYWIg4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["avgs"],"metadata":{"id":"NaIf1N3Gat06"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["stds"],"metadata":{"id":"yKlCeq9uau76"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Moderately diverse group"],"metadata":{"id":"7bx13MdJbq-o"}},{"cell_type":"code","source":["# Observing variation in properties of group G with variations in n1 and n2\n","\n","thresholds = [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000]\n","thresholds_str = []\n","for x in thresholds:\n","    thresholds_str.append(str(x))\n","n = len(thresholds)\n","count = [[0 for j in range(n)] for i in range(n)]\n","support_graph_r = [[0 for j in range(n)] for i in range(n)]\n","dispute_graph_r = [[0 for j in range(n)] for i in range(n)]\n","s_scc = [[0 for j in range(n)] for i in range(n)]\n","d_scc = [[0 for j in range(n)] for i in range(n)]\n","for i in range(n):\n","    for j in range(n):\n","        try:\n","            _1, _2, cnt, support_graph, _4, dispute_graph, _6 = build_graph(moderate_diversity_users, thresholds[i], thresholds[j])\n","            count[i][j] = cnt\n","            support_graph_r[i][j] = nx.algorithms.reciprocity(support_graph)\n","            dispute_graph_r[i][j] = nx.algorithms.reciprocity(dispute_graph)\n","            s_scc[i][j] = nx.number_strongly_connected_components(support_graph)\n","            d_scc[i][j] = nx.number_strongly_connected_components(dispute_graph)\n","        except:\n","            pass"],"metadata":{"id":"pI0OqE1davz9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["fig = plt.figure()\n","ax = fig.add_subplot(111)\n","cax = ax.matshow(dispute_graph_r, interpolation='nearest')\n","fig.colorbar(cax)\n","ax.set_xticks(np.arange(n))\n","ax.set_yticks(np.arange(n))\n","ax.set_xticklabels(thresholds_str)\n","ax.set_yticklabels(thresholds_str)\n","ax.set_ylabel(\"$\\lambda$\", rotation='horizontal')\n","ax.set_xlabel(\"$\\\\rho$\")\n","plt.setp(ax.get_xticklabels(), rotation=90)"],"metadata":{"id":"hkydpc25bxkk"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["avgs, stds = get_stats_from_profile_data(moderate_diversity_users)"],"metadata":{"id":"eAkcUFSdcBUh"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["avgs"],"metadata":{"id":"azSE7NtOcvjw"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["stds"],"metadata":{"id":"b6MPdVR1cwYE"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Temporal variation in diversity"],"metadata":{"id":"c6qf-ybvDgKs"}},{"cell_type":"code","source":["forum_ah = dict()\n","# key: author -> year\n","# value: diversity array\n","\n","ah_score_list = dict()\n","# key: author -> year -> category\n","# value: ah score for comments"],"metadata":{"id":"piT6f5kZcxeM"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for user in user_list:\n","    ah_score_list[user] = dict()\n","    for year in range(2008, 2022):\n","        syear = str(year)\n","        ah_score_list[user][syear] = dict()\n","        for category in categories_selected:\n","            ah_score_list[user][syear][category] = list()"],"metadata":{"id":"JFbZcj9CDpef"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for category in categories_selected:\n","    for comment in tqdm(comments[category]):\n","        year = comment['time'][:4]\n","        try:\n","            int(year)\n","        except:\n","            # Time is not available for given comment\n","            continue\n","        author = comment['author']\n","        score = 1 - comment['score']\n","        ah_score_list[author][year][category].append(score)"],"metadata":{"id":"gKmonglHD0d4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["T = tuple([0, 0, 0, 0, 0, 0])\n","\n","for user in tqdm(user_list):\n","    if not (user in high_diversity_users or user in moderate_diversity_users):\n","        continue\n","    forum_ah[user] = dict()\n","    for year in range(2008, 2022):\n","        syear = str(year)\n","        which = [0 for _ in range(len(categories_selected))]\n","        for i, category in enumerate(categories_selected):\n","            try:\n","                median_score = np.median(ah_score_list[user][syear][category])\n","                if median_score >= EXTREME_AH_LOWER and median_score <= EXTREME_AH_UPPER:\n","                    which[i] = 1\n","                elif median_score >= MODERATE_AH_LOWER and median_score <= MODERATE_AH_UPPER:\n","                    which[i] = 2\n","                elif median_score >= LOW_AH_LOWER and median_score <= LOW_AH_UPPER:\n","                    which[i] = 3\n","                else:\n","                    which[i] = 0\n","            except KeyError:\n","                which[i] = 0\n","        forum_ah[user][syear] = tuple(which) # skbio.diversity.alpha.shannon(freq) / 2\n","        if (T != forum_ah[user][syear]):\n","            print(user, year)"],"metadata":{"id":"SD-HLpOrD5x8"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["pdf = pd.DataFrame(forum_ah)"],"metadata":{"id":"3YqV0Q_SJwpV"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["pdf"],"metadata":{"id":"h8nntP78J4hu"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def plot_change_direction(user_subset, year1, year2, initial_gid, final_gid):\n","    x = list(categories_selected)\n","    y = [0 for _ in range(len(x))]\n","    for user in user_subset:\n","        idx = 0\n","        for initial_gid_it, final_gid_it in zip(forum_ah[user][year1], forum_ah[user][year2]):\n","            if initial_gid_it == initial_gid and final_gid_it == final_gid:\n","                y[idx] += 1\n","            idx += 1\n","    for i in range(len(y)):\n","        y[i] = (y[i] / len(user_subset)) * 100\n","    plt.bar(x, y)\n","    plt.xlabel('Forums')\n","    plt.ylabel('% users')\n","    plt.title(f'{initial_gid}{final_gid}: {year1} -> {year2}')"],"metadata":{"id":"hPVhpEzaEIxr"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_change_direction(high_diversity_users, '2017', '2018', 3, 3)"],"metadata":{"id":"UZfjCAcBIB16"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["diversity = dict()\n","# key: author -> year\n","# value: diversity value\n","\n","ah_score_list = dict()\n","# key: author -> year -> category\n","# value: ah score for comments"],"metadata":{"id":"edFyBxa7IcFt"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for user in user_list:\n","    ah_score_list[user] = dict()\n","    for year in range(2008, 2022):\n","        syear = str(year)\n","        ah_score_list[user][syear] = dict()\n","        for category in categories_selected:\n","            ah_score_list[user][syear][category] = list()"],"metadata":{"id":"XOl6fnPrKtwM"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for category in categories_selected:\n","    for comment in tqdm(comments[category]):\n","        year = comment['time'][:4]\n","        try:\n","            int(year)\n","        except:\n","            # Time is not available for given comment\n","            continue\n","        author = comment['author']\n","        score = 1 - comment['score']\n","        ah_score_list[author][year][category].append(score)"],"metadata":{"id":"j2J9ZbfBKvgV"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for user in tqdm(user_list):\n","    if not (user in high_diversity_users):#or user in moderate_diversity_users):\n","        continue\n","    diversity[user] = dict()\n","    for year in range(2008, 2022):\n","        syear = str(year)\n","        freq = [0, 0, 0, 0]\n","        for category in categories_selected:\n","            try:\n","                median_score = np.median(ah_score_list[user][syear][category])\n","                if median_score >= EXTREME_AH_LOWER and median_score <= EXTREME_AH_UPPER:\n","                    freq[1] += 1\n","                elif median_score >= MODERATE_AH_LOWER and median_score <= MODERATE_AH_UPPER:\n","                    freq[2] += 1\n","                elif median_score >= LOW_AH_LOWER and median_score <= LOW_AH_UPPER:\n","                    freq[3] += 1\n","                else:\n","                    freq[0] += 1\n","            except KeyError:\n","                freq[0] += 1\n","        diversity[user][syear] = freq #skbio.diversity.alpha.shannon(freq) / 2"],"metadata":{"id":"cy9zG3HtKymg"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for k, v in diversity.items():\n","    for k1, v1 in v.items():\n","        if (v1 != [6, 0, 0, 0]):\n","            print(k, k1, v1)"],"metadata":{"id":"U1L8Bs2BLHkw"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def plot_diversity_users():\n","    x = [str(year) for year in range(2008, 2022)]\n","    y1 = [np.average([diversity[user][year] for user in high_diversity_users]) for year in x]\n","    y2 = [np.average([diversity[user][year] for user in moderate_diversity_users]) for year in x]\n","    plt.plot(x, y1, label='High')\n","    plt.plot(x, y2, label='Moderate')\n","    plt.xlabel('Year')\n","    plt.ylabel('Diversity')\n","    plt.legend()\n","    plt.title(f'Avg. Shannon diveristy index for highly and moderately diverse group')"],"metadata":{"id":"PWEK-J2YK5m3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_diversity_users()"],"metadata":{"id":"FDEstfjYLAZG"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"n37_v7T7LB7Y"},"execution_count":null,"outputs":[]}]}