30-User-Overlap-Across-Categories-CreateDebate.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"private_outputs":true,"provenance":[],"authorship_tag":"ABX9TyP5fa+KdKQAy6yFDeohwqTV"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":null,"metadata":{"id":"MI09sOh_yB7g"},"outputs":[],"source":["from google.colab import drive\n","drive.mount('/content/gdrive', force_remount=True)"]},{"cell_type":"code","source":["!git clone https://github.com/utkarsh512/CreateDebate-Scraper.git"],"metadata":{"id":"b002uIR2zMeA"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["%cd CreateDebate-Scraper/src/nested/"],"metadata":{"id":"edFMwmLvzQhH"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from thread import Thread, Comment\n","import pickle as pkl\n","import numpy as np\n","from tqdm import tqdm\n","from copy import deepcopy\n","import matplotlib.pyplot as plt\n","import matplotlib.colors as colors"],"metadata":{"id":"fGFhaurjzTqp"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import numpy as np\n","import pandas as pd\n","import matplotlib\n","from   matplotlib import pyplot as plt\n","import seaborn as sns"],"metadata":{"id":"XOyvSySlfON3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# sns.set(style='darkgrid')\n","# matplotlib.rcParams['figure.dpi'] = 120\n","matplotlib.rcParams['font.size'] = 18\n","# matplotlib.rcParams['figure.figsize'] = (10, 5)\n","matplotlib.rcParams['pdf.fonttype'] = 42\n","matplotlib.rcParams['ps.fonttype'] = 42"],"metadata":{"id":"ZSZ9-oRGfRnh"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["data_dir = {'politics': '/content/gdrive/MyDrive/DL/CreateDebate/politics2/threads.log',\n","            'religion': '/content/gdrive/MyDrive/DL/CreateDebate/religion/threads.log',\n","            'world': '/content/gdrive/MyDrive/DL/CreateDebate/world/threads.log',\n","            'science': '/content/gdrive/MyDrive/DL/CreateDebate/science/threads.log',\n","            'law': '/content/gdrive/MyDrive/DL/CreateDebate/law/threads.log',\n","            'technology': '/content/gdrive/MyDrive/DL/CreateDebate/technology/threads.log'}"],"metadata":{"id":"eHpQJRd0yRnN"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["threads = dict()\n","\n","for k, v in data_dir.items():\n","    threads[k] = list()\n","    with open(v, 'rb') as f:\n","        try:\n","            while True:\n","                threads[k].append(pkl.load(f))\n","        except EOFError:\n","            pass"],"metadata":{"id":"RnbkH4zMzCnB"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["author_comment_count = dict()\n","\n","for k, v in tqdm(threads.items()):\n","    author_comment_count[k] = dict()\n","\n","    # looping over threads\n","    for e in v:\n","\n","        # looping over comments\n","        for cid, cmnt in e.comments.items():\n","            athr = cmnt.author\n","            try:\n","                author_comment_count[k][athr] += 1\n","            except:\n","                author_comment_count[k][athr] = 1"],"metadata":{"id":"JN1tJ7i-znvC"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def get_author_set(category, n):\n","    res = set()\n","    for k, v in author_comment_count[category].items():\n","        if v >= n:\n","            res.add(k)\n","    return res"],"metadata":{"id":"FQ7qf-iL0k_Q"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["author_set = dict()\n","\n","for category in tqdm(author_comment_count.keys()):\n","    author_set[category] = dict()\n","    for n in [1, 2, 5, 10, 20, 50, 100, 200, 500]:\n","        author_set[category][n] = get_author_set(category, n)"],"metadata":{"id":"BOL_TuXv1JXp"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def get_overlap(category, n):\n","    a = deepcopy(author_set['politics'][n])\n","    b = deepcopy(author_set[category][n])\n","    i = (a & b)\n","    return 100 * len(i) / len(b)"],"metadata":{"id":"XN9TPKD018Gd"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["valN = [1, 2, 5, 10, 20, 50, 100, 200]\n","N = len(valN)\n","C = len(threads) - 1"],"metadata":{"id":"v6KyFR024JFc"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["overlap = [[0 for j in range(N)] for i in range(C)]"],"metadata":{"id":"Q7NFyaDs43Fj"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for i, category in enumerate(['religion', 'world', 'science', 'law', 'technology']):\n","    for j, n in enumerate(valN):\n","        try:\n","            overlap[i][j] = get_overlap(category, n)\n","        except:\n","            overlap[i][j] = 0"],"metadata":{"id":"CnXYWEgX5HmO"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from pprint import pprint\n","pprint(overlap)"],"metadata":{"id":"-66FjG4UgAKw"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["fig, ax = plt.subplots()\n","sns.heatmap(overlap, cmap=\"crest\", ax=ax, square=True)\n","ax.set_xticks(np.arange(N), labels=valN)\n","ax.set_yticks(np.arange(C), labels=['religion', 'world', 'science', 'law', 'tech'])\n","# ax.set_xticklabels(valN)\n","# ax.set_yticklabels(['religion', 'world', 'science', 'law', 'tech'])\n","ax.set_xlabel(\"# comments\")\n","plt.setp(ax.get_xticklabels(), rotation=90)\n","plt.setp(ax.get_yticklabels(), rotation=0)\n","fig.savefig('overlap_with_politics.pdf', format='pdf', bbox_inches=\"tight\")"],"metadata":{"id":"kgqDhEcqfoMo"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["fig = plt.figure()\n","ax = fig.add_subplot(111)\n","cax = ax.matshow(overlap, interpolation='nearest')\n","fig.colorbar(cax)\n","ax.set_xticks(np.arange(N))\n","ax.set_yticks(np.arange(C))\n","ax.set_xticklabels(valN)\n","ax.set_yticklabels(['religion', 'world', 'science', 'law', 'tech'])\n","ax.set_xlabel(\"# comments\")\n","plt.setp(ax.get_xticklabels(), rotation=90)\n","plt.savefig('overlap_with_politics.eps', format='eps')"],"metadata":{"id":"H591XDgk52TZ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["threadsByYear = dict()\n","errorCount = 0\n","\n","for e in threads['politics']:\n","    try:\n","        threadTime = list(e.comments.items())[0][1].time\n","        if threadTime == 'Not Available':\n","            continue \n","        threadTime = threadTime[:4]\n","        try:\n","            threadsByYear[threadTime].append(e)\n","        except:\n","            threadsByYear[threadTime] = list()\n","            threadsByYear[threadTime].append(e)\n","    except:\n","        assert len(list(e.comments.items())) == 0\n","        errorCount += 1\n","print(errorCount)"],"metadata":{"id":"Iq1HNM6m6d7A"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["threadsBefore = list()\n","threadsAfter = list()"],"metadata":{"id":"Qa69JaDb_jTm"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for key in ['2016', '2015', '2014', '2013', '2012', '2011', '2010', '2009', '2008']:\n","    for e in threadsByYear[key]:\n","        threadsBefore.append(e)\n","\n","for key in ['2021', '2020', '2019', '2018', '2017']:\n","    for e in threadsByYear[key]:\n","        threadsAfter.append(e)"],"metadata":{"id":"IBZ6ysOZ_n05"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["authorBefore = dict()\n","authorAfter = dict()"],"metadata":{"id":"qwvoLpYn_qe-"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for e in threadsBefore:\n","    for cid, cmnt in e.comments.items():\n","        try:\n","            authorBefore[cmnt.author] += 1\n","        except:\n","            authorBefore[cmnt.author] = 1"],"metadata":{"id":"wG2i8JE3_9a-"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for e in threadsAfter:\n","    for cid, cmnt in e.comments.items():\n","        try:\n","            authorAfter[cmnt.author] += 1\n","        except:\n","            authorAfter[cmnt.author] = 1"],"metadata":{"id":"5HsY4uwAAKsa"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["authorBefore200 = set()\n","authorBefore500 = set()\n","authorAfter200 = set()\n","authorAfter500 = set()"],"metadata":{"id":"KTzpQZ5ZA2hr"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for k, v in authorBefore.items():\n","    if v >= 500:\n","        authorBefore500.add(k)\n","    elif v >= 200:\n","        authorBefore200.add(k)"],"metadata":{"id":"o5nm29Y1BGde"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for k, v in authorAfter.items():\n","    if v >= 500:\n","        authorAfter500.add(k) \n","    elif v >= 200:\n","        authorAfter200.add(k)"],"metadata":{"id":"Vw8LZ7rKBRx0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["len(authorBefore200)"],"metadata":{"id":"FXscSKtsBeGF"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["len(authorBefore500)"],"metadata":{"id":"SZT968I9BjoW"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["len(authorAfter200)"],"metadata":{"id":"IpSxpzMzBl-F"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["len(authorAfter500)"],"metadata":{"id":"yxxcD4XNBoGm"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["a500 = authorBefore500 & authorAfter500"],"metadata":{"id":"tYrpX_AyBqTT"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["len(a500)"],"metadata":{"id":"vCyXgXJWB1Th"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for i in authorBefore200:\n","    print(i)"],"metadata":{"id":"L1JibkICB2i_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for i in authorAfter200:\n","    print(i)"],"metadata":{"id":"B035CdjNCEV4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["len(authorBefore200 & authorAfter200)"],"metadata":{"id":"YkV7bYn2CLzT"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["len(authorBefore200 & authorAfter500)"],"metadata":{"id":"4Docvk6uCVb0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"HBTcGWPhC3zr"},"execution_count":null,"outputs":[]}]}