15a-Annotation-for-Create-Debate-on-Prolific.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"15a-Annotation-for-Create-Debate-on-Prolific.ipynb","private_outputs":true,"provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyNmhx2p8H8bkg42aGqqIW5n"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","metadata":{"id":"7TdpXu8-97Ao"},"source":["# Comment Annotation on Prolific\n","* __Objective__: Create mini-batches (of 20) of comments to be used for annotation on Prolific\n","* __File Management__: Google Drive\n","* __Runtime Type__: GPU"]},{"cell_type":"markdown","metadata":{"id":"iuzhOQozBz0o"},"source":["## Training BERT Model in PyTorch"]},{"cell_type":"code","metadata":{"id":"xcnZlewU91J1"},"source":["!nvidia-smi"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Y8c8vb_p-cGK"},"source":["from google.colab import drive\n","drive.mount('/content/gdrive', force_remount=True)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"b0NGrHPW-gVc"},"source":["import os\n","\n","def read_split(dir):\n","    texts = []\n","    labels = []\n","    with open(dir, 'r', encoding='utf-8') as f:\n","        lines = f.readlines()[1:]\n","        for line in lines:\n","            line = line.strip().split(\",\")\n","            assert(len(line) == 2)\n","            label = line[0].strip()\n","            text = line[1].strip()\n","            texts.append(text)\n","            if label == 'AH':\n","                labels.append(1)\n","            else:\n","                labels.append(0)\n","    return texts, labels"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"lRSg4NpG-sqa"},"source":["train_texts, train_labels = read_split('/content/gdrive/MyDrive/DL/dataset/pytorch/train.csv')\n","test_texts, test_labels = read_split('/content/gdrive/MyDrive/DL/dataset/pytorch/test.csv')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"a2kLmtSv-zsm"},"source":["!pip install transformers\n","from transformers import BertTokenizer, BertForSequenceClassification"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"8TaRj9MC-8k6"},"source":["tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"8BHU_9KM_Klc"},"source":["max_seq_length = 64\n","train_encodings = tokenizer(train_texts, truncation=True, max_length=max_seq_length, padding=\"max_length\")\n","test_encodings = tokenizer(test_texts, truncation=True, max_length=max_seq_length, padding=\"max_length\")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Fqmvipbq_TAf"},"source":["import torch\n","\n","class CustomDataset(torch.utils.data.Dataset):\n","    def __init__(self, encodings, labels):\n","        self.encodings = encodings\n","        self.labels = labels\n","\n","    def __getitem__(self, idx):\n","        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n","        item['labels'] = torch.tensor(self.labels[idx])\n","        return item\n","\n","    def __len__(self):\n","        return len(self.labels)\n","\n","train_dataset = CustomDataset(train_encodings, train_labels)\n","test_dataset = CustomDataset(test_encodings, test_labels)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"h5kjNg3S_a1y"},"source":["from torch.utils.data import DataLoader\n","from transformers import AdamW"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"gW9nYiSG_ejq"},"source":["device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ejyZifQN_hTO"},"source":["model = BertForSequenceClassification.from_pretrained('bert-base-uncased')\n","model.to(device)\n","model.train()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"pr2XinEX_i5p"},"source":["train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)\n","optim = AdamW(model.parameters(), lr=5e-5)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"cnS3Gtl5_2JE"},"source":["from tqdm import tqdm"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"F_ymX9B__4tA"},"source":["for epoch in range(4):\n","    for batch in tqdm(train_loader):\n","        optim.zero_grad()\n","        input_ids = batch['input_ids'].to(device)\n","        attention_mask = batch['attention_mask'].to(device)\n","        labels = batch['labels'].to(device)\n","        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n","        loss = outputs[0]\n","        loss.backward()\n","        optim.step()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"HHB_7LZz_-Go"},"source":["import numpy as np\n","!pip install datasets\n","from datasets import load_metric"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"-LIasZ-zBgs7"},"source":["metric= load_metric(\"accuracy\")\n","model.eval()\n","eval_loader = DataLoader(test_dataset, batch_size=64, shuffle=False)\n","for batch in eval_loader:\n","    batch = {k: v.to(device) for k, v in batch.items()}\n","    with torch.no_grad():\n","        outputs = model(**batch)\n","\n","    logits = outputs.logits\n","    predictions = torch.argmax(logits, dim=-1)\n","    metric.add_batch(predictions=predictions, references=batch[\"labels\"])\n","\n","metric.compute()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"coFhoiJcBnbl"},"source":["import os\n","\n","# Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()\n","\n","output_dir = './utkbert/'\n","\n","# Create output directory if needed\n","if not os.path.exists(output_dir):\n","    os.makedirs(output_dir)\n","\n","print(\"Saving model to %s\" % output_dir)\n","\n","# Save a trained model, configuration and tokenizer using `save_pretrained()`.\n","# They can then be reloaded using `from_pretrained()`\n","model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training\n","model_to_save.save_pretrained(output_dir)\n","tokenizer.save_pretrained(output_dir)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"LlpAsp-hB8Ns"},"source":["## Extracting Attention Scores"]},{"cell_type":"code","metadata":{"id":"fKxNcQMMBwta"},"source":["from transformers import BertModel, BertTokenizer"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"z3xYZnTbCKBN"},"source":["model_version = 'utkbert'\n","do_lower_case = True\n","model = BertModel.from_pretrained(model_version, output_attentions=True)\n","tokenizer = BertTokenizer.from_pretrained(model_version, do_lower_case=do_lower_case)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"knjn2c38CNn0"},"source":["def attention_scores(text, layers=None, heads=None):\n","    sentence_a = text\n","    inputs = tokenizer.encode_plus(sentence_a, None, return_tensors='pt', add_special_tokens=True)\n","    input_ids = inputs['input_ids']\n","    attention = model(input_ids)[-1]\n","    input_id_list = input_ids[0].tolist() # Batch index 0\n","    tokens = tokenizer.convert_ids_to_tokens(input_id_list) \n","    sz = len(tokens)\n","    matrix = [0 for j in range(sz)]\n","    if layers is None:\n","        layers = [x for x in range(12)]\n","    if heads is None:\n","        heads = [x for x in range(12)]\n","    for layer in layers:\n","        for head in heads:\n","            for j in range(sz):\n","                matrix[j] += attention[layer][0, head, 0, j].item()\n","    for j in range(sz):\n","        matrix[j] = (matrix[j]) / (len(layers) * len(heads))\n","    return (tokens, matrix)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"5bVqJdtCCudC"},"source":["## Loading Create Debate Comments"]},{"cell_type":"code","metadata":{"id":"zAadnmpqChgh"},"source":["!git clone https://github.com/utkarsh512/CreateDebate-Scraper.git"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"-0DBKOIHCx_O"},"source":["%cd CreateDebate-Scraper/src/nested/"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"xv6Aj4ItC28t"},"source":["import re\n","import pickle\n","from thread import Thread, Comment\n","import numpy as np\n","import pandas as pd"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ls1NOUpHC7fE"},"source":["dir = '/content/gdrive/MyDrive/DL/CreateDebate/Politics/{}.log'\n","comments_with_score = list()\n","\n","with open(dir.format('comments_with_score'), 'rb') as f:\n","    comments_with_score = pickle.load(f)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"S4TfJVXCDX8F"},"source":["reader_addr = '/content/gdrive/MyDrive/DL/CreateDebate/Politics/threads.log'\n","reader = open(reader_addr, 'rb')\n","threads = list()\n","e = Thread()\n","\n","try:\n","    while True:\n","        e = pickle.load(reader)\n","        threads.append(e)\n","except:\n","    reader.close()\n","\n","authors = dict()\n","tot_comment_cnt = 0\n","idx = -1\n","\n","for thread in threads:\n","    idx += 1\n","    for key in thread.comments.keys():\n","        tot_comment_cnt += 1\n","        comment = thread.comments[key]\n","        cur_text = comment.body\n","        cur_author = comment.author\n","        url = thread.url\n","        try:\n","            authors[cur_author].append((cur_text, url, idx))\n","        except:\n","            authors[cur_author] = list()\n","            authors[cur_author].append((cur_text, url, idx))\n","\n","cur_author_cnt = 0\n","cur_comment_cnt = 0\n","tot_author_cnt = len(authors.keys())\n","comments_with_url = list()\n","\n","for author in authors.keys():\n","    cur_author_cnt += 1\n","    for i in range(len(authors[author])):\n","        cur_comment_cnt += 1\n","        text = [authors[author][i][0]]\n","        url = authors[author][i][1]\n","        idx = authors[author][i][2]\n","        comments_with_url.append((url, text, idx))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"cPxGkPZ8DaYb"},"source":["idx = np.random.randint(len(comments_with_score))\n","print(comments_with_score[idx][1][0])\n","print(comments_with_url[idx][1][0])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"d0KN6fqAD-xr"},"source":["v = list()\n","for i in range(len(comments_with_score)):\n","    score = comments_with_score[i][0]\n","    text = comments_with_score[i][1]\n","    url = comments_with_url[i][0]\n","    idx = comments_with_url[i][2]\n","    v.append((score, text, url, idx))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"EeXCf-DSEMvz"},"source":["v = sorted(v)\n","top_ah_comments = []\n","top_none_comments = []\n","\n","for i in range(1000):\n","    top_ah_comments.append(v[i])\n","    top_none_comments.append(v[-(i + 1)])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Edu2pcQQER8l"},"source":["# Random shuffle of top comments\n","np.random.RandomState(seed=42).shuffle(top_ah_comments)\n","np.random.RandomState(seed=42).shuffle(top_none_comments)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"NPEiALuSElXq"},"source":["small_ah_comments = []\n","medium_ah_comments = []\n","large_ah_comments = []\n","\n","small_none_comments = []\n","medium_none_comments = []\n","large_none_comments = []\n","\n","for x in top_ah_comments:\n","    c = len(x[1][0].strip().split())\n","    if c < 38:\n","        small_ah_comments.append(x)\n","    elif c >= 80:\n","        large_ah_comments.append(x)\n","    else:\n","        medium_ah_comments.append(x)\n","\n","for x in top_none_comments:\n","    c = len(x[1][0].strip().split())\n","    if c < 34:\n","        small_none_comments.append(x)\n","    elif c >= 72:\n","        large_none_comments.append(x)\n","    else:\n","        medium_none_comments.append(x)\n","\n","total_groups_possible = int(len(small_ah_comments) / 5)\n","total_groups_possible = min(total_groups_possible, int(len(small_none_comments) / 5))\n","total_groups_possible = min(total_groups_possible, int(len(large_ah_comments) / 2))\n","total_groups_possible = min(total_groups_possible, int(len(large_none_comments) / 2))\n","total_groups_possible = min(total_groups_possible, int(len(medium_ah_comments) / 3))\n","total_groups_possible = min(total_groups_possible, int(len(medium_none_comments) / 3))\n","print(total_groups_possible)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"nIhlexOpEw40"},"source":["groups = []\n","\n","small_ah_count = 0\n","small_none_count = 0\n","medium_ah_count = 0\n","medium_none_count = 0\n","large_ah_count = 0\n","large_none_count = 0\n","\n","for i in range(96):\n","    group = []\n","    for j in range(5):\n","        group.append(small_ah_comments[small_ah_count])\n","        small_ah_count += 1\n","        group.append(small_none_comments[small_none_count])\n","        small_none_count += 1\n","    for j in range(3):\n","        group.append(medium_ah_comments[medium_ah_count])\n","        medium_ah_count += 1\n","        group.append(medium_none_comments[medium_none_count])\n","        medium_none_count += 1\n","    for j in range(2):\n","        group.append(large_ah_comments[large_ah_count])\n","        large_ah_count += 1\n","        group.append(large_none_comments[large_none_count])\n","        large_none_count += 1  \n","    np.random.RandomState(seed=42).shuffle(group)\n","    groups.append(group)\n","    \n","np.random.RandomState(seed=42).shuffle(groups)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"I6mIAzOPE_mq"},"source":["delimiter = '@#@#@'\n","delimiter2 = '##$$##@@'\n","\n","# Each group is balanced class-wise as well as length-wise and has 20 comments\n","\n","low, high = 0, 1  # index of groups which will be used in the form\n","group_id = 5\n","\n","addr = 'https://utkarsh512.github.io/pages/createdebate_{}/comment_{}.txt'\n","\n","ctr = 0\n","\n","with open('/content/gdrive/MyDrive/DL/CreateDebate/Politics/CommentsForGoogleForm.txt', 'w', encoding='utf-8') as f:\n","    content = []\n","    for i in range(low, high):\n","        content_ = []\n","        for x in groups[i]:\n","            content_.append(f'{x[1][0].strip()}{delimiter2}{addr.format(group_id, ctr + 1)}')\n","            ctr += 1\n","        content_ = delimiter.join([x for x in content_])\n","        content.append(content_)\n","    content = delimiter.join([x for x in content])\n","    content = re.sub(\"\\s+\", \" \", content)\n","    f.write(content)\n","    print(content)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"DvpkUhwyFihp"},"source":["# Constructing static webpages for comments for context\n","\n","addr = '/content/gdrive/MyDrive/DL/CreateDebate/Politics/staticPages/comment_{}.txt'\n","\n","ctr = 0\n","\n","for i in range(low, high):\n","    for x in groups[i]:\n","        with open(addr.format(ctr + 1), 'w', encoding='utf-8') as f:\n","            f.write(str(threads[x[3]]))\n","            ctr += 1"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"XZt4R4KXG5BN"},"source":["def clean_array(w, a):\n","    W = []\n","    A = []\n","    for i in range(len(w)):\n","        if (w[i].startswith('##')):\n","            W[len(W) - 1] += w[i][2:]\n","            A[len(A) - 1] = (A[len(A) - 1] + a[i]) / 2\n","        else:\n","            W.append(w[i])\n","            A.append(a[i])\n","    return (W, A)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"lTq-eab-Jnb1"},"source":["def sanitize(x):\n","    x = x.lower()\n","    x = re.sub(\"\\s+\", \" \", x)  # converting space-like character to single white space\n","    x = ''.join([y for y in x if y.isalnum() or y ==' '])\n","    return x"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ze8BAGFmFzaZ"},"source":["delim3 = '$#$#$#$#$#$#@@@@'\n","delim4 = '@#$$#@@#@@#'\n","\n","def top_three_tokens(text):\n","    text = sanitize(text)\n","    words, attentions = attention_scores(text)\n","    words = words[1:-1] # Remove start and end tags\n","    attentions = attentions[1:-1]\n","    words, attentions = clean_array(words, attentions)\n","    top_tokens = list()\n","    for i in range(len(words)):\n","        top_tokens.append((attentions[i], i))\n","    top_tokens = sorted(top_tokens, reverse=True)\n","    ind = [0]\n","    cur = 1\n","    while len(ind) < 3:\n","        take = True\n","        for ids in ind:\n","            take = take and abs(top_tokens[ids][1] - top_tokens[cur][1]) > 2\n","        if take:\n","            ind.append(cur)\n","        cur += 1\n","    top_three_tkns = list()\n","    for i in range(3):\n","        idx = top_tokens[ind[i]][1]\n","        cur = ''\n","        if idx > 1:\n","            cur += '...'\n","        if idx != 0:\n","            cur += words[idx - 1] + ' '\n","        cur += words[idx]\n","        if idx != len(words) - 1:\n","            cur += ' ' + words[idx + 1]\n","        if idx < len(words) - 2:\n","            cur += '...'\n","        top_three_tkns.append(cur)\n","    return top_three_tkns"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"WRFqW6W6JSIg"},"source":["groups[0][9][1][0]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"WmInVNh0JGH4"},"source":["top_three_tokens(groups[0][4][1][0])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"_H5V5TpzJY7m"},"source":["# Constructing static webpages for comments for context\n","\n","addr = '/content/gdrive/MyDrive/DL/CreateDebate/Politics/staticPages/trigrams.txt'\n","\n","content = []\n","for i in range(low, high):\n","    for x in groups[i]:\n","        trigrams = top_three_tokens(x[1][0])\n","        np.random.shuffle(trigrams)\n","        content_ = delim3.join(trigrams)\n","        print(content_)\n","        content.append(content_)\n","content = delim4.join(content)\n","with open(addr, 'w', encoding='utf-8') as f:\n","    f.write(content)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"c3lb3EFF_0lC"},"source":["## Annotation Analysis (Accuracy Metrics)"]},{"cell_type":"code","metadata":{"id":"cAFjb_asVRhq"},"source":["cols = ['Do you think this is an ad-hominem comment?']\n","for i in range(1, 20):\n","    cols.append(cols[0] + '.' + str(i))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"sBmYBNBdXxT9"},"source":["scores = []\n","y_pred = []\n","\n","low = 1\n","high = 6\n","for i in range(low, high):\n","    for x in groups[i]:\n","        scores.append(x[0])\n","\n","for i in range(len(scores)):\n","    if scores[i] < 0.5:\n","        y_pred.append(1)\n","    else:\n","        y_pred.append(0)\n","y_pred = np.array(y_pred)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"UY6rWnawVI-3"},"source":["addr = '/content/gdrive/MyDrive/DL/CreateDebate/Politics/annotations/batch{}.csv'"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"aD_iKpPSVWcP"},"source":["y = []\n","for i in range(1, 6):\n","    df = pd.read_csv(addr.format(i))\n","    for x in cols:\n","        lbl = df[x].value_counts().idxmax()\n","        if lbl == 'Yes':\n","            y.append(1)\n","        else:\n","            y.append(0)\n","y = np.array(y)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"FeCNYi7CVtL3"},"source":["y.shape"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ZW-A-P2bV5YS"},"source":["y_pred.shape"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"eOUvWOrEV6kn"},"source":["from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"h914ex0v-cyd"},"source":["accuracy_score(y, y_pred)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"lKduRfQv-icU"},"source":["precision_score(y, y_pred)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"mez7v5hH-m1a"},"source":["recall_score(y, y_pred)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"QDTWlMZ3-pmV"},"source":["f1_score(y, y_pred)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"RM7Vxy12AaXO"},"source":["## Annotation Analysis (Annotator Agreement)"]},{"cell_type":"code","metadata":{"id":"-5gjPXYzDPD2"},"source":["from sklearn.metrics import cohen_kappa_score\n","\n","kappa = []\n","acc = []"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"n-Q_6SAM-r_V"},"source":["for i in range(6):\n","    df = pd.read_csv(addr.format(i))\n","    df = df[cols]\n","    for col in cols:\n","        df[col] = df[col].apply(lambda z: 1 if z == 'Yes' else 0)\n","    ann = df.to_numpy()\n","    scores = []\n","    y_pred = []\n","\n","    low = i\n","    high = i + 1\n","    for i in range(low, high):\n","        for x in groups[i]:\n","            scores.append(x[0])\n","\n","    for i in range(len(scores)):\n","        if scores[i] < 0.5:\n","            y_pred.append(1)\n","        else:\n","            y_pred.append(0)\n","    y_pred = np.array(y_pred)\n","    kappa_scores = []\n","    accuracy_scores = []\n","    ann_count = 3 if i else 5\n","    for i in range(ann_count):\n","        kappa_scores.append(cohen_kappa_score(y_pred, ann[i]))\n","        accuracy_scores.append(accuracy_score(y_pred, ann[i]))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"OXwDAiv6DZN4"},"source":["for z in kappa:\n","    print(z)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"bW_UR2tpFLdo"},"source":["## Annotation Analysis (Common Annotators)"]},{"cell_type":"markdown","metadata":{"id":"0yo0JXCHFLav"},"source":[""]},{"cell_type":"code","metadata":{"id":"vh1g0SLLD57s"},"source":["annotators = []\n","for i in range(1, 5):\n","    df = pd.read_csv(addr.format(i))\n","    annotators.append(set(df['Enter your Prolific ID'].to_numpy().squeeze().tolist()))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ZoIJUeeKFvBK"},"source":["print(annotators)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"e9p8-iNRFw2s"},"source":["comm = np.zeros((4, 4))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"a--37k8XF7a_"},"source":["for i in range(0, 4):\n","    for j in range(0, 4):\n","        comm[i][j] = len(annotators[i] & annotators[j])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"7niIEaRkGGAs"},"source":["comm"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"6vcPy5IVGH5b"},"source":["for z in comm.astype(int).tolist():\n","    print(z)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"RguygEj8Gwmd"},"source":[""],"execution_count":null,"outputs":[]}]}