code/WEAT_analysis_peer_reviews.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0aacb818",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "import string\n",
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3da972f4",
   "metadata": {},
   "source": [
    "# Dataset 1a: German → English Argumentation Reviews\n",
    "\n",
    "1. Train a German GloVE model on these embeddings from column 'review'\n",
    "2. WEAT Analysis\n",
    "3. Word Co-occurence analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8fbe055",
   "metadata": {},
   "outputs": [],
   "source": [
    "german_stop_words = stopwords.words('german')\n",
    "eng_stop_words = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aab05e00",
   "metadata": {},
   "outputs": [],
   "source": [
    "def stop_word_removal(x):\n",
    "    token = x.split()\n",
    "    cleaned = ' '.join([w for w in token if not w in german_stop_words])\n",
    "    return cleaned.replace('.', '\\n')\n",
    "\n",
    "def eng_stop_word_removal(x):\n",
    "    token = x.split()\n",
    "    cleaned = ' '.join([w for w in token if not w in eng_stop_words])\n",
    "    return cleaned.replace('.', '\\n')\n",
    "\n",
    "def clean_text(text):\n",
    "    # remove numbers\n",
    "    text_nonum = re.sub(r'\\d+', '', text)\n",
    "    # remove punctuations and convert characters to lower case\n",
    "    text_nopunct = \"\".join([char.lower() for char in text_nonum]) \n",
    "    # substitute multiple whitespace with single whitespace\n",
    "    # Also, removes leading and trailing whitespaces\n",
    "    text_no_doublespace = re.sub('\\s+', ' ', text_nopunct).strip()\n",
    "    return text_no_doublespace\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d3ac18d",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'background/data/'\n",
    "all_files = [i for i in os.listdir('background/data/') if i.split('_')[-1] >= '2017']\n",
    "q_constructive = []\n",
    "q_critical = []\n",
    "q_quality = []\n",
    "q_helpful = []\n",
    "gender = []\n",
    "age = []\n",
    "reviewer_id = []\n",
    "reviewee_id = []\n",
    "for file in all_files:\n",
    "    year = float(file.split('_')[-1].split('.')[0])\n",
    "    data_file = pd.read_csv(data_path + file)\n",
    "    gender.extend(data_file['D2'])\n",
    "    age.extend(year - data_file['D1'])\n",
    "    reviewer_id.extend(data_file['review'])\n",
    "    reviewee_id.extend(data_file['revieweeIDnum'])\n",
    "    q_constructive.extend(data_file['Q_constructive_suggestions'])\n",
    "    q_critical.extend(data_file['Q_critical_aspects'])\n",
    "    q_quality.extend(data_file['Q_high_quality'])\n",
    "    q_helpful.extend(data_file['Q_was_helpful'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0decfb5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "gender_male = np.array(gender) == 'm'\n",
    "gender_female = np.array(gender) == 'f'\n",
    "\n",
    "sum(gender_female)/(sum(gender_male) + sum(gender_female))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adee1251",
   "metadata": {},
   "outputs": [],
   "source": [
    "age = np.array(age)\n",
    "age = age[~np.isnan(age)]\n",
    "np.std(age)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2b1fc30",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.mean(review_lens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db54ecd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "high_quality_mask = np.array(q_quality) > 5\n",
    "high_critical_mask = np.array(q_critical) > 5\n",
    "high_constructive_mask = np.array(q_constructive) > 5\n",
    "high_help_mask = np.array(q_helpful) > 5\n",
    "gender_male = np.array(gender) == 'm'\n",
    "gender_female = np.array(gender) == 'f'\n",
    "\n",
    "def save_to_csv(arr, topic):\n",
    "    pd.Series(arr).to_csv(topic+'.csv', index=False)\n",
    "\n",
    "save_to_csv(gender_male, 'b_gender_male')\n",
    "save_to_csv(gender_female, 'b_gender_female')\n",
    "save_to_csv(high_quality_mask, 'b_quality')\n",
    "save_to_csv(high_critical_mask, 'b_critical')\n",
    "save_to_csv(high_constructive_mask, 'b_constructive')\n",
    "save_to_csv(high_help_mask, 'b_helpfulness')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06498c79",
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus_quality = corpus_reviews[np.array(q_quality) > 5]\n",
    "corpus_critical = corpus_reviews[np.array(q_critical) > 5]\n",
    "corpus_constructive = corpus_reviews[np.array(q_constructive) > 5]\n",
    "corpus_helpful = corpus_reviews[np.array(q_helpful) > 5]\n",
    "corpus_male = corpus_reviews[gender_male]\n",
    "corpus_female = corpus_reviews[gender_female]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b46682d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_corpus(corpus_reviews, filename='corpus_reviews_glove.txt'):\n",
    "    corpus_reviews = corpus_reviews.apply(stop_word_removal)\n",
    "    corpus_reviews = corpus_reviews.apply(clean_text)\n",
    "    full_review = '\\n'.join(corpus_reviews)\n",
    "    with open(filename, \"w\") as text_file:\n",
    "        text_file.write(full_review)\n",
    "\n",
    "clean_corpus(corpus_male, 'male_corpus.txt')\n",
    "clean_corpus(corpus_female, 'female_corpus.txt')\n",
    "clean_corpus(corpus_constructive, 'low_constructive_corpus.txt')\n",
    "clean_corpus(corpus_helpful, 'low_helpful_corpus.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88b9e02a",
   "metadata": {},
   "source": [
    "# Generate GloVE models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77e2f5d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd GloVE\n",
    "!bash GloVe_training.sh"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4a09ee6",
   "metadata": {},
   "source": [
    "# WEAT Co-occurence Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1b57dba",
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m spacy download de_core_news_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a021e2d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "!python weat_cooccurrence_analysis_german.py \\\n",
    "    --data \"reviews_glove.txt\" \\\n",
    "    --output \"output/weat_cooccurrence_analysis\" \\\n",
    "    --processing_cores 7 \\\n",
    "    --tests 1 2 3 4 5 6 7 8 9 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24efad4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "!python accumulate_cooccurrence_counts.py \\\n",
    "    --input \"output/weat_cooccurrence_analysis/weat-cooccurrence-analysis_results-20220419163143.json\" \\\n",
    "    --weat_lexicons \"sbeval/tests/weat_tests_german.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6ccccc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "!bash convert_glove_to_word2vec_format.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e68bf46",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "vectors = [r for r in os.listdir('GloVE/glove_models_trained/') if '_vectors' in r]\n",
    "vectors = ['full_corpus_vectors.txt']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae75b2c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "for v in vectors:\n",
    "    command = 'python3 embedding_bias_evaluation.py \\\n",
    "        --embedding_model \"GloVe/glove_models_trained/\"' + v + ' --output \"output/embedding_model_evaluation\" \\\n",
    "        --lowercase'\n",
    "    os.system(command)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "adaexam",
   "language": "python",
   "name": "adaexam"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}