filter_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "neither-mississippi",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "impaired-essex",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "BIOS_FILE_CRAWLED = \"data/BIOS.pkl\"\n",
    "REVIEW_DIR = \"data/review/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "western-penetration",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with (open(BIOS_FILE_CRAWLED, \"rb\")) as openfile:\n",
    "    full_data = pickle.load(openfile)\n",
    "for i, entry in enumerate(full_data):\n",
    "    entry.update({'review': 0, 'comment': '', 'valid': '', 'label': '', 'stylish valid': ''})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "controversial-assumption",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "full_data[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "small-corrections",
   "metadata": {},
   "source": [
    "## Label potentially ficticious, falsely labeled or samples that are not a biography for review\n",
    "\n",
    "Every sample is checked for keyphrases or patterns that indicate it might not be a bio/ contain info relevant for classification/ refer to a fictitious person/ be falsely labeled. For every suspicion, the review counter is increased and comments are added to make it easier to review these samples."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "identified-straight",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def add_comment(comment, new_comment):\n",
    "    if comment == '':\n",
    "        comment = new_comment\n",
    "    else:\n",
    "        comment += ', '+new_comment\n",
    "    return comment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "talented-processor",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "jobs = []\n",
    "raw_titles = []\n",
    "for entry in full_data:\n",
    "    title =  entry['title'].lower()\n",
    "    if title not in jobs:\n",
    "        jobs.append(title)\n",
    "    raw_title =  entry['raw_title'].lower()\n",
    "    if raw_title not in raw_titles:\n",
    "        raw_titles.append(raw_title)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fifteen-concrete",
   "metadata": {},
   "source": [
    "### Case 1: False entity\n",
    "e.g. name that is not really a name or contains some prefix, or where job and bio refer to different persons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "israeli-finish",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "false_names = [\"Dad\", \"Mom\", \"Brother\", \"Sister\", \"If\", \"The\", \"His\", \"Her\", \"Is\", \"Share\", \"What\", \"Why\", \"Who\", \"Where\", \"Would\"]\n",
    "\n",
    "def filter_entities(bio, name, review, comment):\n",
    "    for wn in false_names:\n",
    "        if wn in name:\n",
    "            review += 1\n",
    "            add_comment(comment, 'entity: check name')\n",
    "    for job in jobs:\n",
    "        if job.capitalize() in name:\n",
    "            review += 1\n",
    "            add_comment(comment, 'entity: check name')\n",
    "    return review, comment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "formal-dining",
   "metadata": {},
   "source": [
    "### Case 2: Ficticious person or movie/book review\n",
    "\n",
    "We simply check for phrases which are more likely to occur in other texts, such as fiction/ move reviews etc., instead of biographies. Furthermore, we identified some URIs that are likely contain movie/book reviews, fan fiction or video descriptions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "revolutionary-brick",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# phrases implying that a book/movie/ficiton is referred, e.g. exaggerations are likely to occur for ficticious persons)\n",
    "fiction_phrases = [\"written by\", \"voiced by\", \"produced by\", \"comments\", \"watch the trailer\", \"starring\", \" in \\\"\", \"M for mature\", \"Crime\", \"Romance\", \"Mystery\", \"Fantasy\", \"played by\",\n",
    "                   \"Rated\", \"Chapters\", \"Words\", \"Reviews\", \"Based on\", \"Realizado por\", \"narrated by\", \"imdb\", \"Avenger\", \"Description: \", \"Vol.\", \"Season \", \"Played by\", \"this DVD\", \"this movie\", \"this book\",\n",
    "                   \", but\", \"Why \", \"and why \", \"But \",\n",
    "                   \"the only problem is\", \"The only problem is\", \"Years later\", \"years later\", \"Until\", \"When\", \"Desperate\", \"paranormal\", \"superhero\", \"dwarv\", \"slutty\", \"cock\", \"stockings\", \"horny\", \"Fetish\", \"boobs\",\n",
    "                   \"mysterious\", \"supernatural\", \"desperately\", \"outrageous\", \"pirate\", \"forced to\", \"Occasionally\", \"Then\", \"one day\", \"One day\", \"with a secret\", \"with a Secret\", \"wannabe\", \"having sex\", \n",
    "                   \"demon\", \"vampire\",\n",
    "                   \"vicious killer\", \"they fall in love\", \"during the night\", \"blowjob\", \"througout this video\", \"this video\", \"what he is not expecting\", \"what she is not expecting\", \"naughty nurse\", \"hot model\"]\n",
    "# first sentence: e.g. is a question, exaggerations, amibious usage of model\n",
    "start_phrases = [\"unpopular\", \"?\", \"with a past\", \"struggling\", \"perfect\", \"famous\", \"brilliant\", \"poor\", \"talented\", \"heavy past\", \"gifted\"]\n",
    "\n",
    "fiction_uri = ['imdb', 'episodecalendar', 'filmschoolrejects', 'fanfiction', 'movie', 'tvtropes', 'marvel', 'idols69', 'sexgaypics', 'fleshandskin', 'lustyguide', 'gotgayporn', 'star-trek', 'criticker', 'goodreads', 'thebestporn', 'madtubes', 'naughty', \n",
    "               'tracktv', 'nude', 'bigboob', 'porn', 'tits', 'story', 'sexy', 'book', 'cinema', 'costume', 'novel', 'cinespider', 'mobileread', 'wikia', 'Characters', 'characters', 'filmjabber']\n",
    "\n",
    "def filter_fiction(bio, name, start_pos, bio_uri, review, comment):\n",
    "    for phrase in start_phrases:\n",
    "        if phrase in bio[:start_pos]:\n",
    "            review += 1\n",
    "            add_comment(comment, 'fiction (start): '+phrase)\n",
    "    for phrase in fiction_phrases:\n",
    "        if phrase in bio:\n",
    "            review += 1\n",
    "            add_comment(comment, 'fiction (bio): '+phrase)\n",
    "    for uri in fiction_uri:\n",
    "        if uri in bio_uri:\n",
    "            review += 1\n",
    "            add_comment(comment, 'fiction (uri)')\n",
    "    return review, comment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "greenhouse-vatican",
   "metadata": {},
   "source": [
    "### Case 3: Mislabeled bios\n",
    "e.g. bios that contain multiple job titles in the first sentence, or bios where the job refers to a third person (\"pastor's wife\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "democratic-insulation",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# other usage of model \n",
    "model_phrases = [\"role model\", \"model student\", \"model citizen\", \"model for\", \"model of\", \"model employee\"]\n",
    "\n",
    "# phrases that indicate change of occupation\n",
    "label_phrases = [\" turned \", \" retired \", \" former \"]\n",
    "\n",
    "# any suffix characters that we accept after jobs (e.g. \"job.\" is fine, \"job's wife\" not because it likely refers to a third person's job)\n",
    "job_suffixes = ['.', ' ', ',', ';', '/', '-']# '\\'s wife', '\\'s husband', '\\'s son', '\\'s daughter']\n",
    "\n",
    "# these raw titles also contain another version of this job, so they would be counted twice\n",
    "raw_titles_to_skip = ['certified public accountant', 'trial attorney', 'plastic surgeon', 'senior software engineer', 'orthopedic surgeon', 'certified personal trainer']\n",
    "\n",
    "def check_label(bio, label, start_pos, review, comment):\n",
    "    start_lower = bio[:start_pos].lower()\n",
    "    for phrase in model_phrases:\n",
    "        if phrase in start_lower:\n",
    "            review += 1\n",
    "            add_comment(comment, 'label (false model)')\n",
    "    \n",
    "    for phrase in label_phrases:\n",
    "        if phrase in start_lower:\n",
    "            review += 1\n",
    "            add_comment(comment, 'label (changed jobs)')\n",
    "\n",
    "    # count jobs\n",
    "    n_jobs = 0\n",
    "    for job in raw_titles:\n",
    "        if job in raw_titles_to_skip:\n",
    "            continue\n",
    "        if (entry['title'] == 'software_engineer' and job == 'architect'): # software engineers/architects cause many false positives\n",
    "            continue\n",
    "        if job == 'cpa' and ' accountant' in start_lower: # often both \"cpa\" and \"certificed public accountant\" are mentioned\n",
    "            continue\n",
    "            \n",
    "        job = job.replace('_', ' ') # for jobs as software_engineer\n",
    "        found = False\n",
    "        for suffix in job_suffixes:\n",
    "            if ' '+job+suffix in start_lower:\n",
    "                found = True\n",
    "                break\n",
    "        if found:\n",
    "            n_jobs += 1\n",
    "    \n",
    "    if not n_jobs == 1:\n",
    "        review += 1\n",
    "        add_comment(comment, 'number of labels: '+str(n_jobs))\n",
    "    \n",
    "    return review, comment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "thick-batch",
   "metadata": {},
   "source": [
    "### Case 4: Filter very short bios and those that contain text passages that do not belong to the bio\n",
    "e.g. some bios are followed by disclaimers or lengthy contact information, other links or contain texts from other web elements that are not part of the bio."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "legislative-treasure",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "web_phrases = ['download', 'copyright', ' Mb ', 'watch stream', ' Min ', 'More about', 'more about', 'facebook:', 'instagram:', 'https://www.', 'http://www.']\n",
    "multi_phrases = ['@', '#']\n",
    "\n",
    "def filter_content(bio, start_pos, review, comment):\n",
    "    bio_lower = bio[start_pos:].lower()\n",
    "    for phrase in web_phrases:\n",
    "        if phrase in bio_lower:\n",
    "            review += 1\n",
    "            add_comment(comment, 'content: '+phrase)\n",
    "    for phrase in multi_phrases:\n",
    "        if bio_lower.count(phrase) > 3:\n",
    "            review += 1\n",
    "            add_comment(comment, 'content: '+phrase)\n",
    "    if len(bio_lower.split(' ')) < 20:\n",
    "        #print(bio_lower)\n",
    "        review += 1\n",
    "        add_comment(comment, 'content very short')\n",
    "        \n",
    "    return review, comment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "skilled-western",
   "metadata": {},
   "source": [
    "### Go through all bios"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "exposed-pilot",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "count = 0\n",
    "for entry in full_data:\n",
    "    entry['review'] = 0\n",
    "    entry['comment'] = ''\n",
    "    \n",
    "    entry['review'], entry['comment'] = filter_entities(entry['raw'], entry['name'], entry['review'], entry['comment'])\n",
    "    entry['review'], entry['comment'] = filter_fiction(entry['raw'], entry['name'], entry['start_pos'], entry['URI'], entry['review'], entry['comment'])\n",
    "    entry['review'], entry['comment'] = check_label(entry['raw'], entry['title'], entry['start_pos'], entry['review'], entry['comment'])\n",
    "    entry['review'], entry['comment'] = filter_content(entry['raw'], entry['start_pos'], entry['review'], entry['comment'])\n",
    "    if entry['review'] > 0:\n",
    "        count += 1\n",
    "\n",
    "        \n",
    "print(\"counted \", count, \" samples with potential problems (of \", len(full_data), \" datapoints in total)\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "color-interstate",
   "metadata": {},
   "source": [
    "## Save automatically labeled data with indices for 10 classes of choice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "graduate-punch",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# these are the largest classes (except for professor, which accounts for a third of the dataset)\n",
    "jobs10 = ['architect', 'surgeon', 'dentist', 'teacher', 'psychologist', 'nurse', 'journalist', 'photographer', 'physician', 'attorney']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "heavy-storm",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data=full_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "million-function",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# sort data by jobs\n",
    "data_by_jobs = {}\n",
    "for job in jobs:\n",
    "    data_by_jobs.update({job: []})\n",
    "    \n",
    "for entry in full_data:\n",
    "    raw_title =  entry['title'].lower()\n",
    "    data_by_jobs[raw_title].append(entry)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "together-richmond",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "n_jobs_review = 0\n",
    "for job in jobs:\n",
    "    if job in jobs10:\n",
    "        dataframe = pd.DataFrame(data=data_by_jobs[job])\n",
    "        df_review = dataframe[dataframe['review'] > 0]\n",
    "        df_noreview = dataframe[dataframe['review'] == 0]\n",
    "        print(job, ':', len(df_review), \" samples for review (of\", len(dataframe),\")\")\n",
    "        n_jobs_review += len(df_review)\n",
    "        df_review.to_csv(REVIEW_DIR+job+\".csv\", index=True, sep='\\t')\n",
    "        \n",
    "print()\n",
    "print(\"found\", n_jobs_review, \"samples for review in the 10 classes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "least-revelation",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fundamental-malaysia",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}