07-fairness-measures.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "141db0c9-b8fb-4dee-945b-1820bed87972",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-10T08:25:33.665099Z",
     "iopub.status.busy": "2022-02-10T08:25:33.664958Z",
     "iopub.status.idle": "2022-02-10T08:25:34.403784Z",
     "shell.execute_reply": "2022-02-10T08:25:34.403133Z",
     "shell.execute_reply.started": "2022-02-10T08:25:33.665056Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn.metrics\n",
    "\n",
    "import src"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b5728d6d-8e4d-4f33-b84a-761922b96cf9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-10T08:25:34.404856Z",
     "iopub.status.busy": "2022-02-10T08:25:34.404638Z",
     "iopub.status.idle": "2022-02-10T08:25:34.407949Z",
     "shell.execute_reply": "2022-02-10T08:25:34.407568Z",
     "shell.execute_reply.started": "2022-02-10T08:25:34.404837Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def equalised_odds(y_true, y_pred, s):\n",
    "    fpr_diff = abs(y_pred[s & ~y_true].mean() - y_pred[~s & ~y_true].mean())\n",
    "    tpr_diff = abs(y_pred[s & y_true].mean() - y_pred[~s & y_true].mean())\n",
    "    return max(fpr_diff, tpr_diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8169adf8-67ba-4e22-81ee-92fcf8408bb5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-10T08:35:32.232780Z",
     "iopub.status.busy": "2022-02-10T08:35:32.231325Z",
     "iopub.status.idle": "2022-02-10T08:35:33.840924Z",
     "shell.execute_reply": "2022-02-10T08:35:33.840025Z",
     "shell.execute_reply.started": "2022-02-10T08:35:32.232739Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>model</th>\n",
       "      <th>baseline</th>\n",
       "      <th>random forest</th>\n",
       "      <th>fair random forest</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>measure</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>precision (non-white)</th>\n",
       "      <td>0.971</td>\n",
       "      <td>0.898</td>\n",
       "      <td>0.890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision (white)</th>\n",
       "      <td>0.952</td>\n",
       "      <td>0.877</td>\n",
       "      <td>0.861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall (non-white)</th>\n",
       "      <td>0.423</td>\n",
       "      <td>0.755</td>\n",
       "      <td>0.825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall (white)</th>\n",
       "      <td>0.052</td>\n",
       "      <td>0.886</td>\n",
       "      <td>0.866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1 (non-white)</th>\n",
       "      <td>0.589</td>\n",
       "      <td>0.820</td>\n",
       "      <td>0.856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1 (white)</th>\n",
       "      <td>0.099</td>\n",
       "      <td>0.882</td>\n",
       "      <td>0.864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>demographic parity</th>\n",
       "      <td>0.317</td>\n",
       "      <td>0.099</td>\n",
       "      <td>0.023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>equalised odds</th>\n",
       "      <td>0.371</td>\n",
       "      <td>0.132</td>\n",
       "      <td>0.040</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "model                  baseline  random forest  fair random forest\n",
       "measure                                                           \n",
       "precision (non-white)     0.971          0.898               0.890\n",
       "precision (white)         0.952          0.877               0.861\n",
       "recall (non-white)        0.423          0.755               0.825\n",
       "recall (white)            0.052          0.886               0.866\n",
       "f1 (non-white)            0.589          0.820               0.856\n",
       "f1 (white)                0.099          0.882               0.864\n",
       "demographic parity        0.317          0.099               0.023\n",
       "equalised odds            0.371          0.132               0.040"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "portcalls = pd.read_pickle('data/portcalls_v3.pkl').astype({'risk': int})\n",
    "ships = np.load('data/not_selected.npy', allow_pickle=True)\n",
    "\n",
    "expert_labels = portcalls.groupby('ship')['risk'].max()\n",
    "y_score = pd.Series({ship: expert_labels.get(ship) for ship in ships}) > 1 # 0 low, 1 medium, 2 high risk\n",
    "\n",
    "inspections = src.get_inspections().groupby('IMO')['WasDetained'].any().replace({False: 1, True: 2})\n",
    "y_true = pd.Series({ship: inspections.get(ship, default=0) for ship in ships}) > 0 # 0 compliant, 1 deficiency, 2 detention\n",
    "y_true_bincount = np.bincount(y_true)\n",
    "\n",
    "sensitive = portcalls.groupby('ship')['flag'].last().astype(int).astype(bool)\n",
    "s = pd.Series({ship: sensitive.get(ship) for ship in ships})\n",
    "\n",
    "assert all(y_score.index == y_score.index)\n",
    "no_instances = len(y_score)\n",
    "assert no_instances == len(y_true)\n",
    "assert no_instances == len(s)\n",
    "\n",
    "result = [\n",
    "    {'model': 'baseline',      'measure': 'precision (non-white)', 'value': sklearn.metrics.precision_score(y_true[s], y_score[s])},\n",
    "    {'model': 'baseline',      'measure': 'precision (white)',     'value': sklearn.metrics.precision_score(y_true[~s], y_score[~s])},  \n",
    "    {'model': 'baseline',      'measure': 'recall (non-white)',    'value': sklearn.metrics.recall_score(y_true[s], y_score[s])}, \n",
    "    {'model': 'baseline',      'measure': 'recall (white)',        'value': sklearn.metrics.recall_score(y_true[~s], y_score[~s])},\n",
    "    {'model': 'baseline',      'measure': 'f1 (non-white)',        'value': sklearn.metrics.f1_score(y_true[s], y_score[s])},\n",
    "    {'model': 'baseline',      'measure': 'f1 (white)',            'value': sklearn.metrics.f1_score(y_true[~s], y_score[~s])},\n",
    "    {'model': 'baseline',      'measure': 'demographic parity',    'value': abs(y_score[s].mean()-y_score[~s].mean())},\n",
    "    {'model': 'baseline',      'measure': 'equalised odds',        'value': equalised_odds(y_true, y_score, s)},\n",
    "]\n",
    "\n",
    "y_score = 1-np.load('cache/y_score_randomforest.npy')\n",
    "y_true = ~np.load('cache/y_true_randomforest.npy')\n",
    "s = np.load('cache/s_randomforest.npy')\n",
    "threshold = np.quantile(y_score, np.mean(~y_true))\n",
    "y_pred = y_score > threshold\n",
    "\n",
    "assert len(y_score) == no_instances\n",
    "assert len(y_true) == no_instances\n",
    "assert (y_true_bincount == np.bincount(y_true)).all()\n",
    "assert (y_true_bincount == np.bincount(y_pred)).all()\n",
    "\n",
    "result_randomforest = [\n",
    "    {'model': 'random forest', 'measure': 'precision (non-white)', 'value': sklearn.metrics.precision_score(y_true[s], y_pred[s])},\n",
    "    {'model': 'random forest', 'measure': 'precision (white)',     'value': sklearn.metrics.precision_score(y_true[~s], y_pred[~s])},  \n",
    "    {'model': 'random forest', 'measure': 'recall (non-white)',    'value': sklearn.metrics.recall_score(y_true[s], y_pred[s])}, \n",
    "    {'model': 'random forest', 'measure': 'recall (white)',        'value': sklearn.metrics.recall_score(y_true[~s], y_pred[~s])},\n",
    "    {'model': 'random forest', 'measure': 'f1 (non-white)',        'value': sklearn.metrics.f1_score(y_true[s], y_pred[s])},\n",
    "    {'model': 'random forest', 'measure': 'f1 (white)',            'value': sklearn.metrics.f1_score(y_true[~s], y_pred[~s])},\n",
    "    {'model': 'random forest', 'measure': 'demographic parity',    'value': abs(y_pred[s].mean()-y_pred[~s].mean())},\n",
    "    {'model': 'random forest', 'measure': 'equalised odds',        'value': equalised_odds(y_true, y_pred, s)},\n",
    "]\n",
    "result.extend(result_randomforest)\n",
    "\n",
    "y_score = 1-np.load('cache/y_score_fairrandomforest.npy')\n",
    "y_true = ~np.load('cache/y_true_fairrandomforest.npy')\n",
    "s = np.load('cache/s_fairrandomforest.npy')\n",
    "threshold = np.quantile(y_score, np.mean(~y_true))\n",
    "y_pred = y_score > threshold\n",
    "\n",
    "assert len(y_score) == no_instances\n",
    "assert len(y_true) == no_instances\n",
    "assert (y_true_bincount == np.bincount(y_true)).all()\n",
    "assert (y_true_bincount == np.bincount(y_pred)).all()\n",
    "\n",
    "result_fairrandomforest = [\n",
    "    {'model': 'fair random forest', 'measure': 'precision (non-white)', 'value': sklearn.metrics.precision_score(y_true[s], y_pred[s])},\n",
    "    {'model': 'fair random forest', 'measure': 'precision (white)',     'value': sklearn.metrics.precision_score(y_true[~s], y_pred[~s])},  \n",
    "    {'model': 'fair random forest', 'measure': 'recall (non-white)',    'value': sklearn.metrics.recall_score(y_true[s], y_pred[s])}, \n",
    "    {'model': 'fair random forest', 'measure': 'recall (white)',        'value': sklearn.metrics.recall_score(y_true[~s], y_pred[~s])}, \n",
    "    {'model': 'fair random forest', 'measure': 'f1 (non-white)',        'value': sklearn.metrics.f1_score(y_true[s], y_pred[s])},\n",
    "    {'model': 'fair random forest', 'measure': 'f1 (white)',            'value': sklearn.metrics.f1_score(y_true[~s], y_pred[~s])},\n",
    "    {'model': 'fair random forest', 'measure': 'demographic parity',    'value': abs(y_pred[s].mean()-y_pred[~s].mean())},\n",
    "    {'model': 'fair random forest', 'measure': 'equalised odds',        'value': equalised_odds(y_true, y_pred, s)},\n",
    "]\n",
    "result.extend(result_fairrandomforest)\n",
    "\n",
    "rows = ['precision (non-white)', 'precision (white)', 'recall (non-white)', 'recall (white)', 'f1 (non-white)', 'f1 (white)', 'demographic parity', 'equalised odds']\n",
    "cols = ['baseline', 'random forest', 'fair random forest']\n",
    "pd.DataFrame(result).pivot(index='measure', columns='model', values='value').loc[rows, cols].round(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f5ea046-cffa-4b23-952b-164170e7fb0f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}