03-baseline-ship-risk-model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9c52d68e-81b8-4f8c-909e-5ade7ef26ef9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-07T11:12:01.742862Z",
     "iopub.status.busy": "2022-02-07T11:12:01.742627Z",
     "iopub.status.idle": "2022-02-07T11:12:40.451909Z",
     "shell.execute_reply": "2022-02-07T11:12:40.451046Z",
     "shell.execute_reply.started": "2022-02-07T11:12:01.742790Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from IPython.display import display\n",
    "import joblib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn.feature_selection\n",
    "import sklearn.metrics\n",
    "import sklearn.model_selection\n",
    "\n",
    "import src\n",
    "from run import filepath_portcalls_processed, filepath_inspections_processed, filepath_ships_classification, filepath_s, filepath_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3e5135f4-989d-4836-b570-ff7dc84eb83d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-07T11:12:40.453367Z",
     "iopub.status.busy": "2022-02-07T11:12:40.453168Z",
     "iopub.status.idle": "2022-02-07T11:13:04.071717Z",
     "shell.execute_reply": "2022-02-07T11:13:04.070858Z",
     "shell.execute_reply.started": "2022-02-07T11:12:40.453342Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Only ships not selected for mobility network.\n",
    "portcalls = pd.read_pickle(filepath_portcalls_processed)\n",
    "inspections = pd.read_pickle(filepath_inspections_processed)\n",
    "expert_labels = portcalls.groupby('ship')['risk'].max()\n",
    "\n",
    "# ALL ships - also selected ones for the mobility network!\n",
    "all_ships = portcalls.ship.unique()\n",
    "y_true_all = pd.Series({ship: inspections.get(ship, default=0) for ship in all_ships})\n",
    "y_score_all = pd.Series({ship: expert_labels.get(ship) for ship in all_ships})\n",
    "sensitive = portcalls.groupby('ship')['flag'].last().astype(int).astype(bool)\n",
    "s_all = pd.Series({ship: sensitive.get(ship) for ship in all_ships})\n",
    "\n",
    "# Ships used for classification\n",
    "ships = np.load(filepath_ships_classification)\n",
    "y_score = pd.Series({ship: expert_labels.get(ship) for ship in ships})\n",
    "y_true = pd.read_pickle(filepath_y)\n",
    "s = pd.read_pickle(filepath_s)\n",
    "ys = y_true.values*3 + s"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79f55193-5151-4266-ab74-ac7ade0c16ca",
   "metadata": {},
   "source": [
    "# Distribution targets on whole data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6d7545d4-026b-47be-9d43-e01354bc4212",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-07T11:13:04.073877Z",
     "iopub.status.busy": "2022-02-07T11:13:04.073729Z",
     "iopub.status.idle": "2022-02-07T11:13:04.091288Z",
     "shell.execute_reply": "2022-02-07T11:13:04.090759Z",
     "shell.execute_reply.started": "2022-02-07T11:13:04.073859Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "compliant                  6743\n",
       "deficiencies              21088\n",
       "detention+deficiencies     1631\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "compliant                 0.228871\n",
       "deficiencies              0.715769\n",
       "detention+deficiencies    0.055359\n",
       "dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(y_true_all.replace({0: 'compliant', 1: 'deficiencies', 2: 'detention+deficiencies'}).value_counts().sort_index())\n",
    "print()\n",
    "display(y_true_all.replace({0: 'compliant', 1: 'deficiencies', 2: 'detention+deficiencies'}).value_counts(normalize=True).sort_index())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ab804d5-f7ba-43fb-a8ed-af249bde344b",
   "metadata": {},
   "source": [
    "# Distribution target\n",
    "## All data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9515854a-c566-4fb3-b769-92dc44fd1d67",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-07T11:13:04.092278Z",
     "iopub.status.busy": "2022-02-07T11:13:04.092157Z",
     "iopub.status.idle": "2022-02-07T11:13:05.006627Z",
     "shell.execute_reply": "2022-02-07T11:13:05.005938Z",
     "shell.execute_reply.started": "2022-02-07T11:13:04.092264Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sensitive</th>\n",
       "      <th>non-sensitive</th>\n",
       "      <th>total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>compliant</th>\n",
       "      <td>0.195664</td>\n",
       "      <td>0.231149</td>\n",
       "      <td>0.228871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>deficiencies</th>\n",
       "      <td>0.554204</td>\n",
       "      <td>0.726851</td>\n",
       "      <td>0.715769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>detention+deficiencies</th>\n",
       "      <td>0.250132</td>\n",
       "      <td>0.042001</td>\n",
       "      <td>0.055359</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        sensitive  non-sensitive     total\n",
       "compliant                0.195664       0.231149  0.228871\n",
       "deficiencies             0.554204       0.726851  0.715769\n",
       "detention+deficiencies   0.250132       0.042001  0.055359"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = {\n",
    "    'sensitive': y_true_all[s_all].replace({0: 'compliant', 1: 'deficiencies', 2: 'detention+deficiencies'}).value_counts(normalize=True).sort_index(),\n",
    "    'non-sensitive': y_true_all[~s_all].replace({0: 'compliant', 1: 'deficiencies', 2: 'detention+deficiencies'}).value_counts(normalize=True).sort_index(),\n",
    "    'total': y_true_all.replace({0: 'compliant', 1: 'deficiencies', 2: 'detention+deficiencies'}).value_counts(normalize=True).sort_index()\n",
    "}\n",
    "pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eaa7f5f0-a1c1-46f7-a512-1aa0ee8d7484",
   "metadata": {},
   "source": [
    "## Only data not used in global cargo ship network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "41383d23-a77f-40d8-9491-9ecb24d8af69",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-07T11:13:05.007866Z",
     "iopub.status.busy": "2022-02-07T11:13:05.007659Z",
     "iopub.status.idle": "2022-02-07T11:13:05.028853Z",
     "shell.execute_reply": "2022-02-07T11:13:05.028270Z",
     "shell.execute_reply.started": "2022-02-07T11:13:05.007842Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sensitive</th>\n",
       "      <th>non-sensitive</th>\n",
       "      <th>total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>compliant</th>\n",
       "      <td>0.172970</td>\n",
       "      <td>0.214599</td>\n",
       "      <td>0.211973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>deficiencies</th>\n",
       "      <td>0.568506</td>\n",
       "      <td>0.742206</td>\n",
       "      <td>0.731250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>detention+deficiencies</th>\n",
       "      <td>0.258524</td>\n",
       "      <td>0.043195</td>\n",
       "      <td>0.056776</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        sensitive  non-sensitive     total\n",
       "compliant                0.172970       0.214599  0.211973\n",
       "deficiencies             0.568506       0.742206  0.731250\n",
       "detention+deficiencies   0.258524       0.043195  0.056776"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = {\n",
    "    'sensitive':     y_true[s ].replace({0: 'compliant', 1: 'deficiencies', 2: 'detention+deficiencies'}).value_counts(normalize=True).sort_index(),\n",
    "    'non-sensitive': y_true[~s].replace({0: 'compliant', 1: 'deficiencies', 2: 'detention+deficiencies'}).value_counts(normalize=True).sort_index(),\n",
    "    'total':         y_true.replace({0: 'compliant', 1: 'deficiencies', 2: 'detention+deficiencies'}).value_counts(normalize=True).sort_index()\n",
    "}\n",
    "pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5738b39-31bf-4ff9-bf5b-f6829d10f194",
   "metadata": {},
   "source": [
    "# Distribution sensitive attribute"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5a6eca3-1ce9-4113-8965-1b105a4c8f98",
   "metadata": {},
   "source": [
    "## All data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e3660260-3e4d-4b13-8652-7aca218e8a1f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-07T11:13:05.029666Z",
     "iopub.status.busy": "2022-02-07T11:13:05.029530Z",
     "iopub.status.idle": "2022-02-07T11:13:05.404976Z",
     "shell.execute_reply": "2022-02-07T11:13:05.404460Z",
     "shell.execute_reply.started": "2022-02-07T11:13:05.029651Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "white flag        0.935816\n",
       "non-white flag    0.064184\n",
       "dtype: float64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(np.bincount(s_all) / len(s_all), index=['white flag', 'non-white flag'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b527a6f3-d145-4eb1-83da-79621ba13524",
   "metadata": {},
   "source": [
    "## Only data not used in global cargo ship network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c31d473c-e585-4747-98f4-3ad6010bf326",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-07T11:13:05.405727Z",
     "iopub.status.busy": "2022-02-07T11:13:05.405614Z",
     "iopub.status.idle": "2022-02-07T11:13:05.463407Z",
     "shell.execute_reply": "2022-02-07T11:13:05.462798Z",
     "shell.execute_reply.started": "2022-02-07T11:13:05.405713Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "white flag        0.936928\n",
       "non-white flag    0.063072\n",
       "dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(np.bincount(s) / len(s), index=['white flag', 'non-white flag'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "339078e7-d00a-4c01-a640-4f25fbb35f44",
   "metadata": {},
   "source": [
    "# Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a6a31479-1f9e-45fb-b7e9-bac15cf459e4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-07T11:51:40.677748Z",
     "iopub.status.busy": "2022-02-07T11:51:40.677413Z",
     "iopub.status.idle": "2022-02-07T11:51:40.684567Z",
     "shell.execute_reply": "2022-02-07T11:51:40.683952Z",
     "shell.execute_reply.started": "2022-02-07T11:51:40.677718Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8606745    0\n",
       "9741516    1\n",
       "9374894    0\n",
       "7212482    1\n",
       "9475428    1\n",
       "          ..\n",
       "9495387    0\n",
       "9404766    1\n",
       "9422366    1\n",
       "9689213    0\n",
       "9630535    1\n",
       "Length: 25574, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5af1c650-efb8-4b18-bea1-7f0037018151",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-07T11:13:05.464259Z",
     "iopub.status.busy": "2022-02-07T11:13:05.464136Z",
     "iopub.status.idle": "2022-02-07T11:13:05.539484Z",
     "shell.execute_reply": "2022-02-07T11:13:05.538917Z",
     "shell.execute_reply.started": "2022-02-07T11:13:05.464243Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC_y: 0.543+-0.006\n"
     ]
    }
   ],
   "source": [
    "skf = sklearn.model_selection.StratifiedKFold(shuffle=True, random_state=42)\n",
    "performance = [sklearn.metrics.roc_auc_score(y_true.iloc[test] > 0, y_score.iloc[test]) for _, test in skf.split(y_true, ys)]\n",
    "print(f\"AUC_y: {np.mean(performance):.3f}+-{np.std(performance):.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6ef764a5-2a95-47a2-8c65-9a20f3a89b93",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-07T11:13:05.540347Z",
     "iopub.status.busy": "2022-02-07T11:13:05.540224Z",
     "iopub.status.idle": "2022-02-07T11:13:05.887249Z",
     "shell.execute_reply": "2022-02-07T11:13:05.886792Z",
     "shell.execute_reply.started": "2022-02-07T11:13:05.540332Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC_y: 0.672+-0.0097\n"
     ]
    }
   ],
   "source": [
    "skf = sklearn.model_selection.StratifiedKFold(shuffle=True, random_state=42)\n",
    "performance = [sklearn.metrics.roc_auc_score(s.iloc[test] > 0, y_score.iloc[test]) for _, test in skf.split(y_true, ys)]\n",
    "print(f\"AUC_y: {np.mean(performance):.3f}+-{np.std(performance):.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28c4eaca-41ee-48bb-8bfe-ab76f90d3737",
   "metadata": {},
   "source": [
    "# Confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c26b1c5b-bf89-4d3e-81b7-177ebd0b3f46",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-07T11:13:05.888023Z",
     "iopub.status.busy": "2022-02-07T11:13:05.887905Z",
     "iopub.status.idle": "2022-02-07T11:13:06.029280Z",
     "shell.execute_reply": "2022-02-07T11:13:06.028855Z",
     "shell.execute_reply.started": "2022-02-07T11:13:05.888009Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">sensitive</th>\n",
       "      <th colspan=\"3\" halign=\"left\">non sensitive</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Expert label</th>\n",
       "      <th>low</th>\n",
       "      <th>medium</th>\n",
       "      <th>high</th>\n",
       "      <th>low</th>\n",
       "      <th>medium</th>\n",
       "      <th>high</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>inspection result</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>compliant</th>\n",
       "      <td>1</td>\n",
       "      <td>261</td>\n",
       "      <td>17</td>\n",
       "      <td>319</td>\n",
       "      <td>4774</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>minor deficiencies</th>\n",
       "      <td>0</td>\n",
       "      <td>620</td>\n",
       "      <td>297</td>\n",
       "      <td>682</td>\n",
       "      <td>16370</td>\n",
       "      <td>732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>detention</th>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>267</td>\n",
       "      <td>1</td>\n",
       "      <td>788</td>\n",
       "      <td>246</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   sensitive             non sensitive            \n",
       "Expert label             low medium high           low medium high\n",
       "inspection result                                                 \n",
       "compliant                  1    261   17           319   4774   49\n",
       "minor deficiencies         0    620  297           682  16370  732\n",
       "detention                  0    150  267             1    788  246"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "assert all(y_score.index == y_score.index)\n",
    "confusion_matrix_sensitive = pd.DataFrame(\n",
    "    data=sklearn.metrics.confusion_matrix(y_true[s], y_score[s]),\n",
    "    index=pd.Index(['compliant', 'minor deficiencies', 'detention'], name='inspection result'),\n",
    "    columns=pd.Index(['low', 'medium', 'high'], name='Expert label')\n",
    ")\n",
    "confusion_matrix_non_sensitive = pd.DataFrame(\n",
    "    data=sklearn.metrics.confusion_matrix(y_true[~s], y_score[~s]),\n",
    "    index=pd.Index(['compliant', 'minor deficiencies', 'detention'], name='inspection result'),\n",
    "    columns=pd.Index(['low', 'medium', 'high'], name='Expert label')\n",
    ")\n",
    "pd.concat({'sensitive': confusion_matrix_sensitive, 'non sensitive': confusion_matrix_non_sensitive}, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "59eb9ad3-abc2-4bdf-9021-c5c6dd90d40f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-07T11:13:06.030486Z",
     "iopub.status.busy": "2022-02-07T11:13:06.029997Z",
     "iopub.status.idle": "2022-02-07T11:13:06.115401Z",
     "shell.execute_reply": "2022-02-07T11:13:06.114967Z",
     "shell.execute_reply.started": "2022-02-07T11:13:06.030467Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True     18819\n",
       "False     5142\n",
       "dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(y_true > 0)[~s].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d2f0e0c9-7d8c-4250-9ce5-94597b82537c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-07T11:13:06.117126Z",
     "iopub.status.busy": "2022-02-07T11:13:06.117001Z",
     "iopub.status.idle": "2022-02-07T11:13:07.676317Z",
     "shell.execute_reply": "2022-02-07T11:13:07.675477Z",
     "shell.execute_reply.started": "2022-02-07T11:13:06.117111Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Flag</th>\n",
       "      <th>True</th>\n",
       "      <th>Predicted</th>\n",
       "      <th>Number</th>\n",
       "      <th>Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Non-white</td>\n",
       "      <td>Compliant</td>\n",
       "      <td>Low</td>\n",
       "      <td>1</td>\n",
       "      <td>Baseline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Non-white</td>\n",
       "      <td>Non-compliant</td>\n",
       "      <td>Low</td>\n",
       "      <td>0</td>\n",
       "      <td>Baseline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>White</td>\n",
       "      <td>Compliant</td>\n",
       "      <td>Low</td>\n",
       "      <td>319</td>\n",
       "      <td>Baseline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>White</td>\n",
       "      <td>Non-compliant</td>\n",
       "      <td>Low</td>\n",
       "      <td>683</td>\n",
       "      <td>Baseline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Non-white</td>\n",
       "      <td>Compliant</td>\n",
       "      <td>Medium</td>\n",
       "      <td>261</td>\n",
       "      <td>Baseline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Non-white</td>\n",
       "      <td>Non-compliant</td>\n",
       "      <td>Medium</td>\n",
       "      <td>770</td>\n",
       "      <td>Baseline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>White</td>\n",
       "      <td>Compliant</td>\n",
       "      <td>Medium</td>\n",
       "      <td>4774</td>\n",
       "      <td>Baseline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>White</td>\n",
       "      <td>Non-compliant</td>\n",
       "      <td>Medium</td>\n",
       "      <td>17158</td>\n",
       "      <td>Baseline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Non-white</td>\n",
       "      <td>Compliant</td>\n",
       "      <td>High</td>\n",
       "      <td>17</td>\n",
       "      <td>Baseline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Non-white</td>\n",
       "      <td>Non-compliant</td>\n",
       "      <td>High</td>\n",
       "      <td>564</td>\n",
       "      <td>Baseline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>White</td>\n",
       "      <td>Compliant</td>\n",
       "      <td>High</td>\n",
       "      <td>49</td>\n",
       "      <td>Baseline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>White</td>\n",
       "      <td>Non-compliant</td>\n",
       "      <td>High</td>\n",
       "      <td>978</td>\n",
       "      <td>Baseline</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Flag           True Predicted  Number     Model\n",
       "0   Non-white      Compliant       Low       1  Baseline\n",
       "1   Non-white  Non-compliant       Low       0  Baseline\n",
       "2       White      Compliant       Low     319  Baseline\n",
       "3       White  Non-compliant       Low     683  Baseline\n",
       "4   Non-white      Compliant    Medium     261  Baseline\n",
       "5   Non-white  Non-compliant    Medium     770  Baseline\n",
       "6       White      Compliant    Medium    4774  Baseline\n",
       "7       White  Non-compliant    Medium   17158  Baseline\n",
       "8   Non-white      Compliant      High      17  Baseline\n",
       "9   Non-white  Non-compliant      High     564  Baseline\n",
       "10      White      Compliant      High      49  Baseline\n",
       "11      White  Non-compliant      High     978  Baseline"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = np.concatenate(\n",
    "    (\n",
    "        sklearn.metrics.confusion_matrix((y_true > 0)[s], y_score[s], normalize=None)[:2, :], \n",
    "        sklearn.metrics.confusion_matrix((y_true > 0)[~s], y_score[~s], normalize=None)[:2, :]\n",
    "    ),\n",
    ")\n",
    "\n",
    "df = pd.DataFrame(\n",
    "    data, \n",
    "    index=pd.MultiIndex.from_product([['Non-white', 'White'], ['Compliant', 'Non-compliant']], names=['Flag', 'True']),\n",
    "    columns=['Low', 'Medium', 'High']\n",
    ")\n",
    "df = (\n",
    "    df\n",
    "    .reset_index()\n",
    "    .melt(id_vars=['Flag', 'True'], var_name='Predicted', value_name='Number')\n",
    "    .assign(Model='Baseline')\n",
    ")\n",
    "display(df)\n",
    "df.to_pickle('models/confusion-matrix-baseline.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "767afcfd-089b-4d65-b381-90f3766e5c81",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-07T11:13:07.677386Z",
     "iopub.status.busy": "2022-02-07T11:13:07.677252Z",
     "iopub.status.idle": "2022-02-07T11:13:07.773136Z",
     "shell.execute_reply": "2022-02-07T11:13:07.772496Z",
     "shell.execute_reply.started": "2022-02-07T11:13:07.677371Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>Flag:</th>\n",
       "      <th colspan=\"3\" halign=\"left\">Non-white</th>\n",
       "      <th colspan=\"3\" halign=\"left\">White</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Predicted:</th>\n",
       "      <th>Low</th>\n",
       "      <th>Medium</th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Medium</th>\n",
       "      <th>High</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True:</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Compliant</th>\n",
       "      <td>1</td>\n",
       "      <td>261</td>\n",
       "      <td>17</td>\n",
       "      <td>319</td>\n",
       "      <td>4774</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Non-compliant</th>\n",
       "      <td>0</td>\n",
       "      <td>770</td>\n",
       "      <td>564</td>\n",
       "      <td>683</td>\n",
       "      <td>17158</td>\n",
       "      <td>978</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Flag:         Non-white             White            \n",
       "Predicted:          Low Medium High   Low Medium High\n",
       "True:                                                \n",
       "Compliant             1    261   17   319   4774   49\n",
       "Non-compliant         0    770  564   683  17158  978"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = np.concatenate(\n",
    "    (\n",
    "        sklearn.metrics.confusion_matrix((y_true > 0)[s], y_score[s], normalize=None)[:2, :], \n",
    "        sklearn.metrics.confusion_matrix((y_true > 0)[~s], y_score[~s], normalize=None)[:2, :]\n",
    "    ),\n",
    "    axis=1\n",
    ")\n",
    "\n",
    "pd.DataFrame(\n",
    "    data,\n",
    "    index=pd.Index(['Compliant', 'Non-compliant'], name='True:'),\n",
    "    columns=pd.MultiIndex.from_product([['Non-white', 'White'],['Low', 'Medium', 'High']], names=['Flag:', 'Predicted:']),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f1d690a-648c-4773-9de6-fa24c1337af3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  },
  "toc-autonumbering": true
 },
 "nbformat": 4,
 "nbformat_minor": 5
}