Handling imbalanced classes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dataset Information:\n",
    "\n",
    "http://archive.ics.uci.edu/ml/datasets/balance+scale\n",
    "\n",
    "The dataset contains information about whether a scale is balanced or not, based on weights and distances of the two arms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('balance-scale.data',names=['Class','LW','LD','RW','RD'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>LW</th>\n",
       "      <th>LD</th>\n",
       "      <th>RW</th>\n",
       "      <th>RD</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>R</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>R</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>R</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Class  LW  LD  RW  RD\n",
       "0     B   1   1   1   1\n",
       "1     R   1   1   1   2\n",
       "2     R   1   1   1   3\n",
       "3     R   1   1   1   4"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 625 entries, 0 to 624\n",
      "Data columns (total 5 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   Class   625 non-null    object\n",
      " 1   LW      625 non-null    int64 \n",
      " 2   LD      625 non-null    int64 \n",
      " 3   RW      625 non-null    int64 \n",
      " 4   RD      625 non-null    int64 \n",
      "dtypes: int64(4), object(1)\n",
      "memory usage: 24.5+ KB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "L    288\n",
       "R    288\n",
       "B     49\n",
       "Name: Class, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['Class'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can convert Target variable into binary classes as follows :\n",
    "    \n",
    "    if Class = 'B' then 1 else 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    576\n",
       "1     49\n",
       "Name: Class, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clist=[]\n",
    "for c in data.Class:\n",
    "    if c=='B':\n",
    "        clist.append(1)\n",
    "    else:\n",
    "        clist.append(0)\n",
    "        \n",
    "data['Class'] = clist\n",
    "\n",
    "data['Class'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see that there is huge imbalance in the target variable Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Strategy Used</th>\n",
       "      <th>Train accuracy</th>\n",
       "      <th>Test accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Model, Strategy Used, Train accuracy, Test accuracy]\n",
       "Index: []"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_details = pd.DataFrame(columns = ['Model','Strategy Used','Train accuracy','Test accuracy'])\n",
    "model_details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model buliding using imbalanced dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of training set:  468\n",
      "length of testing set:  157\n",
      "[0]\n",
      "[0]\n"
     ]
    }
   ],
   "source": [
    "x=data[data.columns.difference(['Class'])]\n",
    "y=data['Class']\n",
    "\n",
    "x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.25,random_state=100)\n",
    "\n",
    "print(\"length of training set: \",len(x_train))\n",
    "print(\"length of testing set: \",len(x_test))\n",
    "\n",
    "clf_logReg1 = LogisticRegression(solver='liblinear').fit(x_train,y_train)\n",
    "\n",
    "y_train_pred = clf_logReg1.predict(x_train)\n",
    "y_test_pred =  clf_logReg1.predict(x_test)\n",
    "\n",
    "train_accuracy = accuracy_score(y_train,y_train_pred)\n",
    "test_accuracy = accuracy_score(y_test,y_test_pred)\n",
    "\n",
    "print(np.unique(y_train_pred))\n",
    "print(np.unique(y_test_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we can see that model is only predicting the majority class that is why it is giving more accuracy in both train and test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Strategy Used</th>\n",
       "      <th>Train accuracy</th>\n",
       "      <th>Test accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>clf_logReg1</td>\n",
       "      <td>NA</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.936306</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Model Strategy Used  Train accuracy  Test accuracy\n",
       "0  clf_logReg1            NA        0.916667       0.936306"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_details = model_details.append({'Model':'clf_logReg1','Strategy Used':'NA','Train accuracy':train_accuracy,'Test accuracy':test_accuracy},\n",
    "                     ignore_index=True)\n",
    "\n",
    "model_details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we will implement different method of handling imbalanced datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Up-Sampling Minority Class"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By Up-sampling minority class means that we will select those records which have target labeled as minority class by random resampling with replacement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import resample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    576\n",
       "1     49\n",
       "Name: Class, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.Class.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    576\n",
       "0    576\n",
       "Name: Class, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##step-1 Separate out data points having minority and majority classes\n",
    "\n",
    "data_majority = data[data['Class']==0] ## majority class\n",
    "data_minority = data[data['Class']==1] ## minority class\n",
    "\n",
    "## step-2 upsampling minority class\n",
    "\n",
    "data_minority_resampled = resample(data_minority,replace = True, n_samples = 576, random_state=100)\n",
    "\n",
    "## step-3 combining dataframe \n",
    "\n",
    "data_upsampled = pd.concat([data_majority,data_minority_resampled],axis=0)\n",
    "\n",
    "data_upsampled['Class'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can see that both classes are balanced in dataset so we will  build model on this dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of training set:  864\n",
      "length of testing set:  288\n",
      "[0 1]\n",
      "[0 1]\n"
     ]
    }
   ],
   "source": [
    "x=data_upsampled[data_upsampled.columns.difference(['Class'])]\n",
    "y=data_upsampled['Class']\n",
    "\n",
    "x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.25,random_state=100)\n",
    "\n",
    "print(\"length of training set: \",len(x_train))\n",
    "print(\"length of testing set: \",len(x_test))\n",
    "\n",
    "clf_logReg2 = LogisticRegression(solver='liblinear').fit(x_train,y_train)\n",
    "\n",
    "y_train_pred = clf_logReg2.predict(x_train)\n",
    "y_test_pred =  clf_logReg2.predict(x_test)\n",
    "\n",
    "train_accuracy = accuracy_score(y_train,y_train_pred)\n",
    "test_accuracy = accuracy_score(y_test,y_test_pred)\n",
    "\n",
    "print(np.unique(y_train_pred))\n",
    "print(np.unique(y_test_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can see that model is predicting both classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Strategy Used</th>\n",
       "      <th>Train accuracy</th>\n",
       "      <th>Test accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>clf_logReg1</td>\n",
       "      <td>NA</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.936306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>clf_logReg2</td>\n",
       "      <td>Up sampling minority class</td>\n",
       "      <td>0.481481</td>\n",
       "      <td>0.458333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Model               Strategy Used  Train accuracy  Test accuracy\n",
       "0  clf_logReg1                          NA        0.916667       0.936306\n",
       "1  clf_logReg2  Up sampling minority class        0.481481       0.458333"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_details = model_details.append({'Model':'clf_logReg2','Strategy Used':'Up sampling minority class','Train accuracy':train_accuracy,'Test accuracy':test_accuracy},\n",
    "                     ignore_index=True)\n",
    "\n",
    "model_details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Down-Sampling Majority Class"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By down sample we means that randomly remove majority class labeled records to match number of reords with minority class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    49\n",
       "0    49\n",
       "Name: Class, dtype: int64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##step-1 : down sample majority class\n",
    "data_majority_resampled  = resample(data_majority, replace=False, n_samples = 49, random_state=100)\n",
    "\n",
    "##step-2 : combine downsampled majority class + minority class to form a new balanced dataset\n",
    "data_downsampled = pd.concat([data_minority,data_majority_resampled],axis=0)\n",
    "\n",
    "data_downsampled['Class'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can see that both classes are balanced in dataset so we will  build model on this dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of training set:  73\n",
      "length of testing set:  25\n",
      "[0 1]\n",
      "[0]\n"
     ]
    }
   ],
   "source": [
    "x=data_downsampled[data_downsampled.columns.difference(['Class'])]\n",
    "y=data_downsampled['Class']\n",
    "\n",
    "x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.25,random_state=100)\n",
    "\n",
    "print(\"length of training set: \",len(x_train))\n",
    "print(\"length of testing set: \",len(x_test))\n",
    "\n",
    "clf_logReg3 = LogisticRegression(solver='liblinear').fit(x_train,y_train)\n",
    "\n",
    "y_train_pred = clf_logReg3.predict(x_train)\n",
    "y_test_pred =  clf_logReg3.predict(x_test)\n",
    "\n",
    "train_accuracy = accuracy_score(y_train,y_train_pred)\n",
    "test_accuracy = accuracy_score(y_test,y_test_pred)\n",
    "\n",
    "print(np.unique(y_train_pred))\n",
    "print(np.unique(y_test_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Strategy Used</th>\n",
       "      <th>Train accuracy</th>\n",
       "      <th>Test accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>clf_logReg1</td>\n",
       "      <td>NA</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.936306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>clf_logReg2</td>\n",
       "      <td>Up sampling minority class</td>\n",
       "      <td>0.481481</td>\n",
       "      <td>0.458333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>clf_logReg3</td>\n",
       "      <td>Down sampling majority class</td>\n",
       "      <td>0.520548</td>\n",
       "      <td>0.400000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Model                 Strategy Used  Train accuracy  Test accuracy\n",
       "0  clf_logReg1                            NA        0.916667       0.936306\n",
       "1  clf_logReg2    Up sampling minority class        0.481481       0.458333\n",
       "2  clf_logReg3  Down sampling majority class        0.520548       0.400000"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_details = model_details.append({'Model':'clf_logReg3','Strategy Used':'Down sampling majority class','Train accuracy':train_accuracy,'Test accuracy':test_accuracy},\n",
    "                     ignore_index=True)\n",
    "\n",
    "model_details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Choose Different Evaluation Metric"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can select different metric such as <b>*AUC-ROC*</b> which is usually used to represent likelihood of a model distinguishing obersavations from two classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5467571644042233\n",
      "0.6133333333333333\n"
     ]
    }
   ],
   "source": [
    "y_train_aucroc = clf_logReg3.predict_proba(x_train)\n",
    "y_train_score = [score[1] for score in y_train_aucroc]\n",
    "\n",
    "print(roc_auc_score(y_train,y_train_score))\n",
    "\n",
    "y_test_aucroc = clf_logReg3.predict_proba(x_test)\n",
    "y_test_score = [score[1] for score in y_test_aucroc]\n",
    "\n",
    "print(roc_auc_score(y_test,y_test_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Strategy Used</th>\n",
       "      <th>Train accuracy</th>\n",
       "      <th>Test accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>clf_logReg1</td>\n",
       "      <td>NA</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.936306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>clf_logReg2</td>\n",
       "      <td>Up sampling minority class</td>\n",
       "      <td>0.481481</td>\n",
       "      <td>0.458333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>clf_logReg3</td>\n",
       "      <td>Down sampling majority class</td>\n",
       "      <td>0.520548</td>\n",
       "      <td>0.400000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>clf_logReg2</td>\n",
       "      <td>AUC-ROC Metric</td>\n",
       "      <td>0.520548</td>\n",
       "      <td>0.400000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Model                 Strategy Used  Train accuracy  Test accuracy\n",
       "0  clf_logReg1                            NA        0.916667       0.936306\n",
       "1  clf_logReg2    Up sampling minority class        0.481481       0.458333\n",
       "2  clf_logReg3  Down sampling majority class        0.520548       0.400000\n",
       "3  clf_logReg2                AUC-ROC Metric        0.520548       0.400000"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_details = model_details.append({'Model':'clf_logReg2','Strategy Used':'AUC-ROC Metric','Train accuracy':train_accuracy,'Test accuracy':test_accuracy},\n",
    "                     ignore_index=True)\n",
    "\n",
    "model_details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Use Penalize Algorithm ( Cost-sensitive Training)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    576\n",
       "1     49\n",
       "Name: Class, dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['Class'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we will build Support vector machine classifier on imbalanced dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of training set:  468\n",
      "length of testing set:  157\n",
      "[0 1]\n",
      "[0 1]\n"
     ]
    }
   ],
   "source": [
    "x=data[data.columns.difference(['Class'])]\n",
    "y=data['Class']\n",
    "\n",
    "x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.25,random_state=100)\n",
    "\n",
    "print(\"length of training set: \",len(x_train))\n",
    "print(\"length of testing set: \",len(x_test))\n",
    "\n",
    "## setting class_weight='balanced' -> it will increase cost of misclassification on minority class\n",
    "\n",
    "clf_svc = SVC(class_weight='balanced',probability=True).fit(x_train,y_train)\n",
    "\n",
    "y_train_pred = clf_svc.predict(x_train)\n",
    "y_test_pred =  clf_svc.predict(x_test)\n",
    "\n",
    "train_accuracy = accuracy_score(y_train,y_train_pred)\n",
    "test_accuracy = accuracy_score(y_test,y_test_pred)\n",
    "\n",
    "print(np.unique(y_train_pred))\n",
    "print(np.unique(y_test_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Strategy Used</th>\n",
       "      <th>Train accuracy</th>\n",
       "      <th>Test accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>clf_logReg1</td>\n",
       "      <td>NA</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.936306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>clf_logReg2</td>\n",
       "      <td>Up sampling minority class</td>\n",
       "      <td>0.481481</td>\n",
       "      <td>0.458333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>clf_logReg3</td>\n",
       "      <td>Down sampling majority class</td>\n",
       "      <td>0.520548</td>\n",
       "      <td>0.400000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>clf_logReg2</td>\n",
       "      <td>AUC-ROC Metric</td>\n",
       "      <td>0.520548</td>\n",
       "      <td>0.400000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>clf_svc</td>\n",
       "      <td>Penalized-SVM Classifier</td>\n",
       "      <td>0.688034</td>\n",
       "      <td>0.713376</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Model                 Strategy Used  Train accuracy  Test accuracy\n",
       "0  clf_logReg1                            NA        0.916667       0.936306\n",
       "1  clf_logReg2    Up sampling minority class        0.481481       0.458333\n",
       "2  clf_logReg3  Down sampling majority class        0.520548       0.400000\n",
       "3  clf_logReg2                AUC-ROC Metric        0.520548       0.400000\n",
       "4      clf_svc      Penalized-SVM Classifier        0.688034       0.713376"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_details = model_details.append({'Model':'clf_svc','Strategy Used':'Penalized-SVM Classifier','Train accuracy':train_accuracy,'Test accuracy':test_accuracy},\n",
    "                     ignore_index=True)\n",
    "\n",
    "model_details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.Use Tree-based Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can use ensemble learning such as random forest classifier "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1]\n",
      "[0 1]\n"
     ]
    }
   ],
   "source": [
    "clf_rf = RandomForestClassifier().fit(x_train,y_train)\n",
    "\n",
    "y_train_pred = clf_rf.predict(x_train)\n",
    "y_test_pred =  clf_rf.predict(x_test)\n",
    "\n",
    "train_accuracy = accuracy_score(y_train,y_train_pred)\n",
    "test_accuracy = accuracy_score(y_test,y_test_pred)\n",
    "\n",
    "print(np.unique(y_train_pred))\n",
    "print(np.unique(y_test_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Strategy Used</th>\n",
       "      <th>Train accuracy</th>\n",
       "      <th>Test accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>clf_logReg1</td>\n",
       "      <td>NA</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.936306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>clf_logReg2</td>\n",
       "      <td>Up sampling minority class</td>\n",
       "      <td>0.481481</td>\n",
       "      <td>0.458333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>clf_logReg3</td>\n",
       "      <td>Down sampling majority class</td>\n",
       "      <td>0.520548</td>\n",
       "      <td>0.400000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>clf_logReg2</td>\n",
       "      <td>AUC-ROC Metric</td>\n",
       "      <td>0.520548</td>\n",
       "      <td>0.400000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>clf_svc</td>\n",
       "      <td>Penalized-SVM Classifier</td>\n",
       "      <td>0.688034</td>\n",
       "      <td>0.713376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>clf_rf</td>\n",
       "      <td>Random forest classifier</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.923567</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Model                 Strategy Used  Train accuracy  Test accuracy\n",
       "0  clf_logReg1                            NA        0.916667       0.936306\n",
       "1  clf_logReg2    Up sampling minority class        0.481481       0.458333\n",
       "2  clf_logReg3  Down sampling majority class        0.520548       0.400000\n",
       "3  clf_logReg2                AUC-ROC Metric        0.520548       0.400000\n",
       "4      clf_svc      Penalized-SVM Classifier        0.688034       0.713376\n",
       "5       clf_rf      Random forest classifier        1.000000       0.923567"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_details = model_details.append({'Model':'clf_rf','Strategy Used':'Random forest classifier','Train accuracy':train_accuracy,'Test accuracy':test_accuracy},\n",
    "                     ignore_index=True)\n",
    "\n",
    "model_details"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we can see that ensemble technique has given better accuracy on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py37]",
   "language": "python",
   "name": "conda-env-py37-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}