Model-Logistic-Regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already up-to-date: sklearn in /opt/anaconda3/lib/python3.7/site-packages (0.0)\n",
      "Requirement already satisfied, skipping upgrade: scikit-learn in /opt/anaconda3/lib/python3.7/site-packages (from sklearn) (0.21.3)\n",
      "Requirement already satisfied, skipping upgrade: scipy>=0.17.0 in /opt/anaconda3/lib/python3.7/site-packages (from scikit-learn->sklearn) (1.4.1)\n",
      "Requirement already satisfied, skipping upgrade: numpy>=1.11.0 in /opt/anaconda3/lib/python3.7/site-packages (from scikit-learn->sklearn) (1.17.2)\n",
      "Requirement already satisfied, skipping upgrade: joblib>=0.11 in /opt/anaconda3/lib/python3.7/site-packages (from scikit-learn->sklearn) (0.13.2)\n"
     ]
    }
   ],
   "source": [
    "# Update sklearn to prevent version mismatches\n",
    "!pip install sklearn --upgrade"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: joblib in /opt/anaconda3/lib/python3.7/site-packages (0.13.2)\n"
     ]
    }
   ],
   "source": [
    "# install joblib. This will be used to save your model. \n",
    "# Restart your kernel after installing \n",
    "!pip install joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter('ignore', FutureWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Pre-Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6991, 40) (6991,)\n"
     ]
    }
   ],
   "source": [
    "# Load and Clean Data\n",
    "\n",
    "df = pd.read_csv(\"exoplanet_data.csv\")\n",
    "\n",
    "# Drop the null columns where all values are null\n",
    "df = df.dropna(axis='columns', how='all')\n",
    "\n",
    "# Drop the null rows\n",
    "df = df.dropna()\n",
    "\n",
    "\n",
    "# Set features(X) and target(y) values\n",
    "X = df.drop(\"koi_disposition\", axis=1)\n",
    "y = df[\"koi_disposition\"]\n",
    "print(X.shape, y.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>koi_fpflag_nt</th>\n",
       "      <th>koi_fpflag_ss</th>\n",
       "      <th>koi_fpflag_co</th>\n",
       "      <th>koi_fpflag_ec</th>\n",
       "      <th>koi_period</th>\n",
       "      <th>koi_period_err1</th>\n",
       "      <th>koi_period_err2</th>\n",
       "      <th>koi_time0bk</th>\n",
       "      <th>koi_time0bk_err1</th>\n",
       "      <th>koi_time0bk_err2</th>\n",
       "      <th>...</th>\n",
       "      <th>koi_steff_err2</th>\n",
       "      <th>koi_slogg</th>\n",
       "      <th>koi_slogg_err1</th>\n",
       "      <th>koi_slogg_err2</th>\n",
       "      <th>koi_srad</th>\n",
       "      <th>koi_srad_err1</th>\n",
       "      <th>koi_srad_err2</th>\n",
       "      <th>ra</th>\n",
       "      <th>dec</th>\n",
       "      <th>koi_kepmag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>4002</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>99.673478</td>\n",
       "      <td>3.463000e-04</td>\n",
       "      <td>-3.463000e-04</td>\n",
       "      <td>219.334830</td>\n",
       "      <td>0.002300</td>\n",
       "      <td>-0.002300</td>\n",
       "      <td>...</td>\n",
       "      <td>-148</td>\n",
       "      <td>4.777</td>\n",
       "      <td>0.040</td>\n",
       "      <td>-0.027</td>\n",
       "      <td>0.492</td>\n",
       "      <td>0.026</td>\n",
       "      <td>-0.027</td>\n",
       "      <td>293.05801</td>\n",
       "      <td>45.248821</td>\n",
       "      <td>15.801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4246</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.592244</td>\n",
       "      <td>9.000000e-08</td>\n",
       "      <td>-9.000000e-08</td>\n",
       "      <td>131.654831</td>\n",
       "      <td>0.000124</td>\n",
       "      <td>-0.000124</td>\n",
       "      <td>...</td>\n",
       "      <td>-146</td>\n",
       "      <td>4.664</td>\n",
       "      <td>0.056</td>\n",
       "      <td>-0.032</td>\n",
       "      <td>0.591</td>\n",
       "      <td>0.045</td>\n",
       "      <td>-0.045</td>\n",
       "      <td>290.28094</td>\n",
       "      <td>45.464260</td>\n",
       "      <td>15.653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>548</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>9.991625</td>\n",
       "      <td>5.360000e-06</td>\n",
       "      <td>-5.360000e-06</td>\n",
       "      <td>137.447816</td>\n",
       "      <td>0.000445</td>\n",
       "      <td>-0.000445</td>\n",
       "      <td>...</td>\n",
       "      <td>-176</td>\n",
       "      <td>4.338</td>\n",
       "      <td>0.153</td>\n",
       "      <td>-0.187</td>\n",
       "      <td>1.096</td>\n",
       "      <td>0.309</td>\n",
       "      <td>-0.206</td>\n",
       "      <td>301.04239</td>\n",
       "      <td>45.022888</td>\n",
       "      <td>14.039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3953</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>178.412990</td>\n",
       "      <td>3.100000e-05</td>\n",
       "      <td>-3.100000e-05</td>\n",
       "      <td>218.225235</td>\n",
       "      <td>0.000127</td>\n",
       "      <td>-0.000127</td>\n",
       "      <td>...</td>\n",
       "      <td>-134</td>\n",
       "      <td>4.346</td>\n",
       "      <td>0.084</td>\n",
       "      <td>-0.126</td>\n",
       "      <td>1.148</td>\n",
       "      <td>0.202</td>\n",
       "      <td>-0.124</td>\n",
       "      <td>288.32785</td>\n",
       "      <td>38.627621</td>\n",
       "      <td>13.944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2362</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>45.294223</td>\n",
       "      <td>5.600000e-05</td>\n",
       "      <td>-5.600000e-05</td>\n",
       "      <td>138.678725</td>\n",
       "      <td>0.000987</td>\n",
       "      <td>-0.000987</td>\n",
       "      <td>...</td>\n",
       "      <td>-68</td>\n",
       "      <td>4.347</td>\n",
       "      <td>0.030</td>\n",
       "      <td>-0.030</td>\n",
       "      <td>1.044</td>\n",
       "      <td>0.057</td>\n",
       "      <td>-0.042</td>\n",
       "      <td>285.67938</td>\n",
       "      <td>50.241299</td>\n",
       "      <td>10.961</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      koi_fpflag_nt  koi_fpflag_ss  koi_fpflag_co  koi_fpflag_ec  koi_period  \\\n",
       "4002              0              0              1              0   99.673478   \n",
       "4246              0              1              0              0    0.592244   \n",
       "548               0              1              1              0    9.991625   \n",
       "3953              0              1              0              0  178.412990   \n",
       "2362              0              0              0              0   45.294223   \n",
       "\n",
       "      koi_period_err1  koi_period_err2  koi_time0bk  koi_time0bk_err1  \\\n",
       "4002     3.463000e-04    -3.463000e-04   219.334830          0.002300   \n",
       "4246     9.000000e-08    -9.000000e-08   131.654831          0.000124   \n",
       "548      5.360000e-06    -5.360000e-06   137.447816          0.000445   \n",
       "3953     3.100000e-05    -3.100000e-05   218.225235          0.000127   \n",
       "2362     5.600000e-05    -5.600000e-05   138.678725          0.000987   \n",
       "\n",
       "      koi_time0bk_err2  ...  koi_steff_err2  koi_slogg  koi_slogg_err1  \\\n",
       "4002         -0.002300  ...            -148      4.777           0.040   \n",
       "4246         -0.000124  ...            -146      4.664           0.056   \n",
       "548          -0.000445  ...            -176      4.338           0.153   \n",
       "3953         -0.000127  ...            -134      4.346           0.084   \n",
       "2362         -0.000987  ...             -68      4.347           0.030   \n",
       "\n",
       "      koi_slogg_err2  koi_srad  koi_srad_err1  koi_srad_err2         ra  \\\n",
       "4002          -0.027     0.492          0.026         -0.027  293.05801   \n",
       "4246          -0.032     0.591          0.045         -0.045  290.28094   \n",
       "548           -0.187     1.096          0.309         -0.206  301.04239   \n",
       "3953          -0.126     1.148          0.202         -0.124  288.32785   \n",
       "2362          -0.030     1.044          0.057         -0.042  285.67938   \n",
       "\n",
       "            dec  koi_kepmag  \n",
       "4002  45.248821      15.801  \n",
       "4246  45.464260      15.653  \n",
       "548   45.022888      14.039  \n",
       "3953  38.627621      13.944  \n",
       "2362  50.241299      10.961  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train and Test split data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Using stratify for fixing proportion of values\n",
    "# Source Link: https://towardsdatascience.com/6-amateur-mistakes-ive-made-working-with-train-test-splits-916fabb421bb\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1, stratify=y)\n",
    "\n",
    "\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MinMaxScaler\n",
    "from sklearn.preprocessing import LabelEncoder, MinMaxScaler\n",
    "\n",
    "X_scaler = MinMaxScaler().fit(X_train)\n",
    "\n",
    "X_train_scaled = X_scaler.transform(X_train)\n",
    "X_test_scaled = X_scaler.transform(X_test)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create and Train(Fit) Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### First Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:929: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='warn', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a Logistic Regression Model\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "classifier1 = LogisticRegression()\n",
    "\n",
    "# Fit (train) or model - First Classifier\n",
    "classifier1.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Data Score: 0.6620255578867061\n",
      "Testing Data Score: 0.6653318077803204\n"
     ]
    }
   ],
   "source": [
    "print(f\"Training Data Score: {classifier1.score(X_train, y_train)}\")\n",
    "print(f\"Testing Data Score: {classifier1.score(X_test, y_test)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Second Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='warn', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a Logistic Regression Model\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "classifier2 = LogisticRegression()\n",
    "\n",
    "# Fit (train) or model - Second Classifier\n",
    "classifier2.fit(X_train_scaled, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Data Score: 0.8411214953271028\n",
      "Testing Data Score: 0.8409610983981693\n"
     ]
    }
   ],
   "source": [
    "print(f\"Training Data Score: {classifier2.score(X_train_scaled, y_train)}\")\n",
    "print(f\"Testing Data Score: {classifier2.score(X_test_scaled, y_test)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameter Tuning - GridSearchCV "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Source Link: https://towardsdatascience.com/logistic-regression-model-tuning-with-scikit-learn-part-1-425142e01af5\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "param_grid = {\n",
    "     'penalty' : ['l1', 'l2'],\n",
    "    'C': [1, 5, 10, 50]}\n",
    "\n",
    "grid = GridSearchCV(classifier2, param_grid = param_grid, cv = 5, verbose=3, n_jobs=-1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  24 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=-1)]: Done  40 out of  40 | elapsed:  4.2min finished\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:929: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
       "                                          fit_intercept=True,\n",
       "                                          intercept_scaling=1, l1_ratio=None,\n",
       "                                          max_iter=100, multi_class='warn',\n",
       "                                          n_jobs=None, penalty='l2',\n",
       "                                          random_state=None, solver='warn',\n",
       "                                          tol=0.0001, verbose=0,\n",
       "                                          warm_start=False),\n",
       "             iid='warn', n_jobs=-1,\n",
       "             param_grid={'C': [1, 5, 10, 50], 'penalty': ['l1', 'l2']},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=3)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit the model\n",
    "\n",
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Quantify our Trained Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 5, 'penalty': 'l1'}\n",
      "Best Grid score: 0.885\n"
     ]
    }
   ],
   "source": [
    "# List the best parameters for this dataset\n",
    "# List the best score\n",
    "\n",
    "print(grid.best_params_)\n",
    "print('Best Grid score: %.3f' % grid.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression\n",
      "Test Accuracy: 0.880\n",
      "                precision    recall  f1-score   support\n",
      "\n",
      "     CANDIDATE       0.82      0.69      0.75       422\n",
      "     CONFIRMED       0.75      0.84      0.79       450\n",
      "FALSE POSITIVE       0.98      0.99      0.98       876\n",
      "\n",
      "      accuracy                           0.88      1748\n",
      "     macro avg       0.85      0.84      0.84      1748\n",
      "  weighted avg       0.88      0.88      0.88      1748\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Make predictions with the hypertuned model\n",
    "predictions = grid.predict(X_test)\n",
    "\n",
    "print('Logistic Regression')\n",
    "print('Test Accuracy: %.3f' % grid.score(X_test, y_test))\n",
    "\n",
    "# Calculate classification report\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Save the Model (If the best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Create a file for your best model and push to GitHub\n",
    "# import joblib\n",
    "# filename = 'logistic-regression.sav'\n",
    "# joblib.dump(grid, filename)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}