Hyperparameter tunning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9801cf77",
   "metadata": {},
   "source": [
    "# import libaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6964adcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from imblearn.metrics import classification_report_imbalanced\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from imblearn.under_sampling import RepeatedEditedNearestNeighbours\n",
    "from imblearn.pipeline import Pipeline \n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "924717ce",
   "metadata": {},
   "source": [
    "# import the dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6b34b4d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['HeartDiseaseorAttack', 'HighBP', 'HighChol', 'CholCheck', 'BMI',\n",
       "       'Smoker', 'Stroke', 'Diabetes', 'PhysActivity', 'Fruits', 'Veggies',\n",
       "       'HvyAlcoholConsump', 'AnyHealthcare', 'NoDocbcCost', 'GenHlth',\n",
       "       'MentHlth', 'PhysHlth', 'DiffWalk', 'Sex', 'Age', 'Education',\n",
       "       'Income'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv('../heart_disease/heart_disease.csv')\n",
    "#df.isnull().sum() #check for null values \n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "474dc98f",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop (labels = ['HeartDiseaseorAttack'], axis =1)\n",
    "y = df['HeartDiseaseorAttack']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1012cc9",
   "metadata": {},
   "source": [
    "# Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1e6c66df",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The optimal number of features: 20\n",
      "Best features: Index(['HighBP', 'HighChol', 'BMI', 'Smoker', 'Stroke', 'Diabetes',\n",
      "       'PhysActivity', 'Fruits', 'Veggies', 'HvyAlcoholConsump',\n",
      "       'AnyHealthcare', 'NoDocbcCost', 'GenHlth', 'MentHlth', 'PhysHlth',\n",
      "       'DiffWalk', 'Sex', 'Age', 'Education', 'Income'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_selection import RFECV\n",
    "\n",
    "#balanced_accuracy, or f1_weighted\n",
    "rfecv = RFECV(estimator= DecisionTreeClassifier(), step = 1, cv = 5, scoring=\"balanced_accuracy\", n_jobs=-1)\n",
    "rfecv = rfecv.fit(X, y)\n",
    "\n",
    "print(\"The optimal number of features:\", rfecv.n_features_)\n",
    "print(\"Best features:\", X.columns[rfecv.support_])\n",
    "\n",
    "\n",
    "X_new = rfecv.transform(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57903dfa",
   "metadata": {},
   "source": [
    "# Resampling the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a4ed72ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Y balanced Counter({0.0: 96091, 1.0: 17692})\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train , y_test = train_test_split(X_new,y,test_size=0.3, random_state=1234) \n",
    "\n",
    "over=  RandomOverSampler(sampling_strategy=0.11)\n",
    "under= RepeatedEditedNearestNeighbours(sampling_strategy='majority', max_iter=100,n_neighbors=7, kind_sel='all', n_jobs=-1)\n",
    "\n",
    "X_balanced, y_balanced = over.fit_resample(X_train, y_train)\n",
    "X_balanced, y_balanced = under.fit_resample(X_balanced, y_balanced)\n",
    "\n",
    "print(f'Y balanced {Counter(y_balanced)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35cf9f1a",
   "metadata": {},
   "source": [
    "# Scaling the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "640c3299",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X_balanced)\n",
    "\n",
    "scaleddata=scaler.transform(X_balanced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9285a5f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.55198945 5.30866781]\n"
     ]
    }
   ],
   "source": [
    "# calculate heuristic class weighting\n",
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "\n",
    "weighting = compute_class_weight(class_weight='balanced', classes=[0,1], y=y)\n",
    "print(weighting)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bec041cb",
   "metadata": {},
   "source": [
    "# Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6b425e4c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Parameters: {'C': 0.001, 'class_weight': {0: 0.55, 1: 5.3}, 'penalty': 'l2'}\n",
      "\n",
      "Best Score: 0.9507572447225197\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#hyperparameter tunning\n",
    "#grid search \n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "params = {'penalty':[\"none\", \"l2\"],\n",
    "          'C':[0.001, 0.01, 0.1, 1, 10],\n",
    "          'class_weight':[{0:0.55,1:5.3},{0:5.3,1:0.55}, {0:5,1:1}, {0:1,1:1}, {0:1,1:5}]}\n",
    "\n",
    "model=LogisticRegression()\n",
    "\n",
    "#f1_weighted, recall, balanced_accuracy\n",
    "param_search=GridSearchCV(estimator=model, param_grid=params, cv=5, scoring=\"recall\", n_jobs=-1) \n",
    "\n",
    "best_model = param_search.fit(scaleddata,y_balanced)\n",
    "\n",
    "print('Best Parameters:',best_model.best_params_,end='\\n\\n')\n",
    "print('Best Score:',best_model.best_score_)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f55656d8",
   "metadata": {},
   "source": [
    "# Decision Tree "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4aa0d5ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Parameters: {'class_weight': {0: 0.55, 1: 5.3}, 'criterion': 'entropy', 'max_depth': 30, 'min_samples_leaf': 2, 'min_samples_split': 4}\n",
      "\n",
      "Best Score: 0.9409134561065271\n"
     ]
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "#hyperparameter tunning\n",
    "#grid search \n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "params = {'max_depth':[15,20,25,30,35],\n",
    "          'criterion':['gini','entropy', 'log_loss'],\n",
    "          'class_weight':[{0:0.55,1:5.3},{0:5.3,1:0.55}, {0:5,1:1}, {0:1,1:1}, {0:1,1:5}]}\n",
    "\n",
    "model=DecisionTreeClassifier()\n",
    "\n",
    "#f1_weighted, recall, balanced_accuracy\n",
    "param_search=GridSearchCV(estimator=model, param_grid=params, cv=5, scoring=\"balanced_accuracy\", n_jobs=-1)\n",
    "\n",
    "#not scaled data\n",
    "best_model = param_search.fit(X_balanced, y_balanced)\n",
    "\n",
    "print('Best Parameters:',best_model.best_params_,end='\\n\\n')\n",
    "print('Best Score:',best_model.best_score_)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "479c1b35",
   "metadata": {},
   "source": [
    "# SVM  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6baf5cbe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Parameters: {'C': 1, 'gamma': 0.1}\n",
      "\n",
      "Best Score: 0.9762674226666912\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "#hyperparameter tunning\n",
    "#grid search \n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "params = {'C': [0.01, 0.1, 1, 10], \n",
    "          'gamma': [0.01, 0.1, 1, 10]\n",
    "          #'kernel': ['rbf'],\n",
    "          #'class_weight':[{0:0.55,1:5.3}, {0:1,1:1}]\n",
    "         } \n",
    "\n",
    "model=SVC()\n",
    "\n",
    "#f1_weighted, recall, balanced_accuracy\n",
    "param_search=GridSearchCV(estimator=model, param_grid=params, cv=5, scoring=\"balanced_accuracy\", n_jobs=-1)\n",
    "\n",
    "best_model = param_search.fit(scaleddata, y_balanced)\n",
    "\n",
    "print('Best Parameters:',best_model.best_params_,end='\\n\\n')\n",
    "print('Best Score:',best_model.best_score_)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}