Grid_Search_AB.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: pip is being invoked by an old script wrapper. This will fail in a future version of pip.\n",
      "Please see https://github.com/pypa/pip/issues/5599 for advice on fixing the underlying issue.\n",
      "To avoid this problem you can invoke Python with '-m pip' instead of running pip directly.\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.7/site-packages (1.1.2)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /opt/conda/lib/python3.7/site-packages (from pandas) (2.8.1)\n",
      "Requirement already satisfied: pytz>=2017.2 in /opt/conda/lib/python3.7/site-packages (from pandas) (2020.1)\n",
      "Requirement already satisfied: numpy>=1.15.4 in /opt/conda/lib/python3.7/site-packages (from pandas) (1.19.2)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil>=2.7.3->pandas) (1.14.0)\n",
      "WARNING: pip is being invoked by an old script wrapper. This will fail in a future version of pip.\n",
      "Please see https://github.com/pypa/pip/issues/5599 for advice on fixing the underlying issue.\n",
      "To avoid this problem you can invoke Python with '-m pip' instead of running pip directly.\n",
      "Requirement already satisfied: matplotlib in /opt/conda/lib/python3.7/site-packages (3.3.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib) (1.2.0)\n",
      "Requirement already satisfied: numpy>=1.15 in /opt/conda/lib/python3.7/site-packages (from matplotlib) (1.19.2)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib) (2.8.1)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in /opt/conda/lib/python3.7/site-packages (from matplotlib) (2.4.7)\n",
      "Requirement already satisfied: certifi>=2020.06.20 in /opt/conda/lib/python3.7/site-packages (from matplotlib) (2020.6.20)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib) (7.2.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.7/site-packages (from matplotlib) (0.10.0)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil>=2.1->matplotlib) (1.14.0)\n",
      "WARNING: pip is being invoked by an old script wrapper. This will fail in a future version of pip.\n",
      "Please see https://github.com/pypa/pip/issues/5599 for advice on fixing the underlying issue.\n",
      "To avoid this problem you can invoke Python with '-m pip' instead of running pip directly.\n",
      "Requirement already satisfied: sklearn in /opt/conda/lib/python3.7/site-packages (0.0)\n",
      "Requirement already satisfied: scikit-learn in /opt/conda/lib/python3.7/site-packages (from sklearn) (0.23.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->sklearn) (2.1.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->sklearn) (0.16.0)\n",
      "Requirement already satisfied: numpy>=1.13.3 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->sklearn) (1.19.2)\n",
      "Requirement already satisfied: scipy>=0.19.1 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->sklearn) (1.5.2)\n",
      "WARNING: pip is being invoked by an old script wrapper. This will fail in a future version of pip.\n",
      "Please see https://github.com/pypa/pip/issues/5599 for advice on fixing the underlying issue.\n",
      "To avoid this problem you can invoke Python with '-m pip' instead of running pip directly.\n",
      "Requirement already satisfied: seaborn in /opt/conda/lib/python3.7/site-packages (0.11.0)\n",
      "Requirement already satisfied: matplotlib>=2.2 in /opt/conda/lib/python3.7/site-packages (from seaborn) (3.3.1)\n",
      "Requirement already satisfied: scipy>=1.0 in /opt/conda/lib/python3.7/site-packages (from seaborn) (1.5.2)\n",
      "Requirement already satisfied: pandas>=0.23 in /opt/conda/lib/python3.7/site-packages (from seaborn) (1.1.2)\n",
      "Requirement already satisfied: numpy>=1.15 in /opt/conda/lib/python3.7/site-packages (from seaborn) (1.19.2)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib>=2.2->seaborn) (7.2.0)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib>=2.2->seaborn) (2.8.1)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in /opt/conda/lib/python3.7/site-packages (from matplotlib>=2.2->seaborn) (2.4.7)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib>=2.2->seaborn) (1.2.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.7/site-packages (from matplotlib>=2.2->seaborn) (0.10.0)\n",
      "Requirement already satisfied: certifi>=2020.06.20 in /opt/conda/lib/python3.7/site-packages (from matplotlib>=2.2->seaborn) (2020.6.20)\n",
      "Requirement already satisfied: pytz>=2017.2 in /opt/conda/lib/python3.7/site-packages (from pandas>=0.23->seaborn) (2020.1)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil>=2.1->matplotlib>=2.2->seaborn) (1.14.0)\n"
     ]
    }
   ],
   "source": [
    "! pip install pandas\n",
    "! pip install matplotlib\n",
    "! pip install sklearn\n",
    "! pip install seaborn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sklearn.ensemble as skle\n",
    "import sklearn.svm as svm\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "from sklearn.model_selection import GridSearchCV, KFold\n",
    "import sklearn.metrics as sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('V5_final_dataset.csv', index_col=0).drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop(columns=['Glu (vein)'])\n",
    "y = df['Glu (vein)']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_osm = df['Osmolarity (vein)']\n",
    "X_osm = df.drop(columns=['Osmolarity (vein)'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_o, X_test_o, y_train_o, y_test_o = train_test_split(X_osm, y_osm, test_size=0.1, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)\n",
    "X_train_s = preprocessing.normalize(X_train)\n",
    "X_test_s = preprocessing.normalize(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = KFold(n_splits=10, shuffle=True, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "params_grid_AB = {'n_estimators': range(300, 1200, 50),\n",
    "                  'loss': ['linear', 'square', 'exponential']\n",
    "                 }\n",
    "Ada_boost = skle.AdaBoostRegressor(random_state=42)\n",
    "GS_AB = GridSearchCV(Ada_boost, params_grid_AB, cv=cv, scoring='neg_mean_squared_error')\n",
    "GS_AB.fit(X_train, y_train)\n",
    "Ada_boost_GS_y = GS_AB.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "Ada_boost = skle.AdaBoostRegressor(random_state=42, n_estimators=1000, loss='square')\n",
    "Ada_boost.fit(X_train_o, y_train_o)\n",
    "Ada_boost_GS_yo = Ada_boost.predict(X_test_o)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.055320616093225e+30, 0.8338702437498802)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sm.mean_squared_error(y_test_o, Ada_boost_GS_yo), sm.r2_score(y_test_o, Ada_boost_GS_yo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.3220126234906975, 'ctHb (vein)'),\n",
       " (0.15251927669675894, 'Cl- (vein)'),\n",
       " (0.1395901929792498, 'Na + (vein)'),\n",
       " (0.08902439878575912, 'pO2 (vein)'),\n",
       " (0.08896709646409703, 'pCO2 (vein)'),\n",
       " (0.04337559807849602, 'HCO3- (P) (vein)'),\n",
       " (0.04278355807781538, 'Ca2 + ( vein)'),\n",
       " (0.03815110739232668, 'sO2 (vein)'),\n",
       " (0.028984830410525287, 'Hct (vein)'),\n",
       " (0.01219077381532502, 'ABE (vein)'),\n",
       " (0.010985560810820706, 'Glu (vein)'),\n",
       " (0.008651710722291572, 'Lac (vein)'),\n",
       " (0.008002664181387611, 'pH (vein)'),\n",
       " (0.0046115911762977455, 'p50 (vein)'),\n",
       " (0.004585119443776119, 'ctO2 (vein)'),\n",
       " (0.0042608722272970126, 'K + (vein)'),\n",
       " (0.0012317818328148938, 'Age'),\n",
       " (7.12434142636114e-05, 'Sex'),\n",
       " (0.0, 'ctBil (vein)'),\n",
       " (0.0, 'Smoke_status'),\n",
       " (0.0, 'SBE (vein)'),\n",
       " (0.0, 'HCO3- (P, st) (vein)'),\n",
       " (0.0, 'FMetHb (vein)'),\n",
       " (0.0, 'FHbF (vein)'),\n",
       " (0.0, 'FHHb (vein)'),\n",
       " (0.0, 'FCOHb (vein)'),\n",
       " (0.0, 'F02Hb (vein)')]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feat_o = {}\n",
    "for i in range(len(X_train_o.columns)):\n",
    "    feat_o[X_train_o.columns[i]] = Ada_boost.feature_importances_[i]\n",
    "    \n",
    "sorted( ((k,v) for v,k in feat_o.items()), reverse=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12.991472614721305, 0.14713787183664018)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sm.mean_squared_error(y_test, Ada_boost_GS_y), sm.r2_score(y_test, Ada_boost_GS_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(AdaBoostRegressor(loss='square', n_estimators=1000, random_state=42),\n",
       " {'loss': 'square', 'n_estimators': 1000})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "GS_AB.best_estimator_, GS_AB.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nastya_env_3",
   "language": "python",
   "name": "nastya_env_3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}