LightGBM_Bayesian.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import packages and data sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install lightgbm\n",
    "!pip install bayesian-optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import pandas as pd\n",
    "#from math import cos, sin\n",
    "import numpy as np\n",
    "import lightgbm as lgbm\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from bayes_opt import BayesianOptimization\n",
    "from sklearn.model_selection import train_test_split\n",
    "import os\n",
    "import sys\n",
    "\n",
    "\n",
    "raw_dir = \"/home/ec2-user/pwp-summer-2019/master_thesis_nhh_2019/raw_data/\" \n",
    "data_dir = \"/home/ec2-user/pwp-summer-2019/master_thesis_nhh_2019/processed_data/\" \n",
    "\n",
    "pd.set_option('display.max_columns', 999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Functions import (feature_engineering)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import joblib\n",
    "\n",
    "df_train = pd.read_pickle(data_dir+'df_train')\n",
    "df_val = pd.read_pickle(data_dir+'df_val')\n",
    "df_test = pd.read_pickle(data_dir+'df_test')\n",
    "\n",
    "formation_dictionary = joblib.load(data_dir+'formation_dictionary.pkl')\n",
    "\n",
    "df_train_val = df_train.append(df_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature engineering and remove outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params_features = {\n",
    "    'outlier_values': {'gr': df_train_val.gr.quantile(0.9995),\n",
    "                       'rmed': df_train_val.rmed.quantile(0.9995),\n",
    "                       'rdep': df_train_val.rdep.quantile(0.9995)\n",
    "                      },\n",
    "    'above_below_variables': ['gr','rdep','rmed'], #,'dt','nphi','rhob'],\n",
    "    'y_variable': 'formation_2',\n",
    "    'num_shifts': 1,\n",
    "    'cols_to_remove' : ['depth', 'dts','hgr', 'hnphi', \n",
    "                        'hrdep', 'hrhob', 'hrmed', 'hrsh','rsh','field','main_area','md'],\n",
    "    'thresh': 7,\n",
    "    'var1_ratio': 'gr'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For home-made stratified split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_class = feature_engineering(df_train,**params_features)\n",
    "\n",
    "train_class.remove_outliers()\n",
    "train_class.above_below()\n",
    "train_class.cleaning()\n",
    "train_class.xyz()\n",
    "\n",
    "df_train = train_class.df\n",
    "columns_class = df_train.columns\n",
    "\n",
    "val_class = feature_engineering(df_val,**params_features)\n",
    "\n",
    "val_class.remove_outliers()\n",
    "val_class.above_below()\n",
    "val_class.cleaning()\n",
    "val_class.xyz()\n",
    "df_val = val_class.df[columns_class]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For sklearn(randomized) stratified split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_class = feature_engineering(df_train_val,**params_features)\n",
    "\n",
    "df_class.remove_outliers()\n",
    "df_class.above_below()\n",
    "df_class.cleaning()\n",
    "df_class.xyz()\n",
    "\n",
    "df = df_class.df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split into train_valid/test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For home-made stratified split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "col = ['formation','title','formation_2','group'] \n",
    "X_train = df_train.drop(col, axis=1)\n",
    "Y_train = df_train['formation_2']\n",
    "\n",
    "X_valid = df_val.drop(col, axis=1)\n",
    "Y_valid = df_val['formation_2']\n",
    "\n",
    "features_list = X_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_valid = X_train.append(X_valid)\n",
    "Y_train_valid = Y_train.append(Y_valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For sklearn(randomized) stratified split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "col = ['formation','title','formation_2','group'] #,'depth','group'\n",
    "X = df.drop(col, axis=1)\n",
    "y = df['formation_2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_and_valid, X_test, Y_train_and_valid, Y_test = train_test_split( X, y, \n",
    "                                                                        test_size=0.10, \n",
    "                                                                        random_state=42, \n",
    "                                                                        stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_valid, Y_train, Y_valid = train_test_split(X_train_and_valid, Y_train_and_valid, \n",
    "                                                      test_size=0.33, \n",
    "                                                      random_state=42, \n",
    "                                                      stratify=Y_train_and_valid)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bayesian"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd;\n",
    "import numpy as np;\n",
    "import lightgbm as lgb\n",
    "from bayes_opt import BayesianOptimization\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "def lgb_evaluate(                \n",
    "                learningRate,\n",
    "                nEstimators,\n",
    "                maxDepth,\n",
    "                numLeaves,\n",
    "                featureFraction,\n",
    "                minDataInLeaf\n",
    "                ):\n",
    "    \n",
    "    clf = lgb.LGBMClassifier(\n",
    "        n_estimators= int(nEstimators),\n",
    "        num_leaves= int(numLeaves),\n",
    "        max_depth= int(maxDepth),\n",
    "        verbose =-1,\n",
    "        learning_rate=float(learningRate),\n",
    "        feature_fraction=float(featureFraction),\n",
    "        min_data_in_leaf=int(minDataInLeaf),\n",
    "        objective = 'multiclass',\n",
    "        metric= 'multi_logloss',\n",
    "        eval_metric= 'multi_logloss')    \n",
    "    \n",
    "    scores = cross_val_score(clf, X_train_valid, Y_train_valid, cv=5, scoring='f1_micro')\n",
    "    print(np.mean(scores))\n",
    "\n",
    "    return np.mean(scores)\n",
    "   \n",
    "def bayesOpt(train_x, train_y):\n",
    "    lgbBO = BayesianOptimization(lgb_evaluate, {\n",
    "                                                'learningRate' : (.05,.5),\n",
    "                                                'nEstimators':(10,150),\n",
    "                                                'numLeaves':  (5, 250),\n",
    "                                                'maxDepth': (2, 90),\n",
    "                                                'featureFraction':(.50,1),\n",
    "                                                'minDataInLeaf':(100,1000)})\n",
    "\n",
    "\n",
    "    lgbBO.maximize(init_points=4, n_iter=8)\n",
    "\n",
    "    \n",
    "    return lgbBO\n",
    "bayes_result=bayesOpt(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results=pd.DataFrame(bayes_result.res)\n",
    "\n",
    "results['learningRate']=results['params'].apply(lambda x: x['maxDepth'])\n",
    "results.plot.scatter(x='learningRate',y='target')\n",
    "bayes_result.max\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bayes_result.set_bounds(new_bounds={\"nEstimators\": (30, 100)})\n",
    "\n",
    "bayes_result.maximize(\n",
    "    init_points=15,\n",
    "    n_iter=30)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_tensorflow_p36",
   "language": "python",
   "name": "conda_tensorflow_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}