Predictive Modeling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7e97fe74",
   "metadata": {},
   "source": [
    "# **Modeling**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1241109",
   "metadata": {},
   "source": [
    "# **Imports**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9e2cd85d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Numpy\n",
    "import numpy as np\n",
    "from numpy import median\n",
    "\n",
    "#Pandas\n",
    "import pandas as pd\n",
    "\n",
    "#Seaborn\n",
    "import seaborn as sns\n",
    "\n",
    "#matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import plotly\n",
    "from matplotlib.ticker import ScalarFormatter\n",
    "\n",
    "#Sklearn\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
    "from sklearn.model_selection import GridSearchCV, train_test_split\n",
    "from sklearn.compose import make_column_selector, make_column_transformer\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from tensorflow.keras.metrics import RootMeanSquaredError\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\n",
    "import lightgbm as lgb\n",
    "\n",
    "#Scipy\n",
    "from scipy.sparse import csr_matrix\n",
    "\n",
    "#Warnings\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa8ba1b4",
   "metadata": {},
   "source": [
    "# **Loading Data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f7dbcc02",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading in the data from the previous notebook\n",
    "fd = pd.read_csv('fd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "080392f2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Order #</th>\n",
       "      <th>Transaction Type</th>\n",
       "      <th>Order Time</th>\n",
       "      <th>Order Date</th>\n",
       "      <th>Delivery Date</th>\n",
       "      <th>Product Total</th>\n",
       "      <th>Delivery</th>\n",
       "      <th>Nontaxable Delivery</th>\n",
       "      <th>Wire Out Fee</th>\n",
       "      <th>Discount</th>\n",
       "      <th>Gift Cards</th>\n",
       "      <th>Tax</th>\n",
       "      <th>Tips</th>\n",
       "      <th>Grand Total</th>\n",
       "      <th>Payment Method</th>\n",
       "      <th>Order Type</th>\n",
       "      <th>Order Method</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100017453</td>\n",
       "      <td>Sale</td>\n",
       "      <td>03:28:16PM Sat</td>\n",
       "      <td>2018-12-29</td>\n",
       "      <td>2018-12-29</td>\n",
       "      <td>815.00</td>\n",
       "      <td>$0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>79.46</td>\n",
       "      <td>0.0</td>\n",
       "      <td>894.46</td>\n",
       "      <td>Credit Card</td>\n",
       "      <td>Taken</td>\n",
       "      <td>Phone</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100017452</td>\n",
       "      <td>Sale</td>\n",
       "      <td>03:16:50PM Sat</td>\n",
       "      <td>2018-12-29</td>\n",
       "      <td>2018-12-29</td>\n",
       "      <td>249.90</td>\n",
       "      <td>$0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-124.96</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>137.13</td>\n",
       "      <td>Credit Card</td>\n",
       "      <td>Taken</td>\n",
       "      <td>Walk-In</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100017451</td>\n",
       "      <td>Sale</td>\n",
       "      <td>02:58:53PM Sat</td>\n",
       "      <td>2018-12-29</td>\n",
       "      <td>2018-12-29</td>\n",
       "      <td>24.95</td>\n",
       "      <td>$0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.43</td>\n",
       "      <td>0.0</td>\n",
       "      <td>27.38</td>\n",
       "      <td>Credit Card</td>\n",
       "      <td>Taken</td>\n",
       "      <td>Walk-In</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100017450</td>\n",
       "      <td>Sale</td>\n",
       "      <td>02:54:45PM Sat</td>\n",
       "      <td>2018-12-29</td>\n",
       "      <td>2018-12-29</td>\n",
       "      <td>635.00</td>\n",
       "      <td>$0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61.91</td>\n",
       "      <td>0.0</td>\n",
       "      <td>696.91</td>\n",
       "      <td>Credit Card</td>\n",
       "      <td>Taken</td>\n",
       "      <td>Phone</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100017202</td>\n",
       "      <td>Sale</td>\n",
       "      <td>02:48:15PM Sat</td>\n",
       "      <td>2018-12-29</td>\n",
       "      <td>2018-12-23</td>\n",
       "      <td>702.00</td>\n",
       "      <td>$0.00</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>68.45</td>\n",
       "      <td>0.0</td>\n",
       "      <td>795.45</td>\n",
       "      <td>Credit Card</td>\n",
       "      <td>Delivery</td>\n",
       "      <td>Walk-In</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Order # Transaction Type      Order Time  Order Date Delivery Date  \\\n",
       "0  100017453             Sale  03:28:16PM Sat  2018-12-29    2018-12-29   \n",
       "1  100017452             Sale  03:16:50PM Sat  2018-12-29    2018-12-29   \n",
       "2  100017451             Sale  02:58:53PM Sat  2018-12-29    2018-12-29   \n",
       "3  100017450             Sale  02:54:45PM Sat  2018-12-29    2018-12-29   \n",
       "4  100017202             Sale  02:48:15PM Sat  2018-12-29    2018-12-23   \n",
       "\n",
       "   Product Total Delivery  Nontaxable Delivery  Wire Out Fee  Discount  \\\n",
       "0         815.00    $0.00                  0.0           0.0      0.00   \n",
       "1         249.90    $0.00                  0.0           0.0   -124.96   \n",
       "2          24.95    $0.00                  0.0           0.0      0.00   \n",
       "3         635.00    $0.00                  0.0           0.0      0.00   \n",
       "4         702.00    $0.00                 25.0           0.0      0.00   \n",
       "\n",
       "   Gift Cards    Tax  Tips  Grand Total Payment Method Order Type  \\\n",
       "0         0.0  79.46   0.0       894.46    Credit Card      Taken   \n",
       "1         0.0  12.19   0.0       137.13    Credit Card      Taken   \n",
       "2         0.0   2.43   0.0        27.38    Credit Card      Taken   \n",
       "3         0.0  61.91   0.0       696.91    Credit Card      Taken   \n",
       "4         0.0  68.45   0.0       795.45    Credit Card   Delivery   \n",
       "\n",
       "  Order Method  Year  \n",
       "0        Phone  2018  \n",
       "1      Walk-In  2018  \n",
       "2      Walk-In  2018  \n",
       "3        Phone  2018  \n",
       "4      Walk-In  2018  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Taking a glance at the data to ensure it was laoded properly\n",
    "fd.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "509e3245",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 46062 entries, 0 to 46061\n",
      "Data columns (total 18 columns):\n",
      " #   Column               Non-Null Count  Dtype  \n",
      "---  ------               --------------  -----  \n",
      " 0   Order #              46062 non-null  int64  \n",
      " 1   Transaction Type     46062 non-null  object \n",
      " 2   Order Time           46062 non-null  object \n",
      " 3   Order Date           46062 non-null  object \n",
      " 4   Delivery Date        46062 non-null  object \n",
      " 5   Product Total        46062 non-null  float64\n",
      " 6   Delivery             46062 non-null  object \n",
      " 7   Nontaxable Delivery  46062 non-null  float64\n",
      " 8   Wire Out Fee         46062 non-null  float64\n",
      " 9   Discount             46062 non-null  float64\n",
      " 10  Gift Cards           46062 non-null  float64\n",
      " 11  Tax                  46062 non-null  float64\n",
      " 12  Tips                 46062 non-null  float64\n",
      " 13  Grand Total          46062 non-null  float64\n",
      " 14  Payment Method       46062 non-null  object \n",
      " 15  Order Type           46062 non-null  object \n",
      " 16  Order Method         46062 non-null  object \n",
      " 17  Year                 46062 non-null  int64  \n",
      "dtypes: float64(8), int64(2), object(8)\n",
      "memory usage: 6.3+ MB\n"
     ]
    }
   ],
   "source": [
    "# Looking at the data types  and ensure there are no missing values\n",
    "fd.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1789ee63",
   "metadata": {},
   "source": [
    "# **Feature Engineering**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f6bed98d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filtering out the years 2016 & 2017 because the data for those years is incomplete.\n",
    "# The data for those years is incomplete and also could be considered irrelevant because of how old it is.\n",
    "filtered_data = fd[(fd['Year'] != 2016) & (fd['Year'] != 2017)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9afd2ca4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Spliting up the 'Delivery Date' column into indiviual columns for year, month, and day to prepare the data for modeling.\n",
    "fd['Delivery Date'] = pd.to_datetime(fd['Delivery Date'])\n",
    "fd['Delivery Year'] = fd['Delivery Date'].dt.year\n",
    "fd['Delivery Day'] = fd['Delivery Date'].dt.day\n",
    "fd['Delivery Month'] = fd['Delivery Date'].dt.month\n",
    "\n",
    "#Spliting up the 'Order Date' column into indiviual columns for year, month, and day to prepare the data for modeling.\n",
    "fd['Order Date'] = pd.to_datetime(fd['Order Date'])\n",
    "fd['Order Year'] = fd['Order Date'].dt.year\n",
    "fd['Order Day'] = fd['Order Date'].dt.day\n",
    "fd['Order Month'] = fd['Order Date'].dt.month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4bedaa9f",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 46062 entries, 0 to 46061\n",
      "Data columns (total 24 columns):\n",
      " #   Column               Non-Null Count  Dtype         \n",
      "---  ------               --------------  -----         \n",
      " 0   Order #              46062 non-null  int64         \n",
      " 1   Transaction Type     46062 non-null  object        \n",
      " 2   Order Time           46062 non-null  object        \n",
      " 3   Order Date           46062 non-null  datetime64[ns]\n",
      " 4   Delivery Date        46062 non-null  datetime64[ns]\n",
      " 5   Product Total        46062 non-null  float64       \n",
      " 6   Delivery             46062 non-null  object        \n",
      " 7   Nontaxable Delivery  46062 non-null  float64       \n",
      " 8   Wire Out Fee         46062 non-null  float64       \n",
      " 9   Discount             46062 non-null  float64       \n",
      " 10  Gift Cards           46062 non-null  float64       \n",
      " 11  Tax                  46062 non-null  float64       \n",
      " 12  Tips                 46062 non-null  float64       \n",
      " 13  Grand Total          46062 non-null  float64       \n",
      " 14  Payment Method       46062 non-null  object        \n",
      " 15  Order Type           46062 non-null  object        \n",
      " 16  Order Method         46062 non-null  object        \n",
      " 17  Year                 46062 non-null  int64         \n",
      " 18  Delivery Year        46062 non-null  int64         \n",
      " 19  Delivery Day         46062 non-null  int64         \n",
      " 20  Delivery Month       46062 non-null  int64         \n",
      " 21  Order Year           46062 non-null  int64         \n",
      " 22  Order Day            46062 non-null  int64         \n",
      " 23  Order Month          46062 non-null  int64         \n",
      "dtypes: datetime64[ns](2), float64(8), int64(8), object(6)\n",
      "memory usage: 8.4+ MB\n"
     ]
    }
   ],
   "source": [
    "#Confirming the new columns were formed properly\n",
    "fd.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ac51040",
   "metadata": {},
   "source": [
    "### **More Feature Engineering**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "00ad69ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "#transformed_data = fd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a0916663",
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of columns to drop\n",
    "columns_to_drop = ['Tax', 'Wire Out Fee', 'Tips', 'Discount', 'Gift Cards', 'Nontaxable Delivery', 'Order Type', 'Delivery', 'Payment Method', 'Order Method', 'Delivery Date', 'Transaction Type', 'Order Time', 'Order Date', 'Year']\n",
    "fd.drop(columns=columns_to_drop, inplace=True)\n",
    "X_reduced = fd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cee101f9",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 46062 entries, 0 to 46061\n",
      "Data columns (total 9 columns):\n",
      " #   Column          Non-Null Count  Dtype  \n",
      "---  ------          --------------  -----  \n",
      " 0   Order #         46062 non-null  int64  \n",
      " 1   Product Total   46062 non-null  float64\n",
      " 2   Grand Total     46062 non-null  float64\n",
      " 3   Delivery Year   46062 non-null  int64  \n",
      " 4   Delivery Day    46062 non-null  int64  \n",
      " 5   Delivery Month  46062 non-null  int64  \n",
      " 6   Order Year      46062 non-null  int64  \n",
      " 7   Order Day       46062 non-null  int64  \n",
      " 8   Order Month     46062 non-null  int64  \n",
      "dtypes: float64(2), int64(7)\n",
      "memory usage: 3.2 MB\n"
     ]
    }
   ],
   "source": [
    "fd = X_reduced\n",
    "fd.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3bb850a8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Order #</th>\n",
       "      <th>Product Total</th>\n",
       "      <th>Grand Total</th>\n",
       "      <th>Delivery Year</th>\n",
       "      <th>Delivery Day</th>\n",
       "      <th>Delivery Month</th>\n",
       "      <th>Order Year</th>\n",
       "      <th>Order Day</th>\n",
       "      <th>Order Month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100017453</td>\n",
       "      <td>815.00</td>\n",
       "      <td>894.46</td>\n",
       "      <td>2018</td>\n",
       "      <td>29</td>\n",
       "      <td>12</td>\n",
       "      <td>2018</td>\n",
       "      <td>29</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100017452</td>\n",
       "      <td>249.90</td>\n",
       "      <td>137.13</td>\n",
       "      <td>2018</td>\n",
       "      <td>29</td>\n",
       "      <td>12</td>\n",
       "      <td>2018</td>\n",
       "      <td>29</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100017451</td>\n",
       "      <td>24.95</td>\n",
       "      <td>27.38</td>\n",
       "      <td>2018</td>\n",
       "      <td>29</td>\n",
       "      <td>12</td>\n",
       "      <td>2018</td>\n",
       "      <td>29</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100017450</td>\n",
       "      <td>635.00</td>\n",
       "      <td>696.91</td>\n",
       "      <td>2018</td>\n",
       "      <td>29</td>\n",
       "      <td>12</td>\n",
       "      <td>2018</td>\n",
       "      <td>29</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100017202</td>\n",
       "      <td>702.00</td>\n",
       "      <td>795.45</td>\n",
       "      <td>2018</td>\n",
       "      <td>23</td>\n",
       "      <td>12</td>\n",
       "      <td>2018</td>\n",
       "      <td>29</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Order #  Product Total  Grand Total  Delivery Year  Delivery Day  \\\n",
       "0  100017453         815.00       894.46           2018            29   \n",
       "1  100017452         249.90       137.13           2018            29   \n",
       "2  100017451          24.95        27.38           2018            29   \n",
       "3  100017450         635.00       696.91           2018            29   \n",
       "4  100017202         702.00       795.45           2018            23   \n",
       "\n",
       "   Delivery Month  Order Year  Order Day  Order Month  \n",
       "0              12        2018         29           12  \n",
       "1              12        2018         29           12  \n",
       "2              12        2018         29           12  \n",
       "3              12        2018         29           12  \n",
       "4              12        2018         29           12  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fd.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22e57a92",
   "metadata": {},
   "source": [
    "# **Preprocessing**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc7e1bc9",
   "metadata": {},
   "source": [
    "### **Validation Split**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "80110fbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop the target column to separate it from the features\n",
    "X = fd.drop(columns='Grand Total')\n",
    "y = fd['Grand Total']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8c43971b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Split training and test\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2b2c7a11",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Imputers\n",
    "freq_imputer = SimpleImputer(strategy='most_frequent')\n",
    "mean_imputer = SimpleImputer(strategy= 'mean')\n",
    "\n",
    "#Scaler\n",
    "scaler = StandardScaler()\n",
    "\n",
    "#OneHotEncoder\n",
    "ohe =OneHotEncoder(handle_unknown='ignore', sparse=True)\n",
    "\n",
    "#Create Selectors\n",
    "cat_selector = make_column_selector(dtype_include= 'object')\n",
    "num_selector = make_column_selector(dtype_include='number')\n",
    "\n",
    "# Numeric pipeline\n",
    "numeric_pipe = make_pipeline(mean_imputer, scaler)\n",
    "numeric_pipe\n",
    "\n",
    "#cat pipe\n",
    "categorical_pipe = make_pipeline(freq_imputer, ohe)\n",
    "categorical_pipe\n",
    "\n",
    "#Make tuples for preprocessing the categorical and numeric columns\n",
    "num_tuple = (numeric_pipe, num_selector)\n",
    "cat_tuple = (categorical_pipe, cat_selector)          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d60b10f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create Column Transformer\n",
    "preprocessor= make_column_transformer(num_tuple, cat_tuple, remainder = 'passthrough')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "32f48e66",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Order #             int64\n",
      "Product Total     float64\n",
      "Delivery Year       int64\n",
      "Delivery Day        int64\n",
      "Delivery Month      int64\n",
      "Order Year          int64\n",
      "Order Day           int64\n",
      "Order Month         int64\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "#Printing data types for the x train\n",
    "print(X_train.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "cfac7986",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocess the data\n",
    "# Fit and transform the preprocessing on the training data\n",
    "X_train_preprocessed = preprocessor.fit_transform(X_train)\n",
    "\n",
    "# Transform the test data using the fitted preprocessor\n",
    "X_test_preprocessed = preprocessor.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "571bdf98",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>ColumnTransformer(remainder=&#x27;passthrough&#x27;,\n",
       "                  transformers=[(&#x27;pipeline-1&#x27;,\n",
       "                                 Pipeline(steps=[(&#x27;simpleimputer&#x27;,\n",
       "                                                  SimpleImputer()),\n",
       "                                                 (&#x27;standardscaler&#x27;,\n",
       "                                                  StandardScaler())]),\n",
       "                                 &lt;sklearn.compose._column_transformer.make_column_selector object at 0x0000020BBB79C2B0&gt;),\n",
       "                                (&#x27;pipeline-2&#x27;,\n",
       "                                 Pipeline(steps=[(&#x27;simpleimputer&#x27;,\n",
       "                                                  SimpleImputer(strategy=&#x27;most_frequent&#x27;)),\n",
       "                                                 (&#x27;onehotencoder&#x27;,\n",
       "                                                  OneHotEncoder(handle_unknown=&#x27;ignore&#x27;))]),\n",
       "                                 &lt;sklearn.compose._column_transformer.make_column_selector object at 0x0000020BBB79C1C0&gt;)])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(remainder=&#x27;passthrough&#x27;,\n",
       "                  transformers=[(&#x27;pipeline-1&#x27;,\n",
       "                                 Pipeline(steps=[(&#x27;simpleimputer&#x27;,\n",
       "                                                  SimpleImputer()),\n",
       "                                                 (&#x27;standardscaler&#x27;,\n",
       "                                                  StandardScaler())]),\n",
       "                                 &lt;sklearn.compose._column_transformer.make_column_selector object at 0x0000020BBB79C2B0&gt;),\n",
       "                                (&#x27;pipeline-2&#x27;,\n",
       "                                 Pipeline(steps=[(&#x27;simpleimputer&#x27;,\n",
       "                                                  SimpleImputer(strategy=&#x27;most_frequent&#x27;)),\n",
       "                                                 (&#x27;onehotencoder&#x27;,\n",
       "                                                  OneHotEncoder(handle_unknown=&#x27;ignore&#x27;))]),\n",
       "                                 &lt;sklearn.compose._column_transformer.make_column_selector object at 0x0000020BBB79C1C0&gt;)])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">pipeline-1</label><div class=\"sk-toggleable__content\"><pre>&lt;sklearn.compose._column_transformer.make_column_selector object at 0x0000020BBB79C2B0&gt;</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">StandardScaler</label><div class=\"sk-toggleable__content\"><pre>StandardScaler()</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">pipeline-2</label><div class=\"sk-toggleable__content\"><pre>&lt;sklearn.compose._column_transformer.make_column_selector object at 0x0000020BBB79C1C0&gt;</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(strategy=&#x27;most_frequent&#x27;)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-7\" type=\"checkbox\" ><label for=\"sk-estimator-id-7\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">OneHotEncoder</label><div class=\"sk-toggleable__content\"><pre>OneHotEncoder(handle_unknown=&#x27;ignore&#x27;)</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-8\" type=\"checkbox\" ><label for=\"sk-estimator-id-8\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">remainder</label><div class=\"sk-toggleable__content\"><pre>[]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-9\" type=\"checkbox\" ><label for=\"sk-estimator-id-9\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">passthrough</label><div class=\"sk-toggleable__content\"><pre>passthrough</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "ColumnTransformer(remainder='passthrough',\n",
       "                  transformers=[('pipeline-1',\n",
       "                                 Pipeline(steps=[('simpleimputer',\n",
       "                                                  SimpleImputer()),\n",
       "                                                 ('standardscaler',\n",
       "                                                  StandardScaler())]),\n",
       "                                 <sklearn.compose._column_transformer.make_column_selector object at 0x0000020BBB79C2B0>),\n",
       "                                ('pipeline-2',\n",
       "                                 Pipeline(steps=[('simpleimputer',\n",
       "                                                  SimpleImputer(strategy='most_frequent')),\n",
       "                                                 ('onehotencoder',\n",
       "                                                  OneHotEncoder(handle_unknown='ignore'))]),\n",
       "                                 <sklearn.compose._column_transformer.make_column_selector object at 0x0000020BBB79C1C0>)])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Looking at how the preprocessor is organized\n",
    "preprocessor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "3bd5102f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train_preprocessed shape: (34546, 8)\n",
      "y_train shape: (34546,)\n"
     ]
    }
   ],
   "source": [
    "# Ensuring the shape is correct\n",
    "print(\"X_train_preprocessed shape:\", X_train_preprocessed.shape)\n",
    "print(\"y_train shape:\", y_train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28444384",
   "metadata": {},
   "source": [
    "# **Linear Regression Model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "41a41759",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a LinearRegression model\n",
    "regressor = LinearRegression()\n",
    "\n",
    "# Train the model using the preprocessed training sets\n",
    "regressor.fit(X_train_preprocessed, y_train)\n",
    "\n",
    "# Make predictions for training data\n",
    "y_predictions_train = regressor.predict(X_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = regressor.predict(X_test_preprocessed)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bda42769",
   "metadata": {},
   "source": [
    "## **Tuning Hyperparameters**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "21f4a1bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function for evaluating Model preformance\n",
    "def evaluate_model(y_true, y_pred, split='training'):\n",
    "  \"\"\" prints RMSE, and R2 metrics, include which data split was evaluated\n",
    "\n",
    "  Args:\n",
    "    y_true: y-train or y-test\n",
    "    y_pred: result of model.predict(X)\n",
    "    split: which data split is being evaluate ['training','test']\n",
    "  \"\"\"\n",
    "\n",
    "  r2 = r2_score(y_true,y_pred)\n",
    "  mae = mean_absolute_error(y_true,y_pred)\n",
    "  mse = mean_squared_error(y_true, y_pred)\n",
    "  rmse = mean_squared_error(y_true,y_pred,squared=False)\n",
    "\n",
    "\n",
    "  print(f'Results for {split} data:')\n",
    "  print(f\"  - R^2 = {round(r2,3)}\")\n",
    "  print(f\"  - MAE = {round(mae,3)}\")\n",
    "  print(f\"  - MSE = {round(mse,3)}\")\n",
    "  print(f\"  - RMSE = {round(rmse,3)}\")\n",
    "  print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "975c4e4f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for training data:\n",
      "  - R^2 = -68730864093.04\n",
      "  - MAE = 31272481.788\n",
      "  - MSE = 2770656803286024.5\n",
      "  - RMSE = 52637028.823\n",
      "\n",
      "Results for testing data:\n",
      "  - R^2 = 0.94\n",
      "  - MAE = 13.926\n",
      "  - MSE = 1812.342\n",
      "  - RMSE = 42.572\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## Evaluate model's performance\n",
    "evaluate_model(y_train, y_predictions_train,split='training')\n",
    "evaluate_model(y_test, y_pred,split='testing')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "0eb4f8c2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Root Mean Squared Error (Validation): 35.49055846857412\n",
      "Root Mean Squared Error (Test): 42.25580940793254\n"
     ]
    }
   ],
   "source": [
    "# Splitting the training set further for hyperparameter tuning\n",
    "X_train_tune, X_val, y_train_tune, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42)\n",
    "\n",
    "# Define the parameter grid to search over\n",
    "param_grid = {\n",
    "    'fit_intercept': [True, False],  # Whether to calculate the intercept for this model\n",
    "    'normalize': [True, False],   # Whether to normalize the features before fitting the model\n",
    "    'copy_X': [True, False],          # Whether to copy the input data; set to False when using larger datasets\n",
    "    'positive': [True, False]         # Restrict coefficients to be positive\n",
    "}\n",
    "\n",
    "# Create the Linear Regression model\n",
    "model = regressor\n",
    "\n",
    "# Initialize the GridSearchCV object\n",
    "grid_search = GridSearchCV(model, param_grid, cv=12, scoring='neg_mean_squared_error')\n",
    "\n",
    "# Perform the grid search on the training data\n",
    "grid_search.fit(X_train_tune, y_train_tune)\n",
    "\n",
    "# Get the best model from the grid search\n",
    "best_model = grid_search.best_estimator_\n",
    "\n",
    "# Predict on the validation set\n",
    "y_pred_val = best_model.predict(X_val)\n",
    "\n",
    "# Calculate RMSE for the validation set\n",
    "rmse_val = mean_squared_error(y_val, y_pred_val, squared=False)\n",
    "print(\"Root Mean Squared Error (Validation):\", rmse_val)\n",
    "\n",
    "# Now, you can use the best_model to predict on the test set and calculate RMSE\n",
    "y_pred_test = best_model.predict(X_test)\n",
    "rmse_test = mean_squared_error(y_test, y_pred_test, squared=False)\n",
    "print(\"Root Mean Squared Error (Test):\", rmse_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7cd7ca3",
   "metadata": {},
   "source": [
    "# **Random Forest Regressor model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "ddaf3bb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Random forrest Regressor Pipeline\n",
    "#Fitting the pipeline\n",
    "rf_tree_pipe = make_pipeline(preprocessor,RandomForestRegressor(random_state = 42))\n",
    "rf_tree_pipe.fit(X_train, y_train)\n",
    "\n",
    "# Get predictions for training and test data\n",
    "y_hat_train = rf_tree_pipe.predict(X_train)\n",
    "y_hat_test = rf_tree_pipe.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "4ee90472",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for training data:\n",
      "  - R^2 = 0.994\n",
      "  - MAE = 3.994\n",
      "  - MSE = 222.182\n",
      "  - RMSE = 14.906\n",
      "\n",
      "Results for testing data:\n",
      "  - R^2 = 0.916\n",
      "  - MAE = 10.446\n",
      "  - MSE = 2549.853\n",
      "  - RMSE = 50.496\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Evaluate model's performance\n",
    "evaluate_model(y_train, y_hat_train,split='training')\n",
    "evaluate_model(y_test, y_hat_test,split='testing')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef36a8c3",
   "metadata": {},
   "source": [
    "## **Tuning Random Forest Regressor**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "054d84d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     Train (max_depth=5)  Test (max_depth=5)  Train (max_depth=10)  \\\n",
      "202             0.974399            0.911653              0.990024   \n",
      "203             0.974406            0.911973              0.990024   \n",
      "204             0.974405            0.912040              0.990017   \n",
      "205             0.974413            0.911922              0.990034   \n",
      "206             0.974423            0.911896              0.990054   \n",
      "207             0.974427            0.911755              0.990068   \n",
      "208             0.974458            0.911664              0.990088   \n",
      "\n",
      "     Test (max_depth=10)  Train (max_depth=15)  Test (max_depth=15)  \n",
      "202             0.915894              0.993594             0.916199  \n",
      "203             0.916189              0.993593             0.916480  \n",
      "204             0.916244              0.993586             0.916565  \n",
      "205             0.916082              0.993601             0.916400  \n",
      "206             0.916026              0.993621             0.916337  \n",
      "207             0.915912              0.993627             0.916355  \n",
      "208             0.915855              0.993650             0.916240  \n"
     ]
    }
   ],
   "source": [
    "#create a range of max_depth values\n",
    "n_estimators = [202, 203, 204, 205, 206, 207, 208]\n",
    "\n",
    "max_depths = [5, 10, 15] \n",
    "\n",
    "#create a dataframe to store train and test scores.\n",
    "scores = pd.DataFrame(index=n_estimators)\n",
    "\n",
    "# Loop over the values in n_estimators\n",
    "for n in n_estimators:\n",
    "    for max_depth in max_depths:\n",
    "        # Fit a new model with max_depth\n",
    "        rf = RandomForestRegressor(random_state=42, n_estimators=n, max_depth=max_depth)\n",
    "\n",
    "        # Put the model into a pipeline\n",
    "        rf_pipe = make_pipeline(preprocessor, rf)\n",
    "\n",
    "        # Fit the model\n",
    "        rf_pipe.fit(X_train, y_train)\n",
    "\n",
    "        # Create prediction arrays\n",
    "        train_pred = rf_pipe.predict(X_train)\n",
    "        test_pred = rf_pipe.predict(X_test)\n",
    "\n",
    "        # Evaluate the model using R2 Score\n",
    "        train_r2score = r2_score(y_train, train_pred)\n",
    "        test_r2score = r2_score(y_test, test_pred)\n",
    "\n",
    "        # Store the scores in the scores dataframe\n",
    "        scores.loc[n, f'Train (max_depth={max_depth})'] = train_r2score\n",
    "        scores.loc[n, f'Test (max_depth={max_depth})'] = test_r2score\n",
    "\n",
    "# Print the scores dataframe\n",
    "print(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d52c01a6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Train (max_depth=5)</th>\n",
       "      <th>Test (max_depth=5)</th>\n",
       "      <th>Train (max_depth=10)</th>\n",
       "      <th>Test (max_depth=10)</th>\n",
       "      <th>Train (max_depth=15)</th>\n",
       "      <th>Test (max_depth=15)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>0.974399</td>\n",
       "      <td>0.911653</td>\n",
       "      <td>0.990024</td>\n",
       "      <td>0.915894</td>\n",
       "      <td>0.993594</td>\n",
       "      <td>0.916199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203</th>\n",
       "      <td>0.974406</td>\n",
       "      <td>0.911973</td>\n",
       "      <td>0.990024</td>\n",
       "      <td>0.916189</td>\n",
       "      <td>0.993593</td>\n",
       "      <td>0.916480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>204</th>\n",
       "      <td>0.974405</td>\n",
       "      <td>0.912040</td>\n",
       "      <td>0.990017</td>\n",
       "      <td>0.916244</td>\n",
       "      <td>0.993586</td>\n",
       "      <td>0.916565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>0.974413</td>\n",
       "      <td>0.911922</td>\n",
       "      <td>0.990034</td>\n",
       "      <td>0.916082</td>\n",
       "      <td>0.993601</td>\n",
       "      <td>0.916400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>0.974423</td>\n",
       "      <td>0.911896</td>\n",
       "      <td>0.990054</td>\n",
       "      <td>0.916026</td>\n",
       "      <td>0.993621</td>\n",
       "      <td>0.916337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>0.974427</td>\n",
       "      <td>0.911755</td>\n",
       "      <td>0.990068</td>\n",
       "      <td>0.915912</td>\n",
       "      <td>0.993627</td>\n",
       "      <td>0.916355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>0.974458</td>\n",
       "      <td>0.911664</td>\n",
       "      <td>0.990088</td>\n",
       "      <td>0.915855</td>\n",
       "      <td>0.993650</td>\n",
       "      <td>0.916240</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Train (max_depth=5)  Test (max_depth=5)  Train (max_depth=10)  \\\n",
       "202             0.974399            0.911653              0.990024   \n",
       "203             0.974406            0.911973              0.990024   \n",
       "204             0.974405            0.912040              0.990017   \n",
       "205             0.974413            0.911922              0.990034   \n",
       "206             0.974423            0.911896              0.990054   \n",
       "207             0.974427            0.911755              0.990068   \n",
       "208             0.974458            0.911664              0.990088   \n",
       "\n",
       "     Test (max_depth=10)  Train (max_depth=15)  Test (max_depth=15)  \n",
       "202             0.915894              0.993594             0.916199  \n",
       "203             0.916189              0.993593             0.916480  \n",
       "204             0.916244              0.993586             0.916565  \n",
       "205             0.916082              0.993601             0.916400  \n",
       "206             0.916026              0.993621             0.916337  \n",
       "207             0.915912              0.993627             0.916355  \n",
       "208             0.915855              0.993650             0.916240  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Train and test scores\n",
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "174e1751",
   "metadata": {},
   "source": [
    "## **Running best Random Forest Regressor model version**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c8cf0499",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Scores for Random Forest Regressor\n",
      "Results for training data:\n",
      "  - R^2 = 0.974\n",
      "  - MAE = 12.19\n",
      "  - MSE = 1029.628\n",
      "  - RMSE = 32.088\n",
      "\n",
      "\n",
      "\n",
      "Testing Scores for Random forest Regressor\n",
      "Results for testing data:\n",
      "  - R^2 = 0.912\n",
      "  - MAE = 12.734\n",
      "  - MSE = 2683.05\n",
      "  - RMSE = 51.798\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Obtaining the traing and test scores for random forest regression model\n",
    "best_rf = RandomForestRegressor(random_state = 42, n_estimators = n, max_depth=5)\n",
    "\n",
    "best_rf_pipe = make_pipeline(preprocessor, best_rf)\n",
    "\n",
    "best_rf_pipe.fit(X_train, y_train)\n",
    "\n",
    "print('Training Scores for Random Forest Regressor')\n",
    "evaluate_model(y_train, best_rf_pipe.predict(X_train), split = 'training')\n",
    "\n",
    "print('\\n')\n",
    "\n",
    "print('Testing Scores for Random forest Regressor')\n",
    "evaluate_model(y_test, best_rf_pipe.predict(X_test), split = 'testing')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b0025b6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error (Test Set): 2544.076786043306\n"
     ]
    }
   ],
   "source": [
    "# Make predictions on the test data (X_test)\n",
    "y_pred_test = rf_pipe.predict(X_test)\n",
    "\n",
    "# Calculate Mean Squared Error on the test set\n",
    "mse_test = mean_squared_error(y_test, y_pred_test)\n",
    "print(\"Mean Squared Error (Test Set):\", mse_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91f208c1",
   "metadata": {},
   "source": [
    "# **Gradient Boosting Regressor**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "825262f1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error: 2141.0031974786943\n"
     ]
    }
   ],
   "source": [
    "gb_regressor = GradientBoostingRegressor(n_estimators=100, learning_rate=0.1, max_depth=3, random_state=42)\n",
    "\n",
    "# Train the regressor on the training data\n",
    "gb_regressor.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test data\n",
    "y_pred = gb_regressor.predict(X_test)\n",
    "\n",
    "# Calculate Mean Squared Error on the test set\n",
    "mse = mean_squared_error(y_test, y_pred)\n",
    "print(\"Mean Squared Error:\", mse)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "400be4a8",
   "metadata": {},
   "source": [
    "## **Tuning Gradient Boosting Regressor Hyperparameters**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "e9b4e4be",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Parameters: {'n_estimators': 50, 'min_samples_split': 4, 'max_depth': 3, 'learning_rate': 0.1}\n",
      "Mean Squared Error: 2198.4788941723173\n"
     ]
    }
   ],
   "source": [
    "# Define the parameter grid to search\n",
    "param_dist = {\n",
    "    'n_estimators': [50, 100, 150],\n",
    "    'learning_rate': [0.01, 0.1, 0.2],\n",
    "    'max_depth': [3, 4, 5],\n",
    "    'min_samples_split': [2, 4, 6]\n",
    "}\n",
    "\n",
    "# Create the RandomizedSearchCV object\n",
    "random_search = RandomizedSearchCV(gb_regressor, param_distributions=param_dist, n_iter=10, cv=5, scoring='neg_mean_squared_error', random_state=42, n_jobs=-1)\n",
    "\n",
    "# Fit the random search to the data\n",
    "random_search.fit(X_train, y_train)\n",
    "\n",
    "# Get the best parameters and best estimator from the random search\n",
    "best_params = random_search.best_params_\n",
    "best_regressor = random_search.best_estimator_\n",
    "\n",
    "# Make predictions on the test data using the best estimator\n",
    "y_pred = best_regressor.predict(X_test)\n",
    "\n",
    "\n",
    "# Calculate Mean Squared Error on the test set\n",
    "mse = mean_squared_error(y_test, y_pred)\n",
    "print(\"Best Parameters:\", best_params)\n",
    "print(\"Mean Squared Error:\", mse)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7062302",
   "metadata": {},
   "source": [
    "## **Running best Gradient Boosting model version**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "175ef52d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Scores for Gradient Boosting Regressor\n",
      "Results for training data:\n",
      "  - R^2 = 0.993\n",
      "  - MAE = 7.929\n",
      "  - MSE = 269.259\n",
      "  - RMSE = 16.409\n",
      "\n",
      "\n",
      "\n",
      "Testing Scores for Gradient Boosting Regressor\n",
      "Results for testing data:\n",
      "  - R^2 = 0.927\n",
      "  - MAE = 10.284\n",
      "  - MSE = 2215.866\n",
      "  - RMSE = 47.073\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Obtaining the traing and test scores for Gradient Boosting regression model\n",
    "best_gb = GradientBoostingRegressor(random_state = 42, n_estimators = n, max_depth=5)\n",
    "\n",
    "best_gb_pipe = make_pipeline(preprocessor, best_gb)\n",
    "\n",
    "best_gb_pipe.fit(X_train, y_train)\n",
    "\n",
    "print('Training Scores for Gradient Boosting Regressor')\n",
    "evaluate_model(y_train, best_gb_pipe.predict(X_train), split = 'training')\n",
    "\n",
    "print('\\n')\n",
    "\n",
    "print('Testing Scores for Gradient Boosting Regressor')\n",
    "evaluate_model(y_test, best_gb_pipe.predict(X_test), split = 'testing')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7201016",
   "metadata": {},
   "source": [
    "# **LightGBM**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "1fec04b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LightGBM Mean Squared Error: 2396.5998992992063\n"
     ]
    }
   ],
   "source": [
    "# LGM model construction\n",
    "lgb_regressor = lgb.LGBMRegressor(n_estimators=100, learning_rate=0.1, max_depth=3, random_state=42)\n",
    "\n",
    "# Train the regressor on the training data\n",
    "lgb_regressor.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test data\n",
    "y_pred_lgb = lgb_regressor.predict(X_test)\n",
    "\n",
    "# Calculate Mean Squared Error on the test set\n",
    "mse_lgb = mean_squared_error(y_test, y_pred_lgb)\n",
    "print(\"LightGBM Mean Squared Error:\", mse_lgb)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3423a3c3",
   "metadata": {},
   "source": [
    "## **Tuning Hyperparameters**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "9f2989d6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LightGBM Best Parameters: {'colsample_bytree': 0.8, 'learning_rate': 0.1, 'max_depth': 3, 'n_estimators': 100, 'subsample': 0.8}\n",
      "LightGBM Mean Squared Error: 2384.6927620415577\n"
     ]
    }
   ],
   "source": [
    "# Creating a parameter grid for LightGBM\n",
    "param_grid_lgb = {\n",
    "    'n_estimators': [100, 150, 200],\n",
    "    'learning_rate': [0.01, 0.1, 0.2],\n",
    "    'max_depth': [3, 4, 5],\n",
    "    'subsample': [0.8, 0.9, 1.0],\n",
    "    'colsample_bytree': [0.8, 0.9, 1.0]\n",
    "}\n",
    "\n",
    "# Creating a LightGBM regressor\n",
    "lgb_regressor = lgb.LGBMRegressor(random_state=42)\n",
    "\n",
    "# Creating the GridSearchCV object for LightGBM\n",
    "grid_search_lgb = GridSearchCV(lgb_regressor, param_grid=param_grid_lgb, cv=5, scoring='neg_mean_squared_error', n_jobs=-1)\n",
    "\n",
    "# Fit the grid search to the data\n",
    "grid_search_lgb.fit(X_train, y_train)\n",
    "\n",
    "# Get the best parameters and best estimator for LightGBM\n",
    "best_params_lgb = grid_search_lgb.best_params_\n",
    "best_regressor_lgb = grid_search_lgb.best_estimator_\n",
    "\n",
    "# Make predictions on the test data using the best LightGBM estimator\n",
    "y_pred_lgb = best_regressor_lgb.predict(X_test)\n",
    "\n",
    "# Calculate Mean Squared Error for LightGBM on the test set\n",
    "mse_lgb = mean_squared_error(y_test, y_pred_lgb)\n",
    "print(\"LightGBM Best Parameters:\", best_params_lgb)\n",
    "print(\"LightGBM Mean Squared Error:\", mse_lgb)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "69c7551d",
   "metadata": {},
   "source": [
    "## **Running best LightGBM model version**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "d1775867",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error: 2409.10 is the best possible outcome for the tuned Light GBM model.\n"
     ]
    }
   ],
   "source": [
    "# Best hyperparameters for LightGBM\n",
    "best_params = {\n",
    "    'colsample_bytree': 0.8,\n",
    "    'learning_rate': 0.1,\n",
    "    'max_depth': 3,\n",
    "    'n_estimators': 100,\n",
    "    'subsample': 0.8\n",
    "}\n",
    "# Create the LightGBM model with the best hyperparameters from the tuning above.\n",
    "lgb_model = lgb.LGBMRegressor(\n",
    "    colsample_bytree=best_params['colsample_bytree'],\n",
    "    learning_rate=best_params['learning_rate'],\n",
    "    max_depth=best_params['max_depth'],\n",
    "    n_estimators=best_params['n_estimators'],\n",
    "    subsample=best_params['subsample']\n",
    ")\n",
    "\n",
    "# Train the LightGBM model on the training data\n",
    "lgb_model.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions using the trained LightGBM model\n",
    "predictions = lgb_model.predict(X_test)\n",
    "\n",
    "# Calculate the Mean Squared Error (MSE) using the true target values and predictions\n",
    "mse = mean_squared_error(y_test, predictions)\n",
    "\n",
    "formatted_mse = \"{:.2f}\".format(mse)\n",
    "print(\"Mean Squared Error:\", formatted_mse, \"is the best possible outcome for the tuned Light GBM model.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6aed433",
   "metadata": {},
   "source": [
    "# **K-Nearest Neighbors (KNN)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "dc74634d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNN Mean Squared Error: 3060.986796922195\n"
     ]
    }
   ],
   "source": [
    "# You can adjust the number of neighbors (n_neighbors) according to your needs\n",
    "knn_regressor = KNeighborsRegressor(n_neighbors=5)\n",
    "\n",
    "# Train the regressor on the training data\n",
    "knn_regressor.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test data\n",
    "y_pred = knn_regressor.predict(X_test)\n",
    "\n",
    "# Calculate Mean Squared Error on the test set\n",
    "mse = mean_squared_error(y_test, y_pred)\n",
    "print(\"KNN Mean Squared Error:\", mse)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e309ca96",
   "metadata": {},
   "source": [
    "## **Tuning Hyperparameters**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "28a472e7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best KNN Mean Squared Error: 2706.0952852630153\n",
      "Best Hyperparameters: {'weights': 'uniform', 'p': 2, 'n_neighbors': 3}\n"
     ]
    }
   ],
   "source": [
    "param_dist = {\n",
    "    'n_neighbors': range(1, 21),  \n",
    "    'weights': ['uniform', 'distance'],\n",
    "    'p': [1, 2]\n",
    "}\n",
    "\n",
    "# Create a KNN regressor\n",
    "knn_regressor = KNeighborsRegressor()\n",
    "\n",
    "# Create a RandomizedSearchCV instance\n",
    "random_search = RandomizedSearchCV(\n",
    "    knn_regressor, param_distributions=param_dist, n_iter=10, cv=5, scoring='neg_mean_squared_error', random_state=42\n",
    ")\n",
    "\n",
    "# Perform the random search on your data\n",
    "random_search.fit(X_train, y_train)\n",
    "\n",
    "# Get the best KNN regressor from the search\n",
    "best_knn_regressor = random_search.best_estimator_\n",
    "\n",
    "# Make predictions on the test data using the best model\n",
    "y_pred_best = best_knn_regressor.predict(X_test)\n",
    "\n",
    "# Calculate Mean Squared Error on the test set for the best model\n",
    "best_mse = mean_squared_error(y_test, y_pred_best)\n",
    "print(\"Best KNN Mean Squared Error:\", best_mse)\n",
    "\n",
    "# Print the best hyperparameters found\n",
    "print(\"Best Hyperparameters:\", random_search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9fdedc7",
   "metadata": {},
   "source": [
    "## **Running best KNN model version**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "ed5a6201",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error: 2706.10 is the best possible outcome for the tuned KNN model.\n"
     ]
    }
   ],
   "source": [
    "# Best hyperparameters\n",
    "best_n_neighbors = 3\n",
    "best_weights = 'uniform'\n",
    "best_p = 2 \n",
    "# Create the KNN regressor with the best parameters\n",
    "knn_model = KNeighborsRegressor(n_neighbors=best_n_neighbors, weights=best_weights, p=best_p)\n",
    "# Train the KNN regressor on the training data\n",
    "knn_model.fit(X_train, y_train)\n",
    "# Make predictions using the trained KNN regressor\n",
    "predictions = knn_model.predict(X_test)\n",
    "# Calculate the Mean Squared Error (MSE) using the true target values and predictions\n",
    "mse = mean_squared_error(y_test, predictions)\n",
    "formatted_mse = \"{:.2f}\".format(mse)\n",
    "print(\"Mean Squared Error:\", formatted_mse, \"is the best possible outcome for the tuned KNN model.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57b31675",
   "metadata": {},
   "source": [
    "# **Decision Tree Regressor**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "df159a9f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree Mean Squared Error: 2966.14\n"
     ]
    }
   ],
   "source": [
    "# Create a Decision Tree Regressor.\n",
    "tree_regressor = DecisionTreeRegressor(max_depth=5, random_state=42)\n",
    "\n",
    "# Training the regressor on the training data.\n",
    "tree_regressor.fit(X_train, y_train)\n",
    "\n",
    "# Making predictions on the test data.\n",
    "y_pred = tree_regressor.predict(X_test)\n",
    "\n",
    "# Calculating Mean Squared Error on the test set.\n",
    "mse = mean_squared_error(y_test, y_pred)\n",
    "\n",
    "formatted_mse = \"{:.2f}\".format(mse)\n",
    "print(\"Decision Tree Mean Squared Error:\", formatted_mse)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfa4d24f",
   "metadata": {},
   "source": [
    "## **Tuning Decision Tree Regressor Hyperparameters**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "41ca8f1e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Decision Tree Mean Squared Error: 1768.1268454062308\n",
      "Best Hyperparameters: {'min_samples_split': 10, 'min_samples_leaf': 4, 'max_features': 'auto', 'max_depth': 20}\n"
     ]
    }
   ],
   "source": [
    "# Define the parameter grid to search\n",
    "param_dist = {\n",
    "    'max_depth': range(1, 21),   \n",
    "    'min_samples_split': [2, 5, 10],\n",
    "    'min_samples_leaf': [1, 2, 4],\n",
    "    'max_features': ['auto', 'sqrt', 'log2']\n",
    "}\n",
    "\n",
    "# Create a Decision Tree regressor\n",
    "tree_regressor = DecisionTreeRegressor(random_state=42)\n",
    "\n",
    "# Create a RandomizedSearchCV instance\n",
    "random_search = RandomizedSearchCV(\n",
    "    tree_regressor, param_distributions=param_dist, n_iter=10, cv=5, scoring='neg_mean_squared_error', random_state=42\n",
    ")\n",
    "\n",
    "# Perform the random search on your data\n",
    "random_search.fit(X_train, y_train)\n",
    "\n",
    "# Get the best Decision Tree regressor from the search\n",
    "best_tree_regressor = random_search.best_estimator_\n",
    "\n",
    "# Make predictions on the test data using the best model\n",
    "y_pred_best = best_tree_regressor.predict(X_test)\n",
    "\n",
    "# Calculate Mean Squared Error on the test set for the best model\n",
    "best_mse = mean_squared_error(y_test, y_pred_best)\n",
    "print(\"Best Decision Tree Mean Squared Error:\", best_mse)\n",
    "\n",
    "# Print the best hyperparameters found\n",
    "print(\"Best Hyperparameters:\", random_search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a234b035",
   "metadata": {},
   "source": [
    "## **Running best Decision Tree Regressor model version**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "5bb0f7a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error: 1767.30 is the best possible outcome for the tuned Decision Tree Regression model.\n"
     ]
    }
   ],
   "source": [
    "#Best hyperparameters for Decision Tree\n",
    "best_params = {\n",
    "    'min_samples_split': 10,\n",
    "    'min_samples_leaf': 4,\n",
    "    'max_features': 'auto',\n",
    "    'max_depth': 20\n",
    "}\n",
    "\n",
    "# Create the Decision Tree model with the best hyperparameters\n",
    "tree_model = DecisionTreeRegressor(\n",
    "    min_samples_split=best_params['min_samples_split'],\n",
    "    min_samples_leaf=best_params['min_samples_leaf'],\n",
    "    max_features=best_params['max_features'],\n",
    "    max_depth=best_params['max_depth']\n",
    ")\n",
    "\n",
    "# Train the Decision Tree model on the training data\n",
    "tree_model.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions using the trained Decision Tree model\n",
    "predictions = tree_model.predict(X_test)\n",
    "\n",
    "# Calculate the Mean Squared Error (MSE) using the true target values and predictions\n",
    "mse = mean_squared_error(y_test, predictions)\n",
    "formatted_mse = \"{:.2f}\".format(mse)\n",
    "print(\"Mean Squared Error:\", formatted_mse, \"is the best possible outcome for the tuned Decision Tree Regression model.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6f2d02d",
   "metadata": {},
   "source": [
    "# **Save Predictions**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "1e0af049",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving the new dataframe to upload in the next notebook\n",
    "fd.to_csv('Predictions', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f2289ef",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f838f7ad",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (dojo-env)",
   "language": "python",
   "name": "dojo-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "303.509px"
   },
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}