20_DL_CNN_starter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "53fbc6b5",
   "metadata": {},
   "source": [
    "# Deep learning - CNN "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e51129f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.keras import Model, Sequential\n",
    "\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.losses import MeanSquaredError\n",
    "from tensorflow.keras.metrics import MeanAbsoluteError\n",
    "\n",
    "from tensorflow.keras.layers import Dense, Conv1D, LSTM, RNN\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea15fd8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams[\"figure.figsize\"] = (9,6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7820ad07",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "tf.keras.utils.set_random_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45672b04",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv('data/train.csv')\n",
    "val_df = pd.read_csv('data/val.csv')\n",
    "test_df = pd.read_csv('data/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54c6cb0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_timeseries_data(df, input_length, output_length, target_names):\n",
    "    \n",
    "    if target_names is not None:\n",
    "        target_indices = {name: i for i, name in enumerate(target_names)}\n",
    "    col_indices = {name: i for i, name in enumerate(df.columns)}\n",
    "    \n",
    "    total_length = input_length + output_length\n",
    "    \n",
    "    input_slice = slice(0, input_length)\n",
    "    output_slice = slice(input_length, None)\n",
    "    \n",
    "    data = np.array(df, dtype=np.float32)\n",
    "    ds = tf.keras.preprocessing.timeseries_dataset_from_array(\n",
    "        data=data,\n",
    "        targets=None,\n",
    "        sequence_length=total_length,\n",
    "        sequence_stride=1,\n",
    "        shuffle=False,\n",
    "        batch_size=32\n",
    "    )\n",
    "    \n",
    "    def split_to_input_output(x):\n",
    "            \n",
    "        inputs = x[:, input_slice, :]\n",
    "        outputs = x[:, output_slice, :]\n",
    "        \n",
    "        if target_names is not None:\n",
    "            outputs = tf.stack(\n",
    "                [outputs[:,:,col_indices[name]] for name in target_names],\n",
    "                axis=-1\n",
    "            )\n",
    "\n",
    "        inputs.set_shape([None, input_length, None])\n",
    "        outputs.set_shape([None, output_length, None])\n",
    "    \n",
    "        return inputs, outputs\n",
    "    \n",
    "    ds = ds.map(split_to_input_output)\n",
    "    \n",
    "    return ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a946b1c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.callbacks import LearningRateScheduler\n",
    "\n",
    "def scheduler(epoch, lr):\n",
    "    if epoch < 2:\n",
    "        return lr\n",
    "    else:\n",
    "        return lr * tf.math.exp(-0.1)\n",
    "    \n",
    "lr_scheduler = LearningRateScheduler(scheduler)\n",
    "\n",
    "def train_model(model, train_ds, val_ds, patience=5, max_epochs=50, use_scheduler=False):\n",
    "    \n",
    "    early_stopping = EarlyStopping(monitor='val_loss', patience=patience, mode='min')\n",
    "    \n",
    "    model.compile(loss=MeanSquaredError(), optimizer=Adam(), metrics=[MeanAbsoluteError()])\n",
    "    \n",
    "    history = model.fit(train_ds, epochs=max_epochs, validation_data=val_ds, callbacks=[early_stopping])\n",
    "    \n",
    "    if use_scheduler == True:\n",
    "        history = model.fit(train_ds, epochs=max_epochs, validation_data=val_ds, callbacks=[early_stopping, lr_scheduler])\n",
    "    \n",
    "    return history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65f55138",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_history(history):\n",
    "\n",
    "    fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2)\n",
    "\n",
    "    ax1.plot(history.history['loss'], label='Train')\n",
    "    ax1.plot(history.history['val_loss'], label='Validation')\n",
    "    ax1.set_xlabel('Epochs')\n",
    "    ax1.set_ylabel('Loss')\n",
    "    ax1.set_title('Loss')\n",
    "    ax1.legend(loc='best')\n",
    "\n",
    "    ax2.plot(history.history['mean_absolute_error'], label='Train')\n",
    "    ax2.plot(history.history['val_mean_absolute_error'], label='Validation')\n",
    "    ax2.set_xlabel('Epochs')\n",
    "    ax2.set_ylabel('MAE')\n",
    "    ax2.set_title('Mean absolute error')\n",
    "    ax2.legend(loc='best')\n",
    "\n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65f1bf11",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_predictions(model, sample_batch, model_type):\n",
    "    \n",
    "    inputs, outputs = sample_batch\n",
    "    preds = model(inputs)\n",
    "    \n",
    "    if model_type == 'single_step':      \n",
    "\n",
    "        fig, ax = plt.subplots()\n",
    "\n",
    "        actual_scaled = outputs.numpy().flatten()\n",
    "        actual = actual_scaled * (OT_max_test - OT_min_test) + OT_min_test\n",
    "        \n",
    "        predictions_scaled = preds.numpy().flatten()\n",
    "        predictions = predictions_scaled * (OT_max_train - OT_min_train) + OT_min_train\n",
    "        \n",
    "        ax.plot(actual, label='Actual')\n",
    "        ax.plot(predictions, label='Predicted')\n",
    "        ax.set_xlabel('Time steps')\n",
    "        ax.set_ylabel('Oil temperature')\n",
    "        ax.set_title('Predictions on a sample batch')\n",
    "        ax.legend(loc='best')\n",
    "        \n",
    "    elif model_type == 'multi_step':\n",
    "\n",
    "        fig, ax = plt.subplots()\n",
    "        \n",
    "        actual_scaled = outputs.numpy().flatten()[:48]\n",
    "        actual = actual_scaled * (OT_max_test - OT_min_test) + OT_min_test\n",
    "        \n",
    "        predictions_scaled = preds.numpy().flatten()[:48]\n",
    "        predictions = predictions_scaled * (OT_max_train - OT_min_train) + OT_min_train\n",
    "\n",
    "        ax.plot(actual, label='Actual')\n",
    "        ax.plot(predictions, label='Predicted')\n",
    "        ax.set_xlabel('Time steps')\n",
    "        ax.set_ylabel('Oil temperature')\n",
    "        ax.set_title('Predictions on a sample batch')\n",
    "        ax.legend(loc='best')\n",
    "        \n",
    "    elif model_type == 'multi_output':\n",
    "        \n",
    "        fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1)\n",
    "        \n",
    "        OT_actual_scaled = outputs.numpy().flatten()[0::2]\n",
    "        OT_actual = OT_actual_scaled * (OT_max_test - OT_min_test) + OT_min_test\n",
    "        \n",
    "        OT_predictions_scaled = preds.numpy().flatten()[0::2]\n",
    "        OT_predictions = OT_predictions_scaled * (OT_max_train - OT_min_train) + OT_min_train\n",
    "\n",
    "        MULL_actual_scaled = outputs.numpy().flatten()[1::2]\n",
    "        MULL_actual = MULL_actual_scaled * (MULL_max_test - MULL_min_test) + MULL_min_test\n",
    "        \n",
    "        MULL_predictions_scaled = preds.numpy().flatten()[1::2]\n",
    "        MULL_predictions = MULL_predictions_scaled * (MULL_max_train - MULL_min_train) + MULL_min_train\n",
    "        \n",
    "        ax1.plot(OT_actual, label='Actual')\n",
    "        ax1.plot(OT_predictions, label='Predicted')\n",
    "        ax1.set_xlabel('Time')\n",
    "        ax1.set_ylabel('Oil temperature')\n",
    "        ax1.legend(loc='best')\n",
    "        ax1.set_title('Predictions on a sample batch')\n",
    "        \n",
    "        ax2.plot(MULL_actual, label='Actual')\n",
    "        ax2.plot(MULL_predictions, label='Predicted')\n",
    "        ax2.set_xlabel('Time')\n",
    "        ax2.set_ylabel('MULL')\n",
    "        ax2.legend(loc='best')\n",
    "        \n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65398e30",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_evaluation(model_list, mae):\n",
    "    \n",
    "    fig, ax = plt.subplots()\n",
    "    \n",
    "    ax.bar(model_list, mae, width=0.3)\n",
    "    ax.set_ylabel('MAE')\n",
    "    ax.set_xlabel('Models')\n",
    "    ax.set_ylim(0, max(mae)+0.15)\n",
    "    for index, value in enumerate(mae):\n",
    "        ax.text(x=index, y=value+0.005, s=str(round(value, 3)), ha='center')\n",
    "    \n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98387460",
   "metadata": {},
   "outputs": [],
   "source": [
    "MULL_max_train = 7.568999767303468\n",
    "MULL_min_train = -5.934000015258789\n",
    "MULL_max_test = 4.690999984741211\n",
    "MULL_min_test = -3.3399999141693115\n",
    "\n",
    "OT_max_train = 46.00699996948242\n",
    "OT_min_train = -4.079999923706056\n",
    "OT_max_test = 17.165000915527347\n",
    "OT_min_test = 3.025000095367432"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0055804b",
   "metadata": {},
   "source": [
    "## Single-step model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "155e24fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a single-step window for a kernel length of 6\n",
    "\n",
    "ss_sample_batch = next(iter(test_ds_ss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69b74c0e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Create a CNN model\n",
    "\n",
    "ss_cnn_history = train_model(ss_cnn, train_ds_ss, val_ds_ss, use_scheduler=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "453fbd0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(ss_cnn_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5ab9098",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_predictions(ss_cnn, ss_sample_batch, 'single_step')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6db0d4ce",
   "metadata": {},
   "source": [
    "### Combine CNN and LSTM "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3008c4d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Combine a CNN with LSTM\n",
    "\n",
    "ss_cnn_lstm_history = train_model(ss_cnn_lstm, train_ds_ss, val_ds_ss, use_scheduler=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ae7f44d",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(ss_cnn_lstm_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e8c1d7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_predictions(ss_cnn_lstm, ss_sample_batch, 'single_step')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18719959",
   "metadata": {},
   "source": [
    "### Evaluation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd627e42",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_list = ['Linear', 'DNN', 'LSTM', 'CNN', 'CNN+LSTM']\n",
    "\n",
    "ss_mae = [0.032, 0.046, 0.036]\n",
    "ss_mae.append(ss_cnn.evaluate(test_ds_ss)[1])\n",
    "ss_mae.append(ss_cnn_lstm.evaluate(test_ds_ss)[1])\n",
    "\n",
    "plot_evaluation(model_list, ss_mae)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ba02d30",
   "metadata": {},
   "source": [
    "## Multi-step model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e92f34a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a multi-step window with a kernel length of 6\n",
    "\n",
    "ms_sample_batch = next(iter(test_ds_ms))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94279dee",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Create a CNN model\n",
    "\n",
    "ms_cnn_history = train_model(ms_cnn, train_ds_ms, val_ds_ms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1e27689",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(ms_cnn_history)\n",
    "# plot_predictions(ms_cnn, ms_sample_batch, 'multi_step')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25487a85",
   "metadata": {},
   "source": [
    "### Combine CNN and LSTM "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f2b5791",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Combine CNN and LSTM\n",
    "\n",
    "ms_cnn_lstm_history = train_model(ms_cnn_lstm, train_ds_ms, val_ds_ms, use_scheduler=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9aa0bf13",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(ms_cnn_lstm_history)\n",
    "# plot_predictions(ms_cnn_lstm, ms_sample_batch, 'multi_step')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70f347fc",
   "metadata": {},
   "source": [
    "### Evaluation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "623e8654",
   "metadata": {},
   "outputs": [],
   "source": [
    "ms_mae = [0.123, 0.154, 0.174]\n",
    "ms_mae.append(ms_cnn.evaluate(test_ds_ms)[1])\n",
    "ms_mae.append(ms_cnn_lstm.evaluate(test_ds_ms)[1])\n",
    "\n",
    "plot_evaluation(model_list, ms_mae)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4995c060",
   "metadata": {},
   "source": [
    "## Multi-output "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4d811f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a multi-output window with a kernel length of 6\n",
    "\n",
    "mo_sample_batch = next(iter(test_ds_mo))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d770cc1a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# CNN model\n",
    "\n",
    "mo_cnn_history = train_model(mo_cnn, train_ds_mo, val_ds_mo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aafe11da",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(mo_cnn_history)\n",
    "# plot_predictions(mo_cnn, mo_sample_batch, 'multi_output')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83bcc378",
   "metadata": {},
   "source": [
    "### Combine CNN and LSTM "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c848e28",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# CNN + LSTM model\n",
    "\n",
    "mo_cnn_lstm_history = train_model(mo_cnn_lstm, train_ds_mo, val_ds_mo, use_scheduler=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6423b1da",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_history(mo_cnn_lstm_history)\n",
    "# plot_predictions(mo_cnn_lstm, mo_sample_batch, 'multi_output')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49747c8f",
   "metadata": {},
   "source": [
    "### Evaluation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4e48615",
   "metadata": {},
   "outputs": [],
   "source": [
    "mo_mae = [0.05, 0.069, 0.093]\n",
    "\n",
    "mo_mae.append(mo_cnn.evaluate(test_ds_mo)[1])\n",
    "mo_mae.append(mo_cnn_lstm.evaluate(test_ds_mo)[1])\n",
    "\n",
    "plot_evaluation(model_list, mo_mae)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6f5d3c0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}