notebooks/Bank_Dataset.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1Sd9FQhDcDhy"
      },
      "source": [
        "# Bank dataset example"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yYLJcIiEOmAm"
      },
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ogunlao/saint/blob/main/notebooks/Bank_Dataset.ipynb)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "This notebook example is made to run seamlessly on colab. You may need to change the paths to run on local."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "c5HdUECdhoNU"
      },
      "outputs": [],
      "source": [
        "%load_ext autoreload\n",
        "%autoreload 2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "B2gTmeVAcPDR"
      },
      "outputs": [],
      "source": [
        "# clone the repository\n",
        "\n",
        "!git clone https://github.com/ogunlao/saint.git"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oc4E8FtdPIGL"
      },
      "outputs": [],
      "source": [
        "!pip install -r saint/requirements.txt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T3RDW8NIPZqQ"
      },
      "source": [
        "**warning: restart the runtime on Colab**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8IRukGupmFCq"
      },
      "source": [
        "## General setup configuration parameters from paper"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FE4DnSbOKZJL"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "\n",
        "import torch\n",
        "import pandas as pd\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BqhYRMvvDyHW"
      },
      "outputs": [],
      "source": [
        "# --- Config Information ---#\n",
        "try:\n",
        "    from ruamel.yaml import YAML\n",
        "except ModuleNotFoundError:\n",
        "    from ruamel_yaml import YAML\n",
        "\n",
        "config_path = 'saint/configs/config.yaml'\n",
        "\n",
        "yaml = YAML(typ='safe')\n",
        "with open(config_path) as f:\n",
        "    args = yaml.load(f)\n",
        "\n",
        "print(args)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VbRuG5AQHiea"
      },
      "source": [
        "## Dataset preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UlBs3LvEdtem"
      },
      "source": [
        "###### Bank dataset [Link](https://archive.ics.uci.edu/ml/datasets/bank+marketing)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VYu-ZwOG_cFi"
      },
      "outputs": [],
      "source": [
        "data_folder = \"saint/data\"\n",
        "\n",
        "# download the data into a data folder\n",
        "URL_LINK = \"https://archive.ics.uci.edu/ml/machine-learning-databases/00222/bank.zip\"\n",
        "\n",
        "!wget -P {data_folder} {URL_LINK}\n",
        "!unzip {data_folder}/bank.zip -d {data_folder}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "G7ISY6V8Qs4U"
      },
      "outputs": [],
      "source": [
        "bank_df = pd.read_csv(os.path.join(data_folder, 'bank-full.csv'),  sep=';')\n",
        "y = bank_df[['y']].copy()\n",
        "\n",
        "bank_df.drop(columns=['y'], inplace=True)\n",
        "bank_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4D5Hkz_e6v5k"
      },
      "outputs": [],
      "source": [
        "len(y) # Data size should be 45211"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8GrHLAdn63xD"
      },
      "outputs": [],
      "source": [
        "y.value_counts() # Data is imbalanced, will require some balanced sampling"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "H_SwEm3TY9jE"
      },
      "outputs": [],
      "source": [
        "# check if there are any missing values\n",
        "bank_df.isna().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zprDn2XVfKi9"
      },
      "outputs": [],
      "source": [
        "from saint.src.dataset import generate_splits, preprocess"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dOZzomB47wYI"
      },
      "outputs": [],
      "source": [
        "# separate 200 samples for supervised training, all others are used in ssl training\n",
        "num_supervised_train_data = 200 \n",
        "\n",
        "# get split indices\n",
        "sup_train_indices, val_indices, test_indices, ssl_train_indices = generate_splits(len(bank_df), \n",
        "                                                                            num_supervised_train_data,\n",
        "                                                                            args['preproc']['validation_split'],\n",
        "                                                                            args['preproc']['test_split'],\n",
        "                                                                            args['seed'],)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sHk_9Ly4ftB0"
      },
      "outputs": [],
      "source": [
        "df_proc, y_proc, no_num, no_cat, cats  = preprocess(bank_df, y, args['transformer']['cls_token_idx'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OAHTOyfQjIN-"
      },
      "outputs": [],
      "source": [
        "# This variables will need to be added to the config files in \"configs/data/bank_*\" before training\n",
        "\n",
        "print('no of numerical columns: ', no_num)\n",
        "print('no of categorical columns: ', no_cat)\n",
        "\n",
        "print('list of categories in each categorical column: ', cats)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8fxrzdv3gH19"
      },
      "outputs": [],
      "source": [
        "#### args.num_features = args.no_num + args.no_cat\n",
        "\n",
        "# split data into train, val and test using generated indices\n",
        "\n",
        "train_df, train_y   = df_proc.iloc[sup_train_indices], y_proc.iloc[sup_train_indices]\n",
        "val_df, val_y       = df_proc.iloc[val_indices], y_proc.iloc[val_indices]\n",
        "test_df, test_y     = df_proc.iloc[test_indices], y_proc.iloc[test_indices]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rPgAlDJylX0Z"
      },
      "outputs": [],
      "source": [
        "# Generate data for self-supervised training if specified\n",
        "\n",
        "train_ssl, train_ssl_y = None, None\n",
        "\n",
        "if num_supervised_train_data != 'all':\n",
        "    train_ssl, train_ssl_y = df_proc.iloc[ssl_train_indices], y_proc.iloc[ssl_train_indices]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Jobqrdnbg8vt"
      },
      "outputs": [],
      "source": [
        "# save dataframes in the data directory\n",
        "\n",
        "train_df.to_csv('saint/data/train.csv' , index=False)\n",
        "train_y.to_csv('saint/data/train_y.csv' , index=False)\n",
        "val_df.to_csv('saint/data/val.csv' , index=False)\n",
        "val_y.to_csv('saint/data/val_y.csv' , index=False)\n",
        "test_df.to_csv('saint/data/test.csv' , index=False)\n",
        "test_y.to_csv('saint/data/test_y.csv' , index=False)\n",
        "\n",
        "if train_ssl is not None:\n",
        "    train_ssl.to_csv('saint/data/train_ssl.csv' , index=False)\n",
        "\n",
        "if train_ssl_y is not None:\n",
        "    train_ssl_y.to_csv('saint/data/train_ssl_y.csv' , index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "upyx0DxZTDTK"
      },
      "source": [
        "### Self-supervised training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# give the number of gpus available if any\n",
        "num_gpus = 1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nOOsb2wrnFKK"
      },
      "outputs": [],
      "source": [
        "# Train saint model in self-supervised settings. \n",
        "# To use gpus, add trainer.gpus=1 where \"1\" is the total no of gpus to the command\n",
        "\n",
        "!python saint/main.py experiment=self-supervised \\\n",
        "    experiment.model=saint \\\n",
        "    data.data_folder=/content/saint/data \\\n",
        "    data=bank_ssl"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "es1Kf9134tGG"
      },
      "outputs": [],
      "source": [
        "# Retrieve the location of self-supervised pretrained model from logs\n",
        "\n",
        "# for example\n",
        "best_ssl_model_ckpt = \"/content/outputs/2021-11-01/10-09-16/lightning_logs/version_0/checkpoints/0-916.ckpt\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FcZFRnjU5zwY"
      },
      "outputs": [],
      "source": [
        "# Train a supervised model initialized from the ssl model\n",
        "\n",
        "!python saint/main.py experiment=supervised \\\n",
        "    experiment.model=saint \\\n",
        "    data.data_folder=/content/saint/data \\\n",
        "    data=bank_sup \\\n",
        "    experiment.pretrained_checkpoint={best_ssl_model_ckpt}\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yTqo6kmoAC2i"
      },
      "outputs": [],
      "source": [
        "# You can also train a supervised model without initializing with a pretrained model i.e. randomly initialized\n",
        "\n",
        "!python saint/main.py experiment=supervised \\\n",
        "    experiment.model=saint \\\n",
        "    data.data_folder=/content/saint/data \\\n",
        "    data=bank_sup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UTVtMFAnADTx"
      },
      "outputs": [],
      "source": [
        "# To predict on a test dataset  # check saint/experiment/predict for params\n",
        "\n",
        "pretrained_checkpoint = \"/content/outputs/2021-11-01/13-30-49/lightning_logs/version_0/checkpoints/2-20.ckpt\"\n",
        "\n",
        "!python saint/predict.py experiment=predict \\\n",
        "    experiment.model=saint \\\n",
        "    data=bank_sup \\\n",
        "    data.data_folder=/content/saint/data \\\n",
        "    experiment.pretrained_checkpoint={pretrained_checkpoint} \\\n",
        "    experiment.pred_sav_path=/content/predict.csv"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gI59qOEYIJii"
      },
      "source": [
        "## View Tensorboard"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wzdymWiKC3EF"
      },
      "outputs": [],
      "source": [
        "# View plots and hyperparameters\n",
        "\n",
        "%load_ext tensorboard\n",
        "%tensorboard --logdir /content/outputs/2021-11-01/10-14-14/lightning_logs # change lightning log path"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tDWTM2j-6K1m"
      },
      "source": [
        "## The End"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "Bank_Dataset.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}