benchmark.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RSVT: Benchmark Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn import datasets, metrics, preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "%run \"RSVT.ipynb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def benchmarks(X_train,X_test, y_train,y_test):\n",
    "    RFC = RandomForestClassifier(random_state=754046)\n",
    "    RFC.fit(X_train, y_train)\n",
    "    preds_RFC = RFC.predict(X_test)\n",
    "    acc_RFC = metrics.accuracy_score(y_test, preds_RFC)\n",
    "\n",
    "    ETC = ExtraTreesClassifier(random_state=754046)\n",
    "    ETC.fit(X_train, y_train)\n",
    "    preds_ETC = ETC.predict(X_test)\n",
    "    acc_ETC = metrics.accuracy_score(y_test, preds_ETC)\n",
    "\n",
    "    partitions = extra_partition(X_train,X_test, y_train, n_estimators=10, random_state=754046)\n",
    "    rsvt = fit_rsvt(partitions)\n",
    "    preds = predict_forest(partitions,rsvt)\n",
    "    acc_RSVT = metrics.accuracy_score(y_test, preds)\n",
    "\n",
    "    print(\"Random Forest: \", acc_RFC, \"\\nExtra-Trees: \",acc_ETC, \"\\nRSVT: \", acc_RSVT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Wine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X,y = datasets.load_wine(return_X_y=True)\n",
    "\n",
    "X_train,X_test, y_train,y_test = train_test_split(X,y, test_size=1/3, stratify=y, random_state=754046)\n",
    "\n",
    "# Rescale input space to [0,1] range\n",
    "scaler = preprocessing.MinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest:  0.9666666666666667 \n",
      "Extra-Trees:  0.95 \n",
      "RSVT:  0.95\n"
     ]
    }
   ],
   "source": [
    "benchmarks(X_train,X_test, y_train,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Breast Cancer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X,y = datasets.load_breast_cancer(return_X_y=True)\n",
    "\n",
    "X_train,X_test, y_train,y_test = train_test_split(X,y, test_size=1/3, stratify=y, random_state=754046)\n",
    "\n",
    "# Rescale input space to [0,1] range\n",
    "scaler = preprocessing.MinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest:  0.9526315789473684 \n",
      "Extra-Trees:  0.9578947368421052 \n",
      "RSVT:  0.968421052631579\n"
     ]
    }
   ],
   "source": [
    "benchmarks(X_train,X_test, y_train,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Land cover"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "landcover_train = pd.read_csv(\"Data/landcover_train.csv\")\n",
    "landcover_test = pd.read_csv(\"Data/landcover_test.csv\")\n",
    "\n",
    "X_train = np.array(landcover_train.iloc[:,1:])\n",
    "X_test = np.array(landcover_test.iloc[:,1:])\n",
    "\n",
    "y_train = landcover_train.iloc[:,0]\n",
    "y_test = landcover_test.iloc[:,0]\n",
    "\n",
    "enc = preprocessing.LabelEncoder()\n",
    "y_train = enc.fit_transform(y_train)\n",
    "y_test = enc.transform(y_test)\n",
    "\n",
    "# Rescale input space to [0,1] range\n",
    "scaler = preprocessing.MinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest:  0.8106508875739645 \n",
      "Extra-Trees:  0.8244575936883629 \n",
      "RSVT:  0.7396449704142012\n"
     ]
    }
   ],
   "source": [
    "benchmarks(X_train,X_test, y_train,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vehicle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "vehicle = pd.read_csv(\"Data/vehicle.txt\", sep=\" \", header=None)\n",
    "vehicle = vehicle.drop([19], axis=1)\n",
    "X = vehicle.iloc[:,0:-1]\n",
    "y = vehicle.iloc[:,-1]\n",
    "enc = preprocessing.LabelEncoder()\n",
    "y = enc.fit_transform(y)\n",
    "\n",
    "X_train,X_test, y_train,y_test = train_test_split(X,y, test_size=1/3, stratify=y, random_state=754046)\n",
    "\n",
    "# Rescale input space to [0,1] range\n",
    "scaler = preprocessing.MinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest:  0.723404255319149 \n",
      "Extra-Trees:  0.7198581560283688 \n",
      "RSVT:  0.7411347517730497\n"
     ]
    }
   ],
   "source": [
    "benchmarks(X_train,X_test, y_train,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Contraceptive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "contraceptive = pd.read_csv(\"Data/cmc.csv\", header=None)\n",
    "X = np.array(contraceptive.iloc[:,:-1])\n",
    "y = np.array(contraceptive.iloc[:,-1])\n",
    "\n",
    "X_train,X_test, y_train,y_test = train_test_split(X,y, test_size=1/3, stratify=y, random_state=754046)\n",
    "\n",
    "# Rescale input space to [0,1] range\n",
    "scaler = preprocessing.MinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest:  0.5132382892057027 \n",
      "Extra-Trees:  0.5010183299389002 \n",
      "RSVT:  0.5234215885947047\n"
     ]
    }
   ],
   "source": [
    "benchmarks(X_train,X_test, y_train,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "segment = np.loadtxt(\"Data/segment.txt\")\n",
    "X = np.array(segment[:,:-1])\n",
    "y = np.array(segment[:,-1])\n",
    "\n",
    "X_train,X_test, y_train,y_test = train_test_split(X,y, test_size=1/3, stratify=y, random_state=754046)\n",
    "\n",
    "# Rescale input space to [0,1] range\n",
    "scaler = preprocessing.MinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest:  0.9688311688311688 \n",
      "Extra-Trees:  0.9701298701298702 \n",
      "RSVT:  0.9207792207792208\n"
     ]
    }
   ],
   "source": [
    "benchmarks(X_train,X_test, y_train,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Madelon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = pd.read_csv(\"Data/madelon_train.csv\", sep=\" \", header=None)\n",
    "X_train = np.array(X_train.drop([500], axis=1))\n",
    "y_train = pd.read_csv(\"Data/madelon_trainlabels.csv\", header=None)\n",
    "y_train = np.array(y_train).reshape(-1,1)\n",
    "\n",
    "train = np.c_[X_train,y_train]\n",
    "X_train = train[:,:-1]\n",
    "y_train = train[:,-1]\n",
    "\n",
    "X_test = pd.read_csv(\"Data/madelon_valid.csv\", sep=\" \", header=None)\n",
    "X_test = np.array(X_test.drop([500], axis=1))\n",
    "y_test = pd.read_csv(\"Data/madelon_validlabels.csv\", header=None)\n",
    "y_test = np.array(y_test).reshape(-1,1)\n",
    "\n",
    "test = np.c_[X_test,y_test]\n",
    "X_test = test[:,:-1]\n",
    "y_test = test[:,-1]\n",
    "\n",
    "# Rescale input space to [0,1] range\n",
    "scaler = preprocessing.MinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest:  0.7033333333333334 \n",
      "Extra-Trees:  0.6733333333333333 \n",
      "RSVT:  0.58\n"
     ]
    }
   ],
   "source": [
    "benchmarks(X_train,X_test, y_train,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Spambase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "spambase = pd.read_csv(\"Data/spambase.csv\", header=None)\n",
    "X = np.array(spambase.iloc[:,:-1])\n",
    "y = np.array(spambase.iloc[:,-1])\n",
    "\n",
    "X_train,X_test, y_train,y_test = train_test_split(X,y, test_size=1/3, stratify=y, random_state=754046)\n",
    "\n",
    "# Rescale input space to [0,1] range\n",
    "scaler = preprocessing.MinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest:  0.9569752281616688 \n",
      "Extra-Trees:  0.9576271186440678 \n",
      "RSVT:  0.939374185136897\n"
     ]
    }
   ],
   "source": [
    "benchmarks(X_train,X_test, y_train,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handwritten Sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "X,y = datasets.load_digits(return_X_y=True)\n",
    "\n",
    "X_train,X_test, y_train,y_test = train_test_split(X,y, test_size=1/3, stratify=y, random_state=754046)\n",
    "\n",
    "# Rescale input space to [0,1] range\n",
    "scaler = preprocessing.MinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest:  0.9666110183639399 \n",
      "Extra-Trees:  0.9849749582637729 \n",
      "RSVT:  0.9666110183639399\n"
     ]
    }
   ],
   "source": [
    "benchmarks(X_train,X_test, y_train,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Isolet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "isolet_train = pd.read_csv(\"Data/isolet-train.csv\", header=None)\n",
    "isolet_test = pd.read_csv(\"Data/isolet-test.csv\", header=None)\n",
    "\n",
    "X_train = np.array(isolet_train.iloc[:,:-1])\n",
    "y_train = isolet_train.iloc[:,-1]\n",
    "\n",
    "X_test = np.array(isolet_test.iloc[:,:-1])\n",
    "y_test = isolet_test.iloc[:,-1]\n",
    "\n",
    "# Rescale input space\n",
    "scaler = preprocessing.MinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest:  0.9422706863373957 \n",
      "Extra-Trees:  0.9512508017960231 \n",
      "RSVT:  0.9583066067992303\n"
     ]
    }
   ],
   "source": [
    "benchmarks(X_train,X_test, y_train,y_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}