Assignment 3. Exercise 1. Best Possible Tree.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Assignment 3. Exercise 1: Best possible tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import precision_score,make_scorer,accuracy_score\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "col_name = ['duration','protocol_type','service','flag','src_bytes','dst_bytes','land','wrong_fragment',\n",
    "            'urgent','hot','num_failed_logins','logged_in','num_compromised','root_shell','su_attempted',\n",
    "            'num_root','num_file_creations','num_shells','num_access_files','num_outbound_cmds',\n",
    "            'is_host_login','is_guest_login','count','srv_count','serror_rate',\n",
    "            'srv_serror_rate','rerror_rate','srv_rerror_rate','same_srv_rate',\n",
    "            'diff_srv_rate','srv_diff_host_rate','dst_host_count',\n",
    "            'dst_host_srv_count','dst_host_same_srv_rate',\n",
    "            'dst_host_diff_srv_rate','dst_host_same_src_port_rate',\n",
    "            'dst_host_srv_diff_host_rate',\n",
    "            'dst_host_serror_rate',\n",
    "            'dst_host_srv_serror_rate',\n",
    "            'dst_host_rerror_rate',\n",
    "            'dst_host_srv_rerror_rate',\n",
    "           'label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = pd.read_csv(\"trainingDecisionTree.csv\",names=col_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = f['label'].apply(lambda x:1 if x=='normal.' else 0)\n",
    "f['label']=label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    401061\n",
       "1     98939\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f[\"label\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = f.iloc[:,:-1]\n",
    "y=f['label']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I use \"LabelEncoder\" for three features: protocol_type, service, flag. In order to convert them to numerical values to be fed into the DecisionTreeClassifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "le1 = preprocessing.LabelEncoder()\n",
    "le1.fit(X['protocol_type'])\n",
    "X['protocol_type'] = le1.transform(X['protocol_type'])\n",
    "\n",
    "le2 = preprocessing.LabelEncoder()\n",
    "le2.fit(X['service'])\n",
    "X['service'] = le2.transform(X['service'])\n",
    "\n",
    "le3 = preprocessing.LabelEncoder()\n",
    "le3.fit(X['flag'])\n",
    "X['flag'] = le3.transform(X['flag'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I decided to use some techniques for feature selection in order to find as good classifier as possible. As you can see below:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Removing features with low variance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I used VarianceThreshold module in sklearn for feature selection/dimensionality reduction on KDD cup dataset, to improve estimators’ accuracy scores and to boost performance on KDD datasets. It removes all features whose variance doesn’t meet some threshold."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import VarianceThreshold\n",
    "\n",
    "pd.options.display.float_format = '{:.4f}'.format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "duration                           546798.4367\n",
       "protocol_type                           0.3271\n",
       "service                               182.1743\n",
       "flag                                    5.0843\n",
       "src_bytes                     95029238872.6983\n",
       "dst_bytes                       329348075.3361\n",
       "land                                    0.0000\n",
       "wrong_fragment                          0.0019\n",
       "urgent                                  0.0000\n",
       "hot                                     0.2020\n",
       "num_failed_logins                       0.0001\n",
       "logged_in                               0.1228\n",
       "num_compromised                         3.5123\n",
       "root_shell                              0.0001\n",
       "su_attempted                            0.0000\n",
       "num_root                                3.8349\n",
       "num_file_creations                      0.0173\n",
       "num_shells                              0.0001\n",
       "num_access_files                        0.0013\n",
       "num_outbound_cmds                       0.0000\n",
       "is_host_login                           0.0000\n",
       "is_guest_login                          0.0008\n",
       "count                               44892.3603\n",
       "srv_count                           60472.6754\n",
       "serror_rate                             0.1456\n",
       "srv_serror_rate                         0.1459\n",
       "rerror_rate                             0.0538\n",
       "srv_rerror_rate                         0.0540\n",
       "same_srv_rate                           0.1514\n",
       "diff_srv_rate                           0.0068\n",
       "srv_diff_host_rate                      0.0198\n",
       "dst_host_count                       4084.3370\n",
       "dst_host_srv_count                  11206.9464\n",
       "dst_host_same_srv_rate                  0.1689\n",
       "dst_host_diff_srv_rate                  0.0116\n",
       "dst_host_same_src_port_rate             0.2311\n",
       "dst_host_srv_diff_host_rate             0.0017\n",
       "dst_host_serror_rate                    0.1456\n",
       "dst_host_srv_serror_rate                0.1459\n",
       "dst_host_rerror_rate                    0.0532\n",
       "dst_host_srv_rerror_rate                0.0533\n",
       "dtype: float64"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.var()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "sel = VarianceThreshold(threshold=0.1)\n",
    "sel.fit(X)\n",
    "X2 = sel.transform(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SelectKBest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I select features according to the 10 highest scores. Based on Sklearn documentation \"chi2\" Chi-squared stats of non-negative features used for classification tasks.so, I choose that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import SelectKBest, chi2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "sel2 = SelectKBest(chi2, k=10)\n",
    "sel2.fit(X2,y)\n",
    "X3 = sel2.transform(X2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(500000, 10)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X3.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(X3,columns=range(10))\n",
    "df[\"label\"] = y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('featurs10.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It generates one .csv file called 'features10.csv' which include 10 best features exracted from SelectKBest in sklearn."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparamter tuning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For Decesion tree classifier, hyperparameters are tuned using K-fold Cross Validation with K = 5. Grid Search is used to obtain the respective hyperparameter values. Also, In DecisionTreeClassifier() to measure the quality of a split, using criterion=’gini’ as default for the Gini impurity. Eventually, since I am running a gridsearch on Decision tree, it will be the best decision tree that it would have selected."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import make_scorer,precision_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtree = DecisionTreeClassifier()\n",
    "\n",
    "depths = np.arange(1, 21)\n",
    "num_splits = np.arange(10,500,20)\n",
    "num_leafs = [1, 5, 10, 20, 50, 100]\n",
    "\n",
    "parameters={'min_samples_split' : num_splits,'max_depth': depths,'min_samples_leaf':num_leafs}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It takes almost 1 hour and 20 minutes to run :(. Note that you don't need to run it, Just call Load model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "       estimator=DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best'),\n",
       "       fit_params=None, iid='warn', n_jobs=3,\n",
       "       param_grid={'min_samples_split': array([ 10,  30,  50,  70,  90, 110, 130, 150, 170, 190, 210, 230, 250,\n",
       "       270, 290, 310, 330, 350, 370, 390, 410, 430, 450, 470, 490]), 'max_depth': array([ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,\n",
       "       18, 19, 20]), 'min_samples_leaf': [1, 5, 10, 20, 50, 100]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs = GridSearchCV(estimator=dtree, param_grid=parameters, scoring='accuracy', cv=5, n_jobs=3)\n",
    "gs.fit(X3,y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Presistence model "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I used dump and load function to presistence model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = 'model.dat'\n",
    "\n",
    "with open(name, 'wb') as f:\n",
    "    pipeline = pickle.dump(gs, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_ml():\n",
    "    with open('model.dat', 'rb') as f:\n",
    "        model = pickle.load(f)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# test Function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is a testing function which take a teset data set as .csv file as test points and labels and returns the classification error of model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_test(test_data):\n",
    "    \n",
    "    X_test = test_data.iloc[:,:-1]\n",
    "    \n",
    "    y_test = test_data['label']\n",
    "    y_test = y_test.apply(lambda x:1 if x=='normal.' else 0)\n",
    "    \n",
    "    X_test['protocol_type'] = le1.transform(X_test['protocol_type'])\n",
    "    X_test['service'] = le2.transform(X_test['service'])\n",
    "    X_test['flag'] = le3.transform(X_test['flag'])\n",
    "    X_test = sel.transform(X_test)\n",
    "    X_test = sel2.transform(X_test)\n",
    "    \n",
    "    model = load_ml()\n",
    "    y_pred = model.predict(X_test)\n",
    "    \n",
    "    acc = accuracy_score(y_test, y_pred, normalize=False)\n",
    "    \n",
    "    error = X_test.shape[0] - acc\n",
    "    \n",
    "    return error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### add your test file path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Please just add the name of csv file below instead \"???.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "testdata = pd.read_csv(\"???.csv\",names=col_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_test(testdata)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}