Dimensionality Reduction A comparison.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "-ZixxRgn_QE2"
      },
      "source": [
        "# Intrusion Detection \n",
        "\n",
        "### Dataset from https://github.com/defcom17/NSL_KDD/\n",
        "* [more info](https://docs.google.com/spreadsheets/d/1oAx320Vo9Z6HrBrL6BcfLH6sh2zIk9EKCv2OlaMGmwY/edit#gid=0)\n",
        "\n",
        "### Sample code used: https://www.kaggle.com/meesterwaffles/nicholas-brougher-neb5211-project4\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "m9hIv2s7_QE6"
      },
      "outputs": [],
      "source": [
        "%config IPCompleter.greedy=True\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import numpy as np\n",
        "import re\n",
        "import sklearn\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib as matplot\n",
        "%matplotlib inline\n",
        "\n",
        "from IPython.core.interactiveshell import InteractiveShell\n",
        "InteractiveShell.ast_node_interactivity = \"all\"\n",
        "\n",
        "from sklearn.model_selection import train_test_split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 216
        },
        "colab_type": "code",
        "id": "xcmNZAzs_Svh",
        "outputId": "2c9b64dc-dcb3-45c7-ea12-941b0348fe24"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(125972, 43)"
            ]
          },
          "execution_count": 2,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        },
        {
          "data": {
            "text/plain": [
              "(22542, 43)"
            ]
          },
          "execution_count": 2,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        },
        {
          "data": {
            "text/plain": [
              "(148514, 43)"
            ]
          },
          "execution_count": 2,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>30</th>\n",
              "      <th>31</th>\n",
              "      <th>32</th>\n",
              "      <th>33</th>\n",
              "      <th>34</th>\n",
              "      <th>35</th>\n",
              "      <th>36</th>\n",
              "      <th>37</th>\n",
              "      <th>38</th>\n",
              "      <th>39</th>\n",
              "      <th>40</th>\n",
              "      <th>41</th>\n",
              "      <th>42</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>udp</td>\n",
              "      <td>other</td>\n",
              "      <td>SF</td>\n",
              "      <td>146</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>13</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.08</td>\n",
              "      <td>0.15</td>\n",
              "      <td>0.00</td>\n",
              "      <td>255</td>\n",
              "      <td>1</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.88</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.00</td>\n",
              "      <td>normal</td>\n",
              "      <td>15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>tcp</td>\n",
              "      <td>private</td>\n",
              "      <td>S0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>123</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.05</td>\n",
              "      <td>0.07</td>\n",
              "      <td>0.00</td>\n",
              "      <td>255</td>\n",
              "      <td>26</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.05</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>1.00</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.00</td>\n",
              "      <td>neptune</td>\n",
              "      <td>19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>tcp</td>\n",
              "      <td>http</td>\n",
              "      <td>SF</td>\n",
              "      <td>232</td>\n",
              "      <td>8153</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "      <td>0.2</td>\n",
              "      <td>0.2</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>30</td>\n",
              "      <td>255</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.03</td>\n",
              "      <td>0.04</td>\n",
              "      <td>0.03</td>\n",
              "      <td>0.01</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.01</td>\n",
              "      <td>normal</td>\n",
              "      <td>21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>tcp</td>\n",
              "      <td>http</td>\n",
              "      <td>SF</td>\n",
              "      <td>199</td>\n",
              "      <td>420</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>30</td>\n",
              "      <td>32</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.09</td>\n",
              "      <td>255</td>\n",
              "      <td>255</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.00</td>\n",
              "      <td>normal</td>\n",
              "      <td>21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>tcp</td>\n",
              "      <td>private</td>\n",
              "      <td>REJ</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>121</td>\n",
              "      <td>19</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.16</td>\n",
              "      <td>0.06</td>\n",
              "      <td>0.00</td>\n",
              "      <td>255</td>\n",
              "      <td>19</td>\n",
              "      <td>0.07</td>\n",
              "      <td>0.07</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.00</td>\n",
              "      <td>neptune</td>\n",
              "      <td>21</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   0    1        2    3    4     5   6   ...    36    37    38   39    40       41  42\n",
              "0   0  udp    other   SF  146     0   0  ...  0.00  0.00  0.00  0.0  0.00   normal  15\n",
              "1   0  tcp  private   S0    0     0   0  ...  0.00  1.00  1.00  0.0  0.00  neptune  19\n",
              "2   0  tcp     http   SF  232  8153   0  ...  0.04  0.03  0.01  0.0  0.01   normal  21\n",
              "3   0  tcp     http   SF  199   420   0  ...  0.00  0.00  0.00  0.0  0.00   normal  21\n",
              "4   0  tcp  private  REJ    0     0   0  ...  0.00  0.00  0.00  1.0  1.00  neptune  21\n",
              "\n",
              "[5 rows x 43 columns]"
            ]
          },
          "execution_count": 2,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train = pd.read_csv('https://raw.githubusercontent.com/defcom17/NSL_KDD/master/KDDTrain%2B.csv')\n",
        "test = pd.read_csv('https://raw.githubusercontent.com/defcom17/NSL_KDD/master/KDDTest%2B.csv')\n",
        "train.shape\n",
        "test.shape \n",
        "train.columns = range(train.shape[1])\n",
        "test.columns = range(test.shape[1])\n",
        "labels = ['duration', 'protocol_type', 'service', 'flag', 'src_bytes',\n",
        "'dst_bytes', 'land', 'wrong_fragment', 'urgent', 'hot',\n",
        "'num_failed_logins', 'logged_in', 'num_compromised', 'root_shell',\n",
        "'su_attempted', 'num_root', 'num_file_creations', 'num_shells',\n",
        "'num_access_files', 'num_outbound_cmds', 'is_host_login',\n",
        "'is_guest_login', 'count', 'srv_count', 'serror_rate',\n",
        "'srv_serror_rate', 'rerror_rate', 'srv_rerror_rate', 'same_srv_rate',\n",
        "'diff_srv_rate', 'srv_diff_host_rate', 'dst_host_count',\n",
        "'dst_host_srv_count', 'dst_host_same_srv_rate', 'dst_host_diff_srv_rate', 'dst_host_same_src_port_rate',\n",
        "'dst_host_srv_diff_host_rate', 'dst_host_serror_rate',\n",
        "'dst_host_srv_serror_rate', 'dst_host_rerror_rate',\n",
        "'dst_host_srv_rerror_rate', 'attack_type', 'difficulty_level']# subclass - > attack_type\n",
        "combined_data = pd.concat([train, test])\n",
        "combined_data.shape\n",
        "combined_data.head(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 158
        },
        "colab_type": "code",
        "id": "iXVbNAVZWG5n",
        "outputId": "b5b8c7f9-0ab2-4ddf-eaf0-80e40f07dcbb"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>duration</th>\n",
              "      <th>protocol_type</th>\n",
              "      <th>service</th>\n",
              "      <th>flag</th>\n",
              "      <th>src_bytes</th>\n",
              "      <th>dst_bytes</th>\n",
              "      <th>land</th>\n",
              "      <th>wrong_fragment</th>\n",
              "      <th>urgent</th>\n",
              "      <th>hot</th>\n",
              "      <th>num_failed_logins</th>\n",
              "      <th>logged_in</th>\n",
              "      <th>num_compromised</th>\n",
              "      <th>root_shell</th>\n",
              "      <th>su_attempted</th>\n",
              "      <th>num_root</th>\n",
              "      <th>num_file_creations</th>\n",
              "      <th>num_shells</th>\n",
              "      <th>num_access_files</th>\n",
              "      <th>num_outbound_cmds</th>\n",
              "      <th>is_host_login</th>\n",
              "      <th>is_guest_login</th>\n",
              "      <th>count</th>\n",
              "      <th>srv_count</th>\n",
              "      <th>serror_rate</th>\n",
              "      <th>srv_serror_rate</th>\n",
              "      <th>rerror_rate</th>\n",
              "      <th>srv_rerror_rate</th>\n",
              "      <th>same_srv_rate</th>\n",
              "      <th>diff_srv_rate</th>\n",
              "      <th>srv_diff_host_rate</th>\n",
              "      <th>dst_host_count</th>\n",
              "      <th>dst_host_srv_count</th>\n",
              "      <th>dst_host_same_srv_rate</th>\n",
              "      <th>dst_host_diff_srv_rate</th>\n",
              "      <th>dst_host_same_src_port_rate</th>\n",
              "      <th>dst_host_srv_diff_host_rate</th>\n",
              "      <th>dst_host_serror_rate</th>\n",
              "      <th>dst_host_srv_serror_rate</th>\n",
              "      <th>dst_host_rerror_rate</th>\n",
              "      <th>dst_host_srv_rerror_rate</th>\n",
              "      <th>attack_type</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>udp</td>\n",
              "      <td>other</td>\n",
              "      <td>SF</td>\n",
              "      <td>146</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>13</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.08</td>\n",
              "      <td>0.15</td>\n",
              "      <td>0.0</td>\n",
              "      <td>255</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.88</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.00</td>\n",
              "      <td>normal</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>tcp</td>\n",
              "      <td>private</td>\n",
              "      <td>S0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>123</td>\n",
              "      <td>6</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.05</td>\n",
              "      <td>0.07</td>\n",
              "      <td>0.0</td>\n",
              "      <td>255</td>\n",
              "      <td>26</td>\n",
              "      <td>0.1</td>\n",
              "      <td>0.05</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>1.00</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.00</td>\n",
              "      <td>neptune</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>tcp</td>\n",
              "      <td>http</td>\n",
              "      <td>SF</td>\n",
              "      <td>232</td>\n",
              "      <td>8153</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "      <td>0.2</td>\n",
              "      <td>0.2</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.0</td>\n",
              "      <td>30</td>\n",
              "      <td>255</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.03</td>\n",
              "      <td>0.04</td>\n",
              "      <td>0.03</td>\n",
              "      <td>0.01</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.01</td>\n",
              "      <td>normal</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   duration protocol_type  ... dst_host_srv_rerror_rate attack_type\n",
              "0         0           udp  ...                     0.00      normal\n",
              "1         0           tcp  ...                     0.00     neptune\n",
              "2         0           tcp  ...                     0.01      normal\n",
              "\n",
              "[3 rows x 42 columns]"
            ]
          },
          "execution_count": 3,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "combined_data.columns = labels\n",
        "combined_data = combined_data.drop('difficulty_level', 1)\n",
        "combined_data.head(3)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "RVdcfGhNG9YD"
      },
      "source": [
        "### Reduce train size for faster trainin, remove when in production"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Xbsr_Auy_QFN"
      },
      "source": [
        "### The following few cells are taken from the 'sample code'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 201
        },
        "colab_type": "code",
        "id": "nl9Yok3f_QFO",
        "outputId": "62bf1d29-5631-482b-f06d-49bc878b91cf"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'mailbomb', 'saint', 'snmpguess', 'guess_passwd', 'udpstorm', 'warezclient', 'neptune', 'normal', 'processtable', 'sendmail', 'teardrop', 'worm', 'xsnoop', 'ps', 'nmap', 'named', 'sqlattack', 'pod', 'portsweep', 'apache2', 'multihop', 'land', 'xterm', 'ftp_write', 'imap', 'httptunnel', 'buffer_overflow', 'rootkit', 'mscan', 'satan', 'phf', 'back', 'warezmaster', 'perl', 'ipsweep', 'smurf', 'spy', 'loadmodule', 'xlock', 'snmpgetattack'}\n",
            "\n",
            "Describing attack_type: \n",
            "min 0\n",
            "max 39\n",
            "mean 15.782242751525109\n",
            "mode 0    16\n",
            "dtype: int64\n",
            "looks like 16 is 'normal' \n"
          ]
        }
      ],
      "source": [
        "from sklearn import preprocessing\n",
        "le = preprocessing.LabelEncoder()\n",
        "\n",
        "print(set(list(combined_data['attack_type']))) # use print to make it print on single line \n",
        "combined_data['attack_type'] = le.fit_transform(combined_data['attack_type'])\n",
        "combined_data['protocol_type'] = le.fit_transform(combined_data['protocol_type'])\n",
        "combined_data['service'] = le.fit_transform(combined_data['service'])\n",
        "combined_data['flag'] = le.fit_transform(combined_data['flag'])\n",
        "\n",
        "print('\\nDescribing attack_type: ')\n",
        "print(\"min\", combined_data['attack_type'].min())\n",
        "print(\"max\", combined_data['attack_type'].max())\n",
        "print(\"mean\", combined_data['attack_type'].mean())\n",
        "print(\"mode\", combined_data['attack_type'].mode())\n",
        "print(\"looks like 16 is 'normal' \")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "LvirvNaV1gT3"
      },
      "source": [
        "# Reduce feature space"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        },
        "colab_type": "code",
        "id": "DMafgVi9ZLEA",
        "outputId": "b0af4db5-a0f0-436b-ebec-2b64c818b836"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "14"
            ]
          },
          "execution_count": 5,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        },
        {
          "data": {
            "text/plain": [
              "{'dst_bytes',\n",
              " 'is_host_login',\n",
              " 'land',\n",
              " 'logged_in',\n",
              " 'num_access_files',\n",
              " 'num_compromised',\n",
              " 'num_file_creations',\n",
              " 'num_outbound_cmds',\n",
              " 'num_root',\n",
              " 'num_shells',\n",
              " 'root_shell',\n",
              " 'srv_rerror_rate',\n",
              " 'su_attempted',\n",
              " 'urgent'}"
            ]
          },
          "execution_count": 5,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# select least correlated\n",
        "corr_matrix = combined_data.corr().abs().sort_values('attack_type')\n",
        "# tmp.head(10) # to view CORR matrix \n",
        "leastCorrelated = corr_matrix['attack_type'].nsmallest(10)\n",
        "leastCorrelated = list(leastCorrelated.index)\n",
        "\n",
        "# select least correlated\n",
        "leastSTD =  combined_data.std().to_frame().nsmallest(5, columns=0)\n",
        "leastSTD = list(leastSTD.transpose().columns)  #fuckin pandas.core.indexes.base.Index   -_-\n",
        "#tmp = tmp.append('num_outbound_cmds')  # might not work...\n",
        "featureElimination = set(leastCorrelated + leastSTD)\n",
        "len(featureElimination)\n",
        "featureElimination"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "60WOhD9X_QFf"
      },
      "source": [
        "## Drop features and preform train_test_split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "x4xaj5Z0ZURo"
      },
      "outputs": [],
      "source": [
        "# dont change combined_data, we will neeed it latter \n",
        "combined_data_reduced = combined_data.drop(featureElimination,axis=1)\n",
        "data_x = combined_data_reduced.drop('attack_type', axis=1)\n",
        "data_y = combined_data_reduced.loc[:,['attack_type']]\n",
        "# del combined_data # free mem\n",
        "X_train, X_test, y_train, y_test = train_test_split(data_x, data_y, test_size=.5, random_state=42) # TODO"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "colab_type": "code",
        "id": "NsFRJ0zE_QFo",
        "outputId": "9680ccea-7242-4993-cbfe-04236cce8ebe"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Thats how to rid rid of 14 dimentions of data, from the 10 lowest STD and 5 lowest correlation\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "(74257, 27)"
            ]
          },
          "execution_count": 7,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        },
        {
          "data": {
            "text/plain": [
              "(74257, 27)"
            ]
          },
          "execution_count": 7,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "print(\"Thats how to rid rid of {0} dimentions of data, from the 10 lowest STD and 5 lowest correlation\".format(len(featureElimination)))\n",
        "\n",
        "X_train.shape\n",
        "X_test.shape"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "EyXt8Id41wlr"
      },
      "source": [
        "# Feature selection with ML "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "colab_type": "code",
        "id": "G4e7NTqd_QFr",
        "outputId": "deb77ef8-a0e4-41c7-a2b6-c50f0b3b1ed7"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "execution_count": 8,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from sklearn import linear_model\n",
        "\n",
        "from sklearn.ensemble import VotingClassifier\n",
        "\n",
        "from sklearn.ensemble import AdaBoostClassifier\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.ensemble import BaggingClassifier\n",
        "from sklearn.ensemble import ExtraTreesClassifier\n",
        "from sklearn.ensemble import GradientBoostingClassifier\n",
        "from sklearn.ensemble import IsolationForest\n",
        "\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "\n",
        "import gc \n",
        "gc.collect()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "colab_type": "code",
        "id": "3gTEKlbI_QFu",
        "outputId": "4ed27606-472a-46fc-e3ca-bef288280b2f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, normalize=False)"
            ]
          },
          "execution_count": 9,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Linear regression processing\n",
            "Linear regression Score: 0.41 %\n"
          ]
        }
      ],
      "source": [
        "LR = linear_model.LinearRegression()\n",
        "LR.fit(X_train, y_train)\n",
        "lr_score = LR.score(X_test, y_test)\n",
        "print('Linear regression processing')\n",
        "print('Linear regression Score: %.2f %%' % lr_score)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "7y-NiCAv_QF9"
      },
      "outputs": [],
      "source": [
        "AB = AdaBoostClassifier(base_estimator=DecisionTreeClassifier(), n_estimators=100, learning_rate=1.0)\n",
        "RF = RandomForestClassifier(n_estimators=10, criterion='entropy', max_features='auto', bootstrap=True)\n",
        "ET = ExtraTreesClassifier(n_estimators=10, criterion='gini', max_features='auto', bootstrap=False)\n",
        "GB = GradientBoostingClassifier(loss='deviance', learning_rate=0.5, n_estimators=75, max_features='auto') # learning_rate=0.1 # very slow "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 308
        },
        "colab_type": "code",
        "id": "OY4ePfZq_QGD",
        "outputId": "9859798d-71f4-4736-b850-e179c7fbfaee"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "AdaBoostClassifier(algorithm='SAMME.R',\n",
              "                   base_estimator=DecisionTreeClassifier(class_weight=None,\n",
              "                                                         criterion='gini',\n",
              "                                                         max_depth=None,\n",
              "                                                         max_features=None,\n",
              "                                                         max_leaf_nodes=None,\n",
              "                                                         min_impurity_decrease=0.0,\n",
              "                                                         min_impurity_split=None,\n",
              "                                                         min_samples_leaf=1,\n",
              "                                                         min_samples_split=2,\n",
              "                                                         min_weight_fraction_leaf=0.0,\n",
              "                                                         presort=False,\n",
              "                                                         random_state=None,\n",
              "                                                         splitter='best'),\n",
              "                   learning_rate=1.0, n_estimators=100, random_state=None)"
            ]
          },
          "execution_count": 11,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "AdaBoostClassifier processing ,,,\n",
            "AdaBoostClassifier Score: 0.994 %\n"
          ]
        }
      ],
      "source": [
        "AB.fit(X_train, y_train)\n",
        "AB_feature = AB.feature_importances_\n",
        "\n",
        "ab_score = AB.score(X_test, y_test)\n",
        "print('AdaBoostClassifier processing ,,,')\n",
        "print('AdaBoostClassifier Score: %.3f %%' % ab_score)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 163
        },
        "colab_type": "code",
        "id": "mGENFPUS_QGN",
        "outputId": "b9ab9e05-cf91-417a-f9e8-cc4a0acc151b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
              "                       max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
              "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "                       min_samples_leaf=1, min_samples_split=2,\n",
              "                       min_weight_fraction_leaf=0.0, n_estimators=10,\n",
              "                       n_jobs=None, oob_score=False, random_state=None,\n",
              "                       verbose=0, warm_start=False)"
            ]
          },
          "execution_count": 12,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "RandomForestClassifier processing ,,,\n",
            "RandomForestClassifier Score: 0.993 %\n"
          ]
        }
      ],
      "source": [
        "RF.fit(X_train, y_train)\n",
        "RF_feature = RF.feature_importances_\n",
        "\n",
        "rf_score = RF.score(X_test, y_test)\n",
        "print('RandomForestClassifier processing ,,,')\n",
        "print('RandomForestClassifier Score: %.3f %%' % rf_score)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 163
        },
        "colab_type": "code",
        "id": "NDZHaC4K_QGV",
        "outputId": "61cea50c-be31-4376-cc92-f725a47f36ba"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "ExtraTreesClassifier(bootstrap=False, class_weight=None, criterion='gini',\n",
              "                     max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
              "                     min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "                     min_samples_leaf=1, min_samples_split=2,\n",
              "                     min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=None,\n",
              "                     oob_score=False, random_state=None, verbose=0,\n",
              "                     warm_start=False)"
            ]
          },
          "execution_count": 13,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "ExtraTreesClassifier processing ,,,\n",
            "ExtraTreeClassifier: 0.991 %\n"
          ]
        }
      ],
      "source": [
        "ET.fit(X_train, y_train)\n",
        "ET_feature = ET.feature_importances_\n",
        "\n",
        "et_score = ET.score(X_test, y_test)\n",
        "print('ExtraTreesClassifier processing ,,,')\n",
        "print('ExtraTreeClassifier: %.3f %%' % et_score)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        },
        "colab_type": "code",
        "id": "G6xhq-V7_QGb",
        "outputId": "7d66ab89-9d3f-4481-a0d8-efd506b96f39"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>features</th>\n",
              "      <th>AdaBoost</th>\n",
              "      <th>RandomForest</th>\n",
              "      <th>ExtraTree</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>duration</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.016136</td>\n",
              "      <td>0.010487</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>protocol_type</td>\n",
              "      <td>0.157608</td>\n",
              "      <td>0.066807</td>\n",
              "      <td>0.042998</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>service</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.050349</td>\n",
              "      <td>0.030385</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>flag</td>\n",
              "      <td>0.006354</td>\n",
              "      <td>0.063868</td>\n",
              "      <td>0.024322</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>src_bytes</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.216365</td>\n",
              "      <td>0.034711</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        features  AdaBoost  RandomForest  ExtraTree\n",
              "0       duration       NaN      0.016136   0.010487\n",
              "1  protocol_type  0.157608      0.066807   0.042998\n",
              "2        service       NaN      0.050349   0.030385\n",
              "3           flag  0.006354      0.063868   0.024322\n",
              "4      src_bytes       NaN      0.216365   0.034711"
            ]
          },
          "execution_count": 14,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "feature_df = pd.DataFrame({'features': X_train.columns.values, # names\n",
        "                           'AdaBoost' : AB_feature,\n",
        "                           'RandomForest' : RF_feature,\n",
        "                           'ExtraTree' : ET_feature\n",
        "                          })\n",
        "feature_df.head(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "colab_type": "code",
        "id": "SOPwTYny_QGe",
        "outputId": "3d9aa2c5-1936-49d1-cac8-e90f04e6ad5a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "18"
            ]
          },
          "execution_count": 15,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "n = 10\n",
        "a_f = feature_df.nlargest(n, 'AdaBoost')\n",
        "r_f = feature_df.nlargest(n, 'RandomForest')\n",
        "e_f = feature_df.nlargest(n, 'ExtraTree')\n",
        "result = pd.concat([a_f, e_f, r_f]).drop_duplicates() \n",
        "len(result.index)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "colab_type": "code",
        "id": "6W8vwtE0_QGh",
        "outputId": "f74ac1d3-aa1e-44f6-bddd-29443e592bfe"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(74257, 18)"
            ]
          },
          "execution_count": 16,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "selected_features = result['features'].values.tolist()\n",
        "X_train_SF = X_train[selected_features]\n",
        "X_test_SF = X_test[selected_features]\n",
        "\n",
        "x = X_train_SF#.reshape(-1, 26)  # 31\n",
        "y = y_train['attack_type'].ravel()\n",
        "\n",
        "x.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        },
        "colab_type": "code",
        "id": "DPbIkY3GNXp6",
        "outputId": "9f1e331f-b30c-4aba-f7a8-e14bd1b06b67"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Acc: 0.98 [DecisionTreeClassifier]\n",
            "Acc: 0.99 [Random Forest]\n",
            "Acc: 0.98 [ExtraTreesClassifier]\n",
            "Acc: 0.99 [Ensemble]\n"
          ]
        }
      ],
      "source": [
        "clf1 = DecisionTreeClassifier() \n",
        "clf2 = RandomForestClassifier(n_estimators=25, random_state=1)# .98\n",
        "# clf3 = GradientBoostingClassifier(learning_rate=.001, n_estimators=200) # .99\n",
        "ET = ExtraTreesClassifier(n_estimators=10, criterion='gini', max_features='auto', bootstrap=False) # .\n",
        "eclf = VotingClassifier(estimators=[('lr', clf1), ('rf', clf2),('et',ET)], voting='hard') # .('gnb', clf3)\n",
        "\n",
        "for clf, label in zip([clf1, clf2,ET, eclf], ['DecisionTreeClassifier', 'Random Forest', 'ExtraTreesClassifier', 'Ensemble']): \n",
        "    _ = clf.fit(x,y)\n",
        "    pred = clf.score(X_test_SF,y_test)\n",
        "    print(\"Acc: %0.2f [%s]\" % (pred,label))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "i_Zp-BxRqwWI"
      },
      "source": [
        "# Comparison"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "bnuRe98NvLGf"
      },
      "outputs": [],
      "source": [
        "DTC = DecisionTreeClassifier() \n",
        "RFC = RandomForestClassifier(n_estimators=25, random_state=1)# .98\n",
        "ETC = ExtraTreesClassifier(n_estimators=10, criterion='gini', max_features='auto', bootstrap=False) # .\n",
        "y   = y_train['attack_type'].ravel()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "XCzMAhAXyeBr"
      },
      "source": [
        "selection the top 2 features from our 3 models, given by `.feature_importances_`, our will be \n",
        "`0.9619160483` and our \n",
        "`number of features` will be 6"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "sKfTTG6exSdE"
      },
      "source": [
        "selection the top 5 features from our 3 models, given by `.feature_importances_`, our will be \n",
        "`0.9832743041` and our \n",
        "`number of features` will be 11"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "colab_type": "code",
        "id": "W9Pd7HpHwN3q",
        "outputId": "784d4487-dfd9-44fc-a3ad-77fefbcd2650"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Acc: 0.9864524557\n",
            "nubmer of features 18\n"
          ]
        }
      ],
      "source": [
        "eclf = VotingClassifier(estimators=[('DecisionTreeClassifier', DTC), ('RandomForestClassifier', RFC),('ExtraTreesClassifier',ETC)], voting='hard')\n",
        "_ = eclf.fit(X_train_SF,y)\n",
        "pred = eclf.score(X_test_SF,y_test)\n",
        "print(\"Acc: %0.10f\" % (pred))\n",
        "\n",
        "print(\"nubmer of features\",X_train_SF.shape[1])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "colab_type": "code",
        "id": "LSEGp7dRwN7S",
        "outputId": "4dd582ee-23f8-472b-d655-69d241e01cc9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Acc: 0.9932262278\n",
            "nubmer of features 27\n"
          ]
        }
      ],
      "source": [
        "eclf = VotingClassifier(estimators=[('DecisionTreeClassifier', DTC), ('RandomForestClassifier', RFC),('ExtraTreesClassifier',ETC)], voting='hard')\n",
        "_ = eclf.fit(X_train, y)\n",
        "pred = eclf.score(X_test,y_test)\n",
        "print(\"Acc: %0.10f\" % (pred))\n",
        "\n",
        "print(\"nubmer of features\",X_train.shape[1])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "BjLm9g4PudEz"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "colab_type": "code",
        "id": "CCmEXEB3wN-6",
        "outputId": "6fbe0793-b80d-4096-9cf7-3e53b90305d3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Acc: 0.9935628964\n",
            "nubmer of features 41\n"
          ]
        }
      ],
      "source": [
        "# shity names, but I dont want to change affect X_train. i like out of order execution  \n",
        "XX_train, XX_test, yy_train, yy_test = train_test_split(combined_data.drop('attack_type', axis=1), combined_data.loc[:,['attack_type']], test_size=.5, random_state=42) \n",
        "eclf = VotingClassifier(estimators=[('DecisionTreeClassifier', DTC), ('RandomForestClassifier', RFC),('ExtraTreesClassifier',ETC)], voting='hard')\n",
        "_ = eclf.fit(XX_train,yy_train)# y would work, but random_state is = 42, a hardcored variable \n",
        "pred = eclf.score(XX_test,yy_test)\n",
        "print(\"Acc: %0.10f\" % (pred))\n",
        "print(\"nubmer of features\",XX_train.shape[1])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "SveyOnPcwOBd"
      },
      "source": [
        "# Conclusion \n",
        "\n",
        "* Acc: 0.9619160483 from Feature Elimination(statistical) + extra feature Elimination(ML); 6 features\n",
        "* Acc: 0.9832743041 from Feature Elimination(statistical) + extra feature Elimination(ML); 11 features\n",
        "* Acc: 0.9865467229 from  Feature Elimination(statistical) + feature Elimination(ML); 18 features\n",
        "* Acc: 0.9890784707 from  Feature Elimination(statistical); 27 features\n",
        "* Acc: 0.9935763632 = from all data; 41 features\n",
        "\n",
        "```\n",
        "eclf = VotingClassifier(estimators=[('lr', clf1), ('rf', clf2),('et',ET)], voting='hard')\n",
        "_ = eclf.fit(X_train, y_train['attack_type'].ravel())\n",
        "pred = eclf.score(X_test,y_test)\n",
        "print(\"Acc: %0.10f\" % (pred))\n",
        "```\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "nlXOJ-SE5HtC"
      },
      "source": [
        "# Bonus \n",
        "## What if we reduce the features will PCA, SVD, RFE "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "iOInYuGjy7z4"
      },
      "outputs": [],
      "source": [
        "from sklearn.feature_selection import RFE\n",
        "from sklearn.decomposition import PCA,TruncatedSVD\n",
        "\n",
        "from sklearn.svm import LinearSVC\n",
        "n= 6 "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "_rbcs6YvDqqA"
      },
      "source": [
        "### Both are realtively fast to fit (\"train\") "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "colab_type": "code",
        "id": "xhNO69jny76Q",
        "outputId": "d215408b-71e8-46fc-a60b-1b60129d52ac"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Acc: 0.9798941514\n",
            "number of features 6\n"
          ]
        }
      ],
      "source": [
        "rfe = RFE(DTC, n) # decision tree classifier\n",
        "\n",
        "# https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.RFE.html\n",
        "rfe = rfe.fit(XX_train, yy_train['attack_type'].ravel())\n",
        "# print(rfe.support_) # bool array, |true| = n, as specified above \n",
        "# print(rfe.ranking_) # not wanted, but you might...  \n",
        "\n",
        "desiredIndices = np.where(rfe.support_==True)[0]\n",
        "whitelist = XX_train.columns.values[desiredIndices]\n",
        "XX_train_RFE,XX_test_RFE = XX_train[whitelist],XX_test[whitelist]\n",
        "\n",
        "eclf = VotingClassifier(estimators=[('DecisionTreeClassifier', DTC), ('RandomForestClassifier', RFC),('ExtraTreesClassifier',ETC)], voting='hard')\n",
        "_ = eclf.fit(XX_train_RFE,yy_train)\n",
        "pred = eclf.score(XX_test_RFE,yy_test)\n",
        "print(\"Acc: %0.10f\" % (pred))\n",
        "print(\"number of features\",XX_train_RFE.shape[1])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "colab_type": "code",
        "id": "HLP94nhEy79I",
        "outputId": "952c40bf-8cee-4ddb-bc1a-92161be49701"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Acc: 0.9750326569\n",
            "number of features 6\n"
          ]
        }
      ],
      "source": [
        "rfe = RFE(RFC, n) # random forest classifier\n",
        "rfe = rfe.fit(XX_train, yy_train['attack_type'].ravel())\n",
        "\n",
        "desiredIndices = np.where(rfe.support_==True)[0]\n",
        "whitelist = XX_train.columns.values[desiredIndices]\n",
        "XX_train_RFE,XX_test_RFE = XX_train[whitelist],XX_test[whitelist]\n",
        "\n",
        "eclf = VotingClassifier(estimators=[('DecisionTreeClassifier', DTC), ('RandomForestClassifier', RFC),('ExtraTreesClassifier',ETC)], voting='hard')\n",
        "_ = eclf.fit(XX_train_RFE,yy_train)\n",
        "\n",
        "pred = eclf.score(XX_test_RFE,yy_test)\n",
        "print(\"Acc: %0.10f\" % (pred))\n",
        "print(\"number of features\",XX_train_RFE.shape[1])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "tlWoITT4D4HT"
      },
      "source": [
        "### PCA is slightly slower to train and gave worst accuracy\n",
        "\n",
        "https://jakevdp.github.io/PythonDataScienceHandbook/05.09-principal-component-analysis.html#PCA-as-dimensionality-reduction"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "colab_type": "code",
        "id": "0X1EGoIfBrNK",
        "outputId": "dcb35b51-184c-48a0-ced7-4afd471d9dd1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Acc: 0.9736186487\n",
            "nubmer of features 6\n"
          ]
        }
      ],
      "source": [
        "pca = PCA(n_components=n)\n",
        "_ = pca.fit(XX_train)  \n",
        "XX_train_pca = pca.transform(XX_train)\n",
        "XX_test_pca = pca.transform(XX_test)\n",
        "# print(pca.explained_variance_ratio_)  \n",
        "# print(pca.singular_values_)  \n",
        "\n",
        "eclf = VotingClassifier(estimators=[('DecisionTreeClassifier', DTC), ('RandomForestClassifier', RFC),('ExtraTreesClassifier',ETC)], voting='hard')\n",
        "_ = eclf.fit(XX_train_pca,yy_train)\n",
        "\n",
        "pred = eclf.score(XX_test_pca,yy_test)\n",
        "print(\"Acc: %0.10f\" % (pred))\n",
        "print(\"nubmer of features\",XX_train_pca.shape[1])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145
        },
        "colab_type": "code",
        "id": "EMxUBoVjJGdn",
        "outputId": "3a4605d3-e3ae-4491-f94c-ef06ff37ebd8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[5.50107175e-01 4.49892716e-01 1.08357432e-07 3.81375180e-10\n",
            " 1.92710577e-10 1.17330222e-10]\n",
            "[1.51474528e+09 1.36984186e+09 6.72272433e+05 3.98833991e+04\n",
            " 2.83510226e+04 2.21218222e+04]\n",
            "6\n",
            "41\n",
            "6 components, 41 in length\n"
          ]
        }
      ],
      "source": [
        "print(pca.explained_variance_ratio_)  \n",
        "print(pca.singular_values_)  \n",
        "\n",
        "print(len(pca.components_))\n",
        "print(len(pca.components_[0]))\n",
        "print(\"6 components, 41 in length\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "bKWd3l9_IArI"
      },
      "source": [
        "### TruncatedSVD"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "UrMyURXxBrK7"
      },
      "outputs": [],
      "source": [
        "svd = TruncatedSVD(n_components=n) \n",
        "_ = svd.fit(XX_train)  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "colab_type": "code",
        "id": "Hq8x6uBQBrDu",
        "outputId": "3f7a56e6-f03b-40c3-fa0c-f6b304be8f5f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Acc: 0.9767698668\n",
            "number of features 6\n"
          ]
        }
      ],
      "source": [
        "XX_train_svd = svd.transform(XX_train)\n",
        "XX_test_svd = svd.transform(XX_test)\n",
        "\n",
        "eclf = VotingClassifier(estimators=[('DecisionTreeClassifier', DTC), ('RandomForestClassifier', RFC),('ExtraTreesClassifier',ETC)], voting='hard')\n",
        "_ = eclf.fit(XX_train_svd,yy_train)\n",
        "\n",
        "pred = eclf.score(XX_test_svd,yy_test)\n",
        "print(\"Acc: %0.10f\" % (pred))\n",
        "print(\"number of features\",XX_train_svd.shape[1])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145
        },
        "colab_type": "code",
        "id": "dLBQSrpsIjQ_",
        "outputId": "63a3728c-fe23-4ead-e1d5-a89db4228976"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[5.50107172e-01 4.49892719e-01 1.08344868e-07 2.10633443e-10\n",
            " 3.37436622e-10 1.49984514e-10]\n",
            "[1.51477859e+09 1.36986009e+09 6.76506292e+05 7.04951620e+04\n",
            " 3.79438065e+04 2.51512669e+04]\n",
            "6\n",
            "41\n",
            "6 components, 41 in length\n"
          ]
        }
      ],
      "source": [
        "print(svd.explained_variance_ratio_)  \n",
        "print(svd.singular_values_)  \n",
        "\n",
        "print(len(svd.components_))\n",
        "print(len(svd.components_[0]))\n",
        "print(\"6 components, 41 in length\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "qkKVQYSxJWYE"
      },
      "source": [
        "# We conclude our conclusion\n",
        "#### RFE; 6\n",
        "* Acc: 0.9798537512\n",
        "\n",
        "#### PCA; 6\n",
        "* Acc: 0.9737667829\n",
        "\n",
        "#### TruncatedSVD; 6\n",
        "* Acc: 0.9765543989\n",
        "  * if dims = 2, Acc: 0.9441938134\n",
        "\n",
        "#### Feature Elimination(statistical) + extra feature Elimination(ML); 6 features\n",
        "* Acc: 0.9619160483  \n",
        "\n",
        "#### Feature Elimination(statistical) + extra feature Elimination(ML); 11 features\n",
        "* Acc: 0.9832743041  \n",
        "\n",
        "#### Feature Elimination(statistical) + feature Elimination(ML); 18 features\n",
        "* Acc: 0.9865467229 \n",
        "\n",
        "#### Feature Elimination(statistical); 27 features\n",
        "* Acc: 0.9890784707 \n",
        "\n",
        "#### from all data; 41 features\n",
        "* Acc: 0.9935763632"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "b_ZmWs8oJWqn"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "D2VRD7C9JWzs"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "5ONsVFt1JWpP"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "17ms4bhFJWoU"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "k0WO4fEBJWnf"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "_d2jMsIyJWeX"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {},
        "colab_type": "code",
        "id": "L1RWZjr3JWbh"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "amonalyDetection/NSL_KDD.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}