iot_dataset_2_percent_processing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "iot_dataset_2_percent_processing.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "79wfr_wpCYmQ",
        "8faZ-2YvUhAN",
        "0nMWjg7bWMVT",
        "zhkVtO-la_D5",
        "URnnRL6jUdpf",
        "AQAlmjVqY-G1",
        "Mg7pczlOymOx",
        "MEFIQwlhKUSA",
        "H6j3RqF0N_rI"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "79wfr_wpCYmQ"
      },
      "source": [
        "##Load"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1gVGGrpNXLeG",
        "outputId": "8199d3a3-9df6-4cb4-960f-1ce910be9f9a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TaIV8RYmXTzZ",
        "outputId": "9e638a85-0fe5-45f5-db40-00996b89b169",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 459
        }
      },
      "source": [
        "!pip install fastparquet"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting fastparquet\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/20/65/557177ac29fa2afe05cf8734283ae297e601e8063509b8305ff6153be7e6/fastparquet-0.4.0.tar.gz (152kB)\n",
            "\u001b[K     |████████████████████████████████| 153kB 4.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas>=0.19 in /usr/local/lib/python3.6/dist-packages (from fastparquet) (1.0.3)\n",
            "Requirement already satisfied: numba>=0.28 in /usr/local/lib/python3.6/dist-packages (from fastparquet) (0.48.0)\n",
            "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.6/dist-packages (from fastparquet) (1.18.4)\n",
            "Collecting thrift>=0.11.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/97/1e/3284d19d7be99305eda145b8aa46b0c33244e4a496ec66440dac19f8274d/thrift-0.13.0.tar.gz (59kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 9.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from fastparquet) (1.12.0)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.19->fastparquet) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.19->fastparquet) (2018.9)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from numba>=0.28->fastparquet) (46.3.0)\n",
            "Requirement already satisfied: llvmlite<0.32.0,>=0.31.0dev0 in /usr/local/lib/python3.6/dist-packages (from numba>=0.28->fastparquet) (0.31.0)\n",
            "Building wheels for collected packages: fastparquet, thrift\n",
            "  Building wheel for fastparquet (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for fastparquet: filename=fastparquet-0.4.0-cp36-cp36m-linux_x86_64.whl size=251893 sha256=a3c299ae3f671115c6c9eba067003d0c7fe54c29301775f28898cb85702b2d39\n",
            "  Stored in directory: /root/.cache/pip/wheels/2f/1d/6f/d10f8d2688469c279b71799a457a1fd8f35e5ba06e91b7d993\n",
            "  Building wheel for thrift (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for thrift: filename=thrift-0.13.0-cp36-cp36m-linux_x86_64.whl size=345240 sha256=a39a25d9d95d1f5d49828e84f37d7fe4677c3d1aa9f5cbfe041874daef29bfb3\n",
            "  Stored in directory: /root/.cache/pip/wheels/02/a2/46/689ccfcf40155c23edc7cdbd9de488611c8fdf49ff34b1706e\n",
            "Successfully built fastparquet thrift\n",
            "Installing collected packages: thrift, fastparquet\n",
            "Successfully installed fastparquet-0.4.0 thrift-0.13.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZMUQ3PP8XUuS"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from fastparquet import ParquetFile\n",
        "pf = ParquetFile(\"drive/My Drive/dataset/iot_dataset_2_percent_decom.parquet\")\n",
        "df = pf.to_pandas()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QxkwUrSQCxMP"
      },
      "source": [
        "##Check"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8faZ-2YvUhAN"
      },
      "source": [
        "####Count + category + dtypes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tyv54mZw4-Rq",
        "outputId": "f0dc06e7-5100-41ba-9418-5296799b721e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 615
        }
      },
      "source": [
        "df.count()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "frame.time_epoch              11252406\n",
              "frame.len                     11252406\n",
              "ip.proto                      11240346\n",
              "ip.src                        11240346\n",
              "ip.dst                        11240346\n",
              "ip.ttl                        11240346\n",
              "ipv6.src                            41\n",
              "ipv6.dst                            41\n",
              "ip.hdr_len                    11240346\n",
              "ip.len                        11240346\n",
              "ip.id                         11240346\n",
              "ip.flags.df                   11240346\n",
              "tcp.srcport                    6979907\n",
              "tcp.dstport                    6979907\n",
              "tcp.stream                     6979907\n",
              "tcp.analysis.initial_rtt       1260503\n",
              "tcp.time_delta                 6979907\n",
              "tcp.len                        6979907\n",
              "tcp.flags                      6979907\n",
              "tcp.window_size_value          6979907\n",
              "tcp.hdr_len                    6979907\n",
              "tcp.time_relative              6979907\n",
              "udp.srcport                    4256583\n",
              "udp.dstport                    4256583\n",
              "udp.length                     4256583\n",
              "udp.stream                     4256583\n",
              "http.response.code               68533\n",
              "http.content_length_header      115296\n",
              "http.content_length             115296\n",
              "http.request.method              52434\n",
              "attack                        11252406\n",
              "category                      11252406\n",
              "subcategory                   11252406\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5_OIAfhFdhXI",
        "outputId": "028a5aee-6c1d-4509-e87a-08916a9400a6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 246
        }
      },
      "source": [
        "df[['category', 'subcategory']].groupby(['category', 'subcategory']).size()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "category  subcategory      \n",
              "DDoS      DDoS_HTTP             194417\n",
              "          DDoS_UDP             1000055\n",
              "          TCP                   999444\n",
              "DoS       DoS_HTTP              287480\n",
              "          DoS_TCP               998703\n",
              "          DoS_UDP               999977\n",
              "Normal    Normal               4631398\n",
              "Scan      OS                    827940\n",
              "          Service               999895\n",
              "Theft     Data_Exfiltration     301710\n",
              "          Keylogging             11387\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h3KqPPdoZM9W",
        "outputId": "19a5695d-9cfe-4dc8-8793-f26e20161fba",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 615
        }
      },
      "source": [
        "df.dtypes"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "frame.time_epoch              float64\n",
              "frame.len                       int32\n",
              "ip.proto                       object\n",
              "ip.src                         object\n",
              "ip.dst                         object\n",
              "ip.ttl                         object\n",
              "ipv6.src                       object\n",
              "ipv6.dst                       object\n",
              "ip.hdr_len                     object\n",
              "ip.len                         object\n",
              "ip.id                          object\n",
              "ip.flags.df                    object\n",
              "tcp.srcport                   float64\n",
              "tcp.dstport                   float64\n",
              "tcp.stream                    float64\n",
              "tcp.analysis.initial_rtt       object\n",
              "tcp.time_delta                float64\n",
              "tcp.len                       float64\n",
              "tcp.flags                      object\n",
              "tcp.window_size_value         float64\n",
              "tcp.hdr_len                   float64\n",
              "tcp.time_relative             float64\n",
              "udp.srcport                   float64\n",
              "udp.dstport                   float64\n",
              "udp.length                    float64\n",
              "udp.stream                    float64\n",
              "http.response.code            float64\n",
              "http.content_length_header    float64\n",
              "http.content_length           float64\n",
              "http.request.method            object\n",
              "attack                          int32\n",
              "category                       object\n",
              "subcategory                    object\n",
              "dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0nMWjg7bWMVT"
      },
      "source": [
        "####Frame"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TkltbYvdWOAZ",
        "outputId": "ffc30add-64c9-4d44-c516-26a0da106f67",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "df['frame.len'].unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 505,  640, 1436, ..., 1048, 1091, 1092], dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zhkVtO-la_D5"
      },
      "source": [
        "####IPv6"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-dfrVa_VYTfT",
        "outputId": "68360575-3ab9-4177-f623-34a3581bef87",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 615
        }
      },
      "source": [
        "#Count IPv6 packets\n",
        "df[~df['ipv6.src'].isna()].count()\n",
        "#dfparq[~dfparq['ipv6.src'].isna()].head(45)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "frame.time_epoch              41\n",
              "frame.len                     41\n",
              "ip.proto                       0\n",
              "ip.src                         0\n",
              "ip.dst                         0\n",
              "ip.ttl                         0\n",
              "ipv6.src                      41\n",
              "ipv6.dst                      41\n",
              "ip.hdr_len                     0\n",
              "ip.len                         0\n",
              "ip.id                          0\n",
              "ip.flags.df                    0\n",
              "tcp.srcport                    0\n",
              "tcp.dstport                    0\n",
              "tcp.stream                     0\n",
              "tcp.analysis.initial_rtt       0\n",
              "tcp.time_delta                 0\n",
              "tcp.len                        0\n",
              "tcp.flags                      0\n",
              "tcp.window_size_value          0\n",
              "tcp.hdr_len                    0\n",
              "tcp.time_relative              0\n",
              "udp.srcport                   24\n",
              "udp.dstport                   24\n",
              "udp.length                    24\n",
              "udp.stream                    24\n",
              "http.response.code             0\n",
              "http.content_length_header     0\n",
              "http.content_length            0\n",
              "http.request.method            0\n",
              "attack                        41\n",
              "category                      41\n",
              "subcategory                   41\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "URnnRL6jUdpf"
      },
      "source": [
        "####IP related"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i9f-mhCSWGGy",
        "outputId": "9fa08040-bdeb-40ea-9012-f209af41d61a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# TCP: 6, UDP: 17, ICMP: 1, IGMP: 2, ESP:50\n",
        "df['ip.proto'].unique() # Categorical"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 6., 17., nan,  1.,  4.,  3.,  2.])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V2CLTW7wbUHo",
        "outputId": "15a0f926-6927-48e6-dcc7-5c66a8cacec0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "df['ip.flags.df'].unique() # Categorical"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 0.,  1., nan,  2.,  3.])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "36TY3XCeY7CQ",
        "outputId": "387fc133-0d62-4e36-84fb-d3f7c9434830",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 334
        }
      },
      "source": [
        "df['ip.ttl'].unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['127.0', '241.0', '64.0', '128.0', None, '50.0', '44.0', '228.0',\n",
              "       '53.0', '127', '241', '64', '114.0', '46.0', '45.0', '52.0', '52',\n",
              "       '53', '255', '57', '45', '56', '38', '50', '46', '39', '43', '41',\n",
              "       '51', '47', '59', '58', '44', '54', '40', '49', '48', '55', '42',\n",
              "       '37', '64,59', '60', '64,60', '64,255', '64,54', '64,55', '128',\n",
              "       '128,255', '61', '64,61', '64,56', '64,57', '64,51', '43.0', '229',\n",
              "       '62', '63', '64,63', '64,64', '128,57', '128,51', '64,62',\n",
              "       '128,58', '128,62', '238.0', '238', '64,58', '128,59', '64,53',\n",
              "       '128,64', '128,60', '64,52', '128,61', '128,56', '229.0', '51.0',\n",
              "       '255.0', '39.0', '64,47', '64,48', '64,41', '64,44', '55.0',\n",
              "       '57.0', '37.0', '38.0', '47.0', '59.0', '41.0', '58.0', '40.0',\n",
              "       '49.0', '54.0', '48.0', '56.0', '42.0', '1.0', '64,45', '64,43',\n",
              "       '64,38', '64,49', '64,40', '64,46', '64,50', '64,39', '64,37',\n",
              "       '64,42', '1', '128,45', '128,41', '128,37', '128,44', '128,48',\n",
              "       '128,53', '128,46', '128,38', '128,47', '128,54', '128,39',\n",
              "       '128,50', '128,43', '128,52', '128,42', '128,49', '128,55',\n",
              "       '224.0', '224', '64,36', '128,40', '228', '240.0', '114'],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LpcoBD7FZFV4",
        "outputId": "c2ec07e3-ed7c-4db6-84bd-7f2923b5bcc5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "df['ip.hdr_len'].unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['20.0', None, '20', '20,20', '24.0', '24'], dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WjRrcenfqAur",
        "outputId": "fff52df3-b359-482f-8a54-18e3cf15594a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "df['ip.id'].unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['0x000004d2', '0x0000c9d2', '0x00006d94', ...,\n",
              "       '0x000038fc,0x000018a5', '0x00003900,0x00001815',\n",
              "       '0x0000a9b5,0x00000000'], dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AF--gNeBbJb5",
        "outputId": "edff6154-0c9b-42d2-b72d-3c6a2acf5860",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "df['ip.id'].str.contains(\",\").any()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6lZcBQ1YWo2s",
        "outputId": "158157ea-046e-4477-f85d-b52ea99ea302",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "df['ip.len'].unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['491.0', '626.0', '1422.0', ..., '1034.0', '1077.0', '1078.0'],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bCsqNsraZwqW",
        "outputId": "dc0fe58a-83d1-451c-f865-fb21f6825bff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "df['ip.len'].str.contains(\",\").any()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lt5AzpPI2v9w",
        "outputId": "b5e0f38e-3caa-4bfb-ad91-6b7aa874b4ad",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# Check non TCP/UDP protocols\n",
        "df[(df['tcp.srcport'].isna()) & (df['udp.srcport'].isna())][\"ip.proto\"].unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([None, '1', '1,50', '17.0', '2.0', '2'], dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7SsrWs2emmYt",
        "outputId": "17cd72da-b2eb-4a12-8dc9-88910b51888d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 615
        }
      },
      "source": [
        "df[~df['udp.srcport'].isna()].count() + df[~df['tcp.srcport'].isna()].count() + df[(df['tcp.srcport'].isna()) & (df['udp.srcport'].isna())].count() "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "frame.time_epoch              11252406\n",
              "frame.len                     11252406\n",
              "ip.proto                      11240346\n",
              "ip.src                        11240346\n",
              "ip.dst                        11240346\n",
              "ip.ttl                        11240346\n",
              "ipv6.src                            41\n",
              "ipv6.dst                            41\n",
              "ip.hdr_len                    11240346\n",
              "ip.len                        11240346\n",
              "ip.id                         11240346\n",
              "ip.flags.df                   11240346\n",
              "tcp.srcport                    6979907\n",
              "tcp.dstport                    6979907\n",
              "tcp.stream                     6979907\n",
              "tcp.analysis.initial_rtt       1260503\n",
              "tcp.time_delta                 6979907\n",
              "tcp.len                        6979907\n",
              "tcp.flags                      6979907\n",
              "tcp.window_size_value          6979907\n",
              "tcp.hdr_len                    6979907\n",
              "tcp.time_relative              6979907\n",
              "udp.srcport                    4256583\n",
              "udp.dstport                    4256583\n",
              "udp.length                     4256583\n",
              "udp.stream                     4256583\n",
              "http.response.code               68533\n",
              "http.content_length_header      115296\n",
              "http.content_length             115296\n",
              "http.request.method              52434\n",
              "attack                        11252406\n",
              "category                      11252406\n",
              "subcategory                   11252406\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GMcxFuZg60yF",
        "outputId": "ae44f0f5-6e69-469f-aa4d-0b70b21c057f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 227
        }
      },
      "source": [
        "# ip.proto = 2: IGMP\n",
        "df[(df['tcp.srcport'].isna() & df['udp.stream'].isna() & ~df['ip.proto'].isna())][(df['ip.proto'] == '2') | (df['ip.proto'] == '2.0')]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>frame.time_epoch</th>\n",
              "      <th>frame.len</th>\n",
              "      <th>ip.proto</th>\n",
              "      <th>ip.src</th>\n",
              "      <th>ip.dst</th>\n",
              "      <th>ip.ttl</th>\n",
              "      <th>ipv6.src</th>\n",
              "      <th>ipv6.dst</th>\n",
              "      <th>ip.hdr_len</th>\n",
              "      <th>ip.len</th>\n",
              "      <th>ip.id</th>\n",
              "      <th>ip.flags.df</th>\n",
              "      <th>tcp.srcport</th>\n",
              "      <th>tcp.dstport</th>\n",
              "      <th>tcp.stream</th>\n",
              "      <th>tcp.analysis.initial_rtt</th>\n",
              "      <th>tcp.time_delta</th>\n",
              "      <th>tcp.len</th>\n",
              "      <th>tcp.flags</th>\n",
              "      <th>tcp.window_size_value</th>\n",
              "      <th>tcp.hdr_len</th>\n",
              "      <th>tcp.time_relative</th>\n",
              "      <th>udp.srcport</th>\n",
              "      <th>udp.dstport</th>\n",
              "      <th>udp.length</th>\n",
              "      <th>udp.stream</th>\n",
              "      <th>http.response.code</th>\n",
              "      <th>http.content_length_header</th>\n",
              "      <th>http.content_length</th>\n",
              "      <th>http.request.method</th>\n",
              "      <th>attack</th>\n",
              "      <th>category</th>\n",
              "      <th>subcategory</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>11153791</th>\n",
              "      <td>1.529381e+09</td>\n",
              "      <td>60</td>\n",
              "      <td>2.0</td>\n",
              "      <td>192.168.100.6</td>\n",
              "      <td>224.0.0.22</td>\n",
              "      <td>1.0</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>24.0</td>\n",
              "      <td>40.0</td>\n",
              "      <td>0x00000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>0</td>\n",
              "      <td>Normal</td>\n",
              "      <td>Normal</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11174904</th>\n",
              "      <td>1.529381e+09</td>\n",
              "      <td>60</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.0.0.0</td>\n",
              "      <td>224.0.0.1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>24.0</td>\n",
              "      <td>36.0</td>\n",
              "      <td>0x00000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>0</td>\n",
              "      <td>Normal</td>\n",
              "      <td>Normal</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11183086</th>\n",
              "      <td>1.529381e+09</td>\n",
              "      <td>60</td>\n",
              "      <td>2</td>\n",
              "      <td>192.168.100.6</td>\n",
              "      <td>224.0.0.22</td>\n",
              "      <td>1</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>24</td>\n",
              "      <td>40</td>\n",
              "      <td>0x00000002</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>0</td>\n",
              "      <td>Normal</td>\n",
              "      <td>Normal</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11249352</th>\n",
              "      <td>1.529382e+09</td>\n",
              "      <td>60</td>\n",
              "      <td>2.0</td>\n",
              "      <td>192.168.100.6</td>\n",
              "      <td>224.0.0.22</td>\n",
              "      <td>1.0</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>24.0</td>\n",
              "      <td>40.0</td>\n",
              "      <td>0x00000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>0</td>\n",
              "      <td>Normal</td>\n",
              "      <td>Normal</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          frame.time_epoch  frame.len ip.proto  ... attack category subcategory\n",
              "11153791      1.529381e+09         60      2.0  ...      0   Normal      Normal\n",
              "11174904      1.529381e+09         60      2.0  ...      0   Normal      Normal\n",
              "11183086      1.529381e+09         60        2  ...      0   Normal      Normal\n",
              "11249352      1.529382e+09         60      2.0  ...      0   Normal      Normal\n",
              "\n",
              "[4 rows x 33 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sZTBdlA87vQH",
        "outputId": "e9d87dc0-41aa-4858-f55a-d0b6d431f226",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        }
      },
      "source": [
        "# ip.proto = 1,50: ICMP + ESP\n",
        "df[(df['tcp.srcport'].isna() & df['udp.stream'].isna() & ~df['ip.proto'].isna())][df['ip.proto'] == '1,50']"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>frame.time_epoch</th>\n",
              "      <th>frame.len</th>\n",
              "      <th>ip.proto</th>\n",
              "      <th>ip.src</th>\n",
              "      <th>ip.dst</th>\n",
              "      <th>ip.ttl</th>\n",
              "      <th>ipv6.src</th>\n",
              "      <th>ipv6.dst</th>\n",
              "      <th>ip.hdr_len</th>\n",
              "      <th>ip.len</th>\n",
              "      <th>ip.id</th>\n",
              "      <th>ip.flags.df</th>\n",
              "      <th>tcp.srcport</th>\n",
              "      <th>tcp.dstport</th>\n",
              "      <th>tcp.stream</th>\n",
              "      <th>tcp.analysis.initial_rtt</th>\n",
              "      <th>tcp.time_delta</th>\n",
              "      <th>tcp.len</th>\n",
              "      <th>tcp.flags</th>\n",
              "      <th>tcp.window_size_value</th>\n",
              "      <th>tcp.hdr_len</th>\n",
              "      <th>tcp.time_relative</th>\n",
              "      <th>udp.srcport</th>\n",
              "      <th>udp.dstport</th>\n",
              "      <th>udp.length</th>\n",
              "      <th>udp.stream</th>\n",
              "      <th>http.response.code</th>\n",
              "      <th>http.content_length_header</th>\n",
              "      <th>http.content_length</th>\n",
              "      <th>http.request.method</th>\n",
              "      <th>attack</th>\n",
              "      <th>category</th>\n",
              "      <th>subcategory</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3808393</th>\n",
              "      <td>1.526352e+09</td>\n",
              "      <td>62</td>\n",
              "      <td>1,50</td>\n",
              "      <td>192.168.100.6</td>\n",
              "      <td>192.168.100.150</td>\n",
              "      <td>128,43</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>20,20</td>\n",
              "      <td>48,28</td>\n",
              "      <td>0x00004386,0x00004bba</td>\n",
              "      <td>0,0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>0</td>\n",
              "      <td>Normal</td>\n",
              "      <td>Normal</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3808499</th>\n",
              "      <td>1.526352e+09</td>\n",
              "      <td>62</td>\n",
              "      <td>1,50</td>\n",
              "      <td>192.168.100.6</td>\n",
              "      <td>192.168.100.150</td>\n",
              "      <td>128,58</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>20,20</td>\n",
              "      <td>48,28</td>\n",
              "      <td>0x00004392,0x0000dc90</td>\n",
              "      <td>0,0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>0</td>\n",
              "      <td>Normal</td>\n",
              "      <td>Normal</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3834065</th>\n",
              "      <td>1.526353e+09</td>\n",
              "      <td>62</td>\n",
              "      <td>1,50</td>\n",
              "      <td>192.168.100.6</td>\n",
              "      <td>192.168.100.149</td>\n",
              "      <td>128,43</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>20,20</td>\n",
              "      <td>48,28</td>\n",
              "      <td>0x00004f77,0x00007460</td>\n",
              "      <td>0,0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>0</td>\n",
              "      <td>Normal</td>\n",
              "      <td>Normal</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3834208</th>\n",
              "      <td>1.526353e+09</td>\n",
              "      <td>62</td>\n",
              "      <td>1,50</td>\n",
              "      <td>192.168.100.6</td>\n",
              "      <td>192.168.100.149</td>\n",
              "      <td>128,45</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>20,20</td>\n",
              "      <td>48,28</td>\n",
              "      <td>0x00004f87,0x0000651e</td>\n",
              "      <td>0,0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>0</td>\n",
              "      <td>Normal</td>\n",
              "      <td>Normal</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3834251</th>\n",
              "      <td>1.526353e+09</td>\n",
              "      <td>62</td>\n",
              "      <td>1,50</td>\n",
              "      <td>192.168.100.6</td>\n",
              "      <td>192.168.100.149</td>\n",
              "      <td>128,37</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>20,20</td>\n",
              "      <td>48,28</td>\n",
              "      <td>0x00004f8b,0x00007db8</td>\n",
              "      <td>0,0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>0</td>\n",
              "      <td>Normal</td>\n",
              "      <td>Normal</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3834539</th>\n",
              "      <td>1.526353e+09</td>\n",
              "      <td>62</td>\n",
              "      <td>1,50</td>\n",
              "      <td>192.168.100.6</td>\n",
              "      <td>192.168.100.147</td>\n",
              "      <td>128,59</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>20,20</td>\n",
              "      <td>48,28</td>\n",
              "      <td>0x00004fad,0x0000b771</td>\n",
              "      <td>0,0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>0</td>\n",
              "      <td>Normal</td>\n",
              "      <td>Normal</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         frame.time_epoch  frame.len ip.proto  ... attack category subcategory\n",
              "3808393      1.526352e+09         62     1,50  ...      0   Normal      Normal\n",
              "3808499      1.526352e+09         62     1,50  ...      0   Normal      Normal\n",
              "3834065      1.526353e+09         62     1,50  ...      0   Normal      Normal\n",
              "3834208      1.526353e+09         62     1,50  ...      0   Normal      Normal\n",
              "3834251      1.526353e+09         62     1,50  ...      0   Normal      Normal\n",
              "3834539      1.526353e+09         62     1,50  ...      0   Normal      Normal\n",
              "\n",
              "[6 rows x 33 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OkqzMdlq8OtD",
        "outputId": "8d3e695d-997b-4322-bb9b-9ea94a28e0b6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 651
        }
      },
      "source": [
        "# ip.proto = 1: ICMP\n",
        "df[(df['tcp.srcport'].isna() & df['udp.stream'].isna() & ~df['ip.proto'].isna())][df['ip.proto'] == '1'].count()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "frame.time_epoch              3868\n",
              "frame.len                     3868\n",
              "ip.proto                      3868\n",
              "ip.src                        3868\n",
              "ip.dst                        3868\n",
              "ip.ttl                        3868\n",
              "ipv6.src                         0\n",
              "ipv6.dst                         0\n",
              "ip.hdr_len                    3868\n",
              "ip.len                        3868\n",
              "ip.id                         3868\n",
              "ip.flags.df                   3868\n",
              "tcp.srcport                      0\n",
              "tcp.dstport                      0\n",
              "tcp.stream                       0\n",
              "tcp.analysis.initial_rtt         0\n",
              "tcp.time_delta                   0\n",
              "tcp.len                          0\n",
              "tcp.flags                        0\n",
              "tcp.window_size_value            0\n",
              "tcp.hdr_len                      0\n",
              "tcp.time_relative                0\n",
              "udp.srcport                      0\n",
              "udp.dstport                      0\n",
              "udp.length                       0\n",
              "udp.stream                       0\n",
              "http.response.code               0\n",
              "http.content_length_header       0\n",
              "http.content_length              0\n",
              "http.request.method              0\n",
              "attack                        3868\n",
              "category                      3868\n",
              "subcategory                   3868\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AQAlmjVqY-G1"
      },
      "source": [
        "####TCP related"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j-8oE7Q8cRL4",
        "outputId": "5a19e789-ffda-4d1e-bdf4-c9093fbf7182",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        }
      },
      "source": [
        "df['tcp.flags'].unique() # Categorical"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['0x00000000', None, '0x00000010', '0x00000018', '0x00000002',\n",
              "       '0x00000012', '0x00000011', '0x00000014', '0x00000004',\n",
              "       '0x00000019', '0x000008c2', '0x00000052', '0x0000002b',\n",
              "       '0x00000029', '0x00000001'], dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "516SxrTEq0Dk",
        "outputId": "38d5c870-c259-472d-a56a-f5e4786deb38",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "df['tcp.analysis.initial_rtt'].astype(float).unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([        nan, 1.30000e-04, 9.80000e-05, ..., 1.55505e-01,\n",
              "       1.55916e-01, 1.53953e-01])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JrlOSDPzrd_z",
        "outputId": "88262155-8361-4db7-b648-ed0bdbbf8d0b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "#df['tcp.time_delta'].unique() \n",
        "df['tcp.time_relative'].unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.00000000e+00,            nan, 2.49200000e-02, ...,\n",
              "       4.85127391e+02, 4.85150747e+02, 4.85164078e+02])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hyGASR8NrhJq",
        "outputId": "4e064671-a398-4fe9-f503-28f1d5013f0d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "df['tcp.window_size_value'].unique() "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1024.,   nan,  422., ..., 5166.,  725.,  770.])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eSU7Dfk0rmii",
        "outputId": "2a8ba4f5-dd9c-4bd6-c288-a4f19328b542",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "df['tcp.hdr_len'].unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([20., nan, 32., 40., 44., 24., 36., 28.])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NTt4u6dFb6TJ",
        "outputId": "3d5275bc-513d-41a9-e3ef-ffd7438845f7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(df['tcp.srcport'].unique())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "65537"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mg7pczlOymOx"
      },
      "source": [
        "####HTTP related"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WDJrIJEEtDeL",
        "outputId": "8cd2f135-6cd1-4bca-a0c1-33a862a19c5e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 253
        }
      },
      "source": [
        "df[~df['http.content_length'].isna()].groupby([\"category\", \"subcategory\"]).count()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>frame.time_epoch</th>\n",
              "      <th>frame.len</th>\n",
              "      <th>ip.proto</th>\n",
              "      <th>ip.src</th>\n",
              "      <th>ip.dst</th>\n",
              "      <th>ip.ttl</th>\n",
              "      <th>ipv6.src</th>\n",
              "      <th>ipv6.dst</th>\n",
              "      <th>ip.hdr_len</th>\n",
              "      <th>ip.len</th>\n",
              "      <th>ip.id</th>\n",
              "      <th>ip.flags.df</th>\n",
              "      <th>tcp.analysis.initial_rtt</th>\n",
              "      <th>tcp.time_delta</th>\n",
              "      <th>tcp.flags</th>\n",
              "      <th>tcp.window_size_value</th>\n",
              "      <th>tcp.hdr_len</th>\n",
              "      <th>tcp.time_relative</th>\n",
              "      <th>http.response.code</th>\n",
              "      <th>http.content_length_header</th>\n",
              "      <th>http.content_length</th>\n",
              "      <th>http.request.method</th>\n",
              "      <th>attack</th>\n",
              "      <th>srcport</th>\n",
              "      <th>dstport</th>\n",
              "      <th>trans.length</th>\n",
              "      <th>trans.stream</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>category</th>\n",
              "      <th>subcategory</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>DDoS</th>\n",
              "      <th>DDoS_HTTP</th>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "      <td>38100</td>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "      <td>19083</td>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "      <td>19178</td>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "      <td>38261</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>DoS</th>\n",
              "      <th>DoS_HTTP</th>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "      <td>56368</td>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "      <td>28491</td>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "      <td>28566</td>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "      <td>57057</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Normal</th>\n",
              "      <th>Normal</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">Scan</th>\n",
              "      <th>OS</th>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>1227</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>240</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "      <td>1467</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Service</th>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "      <td>18501</td>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "      <td>18207</td>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "      <td>292</td>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "      <td>18510</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                      frame.time_epoch  frame.len  ...  trans.length  trans.stream\n",
              "category subcategory                               ...                            \n",
              "DDoS     DDoS_HTTP               38261      38261  ...         38261         38261\n",
              "DoS      DoS_HTTP                57057      57057  ...         57057         57057\n",
              "Normal   Normal                      1          1  ...             1             1\n",
              "Scan     OS                       1467       1467  ...          1467          1467\n",
              "         Service                 18510      18510  ...         18510         18510\n",
              "\n",
              "[5 rows x 27 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "orle_1yIro0x",
        "outputId": "c768e26f-25f1-4fd5-e9ca-8d00bfd5923f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "df['http.response.code'].unique() # Categorical"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ nan, 404., 200., 505., 405., 501., 302., 400., 301., 403., 408.,\n",
              "       401.])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5b8Z7CHHblLc",
        "outputId": "0b145614-9449-4b49-8e39-6abe56988ff2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(df[~df[\"http.request.method\"].isna()][\"http.request.method\"].unique())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "92"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sddGMPtxCcGb",
        "outputId": "c4ed7072-db13-4745-9493-133e49dc6dbd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "df[\"http.request.method\"].unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([None, 'POST', 'GET', 'OPTIONS', 'PROPFIND', 'HEAD', 'TRACE',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET',\n",
              "       'GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,HEAD,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,GET,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'HEAD,HEAD,GET,GET,GET,HEAD,GET,GET,GET,HEAD,HEAD,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,GET,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,HEAD,HEAD,GET,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD', 'HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'GET,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'HEAD,HEAD,GET,GET,GET,HEAD,GET,GET,GET,HEAD,HEAD,GET,GET,HEAD,HEAD,HEAD',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'HEAD,HEAD,HEAD,HEAD', 'GET,GET,GET,GET,GET,GET,GET',\n",
              "       'HEAD,HEAD,HEAD,GET,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,GET,HEAD',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'GET,GET,GET,GET',\n",
              "       'GET,GET,GET,HEAD,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'GET,HEAD,HEAD,GET', 'GET,GET,HEAD,GET', 'GET,GET,GET,HEAD',\n",
              "       'HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,GET,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,GET,GET,GET',\n",
              "       'GET,GET,GET', 'HEAD,GET,GET',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,HEAD',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,GET,GET,GET',\n",
              "       'HEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,GET,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET', 'GET,GET,GET,GET,GET,GET',\n",
              "       'HEAD,HEAD,HEAD,GET,GET,GET,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD',\n",
              "       'HEAD,GET,GET,GET,GET,GET,GET,GET,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,HEAD,GET,GET,GET,GET,GET',\n",
              "       'GET,GET,GET,HEAD,HEAD,HEAD,HEAD', 'GET,GET,GET,GET,GET',\n",
              "       'HEAD,GET,GET,GET,HEAD,HEAD,GET', 'HEAD,HEAD,GET,GET,GET,GET,GET',\n",
              "       'GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET'],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7IKtswVZJBgm",
        "outputId": "cb5e4785-6030-416e-d5a6-d29882c53e98",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 252
        }
      },
      "source": [
        "# Categorical \n",
        "df.groupby([\"category\", \"subcategory\"])[\"http.request.method\"].value_counts().unstack(fill_value=0).add_prefix('flag').reset_index().rename_axis(None, axis=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>subcategory</th>\n",
              "      <th>flagGET</th>\n",
              "      <th>flagGET,GET</th>\n",
              "      <th>flagGET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,HEAD,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagGET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagGET,GET,GET,HEAD</th>\n",
              "      <th>flagGET,GET,GET,HEAD,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagGET,GET,GET,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>...</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,GET,GET,GET,GET</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,GET,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,GET,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,GET,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,GET,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,HEAD,HEAD,GET,GET</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,GET,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,GET,GET,GET,GET,GET,GET,GET,GET,GET,GET</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagHEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD</th>\n",
              "      <th>flagOPTIONS</th>\n",
              "      <th>flagPOST</th>\n",
              "      <th>flagPROPFIND</th>\n",
              "      <th>flagTRACE</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>DDoS</td>\n",
              "      <td>DDoS_HTTP</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>19178</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>DoS</td>\n",
              "      <td>DoS_HTTP</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>28566</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Normal</td>\n",
              "      <td>Normal</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Scan</td>\n",
              "      <td>OS</td>\n",
              "      <td>1731</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>567</td>\n",
              "      <td>240</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Scan</td>\n",
              "      <td>Service</td>\n",
              "      <td>884</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>59</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>112</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>7</td>\n",
              "      <td>59</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>27</td>\n",
              "      <td>32</td>\n",
              "      <td>1</td>\n",
              "      <td>7</td>\n",
              "      <td>1</td>\n",
              "      <td>9</td>\n",
              "      <td>63</td>\n",
              "      <td>4</td>\n",
              "      <td>6</td>\n",
              "      <td>4</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>10</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>14</td>\n",
              "      <td>12</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>7</td>\n",
              "      <td>29</td>\n",
              "      <td>2</td>\n",
              "      <td>57</td>\n",
              "      <td>2</td>\n",
              "      <td>4</td>\n",
              "      <td>53</td>\n",
              "      <td>6</td>\n",
              "      <td>17</td>\n",
              "      <td>5</td>\n",
              "      <td>8</td>\n",
              "      <td>5</td>\n",
              "      <td>9</td>\n",
              "      <td>8</td>\n",
              "      <td>43</td>\n",
              "      <td>21</td>\n",
              "      <td>12</td>\n",
              "      <td>104</td>\n",
              "      <td>266</td>\n",
              "      <td>43</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 94 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "  category subcategory  flagGET  ...  flagPOST  flagPROPFIND  flagTRACE\n",
              "0     DDoS   DDoS_HTTP        0  ...     19178             0          0\n",
              "1      DoS    DoS_HTTP        0  ...     28566             0          0\n",
              "2   Normal      Normal        1  ...         0             0          0\n",
              "3     Scan          OS     1731  ...       240             0          0\n",
              "4     Scan     Service      884  ...       266            43          9\n",
              "\n",
              "[5 rows x 94 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XHftLEvAryqa",
        "outputId": "86fca30d-c05a-456c-d51e-a52c6155f68b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 417
        }
      },
      "source": [
        "df.groupby([\"http.request.method\"])[\"category\"].value_counts().unstack(fill_value=0).add_prefix('flag').reset_index().rename_axis(None, axis=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>http.request.method</th>\n",
              "      <th>flagDDoS</th>\n",
              "      <th>flagDoS</th>\n",
              "      <th>flagNormal</th>\n",
              "      <th>flagScan</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>GET</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2615</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>GET,GET</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>GET,GET,GET</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>GET,GET,GET,GET</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>59</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>GET,GET,GET,GET,GET</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>87</th>\n",
              "      <td>HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,H...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>88</th>\n",
              "      <td>OPTIONS</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>671</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>89</th>\n",
              "      <td>POST</td>\n",
              "      <td>19178</td>\n",
              "      <td>28566</td>\n",
              "      <td>0</td>\n",
              "      <td>506</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>90</th>\n",
              "      <td>PROPFIND</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>43</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91</th>\n",
              "      <td>TRACE</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>92 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                  http.request.method  ...  flagScan\n",
              "0                                                 GET  ...      2615\n",
              "1                                             GET,GET  ...         2\n",
              "2                                         GET,GET,GET  ...         1\n",
              "3                                     GET,GET,GET,GET  ...        59\n",
              "4                                 GET,GET,GET,GET,GET  ...         1\n",
              "..                                                ...  ...       ...\n",
              "87  HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,HEAD,H...  ...        12\n",
              "88                                            OPTIONS  ...       671\n",
              "89                                               POST  ...       506\n",
              "90                                           PROPFIND  ...        43\n",
              "91                                              TRACE  ...         9\n",
              "\n",
              "[92 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ngHS-Uh_Aqbd",
        "outputId": "d6e79beb-ea7a-4e34-9bff-d0f59552ee6d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 703
        }
      },
      "source": [
        "df[\"http.content_length\"].unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([        nan, 0.00000e+00, 2.77000e+02, 8.91000e+02, 9.42000e+02,\n",
              "       4.41000e+02, 2.80000e+02, 3.01000e+02, 9.41000e+02, 2.82000e+02,\n",
              "       1.05400e+03, 2.85000e+02, 3.06000e+02, 9.58000e+02, 1.02100e+03,\n",
              "       2.87000e+02, 9.64000e+02, 1.10960e+04, 3.05000e+02, 1.15100e+04,\n",
              "       8.00000e+00, 2.71000e+02, 1.40000e+02, 1.53000e+02, 1.44000e+02,\n",
              "       1.49000e+02, 1.52000e+02, 1.58000e+02, 1.54000e+02, 1.63000e+02,\n",
              "       1.48000e+02, 8.80000e+01, 2.88700e+03, 7.74000e+02, 3.21000e+02,\n",
              "       2.86000e+02, 5.20000e+01, 3.09000e+02, 2.99000e+02, 2.93000e+02,\n",
              "       2.89000e+02, 3.00000e+02, 2.90000e+02, 3.17000e+02, 3.08000e+02,\n",
              "       3.50000e+01, 2.97000e+02, 2.94000e+02, 2.88000e+02, 2.84000e+02,\n",
              "       2.81000e+02, 9.04100e+03, 2.79000e+02, 2.72000e+02, 3.40000e+01,\n",
              "       3.57000e+02, 2.91000e+02, 2.83000e+02, 3.03000e+02, 1.40000e+01,\n",
              "       3.07000e+02, 3.02000e+02, 3.16000e+02, 2.40000e+01, 3.47000e+02,\n",
              "       1.05000e+02, 3.15000e+02, 5.19000e+02, 3.12000e+02, 2.95000e+02,\n",
              "       9.47900e+03, 8.51300e+03, 9.64900e+03, 6.26000e+02, 9.29000e+03,\n",
              "       8.88400e+03, 6.69900e+03, 2.89700e+03, 1.69580e+04, 3.04000e+02,\n",
              "       3.14000e+02, 2.92000e+02, 8.60000e+01, 1.00000e+02, 3.28000e+02,\n",
              "       5.15000e+02, 3.27000e+02, 9.17300e+03, 9.41600e+03, 9.20600e+03,\n",
              "       3.62000e+02, 3.10000e+02, 3.11000e+02, 1.03600e+03, 1.19500e+03,\n",
              "       7.49800e+03, 9.70000e+02, 9.79000e+02, 1.00000e+03, 1.01800e+03,\n",
              "       9.76000e+02, 9.61000e+02, 9.82000e+02, 9.85000e+02, 3.26000e+02,\n",
              "       9.91000e+02, 9.94000e+02, 1.04800e+03, 1.00300e+03, 9.67000e+02,\n",
              "       7.82000e+02, 2.96000e+02, 9.48000e+02, 3.32000e+02, 1.08100e+03,\n",
              "       1.28900e+03, 2.98000e+02, 3.18000e+02, 2.20000e+01, 1.80000e+01,\n",
              "       6.94000e+02, 1.06300e+03, 3.52000e+02, 1.27000e+02, 1.50000e+01,\n",
              "       1.89020e+04, 1.04000e+02, 1.00600e+03, 4.78800e+03, 1.04200e+03,\n",
              "       9.88000e+02, 2.61179e+05, 9.97000e+02, 1.00900e+03, 1.01500e+03,\n",
              "       9.73000e+02, 1.03900e+03, 1.01200e+03, 1.03000e+03, 1.28200e+03,\n",
              "       3.60700e+03, 1.03300e+03, 1.02700e+03, 1.05100e+03, 1.06600e+03,\n",
              "       1.09000e+03, 1.04500e+03, 1.02400e+03, 1.09600e+03, 1.05700e+03,\n",
              "       1.06000e+03, 1.10200e+03, 1.12000e+03, 1.11100e+03, 1.12900e+03,\n",
              "       1.10800e+03, 9.55000e+02, 8.90000e+01, 3.33000e+02, 1.07500e+03,\n",
              "       3.13000e+02, 3.19000e+02, 3.43000e+02, 1.63890e+04, 2.98600e+03,\n",
              "       3.30000e+01, 1.33800e+03, 1.06460e+04, 2.13890e+04, 3.54000e+02,\n",
              "       1.92500e+03, 5.50000e+01, 6.45000e+02, 1.64100e+03, 2.68900e+03,\n",
              "       1.33100e+03, 2.60000e+01, 7.70000e+01, 1.07200e+03, 1.09900e+03,\n",
              "       1.08400e+03, 1.19800e+03, 1.18900e+03, 1.14400e+03, 1.13200e+03,\n",
              "       9.46000e+02, 3.23000e+02, 8.84000e+02, 4.82000e+04, 9.36000e+02,\n",
              "       9.25500e+03])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mcu8nrmnyoPA",
        "outputId": "e5c0aef8-8d53-48fe-9858-bc11bc4138f0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "df[\"http.content_length_header\"].equals(df[\"http.content_length\"])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MEFIQwlhKUSA"
      },
      "source": [
        "####Category"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jz48PjDPKsDW",
        "outputId": "b3c6ebbf-6572-4f5b-fbaf-3813d321bf30",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "df[\"category\"].unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['Normal', 'DDoS', 'DoS', 'Scan', 'Theft'], dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iogNSHwdKWjP",
        "outputId": "25407eac-e420-4716-ae2a-46849523b607",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "df[\"subcategory\"].unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['Normal', 'DDoS_HTTP', 'DoS_HTTP', 'OS', 'Service', 'DDoS_TCP',\n",
              "       'DoS_TCP', 'DDoS_UDP', 'DoS_UDP', 'Data_Exfiltration',\n",
              "       'Keylogging'], dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H6j3RqF0N_rI"
      },
      "source": [
        "##Process"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RtqtiiH4f8cZ"
      },
      "source": [
        "# Combine TCP and UDP columns: srcport, dstport, length and stream\n",
        "df['srcport'] = np.where(pd.isnull(df['tcp.srcport']),df['udp.srcport'],df['tcp.srcport'])\n",
        "df['dstport'] = np.where(pd.isnull(df['tcp.dstport']),df['udp.dstport'],df['tcp.dstport'])\n",
        "df['trans.length'] = np.where(pd.isnull(df['tcp.len']),df['udp.length'],df['tcp.len'])\n",
        "df['trans.stream'] = np.where(pd.isnull(df['tcp.stream']),df['udp.stream'],df['tcp.stream'])\n",
        "df = df.drop(['tcp.srcport', 'udp.srcport', 'tcp.dstport', 'udp.dstport', 'tcp.len', 'udp.length', 'tcp.stream', 'udp.stream'], axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JE058ehyakrR"
      },
      "source": [
        "# Unify values for categorical columns\n",
        "# Convert values to float\n",
        "df[\"ip.proto\"] = df[\"ip.proto\"].replace({'1,50': 3, '1,17': 4}).astype(float)\n",
        "df['ip.flags.df'] = df['ip.flags.df'].replace({'0,0': 2, '0,1': 3}).astype(float)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cksr978GK1fX"
      },
      "source": [
        "# Unify label column\n",
        "df['subcategory'] = df['subcategory'].replace({'TCP': 'DDoS_TCP'})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lgd2170wfMHf"
      },
      "source": [
        "# Drop irrelevant columns\n",
        "# Split embedded values in IP related columns (some ICMP packets contain the UDP packet inside the ICMP payload) \n",
        "dfp = pd.concat([df['frame.time_epoch'], df['frame.len'], df['ip.proto'], df['srcport'], df['dstport'], df['trans.stream'], df['trans.length'], \n",
        "                 df['tcp.analysis.initial_rtt'].astype(float), df['tcp.time_delta'], df['tcp.time_relative'], \n",
        "                 df['tcp.flags'], df['tcp.window_size_value'], df['tcp.hdr_len'], \n",
        "                 df['http.response.code'], df[\"http.request.method\"], df[\"http.content_length\"],\n",
        "                 df['ip.ttl'].str.split(',', expand=True).rename({0: 'ip.ttl_1', 1: 'ip.ttl_2'}, axis=1), \n",
        "                 df['ip.hdr_len'].str.split(',', expand=True).rename({0: 'ip.hdr_len_1', 1: 'ip.hdr_len_2'}, axis=1),\n",
        "                 df['ip.len'].str.split(',', expand=True).rename({0: 'ip.len_1', 1: 'ip.len_2'}, axis=1), \n",
        "                 df['ip.id'].str.split(',', expand=True).rename({0: 'ip.id_1', 1: 'ip.id_2'}, axis=1),\n",
        "                 df['ip.flags.df'], df['attack'], df['category'], df['subcategory']], axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KoxovdGRLq9E"
      },
      "source": [
        "# Fill nan values with max + 1.0\n",
        "dfp['trans.stream'] = dfp['trans.stream'].fillna(dfp['trans.stream'].max() + 1.0)\n",
        "dfp['trans.length'] = dfp['trans.length'].fillna(dfp['trans.length'].max() + 1.0)\n",
        "dfp['tcp.time_delta'] = dfp['tcp.time_delta'].fillna(dfp['tcp.time_delta'].max() + 1.0)\n",
        "dfp['tcp.time_relative'] = dfp['tcp.time_relative'].fillna(dfp['tcp.time_relative'].max() + 1.0)\n",
        "dfp['tcp.window_size_value'] = dfp['tcp.window_size_value'].fillna(dfp['tcp.window_size_value'].max() + 1.0)\n",
        "dfp[\"http.content_length\"] = dfp[\"http.content_length\"].fillna(dfp[\"http.content_length\"].max() + 1.0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zj0F_P4H2ALv"
      },
      "source": [
        "# Fill none values with '0x000fffff' (max is '0x0000ffff')\n",
        "# Convert to int\n",
        "dfp['ip.id_1'] = dfp['ip.id_1'].fillna('0x00010000').apply(lambda x: int(x, 16))\n",
        "dfp['ip.id_2'] = dfp['ip.id_2'].fillna('0x00010000').apply(lambda x: int(x, 16))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dwfqykeJ8PIA"
      },
      "source": [
        "# Fill none values with '0x000008c3' (max is '0x000008c2')\n",
        "dfp['tcp.flags'] = dfp['tcp.flags'].fillna('0x000008c3').apply(lambda x: int(x, 16))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SqnNRMNaNJ8N"
      },
      "source": [
        "# Convert to float and fill nan values with 0.0\n",
        "dfp['ip.ttl_1'] = dfp['ip.ttl_2'].astype(float).fillna(0.0)\n",
        "dfp['ip.ttl_2'] = dfp['ip.ttl_2'].astype(float).fillna(0.0)\n",
        "dfp['ip.hdr_len_1'] = dfp['ip.hdr_len_1'].astype(float).fillna(0.0)\n",
        "dfp['ip.hdr_len_2'] = dfp['ip.hdr_len_2'].astype(float).fillna(0.0)\n",
        "dfp['ip.len_1'] = dfp['ip.len_1'].astype(float).fillna(0.0)\n",
        "dfp['ip.len_2'] = dfp['ip.len_2'].astype(float).fillna(0.0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AmH7umYaAcME"
      },
      "source": [
        "# Fill none values with special values\n",
        "dfp['ip.flags.df'] = dfp['ip.flags.df'].fillna(4.)\n",
        "dfp['srcport'] = dfp['srcport'].fillna(65536.)\n",
        "dfp['dstport'] = dfp['dstport'].fillna(65536.)\n",
        "\n",
        "dfp['http.request.method'] = dfp['http.request.method'].fillna('NO')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1QVFdd72BDGE"
      },
      "source": [
        "# Fill nan with 0. for tcp.analysis.initial_rtt, tcp.hdr_len, http.response.code\n",
        "dfp = dfp.fillna(0.)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "54ZncdZeIjDs"
      },
      "source": [
        "dfp.to_csv('drive/My Drive/dataset/dataset_2_percent_processed.csv', index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dU5UcU7mdEIW",
        "outputId": "79b1f290-cfa6-462d-800d-6ae3c826cd53",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 527
        }
      },
      "source": [
        "dfp.dtypes"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "frame.time_epoch            float64\n",
              "frame.len                     int32\n",
              "ip.proto                    float64\n",
              "srcport                     float64\n",
              "dstport                     float64\n",
              "trans.stream                float64\n",
              "trans.length                float64\n",
              "tcp.analysis.initial_rtt    float64\n",
              "tcp.time_delta              float64\n",
              "tcp.time_relative           float64\n",
              "tcp.flags                     int64\n",
              "tcp.window_size_value       float64\n",
              "tcp.hdr_len                 float64\n",
              "http.response.code          float64\n",
              "http.request.method          object\n",
              "http.content_length         float64\n",
              "ip.ttl_1                    float64\n",
              "ip.ttl_2                    float64\n",
              "ip.hdr_len_1                float64\n",
              "ip.hdr_len_2                float64\n",
              "ip.len_1                    float64\n",
              "ip.len_2                    float64\n",
              "ip.id_1                       int64\n",
              "ip.id_2                       int64\n",
              "ip.flags.df                 float64\n",
              "attack                        int32\n",
              "category                     object\n",
              "subcategory                  object\n",
              "dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pv7joyV9KlQa",
        "outputId": "3ef73840-51ab-4925-f244-1fa6e4f97b3c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 527
        }
      },
      "source": [
        "dfp.count()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "frame.time_epoch            11252406\n",
              "frame.len                   11252406\n",
              "ip.proto                    11252406\n",
              "srcport                     11252406\n",
              "dstport                     11252406\n",
              "trans.stream                11252406\n",
              "trans.length                11252406\n",
              "tcp.analysis.initial_rtt    11252406\n",
              "tcp.time_delta              11252406\n",
              "tcp.time_relative           11252406\n",
              "tcp.flags                   11252406\n",
              "tcp.window_size_value       11252406\n",
              "tcp.hdr_len                 11252406\n",
              "http.response.code          11252406\n",
              "http.request.method         11252406\n",
              "http.content_length         11252406\n",
              "ip.ttl_1                    11252406\n",
              "ip.ttl_2                    11252406\n",
              "ip.hdr_len_1                11252406\n",
              "ip.hdr_len_2                11252406\n",
              "ip.len_1                    11252406\n",
              "ip.len_2                    11252406\n",
              "ip.id_1                     11252406\n",
              "ip.id_2                     11252406\n",
              "ip.flags.df                 11252406\n",
              "attack                      11252406\n",
              "category                    11252406\n",
              "subcategory                 11252406\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    }
  ]
}