05_4_ml_f_measure_comparison.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature Number    Feature           \n",
      "1                 Bwd Packet Length Std\n",
      "2                 Flow Bytes/s     \n",
      "3                 Total Length of Fwd Packets\n",
      "4                 Fwd Packet Length Std\n",
      "5                 Flow IAT Std     \n",
      "6                 Flow IAT Min     \n",
      "7                 Fwd IAT Total    \n",
      "8                 Flow Duration    \n",
      "9                 Bwd Packet Length Max\n",
      "10                Flow IAT Max     \n",
      "11                Flow IAT Mean    \n",
      "12                Total Length of Bwd Packets\n",
      "13                Fwd Packet Length Min\n",
      "14                Bwd Packet Length Mean\n",
      "15                Flow Packets/s   \n",
      "16                Fwd Packet Length Mean\n",
      "17                Total Backward Packets\n",
      "18                Total Fwd Packets\n",
      "19                Fwd Packet Length Max\n",
      "20                Bwd Packet Length Min\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "ML algorithm      Feature Name                   F1-score    Accuracy   Feature List    \n",
      "Naive Bayes       Bwd Packet Length Std          0.86        0.88       [1,             ------> New feature found!!! \n",
      "Naive Bayes       Flow Bytes/s                   0.84        0.86       [1, 2,          \n",
      "Naive Bayes       Total Length of Fwd Packets    0.86        0.88       [1, 2,          ------> New feature found!!! \n",
      "Naive Bayes       Fwd Packet Length Std          0.85        0.86       [1, 2, 3,       \n",
      "Naive Bayes       Flow IAT Std                   0.85        0.86       [1, 2, 3,       \n",
      "Naive Bayes       Flow IAT Min                   0.86        0.88       [1, 2, 3,       ------> New feature found!!! \n",
      "Naive Bayes       Fwd IAT Total                  0.82        0.83       [1, 2, 3, 4,    \n",
      "Naive Bayes       Flow Duration                  0.82        0.83       [1, 2, 3, 4,    \n",
      "Naive Bayes       Bwd Packet Length Max          0.85        0.87       [1, 2, 3, 4,    \n",
      "Naive Bayes       Flow IAT Max                   0.83        0.85       [1, 2, 3, 4,    \n",
      "Naive Bayes       Flow IAT Mean                  0.85        0.87       [1, 2, 3, 4,    \n",
      "Naive Bayes       Total Length of Bwd Packets    0.07        0.18       [1, 2, 3, 4,    \n",
      "Naive Bayes       Fwd Packet Length Min          0.86        0.88       [1, 2, 3, 4,    ------> New feature found!!! \n",
      "Naive Bayes       Bwd Packet Length Mean         0.85        0.86       [1, 2, 3, 4, 5,  \n",
      "Naive Bayes       Flow Packets/s                 0.86        0.87       [1, 2, 3, 4, 5,  ------> New feature found!!! \n",
      "Naive Bayes       Fwd Packet Length Mean         0.86        0.87       [1, 2, 3, 4, 5, 6,  ------> New feature found!!! \n",
      "Naive Bayes       Total Backward Packets         0.13        0.21       [1, 2, 3, 4, 5, 6, 7,  \n",
      "Naive Bayes       Total Fwd Packets              0.52        0.48       [1, 2, 3, 4, 5, 6, 7,  \n",
      "Naive Bayes       Fwd Packet Length Max          0.84        0.84       [1, 2, 3, 4, 5, 6, 7,  \n",
      "Naive Bayes       Bwd Packet Length Min          0.85        0.85       [1, 2, 3, 4, 5, 6, 7,  \n",
      "F1= 0.86 Naive Bayes  The most efficient feature list = ['Bwd Packet Length Std', 'Total Length of Fwd Packets', 'Flow IAT Min', 'Fwd Packet Length Min', 'Flow Packets/s', 'Fwd Packet Length Mean'] \n",
      "\n",
      "\n",
      "QDA               Bwd Packet Length Std          0.86        0.88       [1,             ------> New feature found!!! \n",
      "QDA               Flow Bytes/s                   0.86        0.88       [1, 2,          ------> New feature found!!! \n",
      "QDA               Total Length of Fwd Packets    0.86        0.87       [1, 2, 3,       ------> New feature found!!! \n",
      "QDA               Fwd Packet Length Std          0.3         0.31       [1, 2, 3, 4,    \n",
      "QDA               Flow IAT Std                   0.84        0.85       [1, 2, 3, 4,    \n",
      "QDA               Flow IAT Min                   0.86        0.88       [1, 2, 3, 4,    ------> New feature found!!! \n",
      "QDA               Fwd IAT Total                  0.84        0.85       [1, 2, 3, 4, 5,  \n",
      "QDA               Flow Duration                  0.84        0.86       [1, 2, 3, 4, 5,  \n",
      "QDA               Bwd Packet Length Max          0.84        0.85       [1, 2, 3, 4, 5,  \n",
      "QDA               Flow IAT Max                   0.84        0.85       [1, 2, 3, 4, 5,  \n",
      "QDA               Flow IAT Mean                  0.85        0.87       [1, 2, 3, 4, 5,  \n",
      "QDA               Total Length of Bwd Packets    0.08        0.18       [1, 2, 3, 4, 5,  \n",
      "QDA               Fwd Packet Length Min          0.85        0.87       [1, 2, 3, 4, 5,  \n",
      "QDA               Bwd Packet Length Mean         0.84        0.86       [1, 2, 3, 4, 5,  \n",
      "QDA               Flow Packets/s                 0.85        0.86       [1, 2, 3, 4, 5,  \n",
      "QDA               Fwd Packet Length Mean         0.84        0.85       [1, 2, 3, 4, 5,  \n",
      "QDA               Total Backward Packets         0.11        0.2        [1, 2, 3, 4, 5,  \n",
      "QDA               Total Fwd Packets              0.1         0.19       [1, 2, 3, 4, 5,  \n",
      "QDA               Fwd Packet Length Max          0.85        0.86       [1, 2, 3, 4, 5,  \n",
      "QDA               Bwd Packet Length Min          0.61        0.56       [1, 2, 3, 4, 5,  \n",
      "F1= 0.86 QDA  The most efficient feature list = ['Bwd Packet Length Std', 'Flow Bytes/s', 'Total Length of Fwd Packets', 'Flow IAT Min'] \n",
      "\n",
      "\n",
      "MLP               Bwd Packet Length Std          0.86        0.88       [1,             ------> New feature found!!! \n",
      "MLP               Flow Bytes/s                   0.86        0.89       [1, 2,          ------> New feature found!!! \n",
      "MLP               Total Length of Fwd Packets    0.76        0.83       [1, 2, 3,       \n",
      "MLP               Fwd Packet Length Std          0.87        0.89       [1, 2, 3,       ------> New feature found!!! \n",
      "MLP               Flow IAT Std                   0.76        0.83       [1, 2, 3, 4,    \n",
      "MLP               Flow IAT Min                   0.88        0.89       [1, 2, 3, 4,    ------> New feature found!!! \n",
      "MLP               Fwd IAT Total                  0.77        0.84       [1, 2, 3, 4, 5,  \n",
      "MLP               Flow Duration                  0.78        0.84       [1, 2, 3, 4, 5,  \n",
      "MLP               Bwd Packet Length Max          0.87        0.89       [1, 2, 3, 4, 5,  \n",
      "MLP               Flow IAT Max                   0.79        0.84       [1, 2, 3, 4, 5,  \n",
      "MLP               Flow IAT Mean                  0.79        0.84       [1, 2, 3, 4, 5,  \n",
      "MLP               Total Length of Bwd Packets    0.78        0.83       [1, 2, 3, 4, 5,  \n",
      "MLP               Fwd Packet Length Min          0.88        0.89       [1, 2, 3, 4, 5,  ------> New feature found!!! \n",
      "MLP               Bwd Packet Length Mean         0.87        0.89       [1, 2, 3, 4, 5, 6,  \n",
      "MLP               Flow Packets/s                 0.87        0.89       [1, 2, 3, 4, 5, 6,  \n",
      "MLP               Fwd Packet Length Mean         0.87        0.89       [1, 2, 3, 4, 5, 6,  \n",
      "MLP               Total Backward Packets         0.88        0.89       [1, 2, 3, 4, 5, 6,  ------> New feature found!!! \n",
      "MLP               Total Fwd Packets              0.88        0.89       [1, 2, 3, 4, 5, 6, 7,  ------> New feature found!!! \n",
      "MLP               Fwd Packet Length Max          0.88        0.9        [1, 2, 3, 4, 5, 6, 7, 8,  ------> New feature found!!! \n",
      "MLP               Bwd Packet Length Min          0.88        0.9        [1, 2, 3, 4, 5, 6, 7, 8, 9,  ------> New feature found!!! \n",
      "F1= 0.88 MLP  The most efficient feature list = ['Bwd Packet Length Std', 'Flow Bytes/s', 'Fwd Packet Length Std', 'Flow IAT Min', 'Fwd Packet Length Min', 'Total Backward Packets', 'Total Fwd Packets', 'Fwd Packet Length Max', 'Bwd Packet Length Min'] \n",
      "\n",
      "\n",
      "mission accomplished!\n",
      "operation time: =  2092.819859981537 secomds\n"
     ]
    }
   ],
   "source": [
    "\n",
    "##  \"all_data.csv\" file is required for the operation of the program.\n",
    "##  \"all_data.csv\" file must be located in the same directory as the program.\n",
    "##The intent of this program is to find the optimal property list for Naive Bayes, and QDA and MLP algorithms.\n",
    "##It follows a kind of trial-and-error method.\n",
    "##The feature list obtained from the file \"04_2_feature_selection_for_attack_files.py\" is placed in the machine learning algorithm to start with the highest importance score.\n",
    "##If the F-measure for each feature is equal to or greater than the highest value obtained, this property is added to the list. Otherwise it is removed from the list.\n",
    "##As a result of the process, the program gives the highest F-measure obtained and the property list that provides it\n",
    "##\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#%matplotlib inline\n",
    "from sklearn import metrics\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis as QDA\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import precision_score\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "            \n",
    "import pandas as pd\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import time\n",
    "seconds = time.time()\n",
    "\n",
    "\n",
    "#list of all columns to be imported\n",
    "# the 20 features selected by the file \"04_2_feature_selection_for_attack_files.py\" are used here. (+ Label Feature)\n",
    "features=[\"Bwd Packet Length Std\",\"Flow Bytes/s\",\"Total Length of Fwd Packets\",\"Fwd Packet Length Std\",\n",
    "\"Flow IAT Std\",\"Flow IAT Min\",\"Fwd IAT Total\",\"Flow Duration\",\"Bwd Packet Length Max\",\"Flow IAT Max\",\n",
    "\"Flow IAT Mean\",\"Total Length of Bwd Packets\",\"Fwd Packet Length Min\",\"Bwd Packet Length Mean\",\n",
    "\"Flow Packets/s\",\"Fwd Packet Length Mean\",\"Total Backward Packets\",\"Total Fwd Packets\",\"Fwd Packet Length Max\",\n",
    "\"Bwd Packet Length Min\",'Label']\n",
    "    \n",
    "df=pd.read_csv('all_data.csv',usecols=features)#CSV rading\n",
    "\n",
    "\n",
    "\n",
    "print ('%-17s %-17s ' % (\"Feature Number\",\"Feature\"))# print output header\n",
    "for i in range(len(features)-1):\n",
    "    print ('%-17s %-17s' % (i+1,features[i]))# print features  and feature numbers \n",
    "\n",
    "\n",
    "print ('\\n\\n\\n')\n",
    "\n",
    "attack_or_not=[]\n",
    "for i in df.iloc[:,-1]:\n",
    "    if i ==\"BENIGN\":#it changes the normal label to \"1\" and the attack tag to \"0\" for use in the machine learning algorithm\n",
    "        attack_or_not.append(1)\n",
    "    else:\n",
    "        attack_or_not.append(0)\n",
    "df.iloc[:,-1]=attack_or_not\n",
    "y = df.iloc[:, -1].values #labes-y\n",
    "my_list=[]\n",
    "\n",
    "\n",
    "least=0\n",
    "\n",
    "\n",
    "\n",
    "ml_list={#The machine learning algorithms to be used are defined in a dictionary (ml_list).\n",
    "\"Naive Bayes\":GaussianNB(),\n",
    "\"QDA\":QDA(),\n",
    "##\"Random Forest\":RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),\n",
    "##\"ID3\" :DecisionTreeClassifier(max_depth=5,criterion=\"entropy\"),\n",
    "##\"AdaBoost\":AdaBoostClassifier(),\n",
    "##\"Nearest Neighbors\":KNeighborsClassifier(3),\n",
    "\"MLP\":MLPClassifier(hidden_layer_sizes=(13,13,13),max_iter=500)}\n",
    "\n",
    "\n",
    "features.pop()#the Label tag is removed, no need any more\n",
    "print ('%-17s %-30s %-10s  %-10s %-15s ' % (\"ML algorithm\",\"Feature Name\",\"F1-score\",\"Accuracy\", \"Feature List\"))# print output header\n",
    "for j in ml_list: # run for every machine learning.  \n",
    "    my_list=[]\n",
    "    for i in features: ## run for every  feature  \n",
    "        my_list.append(i)\n",
    "        X = df.loc[:, my_list].values # data\n",
    "\n",
    "        ## cross-validation\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.20, random_state = 0)\n",
    "       \n",
    "        #machine learning algorithm is applied in this section\n",
    "        clf = ml_list[j]   #                                                                       \n",
    "        clf.fit(X_train, y_train)\n",
    "        predict =clf.predict(X_test)\n",
    "        f1=clf.score(X_test, y_test)\n",
    "        result=f1_score(y_test, predict, average='macro')\n",
    "        accuracy=round(clf.score(X_test, y_test),2)\n",
    "        temp=\"[\"\n",
    "       \n",
    "        for ii in my_list: \n",
    "            temp+=str(my_list.index(ii)+1)+\", \" #translate property list to sequence number for less space\n",
    "\n",
    "       \n",
    "        if result>=least:# If the F-criterion is equal to or greater than the highest value previously accessed, keep the new feature. \n",
    "            least=result\n",
    "            print ('%-17s %-30s %-10s  %-10s %-15s %-15s ' % (j,i,result,accuracy ,temp, \"------> New feature found!!!\"))\n",
    "\n",
    "        else:#If not, remove it from the list\n",
    "            my_list.remove(my_list[len(my_list)-1])\n",
    "            print ('%-17s %-30s %-10s  %-10s %-15s ' % (j,i,result,accuracy ,temp))\n",
    "    print(\"F1=\" ,least,j,\" The most efficient feature list =\",my_list,\"\\n\\n\") #print maximum F1 and the most efficient feature list\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "print(\"mission accomplished!\")\n",
    "print(\"operation time: = \",time.time()- seconds ,\"secomds\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}