NN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "author=\"Jumabek Alikhanov\"\n",
    "date = 'Nov 17,2019'"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download the dataset from \n",
    "- https://www.unb.ca/cic/datasets/ids-2017.html \n",
    "or directly below\n",
    "### Download dataset directly and unzip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!gdown --id 1-t3RdDpmqMs4ABt9oobSapeNYTZJ9tpu"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from os.path import join\n",
    "import glob\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "from preprocessing import load_data\n",
    "%load_ext autoreload"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set dataset path and load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataroot = 'MachineLearningCVE/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MachineLearningCVE/*.pcap_ISCX.csv\n",
      "there are 2830743 flow records with 79 feature dimension\n",
      "stripped column names\n",
      "dropped bad columns\n",
      "There are 0 nan entries\n",
      "converted to numeric\n"
     ]
    }
   ],
   "source": [
    "X,y = load_data(dataroot) # reads csv file and returns np array of X,y -> of shape (N,D) and (N,1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Imbalance\n",
    "1. It is curucial to adress this issue in order to get decent performance\n",
    "2. It also affects evaluation, we should calculate  `balanced accuracy`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "from preprocessing import balance_data, normalize\n",
    "\n",
    "X = normalize(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from models import Classifier\n",
    "\n",
    "def ensure_dir(dir_path):\n",
    "    if not os.path.exists(dir_path):\n",
    "        os.makedirs(dir_path)\n",
    "\n",
    "def getClassifier(args,runs_dir=None):\n",
    "    \n",
    "    (method,optim,lr,reg,batch_size,input_dim,num_class,num_epochs) = args\n",
    "    if runs_dir is not None:\n",
    "        ensure_dir(runs_dir)\n",
    "    \n",
    "    clf = Classifier(method,input_dim,num_class,lr=lr,reg=reg,num_epochs=num_epochs,\n",
    "                        batch_size=batch_size,runs_dir=runs_dir)\n",
    "    return clf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------\n",
      "Fold #0\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from sklearn import metrics\n",
    "K=5\n",
    "skf = StratifiedKFold(n_splits=K,random_state=SEED)\n",
    "for fold_index, (train_index,test_index) in enumerate(skf.split(X,y)):# runs only once \n",
    "        print('---------------------------------------------')\n",
    "        print('Fold #{}'.format(fold_index))    \n",
    "        X_train = X[train_index]\n",
    "        y_train = y[train_index]\n",
    "        X_test = X[test_index]\n",
    "        y_test = y[test_index]\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best epoch 0, best batch 0\n",
      "bst acc  -1\n",
      "Epoch [1/100], Step [50/398], Loss: 0.5975\n",
      "Epoch [1/100], Step [100/398], Loss: 0.2803\n",
      "Epoch [1/100], Step [150/398], Loss: 0.2665\n",
      "Epoch [1/100], Step [200/398], Loss: 0.2261\n",
      "Epoch [1/100], Step [250/398], Loss: 0.2231\n",
      "Epoch [1/100], Step [300/398], Loss: 0.2209\n",
      "Epoch [1/100], Step [350/398], Loss: 0.1969\n",
      "Epoch [2/100], Step [1/398], Loss: 0.1877\n",
      "Epoch [2/100], Step [51/398], Loss: 0.1929\n",
      "Epoch [2/100], Step [101/398], Loss: 0.1947\n",
      "Epoch [2/100], Step [151/398], Loss: 0.1775\n",
      "Epoch [2/100], Step [201/398], Loss: 0.1827\n",
      "Epoch [2/100], Step [251/398], Loss: 0.1677\n",
      "Epoch [2/100], Step [301/398], Loss: 0.1911\n",
      "Epoch [2/100], Step [351/398], Loss: 0.1761\n",
      "Epoch [3/100], Step [2/398], Loss: 0.1726\n",
      "Epoch [3/100], Step [52/398], Loss: 0.1669\n",
      "Epoch [3/100], Step [102/398], Loss: 0.1647\n",
      "Epoch [3/100], Step [152/398], Loss: 0.1654\n",
      "no improvement in accuracy for 10 iterations\n",
      "Loaded MachineLearningCVE/runs/5layer_NN/4th_run/optim_Adam_lr_0.001_reg_1e-06_bs_5120 model trained with batch_size = 5120, seen 1 epochs and 100 mini batches\n",
      "Model is trained in 115.99700736999512 sec\n"
     ]
    }
   ],
   "source": [
    "batch_size = 5*1024\n",
    "optim = 'Adam'\n",
    "\n",
    "input_dim = X_train.shape[1]\n",
    "num_class = len(np.unique(y_train))\n",
    "num_epochs = 100\n",
    "\n",
    "accuracies = {}\n",
    "learning_rates = [1e-3]\n",
    "regularizations = [1e-6]\n",
    "\n",
    "best_model = None\n",
    "best_acc = -1\n",
    "architecture = '5layer_NN'\n",
    "run_number = 4\n",
    "method='nn5'\n",
    "\n",
    "for lr in learning_rates:\n",
    "    for reg in regularizations:\n",
    "\n",
    "        classifier_args = (method,optim,lr,reg,batch_size,input_dim,num_class,num_epochs)\n",
    "        config =  '{}/{}th_run/optim_{}_lr_{}_reg_{}_bs_{}'.format(architecture,run_number,optim,lr,reg,batch_size)\n",
    "        runs_dir = join(dataroot,'runs',config)\n",
    "        \n",
    "        X_train = X_train.astype(float)\n",
    "        y_train = y_train.astype(int)\n",
    "        p = np.random.permutation(len(y_train))\n",
    "        X_train = X_train[p]\n",
    "        y_train = y_train[p]\n",
    "        X_train,y_train = balance_data(X_train,y_train,seed=SEED)\n",
    "\n",
    "        tick = time.time()\n",
    "        clf = getClassifier(classifier_args,runs_dir)\n",
    "        \n",
    "        clf.fit(X_train,y_train)\n",
    "        pred = clf.predict(X_test,eval_mode=True)\n",
    "        \n",
    "        acc = metrics.balanced_accuracy_score(y_test,pred)\n",
    "        if acc >best_acc:\n",
    "            best_model = clf\n",
    "            best_acc = acc\n",
    "        accuracies[(lr,reg)]=acc\n",
    "        tock = time.time()\n",
    "        print(\"Model is trained in {} sec\".format(tock-tick))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(0.001, 1e-06) -> 86.21\n"
     ]
    }
   ],
   "source": [
    "for x in accuracies:\n",
    "    print()\n",
    "    print('{0:} -> {1:.2f}'.format(x,accuracies[x]*100))\n",
    "results = accuracies  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEWCAYAAABi5jCmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de7zlY/3//8dzxiBmzGAmMYyzMhTVUBSR0vApQgeS6EAfRX1Exad+klKOqT5KCREih9LE5FQO5ecwQ06DqSHHcZiJYSZizH5+/7iuZd57WYf33rPW3muved1vt/dtv8/Xa6291rre7+u63tcl24QQQggVwwY7gBBCCJ0lMoYQQgi9RMYQQgihl8gYQggh9BIZQwghhF4iYwghhNBLZAx9IGmBpPU6II5tJM1s4fnOl/ThVp1vaSbpEkk7tencZ0n6bp5v+Bko7tvPtDrisx4GR9dkDJIekvS0pBUL6z4n6bqSxzf9ItkeafvBkuezpA3K7NtXtv9i+42tOJektwCbAb8vrFtd0hmSnpA0X9L9kr5deW+rX5ukjSRdJGmupOck3SXpK5KG10nzOkmfy/PbSerJP0QLJD0m6UJJW1Qds6ukOyQ9n9P5s6R165x/I0m/lzRH0jOSrpT0xqp9DpH0ZD7fmZKWK2z7jqS7Jb0i6aiq4/63EOsCSS/m+MfmXY4D+v2DXFaLPwOv/j8K5y/9WQ/dp2syhmw48OXBDqIMScsMdgzZ54HznJ90lLQKcBPwOmAr26OA9wNjgPWrD5a0PnAL8CjwZtujgY8Ck4BRJWOYbXtk3v+dwP3AXyTtkNPYAPgVcCgwGlgX+AmwqM75xgBTgDcCqwG30jvj+wBwOLADsDawHvDtwvGzgK8Bl1ef2Pb38o/myBzzccB1tufm7bcCK0maVPK1hwGipNt+89rDdldMwEOkL/szwJi87nOkL21lnzcBV+d9ZgIfy+sPABYCLwMLgD/UScPABnn+LNKP0+XAfNKP4/p52w1533/n830c2A54DPg68CRwDrAf8NcGaewM3JvP/zhwWF6/HfBY4ZiNgeuAecAMYJfCtrpx5u0PAu8uLH8XuBsY1uC9LsZ4LnB5H/9X1wGfq/VaCvucAkzP8x8B7liCz8YqOeZV8/Kvge8Vtu8APFnjuHOBoxqcV/n927dq/S+Ab9U55j7gg4XlZYA5wNvy8kX58/Fc/hxtUvW//G6dz8Bbgdvz//g3wAWFfVcGLsvpPJvn18zbjiFlsP/Jn9VTavyPR5My5jnAw8A3K58P8mcYODGf+5/ATg3es8OBB3Kc9wK7VW3fP79Hle2V92Ut4Lc5hn8V4jwKOLdw/Do59mUKn7VjgBuBF4ENgE8X0ngQ+HxVDLsCdwDP51gnky52bqva7yvA7/v7uezkqdtyz+mkD8Jh1RtyMcjVpB+F1wN7Aj+VNNH2acB5wPFOV4IfKpnenqQrzZVJV5nHANjeNm/fLJ/vN3n5DaQfqbVJmVEzZ5A+tKOATYE/13hdI4A/AFfl13UwcF5V0UnNOPN7si4pk6x4H/Bb2z0l4qvsf3HJffvit8Dbcoy3A2+SdLKk7SWN7OO5tiX98P8rL28C3FnYfiewmqRV+3jebUjv+SVV6+8jFc/Vcj6wV2H5A8Bc27fn5T8CG+bz3k76XDYkaVngUtLFxiqkzGWPwi7DgF+SPncTSD+QpwDY/gbwF+Cg/Fk9qEYS/0fKHNYD3gN8ivTjWvEO0mdoLHA8cIYk1Qn3AdL7Npr0mTxX0ur5dXyU9EP/KWAlYBfgX7lI8jJSprQOMJ6U8ZW1D+n7Niqf42nggzmNTwMnS3pbjmFLUib4VdKd57aki84pwLqSNq4676/6EMeQ0W0ZA8CRwMGSxlWt/yDwkO1f2n7F9t9IX+iPLkFav7N9q+1XSF/gzZvs30O6knzJ9oslzr8QmChpJdvPFn48it4JjASOtf2y7T+TvkTFH596cY7Jf+cX9l0VeKJEbP3dv6zZpCvyMU5l3duRfhAuBObmOqGmGYSkNUl3TF8prB5JuiKvqMyXLfqq2Be42PaCqvXzWfzeVvs1sIukFfLyJ0iZBQC2z7Q93/ZLpB/JzSSNbhLHO4ERwA9tL7R9MTCtcM5/2b7E9gu255MuDN5T5gXmH+U9gSNyXA8BJ5F+FCsetv0L24uAs4HVSUV4r2H7ItuzbffkC6Z/AFvmzZ8jXZxNczLL9sN5+xrAV23/2/Z/bP+1TPzZWbZn5O/9QtuX234gp3E96aJqm7zvZ4EzbV+dY3zc9v35//Eb4JP5fdmElEld1oc4hoyuyxhs30P6Zx1etWlt4B2S5lUmYG/SVXx/PVmYf4H0g9PIHNv/6cP59yAVJz0s6XpJW9XYZw3g0aor/IdJP6LN4pyX/xZ/EP9F+mKX1XB/ST8rVNT+bx/OO55UJDAPwPbNtj9mexzpS7wt8I2cRrEyeEIh7XGkL/1PbZ9fOPcC0tViRWW+mEE2lH/YP0r6Iaw2isXvbS+2Z5HuKD6Uz7ELKbNA0nBJx0p6QNLzpCtVSFfijawBPG672CPmw8VYJf1c0sP5vDcAY+o1DqgylpTpPFxYV/fzZfuFPFvzuyDpU7kRQeU7uCmLX99apDuKamuRMp9XSsRby6NVMewk6ebcMGEe6TvWLAZI/+tP5LuhfYALc4bRdbouY8i+RSqrLH54HwWutz2mMI20fWDePhDdzFan8W+gcuWIpF6ZVL5y2pVUrHAp6Wq52mxgrapKtQmkOonGwdj/Jn0JNiqsvgbYrQ+VdNfQu9iiOo3/9uLK2u+VPCfAbsDtOcbqc04jFTVtmpdHFqZHACStTMoUptg+puoUM+hd1LMZ8FShqKlsfM+Qii6rbUzvoqpqleKkXYF7c2YB6e5hV1Lx3GjSFSmkO6dGngDGVxXfTCjMH0qqiH+H7ZVImWrxvI0++3NJd65rV5276eermqS1SfUvB5Hqe8YA9xTieJQaDRzy+gl1Gmz0+g5R+0Lv1deXW59dQqoTWS3HMLVEDNi+mVQPuQ3pf3VOrf26QVdmDPmL9hvgS4XVlwEbSdpH0og8bVEoM3yKVIbaKmXOdyewiaTNJS1PKjoAUrmxpL0ljba9kFQRVqvc/xbSXcDX8mvaDvgQ5ctgp9K7WOEHpCvos/MXGUnjJf1AqWlrtW8BW0s6oZKxSdpA0rmS6hWn1JRbjYyX9C1SscL/5vXvlrS/pNfn5TeRrrRvrnOelYArgRttV985QioX/qykiTnGb5IqdivHj8j/j2HAMpKWr3F1vS/wq6qr9Ir3kOoK6rkA2BE4kHy3kI0CXiLdha0AlM1IbwJeAb6UY9+dxcUzlfO+CMxTanX2rarj635Wc/HQhcAxkkblz8RXSBXzfbUi6Ud6DoCkT5Mz9+x04DBJb8+fhQ1yereSMr9jJa2Y/x/vysfcAWwraUIucjuiSQzLAsvlGF5ReuZkx8L2M4BPS9pB0rD8eXxTYfuvSPUzC/tYnDWkdGXGkB1N+iACkMtWdySVl84m3f4eR/qQQPpATMy3uJe2IP2jSD+u8yR9rNYOtv+e47yGVNZa/UHbB3go3/7/N6noq/ocL5Mygp1IV3c/BT5l+/6ScZ4G7F252rT9DLA16SrxFknzgT+RyuFnVR9s+wFgK9LV7QxJz5GuyKZTvmhmDUkLSEU804A3A9vZvipvn0fKCO7O+10B/I5U0VnLbsAWpC/4a4qZbF+Rj70WeIRUNFL8sfwF6Yd0L1Jx1YsUytQljQfeS42KR6XnLxY4NVutyfYTpB/zrUkXMBW/yrE8TmqRUzPjq3G+l4HdSS2EniG1gvttYZcfkpofz83nvKLqFD8CPiLpWUk/rpHEwaQr8wdJn9FfA2eWia0qzntJ9RM3kTKjN5NaC1W2X0Sq//g16bNzKbBKzpw+RGpR9Aipdd/H8zFXk97Du4DbaFLmn38HvkTK7J4lXflPKWy/lVwhTfrMX0/vu6VzSJlZfzLGIUO1L3jC0kTSr0nlpa3IEJdqki4BzrA9dbBjCa0n6XWkVk1vs/2PwY6nXSJjCCGEkiR9hfQcynsHO5Z26pSnb0MIoaNJeohUSd31/YrFHUMIIYReurnyOYQQQj8sVUVJY8eO9TrrrDPYYYQQhoDbbrttbn6gst/ePmxFP+96fT0uNouXrrQ9eUnSaqWlKmNYZ511mD59+mCHEUIYAiQ93Hyvxp73In64zNpN9/vgK39v9nT7gFqqMoYQQhhQAo1o9uA66fHEDhIZQwghtImGieGvK9ElVZkuNQdQZAwhhNAugmHLlLhj6DCRMYQQQruULUrqMJExhBBCm0iKO4YQQggFQ/SOIR5wCyGEdsl1DM2mUqeSJkuaKWmWpNd0Jy9pbUl/knSXpOuURi8sbl9J0mOSTmmWVmQMIYTQJhIMX3ZY06n5eTScNETtTsBEYC9JE6t2O5E0RshbSN35f79q+3dIo/c1FRlDCCG0jdCw5lMJWwKzbD+Yx9+4gDTaX9FE4M95/tridklvJ43DfRUlRMYQQgjtItDwYU0nYKyk6YXpgKozjaf32NWP0XvoYkgjQu6e53cDRklaNQ/TexJwWNmwo/I5hBDaRMCw4aXuCObanrSEyR0GnCJpP1KR0ePAIuALwFTbj/UeFry+yBhCCKFdRNmiomYeB9YqLK+Z173K9mzyHYOkkcAetudJ2grYRtIXgJHAspIW1BkPHYiMIYQQ2kZSqcrlEqYBG0pal5Qh7Ekar7qY1ljgGds9wBHkcblt713YZz9gUqNMAaKOIYQQ2krDhjWdmrH9CnAQcCVwH2mM9hmSjpa0S95tO2CmpL+TKpqP6W/McccQQgjt0rqiJGxPBaZWrTuyMH8xcHGTc5wFnNUsrcgYQgihbVS28rmjRMYQQghtohbeMQykyBhCCKGNytQhdJpBiVjSUZIel3RHnnaus9+XJd0jaYak/yms31zSzfnY6ZK2HLjoQwihJInhI4Y1nTrNYN4xnGz7xHobJW0K7E96FPxl4ApJl9meBRwPfNv2H3OmcjypRj6EEDrGUC1K6rysarGNgVtsv5Cbal3P4se9DayU50cDswchvhBCaKoVzVUH2mBGdFDuHvZMSSvX2H4P6Wm9VSWtAOzM4if//gc4QdKjpB4Fj6iXiKQDKv2PzJkzp9WvIYQQ6st3DC3oRG9AtS1jkHRNrh+onnYFTgXWBzYHniB18NSL7fuA40i9AV4B3EHq9wPgQOAQ22sBhwBn1IvD9mm2J9meNG7cuFa+xBBCaKJlvasOqLbVMdh+X5n9JP0CuKzOOc4g/+hL+h6pR0GAfYEv5/mLgNOXKNgQQmgDCYYtM3yww+izwWqVtHphcTdSsVGt/V6f/04g1S/8Om+aDbwnz78X+Ed7Ig0hhCUzbLiaTp1msFolHS9pc1Il8kPA5wEkrQGcbrvSfPUSSasCC4Ev2p6X1+8P/EjSMsB/gOq+y0MIYfCpM4uKmhmUjMH2PnXWzyZVMleWt6mz31+Bt7cnuhBCaJ1ObHXUTDz5HEIIbTJUn2OIjCGEENooMoYQQgiLSUOyVVJkDCGE0DYaknUMQy/iEEIYSqTmU6nTaLKkmZJmSXrN0JyS1pb0p9yjxHWS1szrN5d0U+6M9C5JH2+WVmQMIYTQJmpRlxiShgM/AXYCJgJ7SZpYtduJwK9svwU4Gvh+Xv8C8CnbmwCTgR9KGtMovcgYQgihjVrUid6WwCzbD9p+GbgA2LVqn4nAn/P8tZXttv9u+x95fjbwNNCwf6DIGEIIoV1Uuq+ksZXOPvNU/dDueODRwvJjeV3RnSzugXo3YFR+QLgQjrYElgUeaBR2VD6HEEIblWyVNNf2pCVM6jDgFEn7ATcAj7O449FKV0TnAPva7ml0osgYQgihTVr4gNvjLB52AGDNvO5VuZho95SuRgJ7VLoRkrQScDnwDds3N0ssipJCCKFtBMOGNZ+amwZsKGldScsCewJTeqUkjZVUOdkRwJl5/bLA70gV0xeXSSwyhhBCaCNJTadm8iiWBwFXAvcBF9qeIeloSbvk3bYDZkr6O7AacExe/zFgW2A/SXfkafNG6UVRUgghtIta14me7anA1Kp1RxbmLwZec0dg+1zg3L6kFRlDCCG0i4SiS4wQQghF0YleCCGEVwmxuD546CiVMUjaGlinuL/tX7UpphBC6A4CuvGOQdI5wPrAHSx+WMJAZAwhhNDEUOxdtcwdwyRgom23O5gQQug23VrHcA/wBuCJNscSQgjdRULDu6hVkqQ/kIqMRgH3SroVeKmy3fYu9Y4NIYSQdVlR0okDFkUIIXShsk82d5q6GYPt6wEkHWf768Vtko4Drm9zbCGEMPQNwTuGMhG/v8a6nVodSAghdKNWjOA20BrVMRwIfAFYT9JdhU2jgBvbHVgIIQx5qd/twY6izxrVMfwa+CNp3NDiwNPzbT/T1qhCCKFLdFWrJNvPAc9J+mL1NkkjbC9sa2QhhDDUdeuTz8DtpJGDniW9zDHAk5KeAva3fVsb4wshhCFMQ/LJ5zIRXw3sbHus7VVJFc+XkeofftrO4EIIYciTmk8dpkzG8E7bV1YWbF8FbJXHDV2ubZGFEMJQJ1o1tCeSJkuaKWmWpMNrbF9b0p8k3SXpOklrFrbtK+kfedq3WVplipKekPR14IK8/HHgKUnDgZ5SryiEEJZKrekSI//e/oT0+MBjwDRJU2zfW9jtRNK4zmdLei+p4dA+klYBvkXq987AbfnYZ+ulVyar+gSwJnBpnibkdcNJY4mGEEKoRaTmqs2m5rYEZtl+0PbLpAv1Xav2mQj8Oc9fW9j+AeBq28/kzOBqYHKjxJreMdieCxxcZ/OsZseHEMLSS2VbJY2VNL2wfJrt0wrL44FHC8uPAe+oOsedwO7Aj4DdgFGSVq1z7PhGwZQZj2Ej4DBeO1DPe5sdG0IISzNB2RHc5tqetITJHQacImk/4AbgcRaPodMnZeoYLgJ+Bpze30SqSToK2B+Yk1f9r+2pNfb7ct5PwC9s/zCv3yzHNBJ4CNjb9vOtiC2EEFqmdc8xPE56bKBizbzuVbZnk+4YkDQS2MP2PEmPA9tVHXtdo8TKZGWv2D7V9q22b6tMJY5r5mTbm+epVqawKSlT2BLYDPigpA3y5tOBw22/Gfgd8NUWxBNCCC2mVtUxTAM2lLSupGWBPYEpvVKSxmrx7ckRwJl5/kpgR0krS1oZ2DGvq6tMRH+Q9AVJq0tapTKVeSVLaGPgFtsv2H6F1Jvr7nnbRqRbJUgVKXsMQDwhhNB3w4c3n5rIv4EHkX7Q7wMutD1D0tGSKmPjbAfMlPR3YDXgmHzsM8B3SJnLNODoZt0alSlKqrR5LV6VG1ivxLGNHCTpU8B04NAaTafuAY7JlScvAjvnfQFmkGrcLwU+Su9brF4kHQAcADBhwoQlDDmEEPqghZ3o5ZKVqVXrjizMXwxcXOfYM1l8B9FU04htr1tjapopSLpG0j01pl2BU4H1gc1JQ4aeVCPd+4DjgKuAK4A7WFzH8RngC5JuI/X2+nKD+E+zPcn2pHHjxjULO4QQWmuYmk8dpkyrpBWArwATbB8gaUPgjbYva3Sc7feVCUDSL0hdbNQ6xxnAGXm/75GaWWH7flI5WaXV1H+VSSuEEAbcEOx2u0zEvyRdkW+dlx8HvrskiUpavbC4G6nYqNZ+r89/J5DqF35dtX4Y8E1SC6UQQug8XdpX0vq2jwcWAth+gdQIa0kcL+nuPADQ9sAhAJLWkFQsQ7tE0r3AH4Av2p6X1++VK1juB2aTMq8QQugsUsv6ShpIZSqfX5b0OlKFM5LWB15akkRt71Nn/WxSJXNleZs6+/2I9HRfCCF0tmFdNFBPwbdIlb9rSToPeBewXzuDCiGErlC5YxhiGmYMkkQqrtkdeCepCOnLuf+kEEIIzXRgHUIzDTMG25Y0NT9hfPkAxRRCCN2jS1sl3S5pi7ZHEkIIXadEi6QOvKMoU8fwDmBvSQ8D/yYVJ9n2W9oaWQghDHEWuAUD9Qy0MhnDB9oeRQghdKXWdYkxkMpE/F3bDxcnlvABtxBCWGq0pnfVAVXmjmGT4kIee/Tt7QknhBC6izuwDqGZulmVpCMkzQfeIun5PM0HngZ+P2ARhhDCUFXpXXWI3THUjcj2922PAk6wvVKeRtle1fYRAxhjCCEMXUOwVVKZrOoySSsCSPqkpB9IWrvNcYUQQhcQHj686dRpymQMpwIv5HGWDwUeAH7V1qhCCKEbiJYVJUmaLGmmpFmSDq+xfYKkayX9TdJdknbO60dIOjt3XHqfpKYlPmXHfDZpxLRTbP+ENDhOCCGEJqxhTadmcqOfnwA7ARNJPUxPrNrtm6QhP99KGhP6p3n9R4Hlcg8Wbwc+L2mdRumVyRjm5xzmk8DleQyEESWOCyGEpVzLnnzeEphl+0HbLwMXkC7WiwyslOdHk4YkqKxfUdIywOtI4+s83yixMhnDx0ndbH/W9pPAmsAJJY4LIYSlXsk7hrGSphemA6pOMx54tLD8WF5XdBTwSUmPkcaGPjivv5jUa8UTwCPAibafaRRz0+cYcmbwg8LyI0QdQwghNCeVHY9hru1JS5jaXsBZtk+StBVwjqRNSXcbi4A1gJWBv0i6xvaD9U5U5gG3EEII/WBa9oDb48BaheU187qizwKTAWzfJGl5YCzwCeAK2wuBpyXdCEwC6mYMnfdkRQghdJPWtEqaBmwoaV1Jy5Iql6dU7fMIsAOApI2B5YE5ef178/oVSWPr3N8oscgYQgihjYyaTk3PYb8CHARcCdxHan00Q9LRknbJux0K7C/pTuB8YL/covQnwEhJM0gZzC9t39UovaZFSZLeRarUWDvvX+l2e72mryaEEJZqKtUctQzbU0mVysV1Rxbm7yUNvVx93AJSk9XSytQxnAEcAtxGqsAIIYRQVgf2hdRMmYzhOdt/bHskIYTQZSzRU65VUkcpkzFcK+kE4Lek5xkAsH1726IKIYRu0YGd5DVTdmhPSM2bKkyu5Q4hhFBfq+oYBlKZB9y2H4hAQgih+5RrddRpmmZlkkbnrrYrj2qfJGn0QAQXQghDXSs60RtoZSI6E5gPfCxPzwO/bGdQIYTQFcSQHKinTB3D+rb3KCx/W9Id7QoohBC6hRE9GnqtksrcMbwo6d2VhfzA24vtCymEELrHUCxKKnPHcCBwdq5XEPAMsF87gwohhG4xFCufy7RKugPYTNJKebnhAA8hhBAqWtclxkCqmzFI+qTtcyV9pWo9ALZ/UPPAEEIIr2pRt9sDqtEdw4r5b63xnd2GWEIYMAsXLuS5efOYP38+tpHEqFGjGD1mDCNGxMi1oTWsoVn5XDdjsP3zPHuN7RuL23IFdAhD0gv//jdPPfUUqUfixDbPP/888+fPZ7XVVmOFFVdscIYQyhuKdQxlCr/+r+S6PpN0sKT7Jc2QdHydfSZLmilplqTDC+vXlXRLXv+bPHhFCA0tXLjwNZlCkW2eeuopFi5cOMCRhW7VVa2S8pihWwPjquoZVgKW+N5I0vbArsBmtl+S9Poa+wwnDTLxftLg19MkTcn9jh8HnGz7Akk/Iw1rd+qSxhW623Pz5tXNFCps89y8eYwdN26AogrdrNvuGJYFRpIyj1GF6XngIy1I+0DgWNsvAdh+usY+WwKzbD9o+2XgAmBXpRrw9wIX5/3OBj7cgphCl5s/f35L9wuhEedWSV1zx2D7euB6SWfZfrgNaW8EbCPpGOA/wGG2p1XtMx54tLD8GKm311WBeXm4u8r68bUSkXQAcADAhAkTWhd9GJKa3S30db8QmmnVHYOkycCPSCU2p9s+tmr7BNJF8pi8z+F51DckvQX4OanEpwfYwvZ/6qVV5gG3F/J4DJuQBpcGwHbTbrclXQO8ocamb+S0VyENTL0FcKGk9dzib6Tt04DTACZNmhTf9qWcpFI/+hqCTQxDZ+opVZXbWJNi9YpvksaCPlXSRNIwoOtIWgY4F9jH9p2SVgUaVqKVyRjOA34DfBD4b2BfYE6ZF2P7ffW2SToQ+G3OCG6V1AOMrTr348BaheU187p/AWMkLZPvGirrQ2ho1KhRPP9882c0R42q1Uo7hL4SbkHGQKFYHUDSBaQ62mLGYNIdAcBoYHae3xG4y/adALb/1SyxMhGvavsMYKHt621/htYM0nMpsD2ApI1IdRpzq/aZBmyYWyAtC+wJTMmZybUsruvYF/h9C2IKXW70mDFN7wYkMXrMmAGKKHQzk+sZmkzA2MLQBtNzEXhRrWL16uLzo4BPSnqMdLdwcF6/EWBJV0q6XdLXmsVdJmOo3HI8Iem/JL2VVAS0pM4E1pN0D6lSeV/blrSGpKkA+W7gIOBK4D7SbdKMfPzXga9ImkWqczijBTGFLjdixAhWW221upmDJFZbbbV4yC20TMmMYa7tSYXptH4ktRdwlu01gZ2BcyQNI5UMvRvYO//dTdIOjU5Upijpu7kDvUNJzy+sBBzSj6B7ya2MPllj/WzSi6osTyXlftX7PUi6vQqhT1ZYcUXWXGutePI5DIgWVT7XK1Yv+iwwGcD2TZKWJxXPPwbcYHsuQL7wfhvwp3qJlckY7rT9HPAci4t+alUohzBkjBgxgrHjxsWzCqHNWja056vF6qQMYU/gE1X7PALsAJwlaWNSY6E5pBKXr0laAXgZeA9wcqPEyhQl/VPS+fmkFa+5gg8hhNCbgR4Pazo1PU+dYnVJR0vaJe92KLC/pDuB84H9nDwL/ICUudwB3G778kbplbljuBv4C/BXSR+1/QAMwUf5QghhELTqOYZaxeq2jyzM3wvU7MfO9rmkJqullMkYbPunORf6g6SvE72rhhBCKUOxS4wyGYMAbN+Ya7IvBN7U1qhCCKErCLs7M4ZiC6Encud3W7cvpBBC6A4GerrpjqEyghuwV5023ze0LaoQQugGplTlcqfp7whuIYQQSuiqOgbbP88dNz1vu2Gb1xBCCLUMzTqGhvc4theRHrMOIYTQR33oK6mjlKl8vlHSKaQeVv9dWWn79rZFFUIIXWIo3jGUyRg2z3+PLqwzrelhNYQQulrPYAfQD00zBtvbD0QgIYTQbYy6rlXSqyT9F68dwe3o+k97i1wAABXuSURBVEeEEEKALi1KkvQzYAVSz6qnkwbHubXNcYUQQlfoxMrlZsrc42xt+1PAs7a/DWxFGhEohBBCI4aeElOnKVOU9GL++4KkNUjjLa/evpBCCKE7VJqrDjVlMobLJI0BTgBuJ73W09saVQghdImurGOw/Z08e4mky4Dl84huIYQQGhKLuiljkLR7g23Y/m17QgohhO5gWnfHIGky8CNgOHC67WOrtk8AzgbG5H0Oz4P7FLffCxxl+8RGaTW6Y/hQg20GImMIIYQm3ILK5dxv3U+A9wOPAdMkTcmjtlV8kzTk56mSJpJGe1unsP0HwB/LpNeoE71P9zH2EEIIVVpU+bwlMMv2gwCSLgB2Jd0BLE4KVsrzo4HZlQ2SPgz8k0K3Ro2UeY7hyFrr4wG3EEJoonxz1LGSpheWT7N9WmF5PPBoYfkx4B1V5zgKuErSwaRhE94HIGkk8HXS3cZhZYIp0yqpmMMsD3wQuK/MyUMIYWlmoKen1B3DXNuTljC5vYCzbJ8kaSvgHEmbkjKMk20vqDPo2muUaZV0UnFZ0onAlX0OOYQQlkItGtrzcWCtwvKaeV3RZ4HJALZvkrQ8MJZ0Z/ERSceTKqZ7JP3H9in1EivVV1KVFXJQIYQQmmhF5TMwDdhQ0rqkDGFP4BNV+zwC7ACcJWljUgnPHNvbVHaQdBSwoFGmAOXqGO4m3RFBagI1jt5dcIcQQqjBLRrBzfYrkg4ildYMB860PUPS0cB021OAQ4FfSDqE9Ju9n92/bKnMHcMHC/OvAE/ZfqU/iYUQwlKlhX0h5WcSplatO7Iwfy/wribnOKpMWmUyhvlVyytJmm97YZkEQghhadaioqQBVSZjuJ1U6fEsIFLlxZOSngL2t31bG+MLIYQhyzAku8Qo0+321cDOtsfaXhXYCbgM+ALw03YGF0IIQ53dfOo0ZTKGd9p+tXmq7auArWzfDCzXtshCCKELDMWMoUxR0hOSvg5ckJc/DjyV++4YiuNchxDCgLChp0uLkj5Bem7hUuB3pPqGT5CaTH2sfaGFEMLQNxTvGJpmDLbn2j4YeLftt9k+2PYc2y/bntXfhCUdLOl+STPyE3m19pksaaakWZIOL6w/KK+zpLH9jSGEENptUU/zqdM0zRgkbS3pXnL/SJI2k7RElc6Stif1DLiZ7U2A1/QNXuhmdidgIrBX7koW4EZSB1EPL0kcIYTQTpXxGJpNnaZMUdLJwAdIYz1j+05g2yVM90DgWNsv5XM+XWOfV7uZtf0yqY5j17z/32w/tIQxhBBCe5UoRhqSRUkAth+tWrVoCdPdCNhG0i2Srpe0RY19anUzO76vCUk6QNJ0SdPnzJnTz3BDCKF/etx86jRlWiU9KmlrwJJGAF+mRLfbkq4B3lBj0zdyuqsA7wS2AC6UtF5/+/VoJPdpfhrApEmTOvBfEELoVqkoabCj6LsyGcN/k8YZHU/q1e8q4IvNDrL9vnrbJB0I/DZnBLdK6iF1D1u8pC/TzWwIIXS0rssYcgXwPrb3bnG6lwLbA9dK2ghYFphbtU+ZbmZDCKFzuTNbHTXTsI7B9iLa82N8JrCepHtIlcr72rakNSRNzWm/AlS6mb2PNMj1DABJX5L0GOku4i5Jp7chxhBCWCJpBLfmU6cpU5T0V0mnAL+hMMyn7dv7m2huZfTJGutnAzsXll/TzWxe/2Pgx/1NP4QQBkrXFSVlm+e/xcF5DLy39eGEEEJ36cqMwfb2AxFICCF0G7ewOaqkyaSGQMOB020fW7V9AnA2aWiE4cDhtqdKej9wLKku92Xgq7b/3Cit/oz5HEIIoaRWtMIv9ATxftIzXdMkTcmjtlV8k1QXe2ruJWIqsA6pYc+HbM+WtCmp3rbhM2GRMYQQQhstWtLHgZNXe4IAkFTpCaKYMRhYKc+PBmZD6imisM8M4HWSlqv0PFFLZAwhhNAmfejyYqyk6YXl0/LDuRW1eoJ4R9U5jgKuknQwsCKpP7lqewC3N8oUoETGIGn3GqufA+6u08dRCCGErGQdw1zbk5Ywqb2As2yfJGkr4BxJm9ruAZC0CXAcsGOzE5W5Y/gssBVwbV7eDrgNWFfS0bbP6ccLCCGEpUKLWiWV6Qnis8DklKZvkrQ8qUeJpyWtSRpP51O2H2iWWJlO9JYBNra9h+09SF1gm3Qb8/USx4cQwlLLPW46lfBqTxCSliX1BDGlap9HgB0AJG0MLA/MkTQGuJzUSunGMomVyRjWsv1UYfnpvO4ZYGGZREIIYWlkt2agnno9QUg6WtIuebdDgf0l3QmcD+yX+6M7CNgAOFLSHXl6faP0yhQlXSfpMuCivPyRvG5FYF6J40MIYanV06IHGWr1BGH7yML8vcC7ahz3XeC7fUmrTMbwRWB34N15+WzgkpwTxcNvIYRQR9d2u507t/sr6Yk5A7e2Y9yEEELoOh06QlszZcZ8/hhwK6kI6WPALZI+0u7AQghh6DM9bj51mjJFSd8Atqg8syBpHHANcHE7AwshhG7gDuxWu5kyGcOwqgfZ/kXJsaJDCGFpZsOiRZ13R9BMmYzhCklXkpo/AXycGmMkhBBCeK2hWCVbpvL5q5L2YHEzqNNs/669YYUQwtBnWtft9kAq1Yme7UuAS9ocSwghdBdT9snmjlI3Y5A0n5ThvWYTqRXrSjW2hRBCKBiCJUn1MwbbowYykBBC6EatevJ5IMV4DCGE0Ca26enSVkkhhBD6qRMfYGsmMoYQQmijrmyuGkIIoX/sqGMIIYRQZQjeMETGEEII7WKbRWVG4ukw0edRCCG0UYuG9kTSZEkzJc2SdHiN7RMkXSvpb5LukrRzYdsR+biZkj7QLK24YwghhDZqxZPPkoYDPwHeDzwGTJM0JY/aVvFN0pCfp0qaSOrTbp08vyewCbAGcI2kjWwvqpde3DGEEEK7OPWV1GwqYUtglu0Hbb8MXADs+trUqPRIMRqYned3BS6w/ZLtfwKz8vnqijuGEEJoE1P6jmGspOmF5dNsn1ZYHg88Wlh+DHhH1TmOAq6SdDCwIvC+wrE3Vx07vlEwkTGEEELbuOxzDHNtT1rCxPYCzrJ9kqStgHMkbdqfE0XGEEII7WJa1SrpcWCtwvKaeV3RZ4HJALZvkrQ8MLbksb1EHUMIIbRJpSipBa2SpgEbSlpX0rKkyuQpVfs8AuwAIGljYHlgTt5vT0nLSVoX2BC4tVFicccQQgjt0qLxGGy/Iukg4EpgOHCm7RmSjgam254CHAr8QtIhKWX2cyrHmiHpQuBe4BXgi41aJMEgZgy5guSLwCLgcttfq7HPZOBHpDfidNvH5vXnAZOAhaSc7/O2Fw5U7CGEUI5b1ome7alUDats+8jC/L0sHmmz+thjgGPKpjUoRUmStic1odrM9ibAiTX2qbTb3QmYCOyV2+MCnAe8CXgz8DrgcwMRdwgh9FWrHnAbSINVx3AgcKztlwBsP11jn7rtdm1PdUa6Y1hzgOIOIYTSTOoWo9nUaQYrY9gI2EbSLZKul7RFjX1qtdvt1fZW0ghgH+CKeglJOkDSdEnT58yZ04LQQwihJMOiV3qaTp2mbXUMkq4B3lBj0zdyuqsA7wS2AC6UtJ77nnX+FLjB9l/q7ZAfEjkNYNKkSZ2XNYcQulhn3hE007aMwfb76m2TdCDw20pRkKQeUnvb4iV9w7a3kr4FjAM+38q4QwihVWxwT+fdETQzWEVJlwLbA0jaCFgWmFu1T912u5I+B3wA2Mv20HvXQwhLjZ4eN506zWBlDGcC60m6h1SpvK9tS1pD0lRI7XaBSrvd+0i9Bs7Ix/8MWA24SdIdko58bRIhhDD4hmLl86A8x5BbGX2yxvrZwM6F5de0283r48G8EELHs01PB1YuNxM/sCGE0EY9Q7C0OzKGEEJolxZ1iTHQImMIIYQ2MZ35ZHMzkTGEEEIbdWLlcjORMYQQQrsYeobgcwyRMYQQQpsY07OoYQ/XHSkyhhBCaJeofA4hhFBtKGYMMbRnCCG0jelxT9OpDEmTJc2UNEvS4TW2n5x7grhD0t8lzStsO17SDEn3SfqxJDVKK+4YQgihTdyioqTCwGXvJw1BME3SlDxqW07LhxT2Pxh4a57fmjSy21vy5r8C7wGuq5de3DGEEEIbuaen6VRC3YHL6tgLOL8SArA8qbPS5YARwFONEos7hhBCaBe3rFVSrYHL3lFrR0lrA+sCf04h+CZJ1wJPAAJOsX1fo8QiYwghhDYxlO1We6yk6YXl0/IgY/2xJ3Cx7UUAkjYANmbxEMhXS9qm0QBnkTGEEEK7lB+oZ67tSQ22Nxy4rMqewBcLy7sBN9teACDpj8BWQN2MIeoYQgihbVJfSc2mEuoOXFYk6U3AysBNhdWPAO+RtIykEaSK54ZFSZExhBBCG9k9Tafm56g9cJmkoyXtUth1T+AC9+6g6WLgAeBu4E7gTtt/aJReFCWFEEKbpIF6WtMlRq2By2wfWbV8VI3jFgGf70taGoo9//WXpDnAw304ZCyvHYu6k3R6fBAxtkqnx9jp8UHfY1zb9rglSVDSFTndZubanrwkabXSUpUx9JWk6U0qhAZVp8cHEWOrdHqMnR4fDI0YO0XUMYQQQuglMoYQQgi9RMbQWH8fMBkonR4fRIyt0ukxdnp8MDRi7AhRxxBCCKGXuGMIIYTQS2QMIYQQelnqMgZJ35F0Vx7M4ipJa9TZb19J/8jTvoX1x0h6VNKCqv33kzSnMFDG5zowxuUk/SYP9HGLpHUGMca3S7o7x/LqwCGSjpL0eOF93LkDY1xF0tV5/6slrdzP+E6QdH+O8XeSxtTZ78uS7skDrfxPYf1mkm7KMf5B0kp5/TqSXiy8hz/rT3ztjDFvOyK/tzMlfWAQY9xc0s35vZouacu8fjtJzxXexyNrnbcr2V6qJmClwvyXgJ/V2GcV4MH8d+U8v3Le9k5gdWBB1TH7kbqz7eQYv1A5F+nR+d8MYoy35jgF/BHYKa8/CjisQ97HejEeDxye5w8HjutnfDsCy+T542qdB9gUuAdYgdRTwTXABnnbNOA9ef4zwHfy/DrAPS16D9sV40RS9wzLkbqIfgAYPkgxXlX43+4MXJfntwMua8X7ONSmpe6OwfbzhcUVST3jVvsAcLXtZ2w/C1wNTM7H32z7iSEa467A2Xn+YmCHylXwQMYoaXXSj/bNTt/AXwEf7k8cgxRj8X08u7+x277KqQ8cgJtZ3C1y0cbALbZfyPteD+yet20E3JDnrwb26E8cgxTjrqQ+fV6y/U9gFmkwmsGI0UDlTmY0MLs/cXSTpS5jgMVFLcDeQK3bw1qDYowvceo98u3sxZLWar77gMf46jH5y/EcsOogxDg+z1evrzgov49n9reYps0xrlbIeJ8EVluSGLPPkO5Kqt0DbCNpVUkrkK5oK5+tGSwexeuj9O6WeV1Jf5N0vaRtWhBfq2Ps73esHTH+D3BC/pycCBxROG4rSXdK+qOkTVoQ35DQlRmDpGtyWWL1tCuA7W/YXgs4j9RjYSv8AVjH9ltIV0ZnN9p5kGLsk0GK8VRgfWBz0ohTJ3VgjK/KdxN123w3iy/v8w3glRxj9fnvIxWPXAVcAdwBVHpl+wzwBUm3AaOAl/P6J4AJtt8KfAX4dbFsv0Ni7JM2x3ggcEj+nBwCnJHX307qL2kz4P+AS/sT+5A02GVZgzkBE6hRFksaL/XnheWfA3tV7bOgwXmHA891WoykLnu3yvPLkDoU00DHSKr/uL/efoX169Q672DHCMwEVs/zqwMzlyCu/Uh9569Qcv/vAV+osX4j4NY6x1wHTOqkGElX5UfU+mwOdIykO+fKM10Cnq9zzEPA2FZ8Hjt9GvQABvwFw4aF+YNJQ+BV77MK8E9SZeTKeX6Vqn2qf3RXL8xXRkzqtBi/SO/K5wsHK0ZeW7G7c4338RBSOXSnxXgCvSufj+9nfJOBe4FxTfZ7ff47AbgfGFO1fhipDuQzeXkcuSIXWI800tcqHRbjJvSufH6Q/lc+L2mM9wHb5fkdgNvy/BtYnGFsSRrwZokvpIbCNOgBDPgLhktI5Y13kYp/xuf1k4DTC/t9hlQhNgv4dGH98aTy0J7896i8/vuk8tQ7gWuBN3VgjMsDF+X9bwXWG8QYJ+XjHwBOKXwBzyENKHIXaYSq1TswxlWBPwH/ILVu6e+P7ixSOfsdeapk2msAUwv7/YX0w3cnsENh/ZeBv+fp2EJ8e+TP4h2k4pAPLcF72JYY87Zv5Pd2JrlV0CDF+G7gtrz+FuDtef1BLP5O3wxs3d8Yh9oUXWKEEELopSsrn0MIIfRfZAwhhBB6iYwhhBBCL5ExhBBC6CUyhhBCCL1ExhDaSlU9vC7Bec6S9JFWnKtJOv9/u9OoSm+MpC8MZJohNBMZQ1iqSFqm0XbbWw9wmmNIvd6G0DEiYwgDQskJuX+buyV9PK8fJumnuT/9qyVNbXZnoDROwvWSbpN0Ze4JFUn7S5qWOz27JHeWVrnb+JmkW4DjlcZ8OFPSdZIelPSlwrkX5L/b5e0X59jOk14dj2HnvO42pXEaLqsR436Spkj6M/AnSSMl/UnS7fn1V/r4ORZYX6m//xPysV/Nr+MuSd9e0vc+hL5qePUUQgvtTuoYbzNgLDBN0g3Au0h9Ik0EXk/qnuDMeieRNILUodmutufkDOYY0tPLv7X9i7zfd4HP5n0hdcW8te1Fko4C3gRsT+rYbaakU20vrEruraSuG2YDNwLvkjSd1J/Strb/Ken8Bq/5bcBbbD+T7xp2s/28pLHAzZKmkLrU2NT25jnuHYENSV0wCJgiaVvbN9RJI4SWi4whDJR3A+fbXgQ8Jel6YIu8/iLbPcCTkq5tcp43kgZduTpfwA8n9SYKsGnOEMYAI0kds1VclNOuuNz2S8BLkp4mdZ1d7GYbUodvjwFIuoOUgS0AHnQaQwDgfOCAOrFebfuZPC/ge5K2JXVVMp7a3XXvmKe/5eWRpIwiMoYwYCJjCEONgBm2t6qx7Szgw7bvlLQfaQSuin9X7ftSYX4Rtb8LZfZppJjm3qTO7d5ue6Gkh0h9V1UT8H3bP+9jWiG0TNQxhIHyF+DjkoZLGgdsS+rI70bSAEfDJK1G7x/zWmYC4yRtBaloSYsHUBkFPJGLm/Zux4vI6a+nxeNlf7zkcaOBp3OmsD2wdl4/nxR3xZXAZySNBJA0XtLrlzjqEPog7hjCQPkdsBWpp0oDX7P9pKRLSF0d30vqIfN2Uv/4Ndl+OVdO/1jSaNJn+IekXjD/P1LvmHPy31H1ztNftl/MzUuvkPRv0pjGZZwH/EHS3cB0UrfP2P6XpBsl3QP80fZXJW0M3JSLyhYAnwSebvVrCaGe6F01DDpJI20vkLQq6S7iXbafHOy46inEK+AnwD9snzzYcYXQKnHHEDrBZZLGAMsC3+nkTCHbX9K+pHj/RmqlFELXiDuGEEIIvUTlcwghhF4iYwghhNBLZAwhhBB6iYwhhBBCL5ExhBBC6OX/AWZccYjzKuUrAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualize the cross-validation results\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "x_scatter = [math.log10(x[0]) for x in results]\n",
    "y_scatter = [math.log10(x[1]) for x in results]\n",
    "\n",
    "\n",
    "# plot validation accuracy\n",
    "marker_size=100\n",
    "colors = [results[x] for x in results] # default size of markers is 20\n",
    "\n",
    "plt.scatter(x_scatter, y_scatter, marker_size, c=colors, cmap=plt.cm.coolwarm)\n",
    "plt.colorbar()\n",
    "plt.xlabel('log learning rate')\n",
    "plt.ylabel('log regularization strength')\n",
    "plt.title('Net intrusion(CIC-IDS-2017) validation accuracy')\n",
    "#plt.savefig('MachineLearningCVE/5_layers_relu_{}th_run.png'.format(run))\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}