data_loader_multi.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Loader For Multi Files\n",
    "\n",
    "Combine single data loader."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import dgl\n",
    "import torch\n",
    "import collections\n",
    "import numpy as np\n",
    "import networkx as nx\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tool Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Label Function\n",
    "\n",
    "Converge TCP attack type to int. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label(flag: str) -> int:\n",
    "    return {\n",
    "        '-': 0,\n",
    "        'snmpgetattack': 1,\n",
    "        'warez': 2,\n",
    "        'portsweep': 3,\n",
    "        'nmap': 4,\n",
    "        'warezclient': 5,\n",
    "        'ipsweep': 6,\n",
    "        'dict': 7,\n",
    "        'neptune': 8,\n",
    "        'smurf': 9,\n",
    "        'pod': 10,\n",
    "        'snmpguess': 11,\n",
    "        'teardrop': 12,\n",
    "        'satan': 13,\n",
    "        'httptunnel-e': 14,\n",
    "        'ignore': 15,\n",
    "        'mscan': 16,\n",
    "        'guest': 17,\n",
    "        'rootkit': 18,\n",
    "        'back': 19,\n",
    "        'apache2': 20,\n",
    "        'processtable': 21,\n",
    "        'mailbomb': 22,\n",
    "        'smurfttl': 23,\n",
    "        'saint': 24\n",
    "    }.get(flag[:-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Line Split Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def line_split(line: str) -> tuple:\n",
    "    src_ip, dst_ip, port, time, connect_type = line.split('\\t')\n",
    "    return int(src_ip), int(dst_ip), int(port), int(time), label(connect_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data File Path\n",
    "\n",
    "train_set_path = 'data/train'\n",
    "valid_set_path = 'data/valid_query'\n",
    "test_set_path = 'data/test_query'\n",
    "\n",
    "train_set_list = os.listdir(train_set_path)\n",
    "valid_set_list = os.listdir(valid_set_path)\n",
    "test_set_path = os.listdir(test_set_path)\n",
    "\n",
    "file_index = 0\n",
    "\n",
    "# Features\n",
    "\n",
    "src_in_degree = collections.OrderedDict()\n",
    "src_out_degree = collections.OrderedDict()\n",
    "dst_in_degree = collections.OrderedDict()\n",
    "dst_out_degree = collections.OrderedDict()\n",
    "\n",
    "port_num = collections.OrderedDict()\n",
    "port_list = collections.OrderedDict()\n",
    "time_len = collections.OrderedDict()\n",
    "connect_num = collections.OrderedDict()\n",
    "label_list = collections.OrderedDict()\n",
    "\n",
    "# Tool Parameters\n",
    "\n",
    "nodes_origin = []\n",
    "edges_origin = {}\n",
    "\n",
    "edges_trans_src = []\n",
    "edges_trans_dst = []\n",
    "\n",
    "node_in_degree = {}\n",
    "node_out_degree = {}\n",
    "\n",
    "node_in_edges = {}\n",
    "node_out_edges = {}\n",
    "\n",
    "edge_src_list = {}\n",
    "edge_dst_list = {}\n",
    "\n",
    "# Count Parameters\n",
    "\n",
    "edge_index = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in train_set_list:\n",
    "    file_index += 1\n",
    "    if file_index == 114:\n",
    "        continue\n",
    "    print('Loading File {}/{}...'.format(file_index, len(train_set_list)), end='\\r')\n",
    "    \n",
    "    file_path = train_set_path + '/' + file\n",
    "    data = open(file_path)\n",
    "    \n",
    "    for line in data:\n",
    "        src_ip, dst_ip, port, time, connect_type = line_split(line)\n",
    "\n",
    "        # Repeat Connections\n",
    "        if (src_ip, dst_ip) in edges_origin.keys():\n",
    "            edge_index_temp = edges_origin[(src_ip, dst_ip)]\n",
    "\n",
    "            # Assign Features\n",
    "            time_len[edge_index_temp] += time\n",
    "            connect_num[edge_index_temp] += 1\n",
    "            label_list[edge_index_temp][connect_type] += 1\n",
    "            if port in port_list[edge_index_temp]:\n",
    "                pass\n",
    "            else:\n",
    "                port_num[edge_index_temp] += 1\n",
    "                port_list[edge_index_temp].append(port)\n",
    "\n",
    "        # New Connections     \n",
    "        else:\n",
    "            # Record Origin Node and Edge\n",
    "            nodes_origin.append(src_ip)\n",
    "            nodes_origin.append(dst_ip)\n",
    "            edges_origin[(src_ip, dst_ip)] = edge_index\n",
    "\n",
    "            # Update Node Parameters\n",
    "            if src_ip in node_out_degree.keys():\n",
    "                node_out_degree[src_ip] += 1\n",
    "            else:\n",
    "                node_out_degree[src_ip] = 1\n",
    "            if dst_ip in node_in_degree.keys():\n",
    "                node_in_degree[dst_ip] += 1\n",
    "            else:\n",
    "                node_in_degree[dst_ip] = 1\n",
    "\n",
    "            # Store Nodes' In-Out Edges\n",
    "            if src_ip in node_out_edges.keys():\n",
    "                node_out_edges[src_ip].append(edge_index)\n",
    "            else:\n",
    "                node_out_edges[src_ip] = [edge_index]\n",
    "            if dst_ip in node_in_edges.keys():\n",
    "                node_in_edges[dst_ip].append(edge_index)\n",
    "            else:\n",
    "                node_in_edges[dst_ip] = [edge_index]\n",
    "\n",
    "            # Update Edge Parameters\n",
    "            if edge_index in edge_src_list.keys():\n",
    "                edge_src_list[edge_index].append(src_ip)\n",
    "            else:\n",
    "                edge_src_list[edge_index] = [src_ip]\n",
    "            if edge_index in edge_dst_list.keys():\n",
    "                edge_dst_list[edge_index].append(dst_ip)\n",
    "            else:\n",
    "                edge_dst_list[edge_index] = [dst_ip]\n",
    "\n",
    "            # Assign Features\n",
    "            port_num[edge_index] = 1\n",
    "            port_list[edge_index] = [port]\n",
    "            time_len[edge_index] = time\n",
    "            connect_num[edge_index] = 1\n",
    "            label_list[edge_index] = [0 * x for x in range(25)]\n",
    "            label_list[edge_index][connect_type] += 1\n",
    "\n",
    "            edge_index += 1\n",
    "            \n",
    "print('Data Loading Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove Repeats\n",
    "nodes_origin = list(set(nodes_origin))\n",
    "for i in range(len(edge_src_list)):\n",
    "    edge_src_list[i] = list(set(edge_src_list[i]))\n",
    "for i in range(len(edge_dst_list)):\n",
    "    edge_dst_list[i] = list(set(edge_dst_list[i]))\n",
    "    \n",
    "# Create Transform Graph\n",
    "for node in nodes_origin:\n",
    "    if not node in node_in_edges.keys() or not node in node_out_edges.keys():\n",
    "        continue\n",
    "    if not node_in_edges[node] or not node_out_edges[node]:\n",
    "        continue\n",
    "    for src_edge in node_in_edges[node]:\n",
    "        for dst_edge in node_out_edges[node]:\n",
    "            edges_trans_src.append(src_edge)\n",
    "            edges_trans_dst.append(dst_edge)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Features\n",
    "for edge in edge_src_list.keys():\n",
    "    if edge_src_list[edge][0] not in node_in_degree.keys():\n",
    "        src_in_degree[edge] = 0\n",
    "    else:\n",
    "        src_in_degree[edge] = node_in_degree[edge_src_list[edge][0]]\n",
    "    if edge_src_list[edge][0] not in node_out_degree.keys():\n",
    "        src_out_degree[edge] = 0\n",
    "    else:\n",
    "        src_out_degree[edge] = node_out_degree[edge_src_list[edge][0]]\n",
    "    \n",
    "for edge in edge_dst_list.keys():\n",
    "    if edge_dst_list[edge][0] not in node_in_degree.keys():\n",
    "        dst_in_degree[edge] = 0\n",
    "    else:\n",
    "        dst_in_degree[edge] = node_in_degree[edge_dst_list[edge][0]]\n",
    "    if edge_dst_list[edge][0] not in node_out_degree.keys():\n",
    "        dst_out_degree[edge] = 0\n",
    "    else:\n",
    "        dst_out_degree[edge] = node_out_degree[edge_dst_list[edge][0]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normalization Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "src_in_degree_norm = list(src_in_degree.values()) / np.linalg.norm(list(src_in_degree.values()))\n",
    "src_out_degree_norm = list(src_out_degree.values()) / np.linalg.norm(list(src_out_degree.values()))\n",
    "dst_in_degree_norm = list(dst_in_degree.values()) / np.linalg.norm(list(dst_in_degree.values()))\n",
    "dst_out_degree_norm = list(dst_out_degree.values()) / np.linalg.norm(list(dst_out_degree.values()))\n",
    "\n",
    "port_num_norm = list(port_num.values()) / np.linalg.norm(list(port_num.values()))\n",
    "time_len_norm = list(time_len.values()) / np.linalg.norm(list(time_len.values()))\n",
    "connect_num_norm = list(connect_num.values()) / np.linalg.norm(list(connect_num.values()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Feature Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_matrix = np.empty([1, 7])\n",
    "for i in range(len(edges_origin)):\n",
    "    feature_matrix = np.vstack((feature_matrix, [src_in_degree[i], src_out_degree[i], dst_in_degree[i], dst_out_degree[i], \\\n",
    "                                                 port_num[i], time_len[i], connect_num[i]]))\n",
    "feature_matrix = np.delete(feature_matrix, obj=0, axis=0)    \n",
    "feature_matrix = feature_matrix.astype(np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Normalized Feature Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_matrix = np.empty([1, 7])\n",
    "for i in range(len(edges_origin)):\n",
    "    feature_matrix = np.vstack((feature_matrix, [src_in_degree_norm[i], src_out_degree_norm[i], dst_in_degree_norm[i], dst_out_degree_norm[i], \\\n",
    "                                                 port_num_norm[i], time_len_norm[i], connect_num_norm[i]]))\n",
    "feature_matrix = np.delete(feature_matrix, obj=0, axis=0)    \n",
    "feature_matrix = feature_matrix.astype(np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use Normalized Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = dgl.DGLGraph()\n",
    "\n",
    "# Create Nodes and Edges\n",
    "\n",
    "g.add_nodes(len(edges_origin))\n",
    "g.add_edges(edges_trans_src, edges_trans_dst)\n",
    "\n",
    "# Assign Features\n",
    "\n",
    "g.ndata['src_in_degree'] = torch.tensor(src_in_degree_norm)\n",
    "g.ndata['src_out_degree'] = torch.tensor(src_out_degree_norm)\n",
    "g.ndata['dst_in_degree'] = torch.tensor(dst_in_degree_norm)\n",
    "g.ndata['dst_out_degree'] = torch.tensor(dst_out_degree_norm)\n",
    "g.ndata['port_num'] = torch.tensor(port_num_norm)\n",
    "g.ndata['time_len'] = torch.tensor(time_len_norm)\n",
    "g.ndata['connect_num'] = torch.tensor(connect_num_norm)\n",
    "\n",
    "g = dgl.add_self_loop(g)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Graph Status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nodes Number\t39899\n",
      "Edges Number\t1495966\n"
     ]
    }
   ],
   "source": [
    "print('Nodes Number\\t{}\\nEdges Number\\t{}'.format(g.number_of_nodes(), g.number_of_edges()))\n",
    "# print(time_len_norm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Show Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "options = {\"node_size\": 10}\n",
    "nx.draw(g.to_networkx(), with_labels=False, **options)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test on GCN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from dgl.nn.pytorch import GraphConv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GCN(nn.Module):\n",
    "    def __init__(self, in_feats, hidden_size, num_classes):\n",
    "        super(GCN, self).__init__()\n",
    "        self.conv1 = GraphConv(in_feats, hidden_size)\n",
    "        self.conv2 = GraphConv(hidden_size, num_classes)\n",
    "\n",
    "    def forward(self, g, inputs):\n",
    "        h = self.conv1(g, inputs)\n",
    "        h = torch.relu(h)\n",
    "        h = self.conv2(g, h)\n",
    "        return h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = GCN(7, 16, 25)\n",
    "\n",
    "inputs = torch.tensor(feature_matrix)\n",
    "labels = torch.tensor(list(label_list.values()), dtype=torch.float32)\n",
    "\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr=0.1)\n",
    "loss_func = nn.MSELoss()\n",
    "\n",
    "# Recoder\n",
    "loss_list = []\n",
    "\n",
    "for epoch in range(3000):\n",
    "    outputs = net(g, inputs)\n",
    "    loss = loss_func(outputs, labels)\n",
    "    \n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    loss_list.append(loss.item() / 39899)\n",
    "    \n",
    "#     print('Epoch %d | Loss: %.4f' % (epoch, loss.item()))\n",
    "    if epoch % 100 == 0:\n",
    "        print('Epoch %d | Loss: %.4f' % (epoch, loss.item()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAfPUlEQVR4nO3deXhU9b3H8fd3si9AAgmRPewQFgWDgtXiriju1l1p61Jtr1vvrdW61Vqt2lbt9VqtikvdW9RK3SrudYOGfd8RErawJCwJZPvdP3KAEImEzGTOnOTzeh6eOXPmkPPJeYYPM7/5zTnmnENERIIn5HcAERFpGhW4iEhAqcBFRAJKBS4iElAqcBGRgIqP5s6ysrJcbm5uNHcpIhJ4U6dO3eCcy66/PqoFnpubS0FBQTR3KSISeGb2zb7WawhFRCSgVOAiIgGlAhcRCSgVuIhIQKnARUQCar8FbmZPm9l6M5tTZ93vzWyBmc0yszfMLKNZU4qIyLc05hX4s8DJ9dZNAgY754YCi4BbIpxLRET2Y7/zwJ1zn5lZbr1179e5+zVwboRz7eXD+euYXVRKSkIcKYlxpCTE0aldCl0yU+iamUJCnEaCRKT1icQXeX4MvNrQg2Z2FXAVQPfu3Zu0g08WFvP81/ucx05SfIihXdtxaI/2nJCXw7BuGYRC1qT9iIgEiTXmgg7eK/C3nHOD662/FcgHznaN+EH5+fmuqd/ErK5x7Kispryymu07q1hTuoPCzeUsWLOFaSs3M7uolMpqR07bJM4Z3pWLR/agS0ZKk/YlIhJLzGyqcy6//vomvwI3sx8CY4HjGlPe4YoLGWlJ8aQlxZOVnkSPDml7Pb5lRyUfzV/PW7NW8/inS3n806WMGdKJG4/vR5+O6c0dT0Qk6ppU4GZ2MnATMNo5VxbZSE3TNjmBM4d14cxhXSjcXMbzX3/D8199w7uz13DO8K784uT+dGyT7HdMEZGI2e8Qipm9DBwNZAHrgDupnXWSBGz0NvvaOXf1/nYWzhBKU2zYtpPHPlnK8199Q1JCiJvHDODCEd01Ri4igdLQEEqjxsAjJdoFvsvS4m3c9sYcvlq2kcN6tueh8w/R+LiIBEZDBd4q5t/1zk7npSsP54FzhzK3qJQxD3/Gu7PX+B1LRCQsraLAAcyM8/K78fZ1R5GblcY1L07jt2/No7omeu9AREQiqdUU+C65WWlMuPoIxo3qwVOfL+fHz/6HLTsq/Y4lInLAWl2BAyTGh7jrjMHcc9ZgvliygbMe/YLCzTExmUZEpNFaZYHvcvHhPXj+8sNZv3UnP3j8K5as3+p3JBGRRmvVBQ4wqncHXr1qFJXVjh88/hWzCkv8jiQi0iitvsAB8jq3ZcLVo0hLiueiJyczbeVmvyOJiOyXCtyz68PNDumJjHt6CrMLS/2OJCLynVTgdRzULpmXrhxJ2+QELn16MvNWb/E7kohIg1Tg9XTJSOHlK0eSkhDHpeMns2LDdr8jiYjskwp8H7p3SOXFKw6nxjnGPTOFDdt2+h1JRORbVOAN6JWdzvgfjmBt6Q4uf/Y/lFVU+R1JRGQvKvDvMLx7Jo9cOIzZRaVc+9J0qqpr/I4kIrKbCnw/Thx0EHedPogPF6znd+8u8DuOiMhukbgmZot36ahclhZvZ/znyxnYqS3nHtrV70giInoF3li3njqQI3p34FdvzGa6vugjIjFABd5ICXEhHr1oODltk/jJ81NZt2WH35FEpJVTgR+AzLREnrwsn207q7jmhalU6kNNEfGRCvwADTioLfefM5RpK0v4/b8W+h1HRFoxFXgTnHZwZy4Z2Z0nPlvGB/PW+R1HRFopFXgT3XZqHoM6t+W//z5TF4MQEV+owJsoOSGORy8aTnWN42cvTaeiSuPhIhJdKvAw5Gal8cC5Q5m5qoQHJy3yO46ItDIq8DCdMqQTF4zoxl8+W8rkZRv9jiMirYgKPAJuH5tH9/ap/PxvM3WFexGJGhV4BKQlxfPQ+YewdssO7nxzrt9xRKSV2G+Bm9nTZrbezObUWdfezCaZ2WLvNrN5Y8a+4d0zufbYPrwxvYiJM1f7HUdEWoHGvAJ/Fji53rqbgQ+dc32BD737rd5/HdOHQ7plcNsbs1ldUu53HBFp4fZb4M65z4BN9VafATznLT8HnBnZWMEUHxfi4fMPobLaccvrs3HO+R1JRFqwpo6B5zjn1njLa4GchjY0s6vMrMDMCoqLi5u4u+DIzUrjppP78+miYiZMLfQ7joi0YGF/iOlqX2Y2+FLTOfeEcy7fOZefnZ0d7u4CYdyoXEbkZnL3W/N01kIRaTZNLfB1ZtYJwLtdH7lIwRcKGQ+cezA7q2q49Q0NpYhI82hqgU8ExnnL44A3IxOn5eiZlcYvTurPB/PXa1aKiDSLxkwjfBn4CuhvZoVmdjlwH3CCmS0GjvfuSz0/+l5PhnfP4M6JcyneutPvOCLSwjRmFsqFzrlOzrkE51xX59x459xG59xxzrm+zrnjnXP1Z6kIEOcNpZRVVPPrifqCj4hElr6J2cz6dEzn+uP68vbsNXy0QOcOF5HIUYFHwZVH9aJvx3Ru/8dcyiqq/I4jIi2ECjwKEuND3Hv2EIpKyvnTB4v9jiMiLYQKPEpG5LbnwsO68dTny5m3eovfcUSkBVCBR9EvTx5AZmoCv3pjNtU1mhsuIuFRgUdRRmoit4/NY8aqEl6a/I3fcUQk4FTgUXb6wZ05qm8WD7y3UF+zF5GwqMCjzMz47ZmDqaiu4e635vkdR0QCTAXugx4d0vjp0X14a9Yavly6we84IhJQKnCf/GR0L7q1T+HON+dSWV3jdxwRCSAVuE+SE+K4c+wgFq/fxrNfrPA7jogEkArcR8fn5XDsgI48/MEi1usDTRE5QCpwn90xNo/Kase978z3O4qIBIwK3Ge5WWn8ZHQv/jFjNZOXbfQ7jogEiAo8Bvz06D50yUjhzolzqdIHmiLSSCrwGJCSGMftY/NYsHYrf/1K39AUkcZRgceIkwblcFTfLB6atEhX7xGRRlGBxwgz49enD6K8spo/vr/Q7zgiEgAq8BjSOzudcUfk8mrBKuYUlfodR0RinAo8xlx3XF8yUxP5zT/n4ZxOOSsiDVOBx5h2KQn894n9mLJiE+/MXut3HBGJYSrwGHTBiO4MOKgN974znx2V1X7HEZEYpQKPQXEh447T8igqKeepfy/zO46IxCgVeIw6oncWJw3K4c+fLNWFH0Rkn1TgMezWU/Koqnbc/94Cv6OISAxSgcew7h1Sufyonrw+rYjpKzf7HUdEYowKPMb97Jg+ZLdJ4jdvaVqhiOwtrAI3sxvNbK6ZzTGzl80sOVLBpFZ6Ujy/OKk/01eW8OaM1X7HEZEY0uQCN7MuwHVAvnNuMBAHXBCpYLLHucO7MqRLO+57dwFlFVV+xxGRGBHuEEo8kGJm8UAqoJeIzSAUMu48LY+1W3bw+KeaVigitZpc4M65IuAPwEpgDVDqnHu//nZmdpWZFZhZQXFxcdOTtnL5ue0ZO7QTT3y2lNUl5X7HEZEYEM4QSiZwBtAT6Aykmdkl9bdzzj3hnMt3zuVnZ2c3Palw85gB1Dh4QNMKRYTwhlCOB5Y754qdc5XA68ARkYkl+9I1M5Wrjqq9/No0TSsUafXCKfCVwEgzSzUzA44DdGXeZnbN0b1rpxXqbIUirV44Y+CTgQnANGC297OeiFAuaUBaUjw3ndSfGatKmDhTnxmLtGZhzUJxzt3pnBvgnBvsnLvUOadrgUXBOcO7MrhLW+5/dwHlFTpboUhrpW9iBlAoZNx+ah6rS3fwpM5WKNJqqcAD6vBeHThlyEE89slS1pbqbIUirZEKPMBuGTOQ6hrHA//StEKR1kgFHmDd2u85W+HMVSV+xxGRKFOBB9xPj+5NVnoSd+tshSKtjgo84NokJ/A/J/aj4JvNvD17jd9xRCSKVOAtwA/yuzGwU1t+984CXQRZpBVRgbcAcSHj9rEDKSopZ/zny/2OIyJRogJvIXZdBPnRj5ewXhdBFmkVVOAtyK9OGUhldQ2//9dCv6OISBSowFuQHh3S+PH3ejJhWiGzC0v9jiMizUwF3sL87Ng+tE9N1LRCkVZABd7CtE1O4Ocn9mPKik28N2et33FEpBmpwFug8/O7MeCgNtz77nxNKxRpwVTgLVB8XIjbx+axalM5z3yxwu84ItJMVOAt1Pf6ZHH8QG9a4VZNKxRpiVTgLditpw5kZ1U1D76/yO8oItIMVOAtWM+sNMaNyuXVglXMXa1phSItjQq8hbv2uL5kpCRoWqFIC6QCb+HapSTw8xP68fWyTbw/b53fcUQkglTgrcCFh3Wnb8d07n1nPjurNK1QpKVQgbcCu6YVfrOxjOe+XOF3HBGJEBV4K/H9ftkcO6Ajj3y4hA3bdvodR0QiQAXeivzqlIGUV1bz4CRNKxRpCVTgrUifjulcMrIHr0xZyYK1W/yOIyJhUoG3Mjcc35c2yZpWKNIShFXgZpZhZhPMbIGZzTezUZEKJs0jIzWRG4/vyxdLNvLh/PV+xxGRMIT7CvxPwHvOuQHAwcD88CNJc7t4ZA96Z6dxzzvzqaiq8TuOiDRRkwvczNoB3wfGAzjnKpxzJRHKJc0oIS7EbWPzWL5hO3/9aoXfcUSkicJ5Bd4TKAaeMbPpZvaUmaXV38jMrjKzAjMrKC4uDmN3EknH9O/I6H7Z/OnDxWzaXuF3HBFpgnAKPB4YDjzmnBsGbAdurr+Rc+4J51y+cy4/Ozs7jN1JpN126kDKKqp5SNMKRQIpnAIvBAqdc5O9+xOoLXQJiL45bbj48O68OPkbFq3b6nccETlATS5w59xaYJWZ9fdWHQfMi0gqiZobju9HelK8phWKBFC4s1CuBV40s1nAIcC9YSeSqGqflsj1x/fj34s38MlCfUYhEiRhFbhzboY3vj3UOXemc25zpIJJ9Fw6sge9stK4++15VFZrWqFIUOibmEJifIhbTx3IsuLtOluhSICowAWAYwfUTit8+IPFrNuiiyCLBIEKXAAwM+46fRAV1TXc87a+UCsSBCpw2S03K41rRvdm4szVfLFkg99xRGQ/VOCyl2uO7k339qnc8eYcnSdFJMapwGUvyQlx3HX6IJYWb+epz5f5HUdEvoMKXL7lmAEdOTEvh0c+XELh5jK/44hIA1Tgsk93nJYHwG/+qS/XisQqFbjsU9fMVK49rg/vz1vHxwt04QeRWKQClwZdcWQvemencefEueyorPY7jojUowKXBiXGh7j7jMGs3FTGnz9e4nccEalHBS7f6Yg+WZw1rAuPfbpUp5wViTEqcNmv204dSHpSPDe/NouaGp1yViRWqMBlvzqkJ3HbqXlMW1nCi5O/8TuOiHhU4NIoZw/vwlF9s7j/vYWsKS33O46IoAKXRjIz7jlzCFU1Ndzx5lxdvUckBqjApdG6d0jlxuP7MWneOt6bs9bvOCKtngpcDsjlR/ZkUOe23DFxLqXllX7HEWnVVOByQOLjQtx39lA2btvJfe8u8DuOSKumApcDNqRrO644qhcvT1mp84aL+EgFLk3y8xP60SsrjZsmzGLrDg2liPhBBS5NkpwQxx/OO5g1peXc+46GUkT8oAKXJhvePZMrvaGUzxYV+x1HpNVRgUtYbjyhH72z0/jla7PYoqEUkahSgUtYkhPi+ON5h7Buyw5++5Yu/iASTSpwCdsh3TK4enRv/lZQqIs/iESRClwi4vrj+9IvJ51fvjaLTdsr/I4j0iqEXeBmFmdm083srUgEkmBKio/jwfMOYXNZBbe8PkvnShGJgki8Ar8emB+BnyMBN7hLO35xUn/+NXcdr/5nld9xRFq8sArczLoCpwJPRSaOBN0VR/biiN4duOuf81hWvM3vOCItWrivwB8GbgJqGtrAzK4yswIzKygu1lzhli4UMh487xAS40Pc8OoMKqsbfGqISJiaXOBmNhZY75yb+l3bOeeecM7lO+fys7Ozm7o7CZCD2iVz39lDmFVYysMfLPI7jkiLFc4r8O8Bp5vZCuAV4FgzeyEiqSTwxgzpxPn53fjzJ0v5etlGv+OItEhNLnDn3C3Oua7OuVzgAuAj59wlEUsmgXfHaXnkdkjj+lems2HbTr/jiLQ4mgcuzSYtKZ5HLxpOSVklN746g2pd0V4koiJS4M65T5xzYyPxs6RlyevclrtOH8S/F2/g0Y+X+B1HpEXRK3BpdueP6MZZw7rw0AeL+FIXgBCJGBW4NDsz47dnDqZ3djrXvTKD9Vt2+B1JpEVQgUtUpCXF89jFw9m+s4prX56u+eEiEaACl6jpm9OGe88ezOTlm7jnbZ19QSRc8X4HkNblrGFdmVO0hfGfLyevc1vOy+/mdySRwNIrcIm6W8YM4Mg+Wdz2xhymr9zsdxyRwFKBS9TFx4V45MJh5LRL4uoXpupDTZEmUoGLLzLTEnnysny2lFdx9QtT2VlV7XckkcBRgYtvBhzUlgfPO5hpK0u4aYIuAiFyoFTg4qsxQzpx08n9eXPGah6cpDMXihwIzUIR310zujcrN5bxyEdL6JaZynkjNDNFpDFU4OI7M+PuMwdTVFLOr96YTeeMFI7sm+V3LJGYpyEUiQkJcSH+fPFw+nRM55oXpjKnqNTvSCIxTwUuMaNNcgJP/3AEbZLjGff0FF1TU2Q/VOASUzpnpPDCFYcDcMlTk1ldUu5zIpHYpQKXmNMrO53nfnwYW3dUccn4yWzU1XxE9kkFLjFpcJd2PP2jEawuKWfcM1MoLa/0O5JIzFGBS8wakduexy45lIVrt3Lp+MmUlqnERepSgUtMO6Z/Rx6/5FAWrNnKxeO/pqSswu9IIjFDBS4x77iBOfzl0kNZtG4bFz05mc3bVeIioAKXgDhmQEeevCyfJcXbuOipyWzQB5siKnAJjtH9shk/Lp/lG7Zx7mNfsmpTmd+RRHylApdAOapvNi9eMZLNZZWc/diXzFu9xe9IIr5RgUvgHNojkwlXjyI+ZJz/l6/4etlGvyOJ+EIFLoHUN6cNr11zBDntkrls/BRen1bodySRqFOBS2B1zkhhwtWjGN4jg5//bSb3vbuAmhpdFEJajyYXuJl1M7OPzWyemc01s+sjGUykMTJSE3n+8sO56PDuPP7pUq56firbdlb5HUskKsJ5BV4F/LdzLg8YCfzMzPIiE0uk8RLiQtxz5mDuOn0QHy9cz9l//oKlOpOhtAJNLnDn3Brn3DRveSswH+gSqWAiB8LMGHdELs/96DA2bKvgtEc+580ZRX7HEmlWERkDN7NcYBgweR+PXWVmBWZWUFxcHIndiTToyL5ZvHPdUQzq3JbrX5nBLa/PZkelrngvLVPYBW5m6cBrwA3OuW9NynXOPeGcy3fO5WdnZ4e7O5H9OqhdMi9fOZKrR/fm5SkrOfPRL5i7Wlf4kZYnrAI3swRqy/tF59zrkYkkEr74uBA3jxnAMz8awcbtFZzxf1/wyIeLqaqu8TuaSMSEMwvFgPHAfOfcg5GLJBI5x/TvyPs3fJ9ThnTij5MWcc5jX7J43Va/Y4lERDivwL8HXAoca2YzvD+nRCiXSMRkpiXyvxcO49GLhrNyUxlj/vRv7n9vAeUVGhuXYItv6l90zn0OWASziDSrU4d2YmSv9vzu3QU89slSJs5Yza9PH8QJeTl+RxNpEn0TU1qVDulJ/OEHB/O3n4wiLSmOK/9awKXjJzOnSB9ySvCowKVVOqxne96+7ijuGJvH7KJSTvu/z7nx1RkUbtYpaiU4zLnonTsiPz/fFRQURG1/Io1RWl7J458u5enPl+McnHNoV64Z3ZvuHVL9jiYCgJlNdc7lf2u9Clyk1uqSch79eAl/Lyik2jlOP7gz1xzdm345bfyOJq2cClykkdZt2cFT/17Gi5NXUlZRzdH9s7lsVA9G9+tIXEif20v0qcBFDtDm7RU899UKXpq8kvVbd9KtfQoXH96Ds4d1oWPbZL/jSSuiAhdposrqGt6fu46/frWCycs3ETI4sm82Zw/rwomDckhNbPJsXJFGUYGLRMDS4m38Y3oRb0wvonBzOSkJcYzul80JeTkcO6AjmWmJfkeUFkgFLhJBNTWOgm82M3FmEZPmrWPdlp3EhYz8Hpkc3b8j3+vTgUGd22nMXCJCBS7STGpqHLOLSpk0bx0fzF/HgrW151ppmxzPyF4dOKxnew7ulsGgzm013CJNogIXiZL1W3fw1dKNfLlkI18s3UDh5nIAQgb9ctowtGs7hnbNYGjXdvTpmK5Sl/1SgYv4ZP3WHcwuLGVmYSmzCkuYVVjKpu0Vux/vkpFC747p9MlOp2d2Gl0zU+iWmUrXzBSSE+J8TC6xoqEC13/9Is2sY5tkjhuYzHEDa0+a5ZyjcHM5c1eXsnjdNpYUb2PJ+m1MWb6RHZV7n688Kz2JTu2S6dgmiY5tk8huU7uc3SaJjJQE2qUmkJGSSEZqgsq+FVKBi0SZmdGtfSrd2qdy8uA962tqHMXbdrJqUxmFm8sp3FzGqk3lrN2yg9WlO5hZWMLG7RU09KY5KT5Eu5QE2iTHk5oYT0pCHCmJcaQm1t6mJOxajq+9TYgjIS5EfMiIjzPidy2HjIS4EHHe+t3bhELefSMuVLsuFDJCBiEzzPvdrM79kBkWYs+y9xje/ZDV/p1dt3JgVOAiMSIUMnLaJpPTNpn83H1vU1Vdw8btFRRv3cmW8kpKyispKaukpLyC0rJKNpdVsH1nNWUVVZRVVLO5rILVJdWUVVRTXllNuXcb68z2nKvadv/n4N2n9sH66/b1d9j9+J7/XPa6X2d/fOtn7Pm5df/OXhm9bepm3rXNrv3vWnfvWUM4rGf7cA/NXlTgIgESHxfaXfJNVVPj2FFVW+ZVNY7K6hqqqh1VNTVU1TiqqmvXVdc4Kuutr6r2lmtqqKx2OOeoceAc1DiHo3aIaPf9OreOvbfdlaWmzn0H4P0cbxGH2/2uw9VZx17rGtjGW8bbpqHH3e7t6qxrcJs9+9+T09V7fM+6XRulJUV+iEsFLtLKhEJGamK8Zr+0ADofuIhIQKnARUQCSgUuIhJQKnARkYBSgYuIBJQKXEQkoFTgIiIBpQIXEQmoqJ6N0MyKgW+a+NezgA0RjNPcgpQ3SFkhWHmDlBWClTdIWSG8vD2cc9n1V0a1wMNhZgX7Op1irApS3iBlhWDlDVJWCFbeIGWF5smrIRQRkYBSgYuIBFSQCvwJvwMcoCDlDVJWCFbeIGWFYOUNUlZohryBGQMXEZG9BekVuIiI1KECFxEJqEAUuJmdbGYLzWyJmd3sdx4AM1thZrPNbIaZFXjr2pvZJDNb7N1meuvNzP7Xyz/LzIZHId/TZrbezObUWXfA+cxsnLf9YjMbF8WsvzazIu/4zjCzU+o8douXdaGZnVRnfbM/T8ysm5l9bGbzzGyumV3vrY/VY9tQ3pg7vmaWbGZTzGyml/Uub31PM5vs7fdVM0v01id595d4j+fu73eIUt5nzWx5nWN7iLc+8s+F2ksRxe4fIA5YCvQCEoGZQF4M5FoBZNVb9wBws7d8M3C/t3wK8C61l8cbCUyOQr7vA8OBOU3NB7QHlnm3md5yZpSy/hr4n31sm+c9B5KAnt5zIy5azxOgEzDcW24DLPIyxeqxbShvzB1f7xile8sJwGTvmP0NuMBb/zhwjbf8U+Bxb/kC4NXv+h2a4dg2lPdZ4Nx9bB/x50IQXoEfBixxzi1zzlUArwBn+JypIWcAz3nLzwFn1ln/V1frayDDzDo1ZxDn3GfApjDznQRMcs5tcs5tBiYBJ0cpa0POAF5xzu10zi0HllD7HInK88Q5t8Y5N81b3grMB7oQu8e2obwN8e34esdom3c3wfvjgGOBCd76+sd21zGfABxnZvYdv0NEfUfehkT8uRCEAu8CrKpzv5DvfgJGiwPeN7OpZnaVty7HObfGW14L5HjLsfI7HGg+v3P/l/dW8+ldQxLfkSnqWb237MOofeUV88e2Xl6IweNrZnFmNgNYT22RLQVKnHNV+9jv7kze46VAh2hl3Vde59yuY3uPd2wfMrOk+nnr5Wpy3iAUeKw60jk3HBgD/MzMvl/3QVf73ihm52jGej7gMaA3cAiwBvijr2nqMbN04DXgBufclrqPxeKx3UfemDy+zrlq59whQFdqXzUP8DfRd6uf18wGA7dQm3sEtcMiv2yu/QehwIuAbnXud/XW+co5V+TdrgfeoPbJtm7X0Ih3u97bPFZ+hwPN51tu59w67x9HDfAke94C+57VzBKoLcMXnXOve6tj9tjuK28sH18vXwnwMTCK2qGG+H3sd3cm7/F2wMZoZ62X92Rv2Mo553YCz9CMxzYIBf4foK/3SXQitR9WTPQzkJmlmVmbXcvAicAcL9euT5DHAW96yxOBy7xPoUcCpXXebkfTgeb7F3CimWV6b7FP9NY1u3qfEZxF7fHdlfUCbwZCT6AvMIUoPU+8MdbxwHzn3IN1HorJY9tQ3lg8vmaWbWYZ3nIKcAK1Y/YfA+d6m9U/truO+bnAR967n4Z+h4hqIO+COv+RG7Xj9XWPbWSfCwf6yasff6j99HYRteNht8ZAnl7Ufso9E5i7KxO1428fAouBD4D2bs+n1Y96+WcD+VHI+DK1b40rqR1Tu7wp+YAfU/sh0BLgR1HM+ryXZZb3xO9UZ/tbvawLgTHRfJ4AR1I7PDILmOH9OSWGj21DeWPu+AJDgelepjnAHXX+vU3xjtPfgSRvfbJ3f4n3eK/9/Q5RyvuRd2znAC+wZ6ZKxJ8L+iq9iEhABWEIRURE9kEFLiISUCpwEZGAUoGLiASUClxEJKBU4CIiAaUCFxEJqP8HUytFhxpdCwgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss_list[:3500])\n",
    "plt.savefig('train-3500-0.1.png')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}