model_gcn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implement of GCN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using backend: pytorch\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import dgl\n",
    "import torch\n",
    "import collections\n",
    "import numpy as np\n",
    "import networkx as nx\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from dgl.nn.pytorch import GraphConv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Train Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Init variables done.\n",
      "Loading File 637/856...\r"
     ]
    },
    {
     "ename": "IsADirectoryError",
     "evalue": "[Errno 21] Is a directory: 'data/train/.ipynb_checkpoints'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIsADirectoryError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-da29fa6ed23c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     88\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m     \u001b[0mfile_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_set_path\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'/'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m     \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIsADirectoryError\u001b[0m: [Errno 21] Is a directory: 'data/train/.ipynb_checkpoints'"
     ]
    }
   ],
   "source": [
    "def label(flag: str) -> int:\n",
    "    return {\n",
    "        '-': 0,\n",
    "        'snmpgetattack': 1,\n",
    "        'warez': 2,\n",
    "        'portsweep': 3,\n",
    "        'nmap': 4,\n",
    "        'warezclient': 5,\n",
    "        'ipsweep': 6,\n",
    "        'dict': 7,\n",
    "        'neptune': 8,\n",
    "        'smurf': 9,\n",
    "        'pod': 10,\n",
    "        'snmpguess': 11,\n",
    "        'teardrop': 12,\n",
    "        'satan': 13,\n",
    "        'httptunnel-e': 14,\n",
    "        'ignore': 15,\n",
    "        'mscan': 16,\n",
    "        'guest': 17,\n",
    "        'rootkit': 18,\n",
    "        'back': 19,\n",
    "        'apache2': 20,\n",
    "        'processtable': 21,\n",
    "        'mailbomb': 22,\n",
    "        'smurfttl': 23,\n",
    "        'saint': 24\n",
    "    }.get(flag[:-1])\n",
    "\n",
    "\n",
    "def line_split(line: str) -> tuple:\n",
    "    src_ip, dst_ip, port, time, connect_type = line.split('\\t')\n",
    "    return int(src_ip), int(dst_ip), int(port), int(time), label(connect_type)\n",
    "\n",
    "# Data File Path\n",
    "\n",
    "train_set_path = 'data/train'\n",
    "valid_set_path = 'data/valid_query'\n",
    "test_set_path = 'data/test_query'\n",
    "\n",
    "train_set_list = os.listdir(train_set_path)\n",
    "valid_set_list = os.listdir(valid_set_path)\n",
    "test_set_path = os.listdir(test_set_path)\n",
    "\n",
    "file_index = 0\n",
    "\n",
    "# Features\n",
    "\n",
    "src_in_degree = collections.OrderedDict()\n",
    "src_out_degree = collections.OrderedDict()\n",
    "dst_in_degree = collections.OrderedDict()\n",
    "dst_out_degree = collections.OrderedDict()\n",
    "\n",
    "port_num = collections.OrderedDict()\n",
    "port_list = collections.OrderedDict()\n",
    "time_len = collections.OrderedDict()\n",
    "connect_num = collections.OrderedDict()\n",
    "label_list = collections.OrderedDict()\n",
    "\n",
    "# Tool Parameters\n",
    "\n",
    "nodes_origin = []\n",
    "edges_origin = {}\n",
    "\n",
    "edges_trans_src = []\n",
    "edges_trans_dst = []\n",
    "\n",
    "node_in_degree = {}\n",
    "node_out_degree = {}\n",
    "\n",
    "node_in_edges = {}\n",
    "node_out_edges = {}\n",
    "\n",
    "edge_src_list = {}\n",
    "edge_dst_list = {}\n",
    "\n",
    "# Count Parameters\n",
    "\n",
    "edge_index = 0\n",
    "\n",
    "print('Init variables done.')\n",
    "\n",
    "for file in train_set_list:\n",
    "    file_index += 1\n",
    "    if file_index == 114:\n",
    "        continue\n",
    "    print('Loading File {}/{}...'.format(file_index, len(train_set_list)), end='\\r')\n",
    "    \n",
    "    file_path = train_set_path + '/' + file\n",
    "    data = open(file_path)\n",
    "    \n",
    "    for line in data:\n",
    "        src_ip, dst_ip, port, time, connect_type = line_split(line)\n",
    "\n",
    "        # Repeat Connections\n",
    "        if (src_ip, dst_ip) in edges_origin.keys():\n",
    "            edge_index_temp = edges_origin[(src_ip, dst_ip)]\n",
    "\n",
    "            # Assign Features\n",
    "            time_len[edge_index_temp] += time\n",
    "            connect_num[edge_index_temp] += 1\n",
    "            label_list[edge_index_temp][connect_type] += 1\n",
    "            if port in port_list[edge_index_temp]:\n",
    "                pass\n",
    "            else:\n",
    "                port_num[edge_index_temp] += 1\n",
    "                port_list[edge_index_temp].append(port)\n",
    "\n",
    "        # New Connections     \n",
    "        else:\n",
    "            # Record Origin Node and Edge\n",
    "            nodes_origin.append(src_ip)\n",
    "            nodes_origin.append(dst_ip)\n",
    "            edges_origin[(src_ip, dst_ip)] = edge_index\n",
    "\n",
    "            # Update Node Parameters\n",
    "            if src_ip in node_out_degree.keys():\n",
    "                node_out_degree[src_ip] += 1\n",
    "            else:\n",
    "                node_out_degree[src_ip] = 1\n",
    "            if dst_ip in node_in_degree.keys():\n",
    "                node_in_degree[dst_ip] += 1\n",
    "            else:\n",
    "                node_in_degree[dst_ip] = 1\n",
    "\n",
    "            # Store Nodes' In-Out Edges\n",
    "            if src_ip in node_out_edges.keys():\n",
    "                node_out_edges[src_ip].append(edge_index)\n",
    "            else:\n",
    "                node_out_edges[src_ip] = [edge_index]\n",
    "            if dst_ip in node_in_edges.keys():\n",
    "                node_in_edges[dst_ip].append(edge_index)\n",
    "            else:\n",
    "                node_in_edges[dst_ip] = [edge_index]\n",
    "\n",
    "            # Update Edge Parameters\n",
    "            if edge_index in edge_src_list.keys():\n",
    "                edge_src_list[edge_index].append(src_ip)\n",
    "            else:\n",
    "                edge_src_list[edge_index] = [src_ip]\n",
    "            if edge_index in edge_dst_list.keys():\n",
    "                edge_dst_list[edge_index].append(dst_ip)\n",
    "            else:\n",
    "                edge_dst_list[edge_index] = [dst_ip]\n",
    "\n",
    "            # Assign Features\n",
    "            port_num[edge_index] = 1\n",
    "            port_list[edge_index] = [port]\n",
    "            time_len[edge_index] = time\n",
    "            connect_num[edge_index] = 1\n",
    "            label_list[edge_index] = [0 * x for x in range(25)]\n",
    "            label_list[edge_index][connect_type] += 1\n",
    "\n",
    "            edge_index += 1\n",
    "            \n",
    "print('Data Loading Done')\n",
    "\n",
    "# Remove Repeats\n",
    "nodes_origin = list(set(nodes_origin))\n",
    "for i in range(len(edge_src_list)):\n",
    "    edge_src_list[i] = list(set(edge_src_list[i]))\n",
    "for i in range(len(edge_dst_list)):\n",
    "    edge_dst_list[i] = list(set(edge_dst_list[i]))\n",
    "    \n",
    "# Create Transform Graph\n",
    "for node in nodes_origin:\n",
    "    if not node in node_in_edges.keys() or not node in node_out_edges.keys():\n",
    "        continue\n",
    "    if not node_in_edges[node] or not node_out_edges[node]:\n",
    "        continue\n",
    "    for src_edge in node_in_edges[node]:\n",
    "        for dst_edge in node_out_edges[node]:\n",
    "            edges_trans_src.append(src_edge)\n",
    "            edges_trans_dst.append(dst_edge)\n",
    "            \n",
    "print('Data clean up done.')\n",
    "    \n",
    "# Create Features\n",
    "for edge in edge_src_list.keys():\n",
    "    if edge_src_list[edge][0] not in node_in_degree.keys():\n",
    "        src_in_degree[edge] = 0\n",
    "    else:\n",
    "        src_in_degree[edge] = node_in_degree[edge_src_list[edge][0]]\n",
    "    if edge_src_list[edge][0] not in node_out_degree.keys():\n",
    "        src_out_degree[edge] = 0\n",
    "    else:\n",
    "        src_out_degree[edge] = node_out_degree[edge_src_list[edge][0]]\n",
    "    \n",
    "for edge in edge_dst_list.keys():\n",
    "    if edge_dst_list[edge][0] not in node_in_degree.keys():\n",
    "        dst_in_degree[edge] = 0\n",
    "    else:\n",
    "        dst_in_degree[edge] = node_in_degree[edge_dst_list[edge][0]]\n",
    "    if edge_dst_list[edge][0] not in node_out_degree.keys():\n",
    "        dst_out_degree[edge] = 0\n",
    "    else:\n",
    "        dst_out_degree[edge] = node_out_degree[edge_dst_list[edge][0]]\n",
    "    \n",
    "print('Feature create done.')\n",
    "    \n",
    "src_in_degree_norm = list(src_in_degree.values()) / np.linalg.norm(list(src_in_degree.values()))\n",
    "src_out_degree_norm = list(src_out_degree.values()) / np.linalg.norm(list(src_out_degree.values()))\n",
    "dst_in_degree_norm = list(dst_in_degree.values()) / np.linalg.norm(list(dst_in_degree.values()))\n",
    "dst_out_degree_norm = list(dst_out_degree.values()) / np.linalg.norm(list(dst_out_degree.values()))\n",
    "\n",
    "port_num_norm = list(port_num.values()) / np.linalg.norm(list(port_num.values()))\n",
    "time_len_norm = list(time_len.values()) / np.linalg.norm(list(time_len.values()))\n",
    "connect_num_norm = list(connect_num.values()) / np.linalg.norm(list(connect_num.values()))\n",
    "\n",
    "print('Normalize done.')\n",
    "\n",
    "feature_matrix = np.empty([1, 7])\n",
    "for i in range(len(edges_origin)):\n",
    "    feature_matrix = np.vstack((feature_matrix, [src_in_degree_norm[i], src_out_degree_norm[i], dst_in_degree_norm[i], dst_out_degree_norm[i], \\\n",
    "                                                 port_num_norm[i], time_len_norm[i], connect_num_norm[i]]))\n",
    "feature_matrix = np.delete(feature_matrix, obj=0, axis=0)    \n",
    "feature_matrix = feature_matrix.astype(np.float32)\n",
    "\n",
    "print('Data process done.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Loading Done\n",
      "Data clean up done.\n",
      "Feature create done.\n",
      "Normalize done.\n",
      "Data process done.\n"
     ]
    }
   ],
   "source": [
    "print('Data Loading Done')\n",
    "\n",
    "# Remove Repeats\n",
    "nodes_origin = list(set(nodes_origin))\n",
    "for i in range(len(edge_src_list)):\n",
    "    edge_src_list[i] = list(set(edge_src_list[i]))\n",
    "for i in range(len(edge_dst_list)):\n",
    "    edge_dst_list[i] = list(set(edge_dst_list[i]))\n",
    "    \n",
    "# Create Transform Graph\n",
    "for node in nodes_origin:\n",
    "    if not node in node_in_edges.keys() or not node in node_out_edges.keys():\n",
    "        continue\n",
    "    if not node_in_edges[node] or not node_out_edges[node]:\n",
    "        continue\n",
    "    for src_edge in node_in_edges[node]:\n",
    "        for dst_edge in node_out_edges[node]:\n",
    "            edges_trans_src.append(src_edge)\n",
    "            edges_trans_dst.append(dst_edge)\n",
    "            \n",
    "print('Data clean up done.')\n",
    "    \n",
    "# Create Features\n",
    "for edge in edge_src_list.keys():\n",
    "    if edge_src_list[edge][0] not in node_in_degree.keys():\n",
    "        src_in_degree[edge] = 0\n",
    "    else:\n",
    "        src_in_degree[edge] = node_in_degree[edge_src_list[edge][0]]\n",
    "    if edge_src_list[edge][0] not in node_out_degree.keys():\n",
    "        src_out_degree[edge] = 0\n",
    "    else:\n",
    "        src_out_degree[edge] = node_out_degree[edge_src_list[edge][0]]\n",
    "    \n",
    "for edge in edge_dst_list.keys():\n",
    "    if edge_dst_list[edge][0] not in node_in_degree.keys():\n",
    "        dst_in_degree[edge] = 0\n",
    "    else:\n",
    "        dst_in_degree[edge] = node_in_degree[edge_dst_list[edge][0]]\n",
    "    if edge_dst_list[edge][0] not in node_out_degree.keys():\n",
    "        dst_out_degree[edge] = 0\n",
    "    else:\n",
    "        dst_out_degree[edge] = node_out_degree[edge_dst_list[edge][0]]\n",
    "    \n",
    "print('Feature create done.')\n",
    "    \n",
    "src_in_degree_norm = list(src_in_degree.values()) / np.linalg.norm(list(src_in_degree.values()))\n",
    "src_out_degree_norm = list(src_out_degree.values()) / np.linalg.norm(list(src_out_degree.values()))\n",
    "dst_in_degree_norm = list(dst_in_degree.values()) / np.linalg.norm(list(dst_in_degree.values()))\n",
    "dst_out_degree_norm = list(dst_out_degree.values()) / np.linalg.norm(list(dst_out_degree.values()))\n",
    "\n",
    "port_num_norm = list(port_num.values()) / np.linalg.norm(list(port_num.values()))\n",
    "time_len_norm = list(time_len.values()) / np.linalg.norm(list(time_len.values()))\n",
    "connect_num_norm = list(connect_num.values()) / np.linalg.norm(list(connect_num.values()))\n",
    "\n",
    "print('Normalize done.')\n",
    "\n",
    "feature_matrix = np.empty([1, 7])\n",
    "for i in range(len(edges_origin)):\n",
    "    feature_matrix = np.vstack((feature_matrix, [src_in_degree_norm[i], src_out_degree_norm[i], dst_in_degree_norm[i], dst_out_degree_norm[i], \\\n",
    "                                                 port_num_norm[i], time_len_norm[i], connect_num_norm[i]]))\n",
    "feature_matrix = np.delete(feature_matrix, obj=0, axis=0)    \n",
    "feature_matrix = feature_matrix.astype(np.float32)\n",
    "\n",
    "print('Data process done.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GCN(nn.Module):\n",
    "    def __init__(self, in_feats, hidden_size, num_classes):\n",
    "        super(GCN, self).__init__()\n",
    "        self.conv1 = GraphConv(in_feats, hidden_size)\n",
    "        self.conv2 = GraphConv(hidden_size, num_classes)\n",
    "\n",
    "    def forward(self, g, inputs):\n",
    "        h = self.conv1(g, inputs)\n",
    "        h = torch.relu(h)\n",
    "        h = self.conv2(g, h)\n",
    "        return h"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dgl/base.py:45: DGLWarning: Recommend creating graphs by `dgl.graph(data)` instead of `dgl.DGLGraph(data)`.\n",
      "  return warnings.warn(message, category=category, stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "g = dgl.DGLGraph()\n",
    "\n",
    "# Create Nodes and Edges\n",
    "\n",
    "g.add_nodes(len(edges_origin))\n",
    "g.add_edges(edges_trans_src, edges_trans_dst)\n",
    "\n",
    "# Assign Features\n",
    "\n",
    "g.ndata['src_in_degree'] = torch.tensor(src_in_degree_norm)\n",
    "g.ndata['src_out_degree'] = torch.tensor(src_out_degree_norm)\n",
    "g.ndata['dst_in_degree'] = torch.tensor(dst_in_degree_norm)\n",
    "g.ndata['dst_out_degree'] = torch.tensor(dst_out_degree_norm)\n",
    "g.ndata['port_num'] = torch.tensor(port_num_norm)\n",
    "g.ndata['time_len'] = torch.tensor(time_len_norm)\n",
    "g.ndata['connect_num'] = torch.tensor(connect_num_norm)\n",
    "\n",
    "g = dgl.add_self_loop(g)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nodes Number\t39899\n",
      "Edges Number\t1495966\n"
     ]
    }
   ],
   "source": [
    "print('Nodes Number\\t{}\\nEdges Number\\t{}'.format(g.number_of_nodes(), g.number_of_edges()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = GCN(7, 16, 25)\n",
    "\n",
    "inputs = torch.tensor(feature_matrix)\n",
    "labels = torch.tensor(list(label_list.values()), dtype=torch.float32)\n",
    "\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr=0.1)\n",
    "loss_func = nn.MSELoss()\n",
    "\n",
    "# Recoder\n",
    "loss_list = []\n",
    "\n",
    "for epoch in range(3000):\n",
    "    outputs = net(g, inputs)\n",
    "    loss = loss_func(outputs, labels)\n",
    "    \n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    loss_list.append(loss.item() / 39899)\n",
    "    \n",
    "    if epoch % 100 == 0:\n",
    "        print('Epoch %d | Loss: %.4f' % (epoch, loss.item()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Model With Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 | Loss: 11.9851 | Accu: 0.0014\n",
      "Epoch 100 | Loss: 11.9294 | Accu: 0.0098\n",
      "Epoch 200 | Loss: 11.7181 | Accu: 0.3093\n",
      "Epoch 300 | Loss: 11.3541 | Accu: 0.4404\n",
      "Epoch 400 | Loss: 10.8571 | Accu: 0.4705\n",
      "Epoch 500 | Loss: 10.2475 | Accu: 0.4709\n",
      "Epoch 600 | Loss: 9.5484 | Accu: 0.4710\n",
      "Epoch 700 | Loss: 8.7842 | Accu: 0.4713\n",
      "Epoch 800 | Loss: 7.9794 | Accu: 0.4717\n",
      "Epoch 900 | Loss: 7.1574 | Accu: 0.4723\n",
      "Epoch 1000 | Loss: 6.3407 | Accu: 0.4724\n",
      "Epoch 1100 | Loss: 5.5490 | Accu: 0.4724\n",
      "Epoch 1200 | Loss: 4.7994 | Accu: 0.4725\n",
      "Epoch 1300 | Loss: 4.1063 | Accu: 0.4726\n",
      "Epoch 1400 | Loss: 3.4797 | Accu: 0.4735\n",
      "Epoch 1500 | Loss: 2.9274 | Accu: 0.4748\n",
      "Epoch 1600 | Loss: 2.4527 | Accu: 0.4753\n",
      "Epoch 1700 | Loss: 2.0557 | Accu: 0.4756\n",
      "Epoch 1800 | Loss: 1.7329 | Accu: 0.4759\n",
      "Epoch 1900 | Loss: 1.4785 | Accu: 0.4760\n",
      "Epoch 2000 | Loss: 1.2846 | Accu: 0.4763\n",
      "Epoch 2100 | Loss: 1.1419 | Accu: 0.4763\n",
      "Epoch 2200 | Loss: 1.0407 | Accu: 0.4765\n",
      "Epoch 2300 | Loss: 0.9717 | Accu: 0.4765\n",
      "Epoch 2400 | Loss: 0.9264 | Accu: 0.4765\n",
      "Epoch 2500 | Loss: 0.8978 | Accu: 0.4765\n",
      "Epoch 2600 | Loss: 0.8804 | Accu: 0.4770\n",
      "Epoch 2700 | Loss: 0.8702 | Accu: 0.4772\n",
      "Epoch 2800 | Loss: 0.8642 | Accu: 0.4773\n",
      "Epoch 2900 | Loss: 0.8607 | Accu: 0.4876\n"
     ]
    }
   ],
   "source": [
    "net = GCN(7, 16, 25)\n",
    "\n",
    "inputs = torch.tensor(feature_matrix)\n",
    "labels = torch.tensor(list(label_list.values()), dtype=torch.float32)\n",
    "\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr=0.1)\n",
    "loss_func = nn.MSELoss()\n",
    "\n",
    "# Recoder\n",
    "loss_list = []\n",
    "accu_list = []\n",
    "\n",
    "for epoch in range(3000):\n",
    "    outputs = net(g, inputs)\n",
    "    \n",
    "    total_number = 0\n",
    "    correct_number = 0\n",
    "    \n",
    "    outputs_max = np.argmax(outputs.detach().numpy(), axis=1)\n",
    "    labels_max = np.argmax(labels.detach().numpy(), axis=1)\n",
    "    \n",
    "    for i in range(len(outputs_max)):\n",
    "        if outputs_max[i] == labels_max[i]:\n",
    "            correct_number += 1\n",
    "        total_number += 1\n",
    "    \n",
    "    loss = loss_func(outputs, labels)\n",
    "    \n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    loss_list.append(loss.item()/g.number_of_nodes())\n",
    "    accu_list.append(correct_number / total_number)\n",
    "    \n",
    "    if epoch % 100 == 0:\n",
    "        print('Epoch {} | Loss: {:.4f} | Accu: {:.4f}'.format(epoch, loss.item()/g.number_of_nodes(), correct_number / total_number))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 34400, 1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 5499, 9: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0, 16: 0, 17: 0, 18: 0, 19: 0, 20: 0, 21: 0, 22: 0, 23: 0, 24: 0}\n",
      "{0: 21264, 1: 2, 2: 71, 3: 11, 4: 3, 5: 8, 6: 7251, 7: 8, 8: 5, 9: 10154, 10: 106, 11: 0, 12: 4, 13: 14, 14: 1, 15: 14, 16: 55, 17: 0, 18: 1, 19: 0, 20: 2, 21: 1, 22: 1, 23: 669, 24: 254}\n"
     ]
    }
   ],
   "source": [
    "count = {x: 0 for x in range(25)}\n",
    "for i in range(len(outputs_max)):\n",
    "    count[outputs_max[i]] += 1\n",
    "        \n",
    "print(count)\n",
    "count = {x: 0 for x in range(25)}\n",
    "\n",
    "for i in range(len(labels_max)):\n",
    "    count[labels_max[i]] += 1\n",
    "    \n",
    "print(count)\n",
    "\n",
    "# count_2 = 0\n",
    "# for i in range(len(labels_max)):\n",
    "#     if labels_max[i] >= 1:\n",
    "#         count_2 += 1\n",
    "        \n",
    "# print(len(labels_max))\n",
    "# print(count_2)\n",
    "\n",
    "# plt.plot(labels_max, 'bo')\n",
    "# plt.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Show Train Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEGCAYAAABvtY4XAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAg9UlEQVR4nO3dd5hU5d3G8e9vZnulLB1pAsJKdzEqalRMQmxgATXoq6YYExNr1MS8xhhLjLFE7EaD+saGBaOxRKOC2FB6773uUpbdZdkyO8/7xwxICCiwc/bsnL0/17XXzJyZ2XMfHe89PnPOc8w5h4iIBE/I7wAiIuINFbyISECp4EVEAkoFLyISUCp4EZGASvE7wO4KCgpcly5d/I4hIpI0pk6dusk512pvzzWqgu/SpQtTpkzxO4aISNIws5X7ek5DNCIiAaWCFxEJKBW8iEhAqeBFRAJKBS8iElAqeBGRgFLBi4gEVNIXfDTqeOjDJcxaU+p3FBGRRiXpC768OsKzn6/kF89Np6yq1u84IiKNRtIXfH5mKmPOH8ja0h385tXZ6AImIiIxSV/wAEVdWnDtd3vy5qz1PPfFKr/jiIg0CoEoeIDLjj+U43oUcMsb85i/vszvOCIivgtMwYdCxn3nDiA/M5XLn5vG9uqI35FERHwVmIIHKMhJ5/7zBrB803Zu+sccv+OIiPgqUAUPcMyhBVxxUg9enbaWl6eu8TuOiIhvAlfwAFcM7cFR3Vpw02tzWFJc7nccERFfBLLgwyHj/vMGkpUW5vJnp1NVW+d3JBGRBhfIggdok5fBPaP6s3BjOX98a77fcUREGlxgCx7ghMNa88MhXXn6s5V8sGCj33FERBpUoAse4Pphh9GrbS7XvTSL4vIqv+OIiDSYwBd8RmqYB84fSEV1hF+9NItoVFMZiEjTEPiCB+jRJpf/PbU3Hy0qYeynK/yOIyLSIJpEwQNccFRnTu7dmj+9vYB56zSVgYgEX5MpeDPjT2f3Iz8rlStf0KGTIhJ8TabgAVrmpHPvqP4sLq7g9jd16KSIBFuTKniA43q04sfHduX/Pl/Jv+fp0EkRCa4mV/AA1w07jMJ2eVz/yiyKy3TopIgEU5Ms+PSUMGPOH0BlTYTrX5mlq0CJSCA1yYIH6N46l998vzcTFpbw7GRdBUpEgqfJFjzAhUd15rgeBdz+5nyWb9rudxwRkYRq0gUfChl/Pqc/aSkhrn5xBpG6qN+RREQSpkkXPEDb/AxuG9GHGatLeXjCUr/jiIgkTJMveIDT+7dn+ID23P/+YmatKfU7johIQqjg4/5wRh9a56Zz9Ysz2FGjs1xFJPmp4OPys1K5e2R/lpZs50/vLPA7johIvangdzOkewGXDOnCU5+uYNLiEr/jiIjUiwp+DzcM60X31jn86qWZlFbW+B1HROSgqeD3kJEa5r5RA9hcUcNN/5jrdxwRkYOmgt+Lvh3zuerkHrwxcx3/mLHW7zgiIgdFBb8Pl337UAZ2asZNr81h/bYdfscRETlgnha8mV1tZnPNbI6ZPW9mGV6uL5FSwiHuGzWA2jrH9S9rQjIRST6eFbyZdQCuAIqcc32AMHCeV+vzQpeCbG48tTeTFm/ShGQiknS8HqJJATLNLAXIAtZ5vL6Eu+BbnTiuRwF3vDWflZs1IZmIJA/PCt45txa4G1gFrAe2Oefe3fN1ZnapmU0xsyklJY3v2POd13INm3HdS7Ooi2qoRkSSg5dDNM2B4UBXoD2QbWYX7Pk659zjzrki51xRq1atvIpTL+2bZXLzGYfzxYotjP1kud9xRET2i5dDNCcDy51zJc65WuBV4BgP1+epswd14OTebbjrXwtZUlzudxwRkW/kZcGvAo4ysywzM2AoMN/D9XnKzLjjrD5kp4W5ZtxMzR0vIo2el2Pwk4GXgWnA7Pi6HvdqfQ2hdW4Gt43oy6w123hEc8eLSCPn6VE0zrmbnXO9nHN9nHMXOueqvVxfQzi1XztO7x+bO37uum1+xxER2SedyXoQ/nDG4TTPTuPacTOpjmjueBFpnFTwB6F5dhp3ntWXBRvKuf/fi/2OIyKyVyr4gzS0dxtGFXXk0YlLmb5qq99xRET+iwq+Hm46rZB2+ZlcO26mLvMnIo2OCr4ecjNSueucfizbtJ0//2uh33FERP6DCr6ehnQv4KKjO/O3T5bz2dLNfscREdlFBZ8AN3y/F11aZnHdyzOpqI74HUdEBFDBJ0RWWgr3jOrPutId3P5m0p6sKyIBo4JPkCM6t+Anx3fj+S9WMWFhsd9xRERU8Il09ck96dkmhxtemcW2ylq/44hIE6eCT6CM1DD3jBzA5ooafv/GXL/jiEgTp4JPsL4d8/nFSd0ZP30t78xZ73ccEWnCVPAeuPzE7vTpkMdvx89hU0XSz68mIklKBe+B1HCIe0cNoLwqwm/Hz8Y5XeZPRBqeCt4jPdvkcu13e/KvuRsZP32t33FEpAlSwXvox8d1Y3CX5tz8+lzWle7wO46INDEqeA+FQ8bdI/tTF3Vc//IsolEN1YhIw1HBe6xzy2xuPKU3Hy/ZxN8nr/Q7jog0ISr4BjD6W504vmcr7nhrPss3bfc7jog0ESr4BmBm3HV2P9LCIa4dN4M6DdWISANQwTeQtvkZ/GF4H6atKuWxj5b6HUdEmgAVfAMaPqA9p/Rty33vLWL++jK/44hIwKngG5CZcduIvuRnpnHNuJnURKJ+RxKRAFPBN7AW2WnceVZf5q8v4/73F/kdR0QCTAXvg5ML2zDyiI48MmEp01Zt9TuOiASUCt4nvzu9kHb5mfxq3Ex21NT5HUdEAkgF75PcjFT+PLIfyzZt50/vLPA7jogEkAreR8ccWsDFx3ThqU9X8MmSTX7HEZGAUcH77IZhvehWkM11L82krEqX+RORxFHB+ywzLcw9o/qzoayKW16f53ccEQkQFXwjMLBTc35+QndembaGd+du8DuOiASECr6RuGJoDwrb5XHj+Nls1mX+RCQBVPCNRFpKiHvP7U/Zjgi/HT9Hl/kTkXpTwTcivdrmcfV3evLO3A28NkOX+ROR+lHBNzKXHt+NIzo353f/mMtaXeZPROpBBd/IhEPGvaP6E406rnlRc8eLyMFTwTdCnVtmc/MZhzN5+RaemLTM7zgikqQ8LXgza2ZmL5vZAjObb2ZHe7m+IBl5REeGHd6Wu99dyNx12/yOIyJJyOs9+PuBd5xzvYD+wHyP1xcYZsYdZ/WleVYaV70wg6paTUgmIgfGs4I3s3zgeOBJAOdcjXOu1Kv1BVGL7DTuHtmfxcUV3Pm2JiQTkQPj5R58V6AEGGtm083sCTPL3vNFZnapmU0xsyklJSUexklOx/dsxSVDYhOSTVhY7HccEUkiXhZ8CjAIeMQ5NxDYDvx6zxc55x53zhU554patWrlYZzkdcOwXvRsk8N1L89iy/Yav+OISJLwsuDXAGucc5Pjj18mVvhygDJSw/zl3IFsq6zl16/M0lmuIrJfPCt459wGYLWZHRZfNBTQdIkHqbB9Hr/6Xk/enbeRcVNW+x1HRJKA10fR/BJ41sxmAQOAOzxeX6D9+NhuHN2tJbe8MY8Vm7b7HUdEGjlPC945NyM+vt7POTfCOacrTNdDKGTcM6o/KSHjqhdnEKmL+h1JRBoxncmaZNo3y+T2M/syY3UpD364xO84ItKIqeCT0On923PWwA488MESpq7c4nccEWmkVPBJ6vfDD6d9swyueH4G23boWq4i8t9U8EkqLyOVMecNZGNZFTe+OluHTorIf9mvgjezbDMLxe/3NLMzzCzV22jyTQZ2as613z2MN2ev58UvdeikiPyn/d2D/wjIMLMOwLvAhcBTXoWS/ffT47txbPcCfv/GXBZvLPc7jog0Ivtb8OacqwTOAh52zo0EDvculuyvUPwCIdlpKfzy+emadVJEdtnvgo/P5T4aeDO+LOxNJDlQrfMyuHtkfxZsKOeOtzQjs4jE7G/BXwX8BhjvnJtrZt2ADz1LJQfsxF6t+dGxXXnms5W8O3eD33FEpBFI2Z8XOecmAhMB4l+2bnLOXeFlMDlw1w87jMnLN3P9K7Po2zGfdvmZfkcSER/t71E0z5lZXnw+9znAPDO7zttocqDSU8KMOW8gNZEoV76gC3aLNHX7O0RT6JwrA0YAbxO7mMeFXoWSg9etVQ63Du/DF8u38OAHmspApCnb34JPjR/3PgJ43TlXC2j3sJE6a1AHRgxoz/3vL+KzpZv9jiMiPtnfgn8MWAFkAx+ZWWegzKtQUj9mxm1n9qVLy2yueGE6JeXVfkcSER/sV8E758Y45zo4505xMSuBEz3OJvWQk57CQ6MHUbajlitfmK7xeJEmaH+/ZM03s3t3XhzbzO4htjcvjVjvdnncOrwPny7dzJj3F/sdR0Qa2P4O0fwNKAdGxX/KgLFehZLEGVnUkbMHdWTMB4uZtLjE7zgi0oD2t+APdc7d7JxbFv+5BejmZTBJDDPj1hGH06N1Dle9MIMN26r8jiQiDWR/C36HmR2784GZDQF2eBNJEi0rLYWHRw9iR20dv3x+mi71J9JE7G/BXwY8ZGYrzGwF8CDwU89SScJ1b53LHWf25csVW7n73UV+xxGRBrC/R9HMdM71B/oB/ZxzA4GTPE0mCTdiYAfOP7ITj05cygcLNvodR0Q8dkBXdHLOlcXPaAW4xoM84rGbTy+ksF0eV784k9VbKv2OIyIeqs8l+yxhKaTBZKSGeXj0IKLOcdnfp2r+eJEAq0/B68yZJNWlIJu/nDuAuevKuHG8rucqElRfW/BmVm5mZXv5KQfaN1BG8cDQ3m24cmgPXp22lv/7fKXfcUTEA187H7xzLrehgkjDu3JoD+as3cYf3phHYbs8irq08DuSiCRQfYZoJMmFQsa95w6gQ/NMfvbsNIrLdBKUSJCo4Ju4/MxUHr+wiIqqCD97dho1EZ0EJRIUKnjhsLa53HVOP6au3Mptb87zO46IJMh+XZNVgu/0/u2ZtaaUv05aTt8O+YwsOsTvSCJST9qDl11uGNaLo7u15Lfj5zBt1Va/44hIPangZZeUcIiHRw+ibX4Glz4zlXWlmk9OJJmp4OU/NM9O44mLiqiqreMnz0yhsibidyQROUgqePkvPdvkMub8AcxbX8Z1L83Sma4iSUoFL3t1Uq82/HpYL96cvZ4x7y/xO46IHAQdRSP7dOnx3Vi4sZz7/r2IHm1yOKVvO78jicgB8HwP3szCZjbdzP7p9bokscyMO87sy6BOzbhm3AzmrN3mdyQROQANMURzJTC/AdYjHshIDfPohUfQPCuNnzwzhY2azkAkaXha8GbWETgVeMLL9Yi3Wudm8MRFRZTtqOWSsV9SUa0ja0SSgdd78H8Brgf2OcGJmV1qZlPMbEpJSYnHceRgHd4+nwdHD2LhxnJ+8Zwu3C2SDDwreDM7DSh2zk39utc55x53zhU554patWrlVRxJgBMPa82tw/swYWEJv3t9rg6fFGnkvDyKZghwhpmdAmQAeWb2d+fcBR6uUzz2g291YtWWSh6duJROLbK47NuH+h1JRPbBsz1459xvnHMdnXNdgPOAD1TuwXD99w7jtH7tuPPtBbwxc53fcURkH3QcvBywUMi4e2R/NpZVce1LM2mbn8FgXQ1KpNFpkDNZnXMTnHOnNcS6pGFkpIZ5/MIiOjTL5CfPTGFJcbnfkURkD5qqQA5a8+w0nrpkMCmhEBc++YVmnxRpZFTwUi+dW2bz9A8HU1EV4cInJ7Nle43fkUQkTgUv9XZ4+3z+elERq7fu4JKnvmS7ToQSaRRU8JIQR3VryYPnD2T2mlIu+/tUXbxbpBFQwUvCfPfwttx5Vj8mLd7ENeNmEI3qRCgRP+kwSUmoUYMPYUtlDXe+vYBmWancOrwPZuZ3LJEmSQUvCXfZtw9la2UNj01cRnpKmP89tbdKXsQHKnjxxK+H9aK6NsqTHy8nPSXEdd87TCUv0sBU8OIJM+Pm0wupjkR5eMJSMlLDXDG0h9+xRJoUFbx4xsy4fUQfaiJR7n1vEWkpIU1OJtKAVPDiqVDIuOucflRH6rjz7QWkp4S4ZEhXv2OJNAkqePFcOGTcd+4AaiJRbnljHiEzLjqmi9+xRAJPx8FLg0gNh3jgBwP5TmEbbn59Lk9MWuZ3JJHAU8FLg0lPCfPw6EGc0rctt705n4c+XOJ3JJFA0xCNNKjUcIgx5w0kNTyTP/9rIbV1Ua4c2kOHUIp4QAUvDS4lHOLeUQNIDYf4y78XUxOJ6jh5EQ+o4MUX4ZBx19n9SA2HeHjCUnbU1nHTqYWEQip5kURRwYtvQiHjjjP7kJEaYuwnKyitrOWuc2KlLyL1p4IXX5kZvzutkJbZadz97iJKK2t4ePQRZKaF/Y4mkvS0qyS+MzN+cVIPbj+zDxMWlXDBk5MprdSVoUTqSwUvjcbob3XmoR8MYvaabZz72Ods2FbldySRpKaCl0bllL7tGHvJYNZsreSshz9hwYYyvyOJJC0VvDQ6Q7oX8OJPjyYSdZzzyGdMXFTidySRpKSCl0apT4d8Xrt8CB2bZ/LDp77kucmr/I4kknRU8NJotW+Wycs/O4bjehRw4/jZ/PGt+brOq8gBUMFLo5aTnsIT/1PEBUd14rGPlvHzZ6exvTridyyRpKCCl0YvJRzi1uF9+N9Te/PuvA2c/cinrNy83e9YIo2eCl6Sgpnx4+O68fQPj2T9tirOePATffkq8g1U8JJUjuvRijd+cSzt8jO4ZOwXPDpxKc5pXF5kb1TwknQ6tczi1Z8fw/f7tuPOtxfwy+ena1xeZC9U8JKUstJSePD8gdwwrBdvzV7P6Q98zLx1OilKZHcqeElaZsbPTjiUZ398FBXVEUY8/AnPTl6pIRuROBW8JL2jD23JW1cex7e6tuC34+fwi+enU15V63csEd+p4CUQCnLSefqSI7l+2GG8M2cDpz3wMdNXbfU7loivVPASGKGQ8fMTuvPCpUcRqXOc/cin3PPuQmoiUb+jifhCBS+BM7hLC96+6jjOGtSRBz5YwlmPfMKijeV+xxJpcCp4CaS8jFTuHtmfxy48gvWlVZz2wMc8MWmZ5rKRJsWzgjezQ8zsQzObZ2ZzzexKr9Ylsi/fO7wt/7r6eL7dsxW3vTmfUY99xmLtzUsT4eUefAS41jlXCBwFXG5mhR6uT2SvCnLSefzCI7hnZH+WlFRwyphJ3PfeIqojdX5HE/GUZwXvnFvvnJsWv18OzAc6eLU+ka9jZpx9REf+fc23+X6fdtz//mJOHfMxX67Y4nc0Ec80yBi8mXUBBgKT9/LcpWY2xcymlJRo8ijxVkFOOmPOH8jYiwezo6aOkY9+xjUvzqC4TNd/leAxr8/6M7McYCJwu3Pu1a97bVFRkZsyZYqneUR22l4d4aEPl/DEpOWkpYS4Ymh3Lj6mK2kpOvZAkoeZTXXOFe3tOU8/yWaWCrwCPPtN5S7S0LLTU7h+WC/evfp4vtW1BXe8tYBh93/EhIXFfkcTSQgvj6Ix4ElgvnPuXq/WI1JfXQqyefLiwYy9eDDOwcVjv2T0E58za02p39FE6sXLPfghwIXASWY2I/5ziofrE6mXE3u15p2rjuPm0wuZv76cMx78hMufncaykgq/o4kcFM/H4A+ExuClsaiojvDXj5bx10nLqI5EGVV0CFcM7U67/Ey/o4n8h68bg1fBi3yNkvJqHvxgMc99sQrDOKeoIz/79qEc0iLL72gigApepN5Wb6nk0YlLeWnKGqLOcebADvz8xO50Lcj2O5o0cSp4kQRZv20Hj01cxvNfrKKmLsrQXm340bFdOapbC2LHFYg0LBW8SIIVl1fx989W8vfJq9iyvYbCdnn86NiunN6/vY6jlwalghfxSFVtHa9NX8uTHy9ncXEFrXPTOW/wIYwafAgdm2ucXrynghfxmHOOjxZvYuwny5m4KDblxvE9WnH+kZ0Y2rs1qWHt1Ys3VPAiDWjN1krGTVnDuC9Xs6Gsila56Qzv354RAztwePs8jdVLQqngRXwQqYsyYWEJL3y5momLiqmtc3Rrlc2IAR0YPqA9nVvqCBypPxW8iM9KK2t4a/YG/jFjLZOXx6Yo7tMhj+8WtuU7hW3o1TZXe/ZyUFTwIo3IutId/HPWOt6Zs4Hpq0txDjo2z+Q7hW34TmEbjuzSghSN2ct+UsGLNFIl5dW8P38j783byKQlm6iJRMnNSOHobi05tkcBQ7oX0K0gW3v3sk8qeJEksL06wqTFJUxYWMLHSzaxZusOANrmZTCkewFDurdkcJcWdGyeqcKXXVTwIklo1eZKPl6yiU+WbOLTpZvYWlkLQKvcdAZ1asagTs0Z1Lk5fTvkk5Ea9jmt+EUFL5LkolHH/A1lTFu5lWmrSpm2aisrN1cCkBo2erfLo7BdHoXt8+jdLo9ebXPJzUj1ObU0BBW8SABtqqhmerzsZ64uZf76sl17+QCdWmTRu10uPdvk0r11Doe2yqFbq2yy0lJ8TC2J9nUFr3/TIkmqICd915E3EDubdkNZFfPXlzFvXRnz15czb30Z783bSHS3/bgOzTLp3DKLTi2yOGTnT/NMOrXIokV2msb3A0QFLxIQZka7/Eza5WdyUq82u5ZXR+pYubmSpcUVLCmuYGlJBSu3VPLv+cVsqqj+j9+RlRamXX4GbfJ2/0nfdds6N4OCnHQy0zTmnwxU8CIBl54Spmeb2FDNniprIqzZuoNVmytZvbWS1Vt2sKFsBxvLqvli+RaKy6uorfvvYdyM1BAtstJolpVGi+w0mmen0SIrddfjvMwUctJTyU4Pk5ueSk5GCjnpKeRmpJCeEtL/JTQQFbxIE5aVlrLP8ofYsM/Wylo2bKtiY3kVG7dVsaWyhq3ba9iyvZbSyhq2VNawZmslW7bXUFYV+cZ1poSMnIwUstNipZ+RGiI9NUx6SoiM1HDsZ9f9r5alp8RelxoyUsIhUsNGajhESih2mxoOkRI2UsNGSih2Py0cIiX+mnD8xwzCtvN+7DZkENp1P/Y4CH+EVPAisk9mRovs2F55IXnf+PpIXZTSHbWUV0WoqIpQXl1LRVWE7TU7H8duK+K322siVNVGqaqto6wqQkl5NVW1dbFlkbpd9/2w8w9BKP4HIGzx8o//kTBi/3x23e5aBsYer7F9LAcwKMhOZ9xlRyd8G1TwIpIwKeEQBTnpFOSkJ+x3OueojkSpro1SXVdHpM4RqXPU1EWJRKNE6hy1dVEiUUdtJEpt1BGpi1K7a3nsfjTqiDqoczvvO+qiDhdfFrvvqItC1H31fNTFH0fdrvc6wDlwuPhtLKdze1u+22O3+3u/epyX4U0Vq+BFpFEzs13DNKBj+w+EZjQSEQkoFbyISECp4EVEAkoFLyISUCp4EZGAUsGLiASUCl5EJKBU8CIiAdWo5oM3sxJg5UG+vQDYlMA4yUDbHHxNbXtB23ygOjvnWu3tiUZV8PVhZlP2Nel9UGmbg6+pbS9omxNJQzQiIgGlghcRCaggFfzjfgfwgbY5+Jra9oK2OWECMwYvIiL/KUh78CIishsVvIhIQCV9wZvZMDNbaGZLzOzXfufxgpn9zcyKzWzObstamNl7ZrY4ftvcz4yJZmaHmNmHZjbPzOaa2ZXx5YHdbjPLMLMvzGxmfJtviS/vamaT45/xF80sze+siWRmYTObbmb/jD8O9PYCmNkKM5ttZjPMbEp8WcI/20ld8GYWBh4Cvg8UAuebWaG/qTzxFDBsj2W/Bt53zvUA3o8/DpIIcK1zrhA4Crg8/u82yNtdDZzknOsPDACGmdlRwJ+A+5xz3YGtwI/8i+iJK4H5uz0O+vbudKJzbsBux78n/LOd1AUPHAkscc4tc87VAC8Aw33OlHDOuY+ALXssHg48Hb//NDCiITN5zTm33jk3LX6/nFgBdCDA2+1iKuIPU+M/DjgJeDm+PFDbbGYdgVOBJ+KPjQBv7zdI+Gc72Qu+A7B6t8dr4suagjbOufXx+xuANn6G8ZKZdQEGApMJ+HbHhytmAMXAe8BSoNQ5F4m/JGif8b8A1wPR+OOWBHt7d3LAu2Y21cwujS9L+GdbF90OAOecM7NAHu9qZjnAK8BVzrmy2A5eTBC32zlXBwwws2bAeKCXv4m8Y2anAcXOualmdoLPcRrasc65tWbWGnjPzBbs/mSiPtvJvge/Fjhkt8cd48uago1m1g4gflvsc56EM7NUYuX+rHPu1fjiwG83gHOuFPgQOBpoZmY7d8aC9BkfApxhZiuIDa+eBNxPcLd3F+fc2vhtMbE/5EfiwWc72Qv+S6BH/Fv3NOA84HWfMzWU14GL4vcvAv7hY5aEi4/FPgnMd87du9tTgd1uM2sV33PHzDKB7xD77uFD4Jz4ywKzzc653zjnOjrnuhD7b/cD59xoArq9O5lZtpnl7rwPfBeYgwef7aQ/k9XMTiE2jhcG/uacu93fRIlnZs8DJxCbUnQjcDPwGjAO6ERsiuVRzrk9v4hNWmZ2LDAJmM1X47M3EhuHD+R2m1k/Yl+uhYntfI1zzv3BzLoR28NtAUwHLnDOVfuXNPHiQzS/cs6dFvTtjW/f+PjDFOA559ztZtaSBH+2k77gRURk75J9iEZERPZBBS8iElAqeBGRgFLBi4gElApeRCSgdCarJJ344WTvxx+2BeqAkvjjI+PzEu3rvUXA/zjnrjiA9a0AyuPrAfjoQN6/H7+/wjmXk6jfJ7KTDpOUpGZmvwcqnHN377YsZbe5TBKxjhVAkXNuU6J+5x6/XwUvntAQjQSCmT1lZo+a2WTgLjM70sw+i88z/qmZHRZ/3Qm7zTv++/hc+xPMbJmZHdBeefx998fn9J5jZkfGl7cws9fMbJaZfR4/gQkzyzGzsfF5wGeZ2dm7/a7b4/PAf25mgZpATfyjgpcg6Qgc45y7BlgAHOecGwj8DrhjH+/pBXyP2FwgN8fnv9mbD+NFPsPMrt5teZZzbgDwc+Bv8WW3ANOdc/2InX37THz5TcA251zf+HMfxJdnA5/H54H/CPjJAW21yD5oDF6C5KX4bIwA+cDTZtaD2NSs+yruN+OnwVebWTGxKVrX7OV1J+5jiOZ5iM3Zb2Z58blkjgXOji//wMxamlkecDKxOVeIP7c1frcG+Gf8/lRic9CI1Jv24CVItu92/1bgQ+dcH+B0IGMf79l9jpM6DnynZ88vsQ7mS61a99WXYQeTQWSvVPASVPl8Nc3sxR6u51zYNTnaNufcNmKTpI2OLz8B2OScKyN2AY/Ld74xEdfcFPk6KngJqruAP5rZdBKzR7z7GPwzuy2viq/jUb66dujvgSPMbBZwJ19NAXsb0Dz+hexM4MQE5BLZJx0mKXKQzGwCsSlup/idRWRvtAcvIhJQ2oMXEQko7cGLiASUCl5EJKBU8CIiAaWCFxEJKBW8iEhA/T9XdnEyH31v4AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# loss_list = np.array(loss_list) * 10\n",
    "x = np.linspace(0, 50, num=2400)\n",
    "plt.plot(x, loss_list[600:], label='Loss')\n",
    "y1 = loss_list[600:]\n",
    "# plt.title('Loss')\n",
    "plt.xlabel('Train Epoch')\n",
    "plt.ylabel('Loss')\n",
    "plt.savefig('train-3000-0.1.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAg60lEQVR4nO3deXgc1Znv8e/bLcmSZXmXjfcFG4zZQTFLCDFLCGSBuQ8QIMmEzM2NbxayTJJJGJIQwgzcwNwkQ27IEE+AQBYIgSxOcFjCTsJim93Gxgs2lhcs2bJk7VL3e//oktQWkrW4S6Xu/n2ex4+qTldVvwXd/dY5p+occ3dERCR/xaIOQEREoqVEICKS55QIRETynBKBiEieUyIQEclzBVEHMFATJ0702bNnRx2GiEhWWbVqVbW7l/f0WtYlgtmzZ7Ny5cqowxARySpmtqW319Q0JCKS55QIRETynBKBiEieUyIQEclzSgQiInlOiUBEJM8pEYiI5Lmse45ARMLj7iSSTsIdd0gknaQ7ySQkPVW+33oy2C4oT+3ftV970mlPJGlPprZtTzrJYDnpTtJT75kMjuHdjt9x7FRsQYykFlrakiSSTnFhDMzST4LmtiStieRQ/+c7oPQQB+v0+eUcNW3MwR+oGyUCkRyzYVc9q7bsIelQ09jKnvpWnlxfRW1TG0mHZPAjnPoxhvZkkmQy+KvpSYa10cWFSgQicmDb9zbxwR89RUt719VwQcw4ee4ETpg5DjMjHoOYGTEz4jGjIGbEOv4GZfFgOWYQj1lqP4NYZ3nqOKlyI5Z2zI7X4rEYBbGu9yiIx4Ljknb89H33P4Z1vh8YqcvpjqtqAwrjMQriRnPbO6/8iwtjFBXEOvcbrI7ax8HK1PxfBbEMVCt6Om4oRxWRjLp3VSV/21DNDy85jr+8uoNjZ4xl6tiSztdvfmwDa3bUceqhE2hpT3LDhUdz2vxyJpQWdf4I56qy4qgjyH5KBCJZ4OE1O/nr67u45sNH8tlfvcDc8lIe/epiINXU8x8PrgPg9R11AJy5YDLlZSOiCleyTO5eJojkkMqaJhJJ56kNVQBsqmrofG1TddpyVQMjCmJMHFU05DFK9lIiEMkClTVNADy2tuodr63asgeAUSNSFfxpY0uwTNyiInlDiUAkAyprGvdb/9PL27nr+bfesV0y6dy7qpK2Hm5tbG1P8o+3PseqLTX8+ZXtNLa288QbVWzd00htUxsAj6/b1bn9jQ+sZemTG/nPv65nzsRSzjlyMsB+fQci/aE+ApFBam5L8L2/rOWoaWP42m9f5vZ/ehdnHD4JgC/c9SIAly2aud8+y17eztd++zK79jXzucXz+PGj6znl0Inc+MBaSkcU8NT6ap5aXw3ArAkj2bJ7/wSzu6G1c/knj28EUnfg3PzR4wF4en01ZyyYFM4JS85SIhAZpOWv7uDnf9/cub52x77ORJDuxbdq+NVzb3HjhcdQXd8CwMZdDXznj69xxzNbgDd6PH73JNDhPfMnctUHjmDqmBIcJxYzRhcXAnDuUVMO7qQkLykRiGRIcWGqpTWZ9lRWU2uCT9+5kur6Vq44Y17n/f33vVA54OPPnVjKpuoGpowp5ogpozMTtAjqIxAZtPZuj+G2Bj/yexq7mm8eW7eLluCBp41V9eysbR70+x0xNfXjf8ho3TgvmaUageS0lvYEIwrioRx7b9oPPsDeoEM3/cf+c796oXN5Y1U9Ow4iEYwoSF236fkAyTTVCCRnvVpZy+HfeoClT27k6O88yMaqeu5bVYn343n/uuY2vvun1TS1JgDYXN3AZ36xit+u3EpDSzufvnMl1y9fu98+//X4Rj7zi1X871+s6vGY1y9fy19ff7tz/fTDynn+m2f1uO2CQ8reUVYY09dVwqEageSk7XubuH756wCdP9hnff8JAKaMKebUeRN73fftumb+5d5XePKNKmob23hj1z5OnDmOB1bv5JXKvZgZD695u8d9H1i9kxEFMeaWl7KpqoEZ40vY19zOwimjaWxNYJa6vbOmoZVPnjqLSWXF3HTpcVTXt3LLExup2tfC+NIizlwwiTkTSzlschmbdzdQMWsch00u4zcrt3LKoRMy/F9L8p315+poOKmoqPCVK1dGHYYMcydf/wg763puhvnEKbO485ktPH/VWUzq1t6eSDqHXrW81+PGY8ax08fwwlt7e3z9xx89nrOPmExxYZwn3qjihJljKQvu6OlLc1sCdygpCqcpS/Kbma1y94qeXlNdU3JSb0kA4M5ntgCwObg989an32T2lffTlkjy0taaAx43kXReeGsvS06f21l2/xdP61w+YspoigtTP+TvPay830kAoLgwriQgkVDTkOSt371QyUd++kznenV9S787c7/yvsNY+uQmAI6c2jU+/KzxIzMbpMgQUCKQvHX3iq37rW/d08SrlbXv2O7K8xbwvb+s5dgZY5lXPopPnDKL4sI4V563gJnBD//dS05mzfa6nB7uWXKXEoFI4LL/fpZEt2cDvvXBI/jUaXMoKYxz5oJJzEi74v/Mew/tXD557gROnqtOXMlOSgQige5JAOB/vSfVF3D5qbOHOBqRoaN6rIhInlMikJyzr7kt6hBEsooSgeSUvY2tHH3NQ/3evmOguHSXVMzIZEgiw576CCRn7KprZtH1jwxon98sOYXCeIwFh5SRcKcwuOtn5Ig47z6096ePRXKJEoHkhH3NbdyeNjdAf82fPIqRRamvQYyu6R2/8+EjMxWayLCnRCA54Zu/f41lL28f8H4lhXqSV0R9BJIT3trT82xefdEk7yIhJwIzO9fM1pnZBjO78gDbXWhmbmY9Dogk0pfCuH7QRQYrtKYhM4sDNwPvAyqBFWa2zN3XdNuuDPgS8FxYsUjui8cGlgg+ePQU3n2AoahF8kmYNYJFwAZ33+TurcDdwAU9bPdvwA3A4Kdukrz2yduf59lNewa0z0UnTuejJ80MKSKR7BJmIpgGpI/qVRmUdTKzE4AZ7n7/gQ5kZkvMbKWZrayqqsp8pJK1nl5fzePrBv6ZKCpQ95hIh8i+DWYWA34AfLWvbd19qbtXuHtFeXl5+MFJVkgknY/fOrgWRSUCkS5hfhu2AemPaE4PyjqUAUcBj5vZZuBkYJk6jKW/mtoSg963SMNFi3QK89uwAphvZnPMrAi4FFjW8aK717r7RHef7e6zgWeB891d81BKn1raE50TwwyGagQiXUL7Nrh7O3AF8CDwOnCPu682s2vN7Pyw3lfyw82PbuBHj6zfr6y4MMZTXz+Df/+Ho/rcX4lApEuoTxa7+3Jgebeyq3vZdnGYsUhu2d3Q+o6yCaUjmDF+JGXFfX+s1TQk0kXfBslKsR6eCL7m/NT4QP15pmCEagQinfRtkKzUPQ/ceNExvG/hZAAK+pEI1DQk0kXfBslK3X/qy0Z0NQedMrfvJ4aVCES66NsgWeeaZau545kt+5UVp40iOmZkIY989b0HPIb6CES66NsgWefnPcw74Ow/8Xy8j1FFBzo2kUguUyKQnDCmpHC/9b5+6DX8tEgXJQLJej/7RAUnzhq/X5mu+EX6T4lAst7Zwd1C6fpz55CIpCgRSE5SjUCk/5QIJKs093OgOSUCkf7T5PWSVfY2tu233tu4Qr0lgo9UTOeNt+szHpdINlMikKyxfW8TdzyzGYAFh5Tx3fOP5KS5E3rctrdEcMOFx+iOIZFulAgka9z+tzf576fexAx+eMlxHDFldK/b9pYIlARE3kl9BJI1dte3MnVMMS9dfc4BkwDs/0DZh46ZEnZoIllNiUCyxt6mNsaPKnrHw2M9Sa8R/PijJ4QZlkjWUyKQrFHT2MrYkqJ+basmIJH+UyKQrLG3sY2xI/uuDYjIwCgRSFb40P97ijerGxg3sn81AhHpP901JMNeIum8tq0OYNA1gvs+eyqbqvT8gEhPlAhk2NtW09S5XFIUP8CWvTtx1jhOnDUuUyGJ5BQ1Dcmwd++qrZ3LVftaIoxEJDcpEciw9mZ1Az96dAMApUVxLls0M+KIRHKPmoZkWHttWy0AXzhzHl895/CIoxHJTaoRyLC2buc+4jHjijPnRR2KSM5SIpBhbe3OfcyZWMqIgsF1EotI39Q0JMPWjtomnlxfxUUnTh/U/g/98+mMLtYDaCJ9USKQYam5LcElP30Wd2fJe+YO6hiHTS7LcFQiuUlNQzIsbdhVz1t7GrnyvCOYPbE06nBEcpoSgQxLb1Y3APDueT1PPCMimaNEIMPSm9UNmMHsCaoNiIRNiUCGndqmNu58ZjNTx5RQXKi7hUTCpkQgw86//u4VqutbOefIyVGHIpIXdNeQDCu1jW0sf3Unn118KN84d0HU4YjkhVBrBGZ2rpmtM7MNZnZlD69/xsxeNbOXzOxpM1sYZjwy/L21pxGAY6ePjTYQkTwSWiIwszhwM3AesBC4rIcf+l+7+9HufhxwI/CDsOKR7LBlT+puoVkTRkYciUj+CLNGsAjY4O6b3L0VuBu4IH0Dd69LWy0FPMR4JAt01AhmjFciEBkqYfYRTAO2pq1XAid138jMPg98BSgCzuzpQGa2BFgCMHOmhiHOVdv2NnHjA+sYU1LIqBHqvhIZKpHfNeTuN7v7ocA3gG/1ss1Sd69w94ry8vKhDVBC5+6s2V7H3zZUA7D4cP0/FhlKYV52bQNmpK1PD8p6czfwXyHGI8PU429U8U+3r2D6uBIAbrjwmIgjEskvYdYIVgDzzWyOmRUBlwLL0jcws/lpqx8E1ocYjwxTza0JACprmhhdXKCHyESGWGg1AndvN7MrgAeBOHCbu682s2uBle6+DLjCzM4G2oAa4PKw4pHsUF42IuoQRPJOqD1y7r4cWN6t7Oq05S+F+f6SHVoTyR6XRWRoRN5ZLNKW6LpruHpfa4SRiOQnJQKJVENLO1t2N3SuHzVtdITRiOQn3awtkbr4lmdYsyP1XOHPPlHBibPGRRyRSP5RIpBIdSQBgHfNGc+YEs0xLDLU1DQkw0ZRXB9HkSjomyfDRmHcog5BJC8pEciwEY8pEYhEQYlAhg0zJQKRKCgRiIjkOSUCEZE81+9EYGYlZnZ4mMGIiMjQ61ciMLMPAy8BDwTrx5nZsgPuJNKHNo0rJDIs9LdGcA2pqSf3Arj7S8CcUCKSvNHYkog6BBGh/4mgzd1ru5VpfmE5KA2t7VGHICL0f4iJ1Wb2USAeTCbzReDv4YUl+aChRYlAZDjob43gC8CRQAvwa6AW+HJIMUkeWLF5D1f9/tWowxAR+lEjMLM4cL+7nwF8M/yQJB9cfMszUYcgIoE+awTungCSZjZmCOIREZEh1t8+gnrgVTN7GOicRcTdvxhKVCIiMmT6mwh+F/wTEZEc069E4O53mFkRcFhQtM7d28ILS0REhkq/EoGZLQbuADYDBswws8vd/cnQIhMRkSHR36ah7wPnuPs6ADM7DLgLODGswEREZGj09zmCwo4kAODubwCaXFZEJAf0t0aw0sx+BvwyWP8YsDKckCQffeykmVGHIJK3+psIPgt8ntTQEgBPAT8JJSLJOyu/dTYTR42IOgyRvNXfRFAA3OTuP4DOp431zZWMKIxpfiSRKPX3G/gIUJK2XgL8NfPhSD4qiGuuYpEo9TcRFLt7fcdKsDwynJAk3ygRiESrv4mgwcxO6FgxswqgKZyQJN+oaUgkWv3tI/gy8Fsz2x6sTwEuCSUiyTuxmGoEIlE64KWYmb3LzA5x9xXAAuA3QBupuYvfHIL4REQkZH3VyX8KtAbLpwBXATcDNcDSEOMSEZEh0lciiLv7nmD5EmCpu9/n7t8G5vV1cDM718zWmdkGM7uyh9e/YmZrzOwVM3vEzGYN/BRERORg9JkIzKyjH+Es4NG01w7YvxA8a3AzcB6wELjMzBZ22+xFoMLdjwHuBW7sb+AiIpIZfSWCu4AnzOyPpO4SegrAzOaRmrf4QBYBG9x9k7u3AncDF6Rv4O6PuXtjsPosMH2A8YuIyEE64FW9u19nZo+QukvoIXf34KUYqQntD2QasDVtvRI46QDbfwr4S08vmNkSYAnAzJkak0ZEJJP6vH3U3Z/toeyNTAZhZh8HKoD39hLDUoLO6YqKCu9pGxERGZz+PkcwGNuAGWnr04Oy/ZjZ2cA3gfe6e0uI8YiISA/CfKRzBTDfzOYE01xeCixL38DMjid1i+r57r4rxFhERKQXoSUCd28HrgAeBF4H7nH31WZ2rZmdH2z2H8AoUk8tv2Rmy3o5nIiIhCTMpiHcfTmwvFvZ1WnLZ4f5/iIi0jeN9iVDLplUf7/IcKJEIEOuLZmMOgQRSaNEIEOuPaEagchwokQgQ65dTUMiw4oSgQy59oSahkSGEyUCGVKt7Um27Gnse0MRGTKh3j4q0t23//Aav1nZNQTVl86aH2E0IgJKBDLEHlnb9QD59y8+lgtP1ICzIlFT05AMqeLCro9cQVxzFYsMB0oEMqRGFHR95MqKVSEVGQ6UCGRIFRfGO5fnTyqLMBIR6aBEIEMqHutqDpo2tiTCSESkg+rmMiR27Wvmgdd2UtvU1lkWi6mPQGQ4UCKQIXHb05u55YmNnev/8v7DI4xGRNKpaUiGRNK7hpX43OJD+fwZ8yKMRkTSKRHIkKje1zULaWNrIsJIRKQ7JQIJ1d83VDP7yvtZvb2Oji6BCaVF0QYlIvtRH4GE6pfPbQFg3dv7OGfhZL5w5nwOO2RUxFGJSDolAgnVuJFdV//lZSM4evqYCKMRkZ6oaUhClf7cQHnZiAgjEZHeKBFIaPY2tlLT2PXcwMlzJ0QYjYj0Rk1DEooHXtvJZ365ilEjUh+xn3zsBCUCkWFKNQIJxbObdgNQ39LOWQsm8YGjp0QckYj0RolAQpE+yug43S4qMqwpEUgodtY1dy6PVyIQGdaUCCQUO/amEsH8SaM45VD1DYgMZ+oslozasruBz/7yBdbsqOOC46Zy06XHRx2SiPRBNQLJqNv/tpk1O+oAmDOxNOJoRKQ/lAgko557c0/n8oUnaGJ6kWygpiHJiC/e9SKrttSwbW8THz52KifPHc+M8SOjDktE+kGJQDJi2cvbgdSE9Nf9j6MYXVwYcUQi0l9qGpKDlkw6MYOT5oznwS+friQgkmWUCOSg1TW3kXQ458hDmKoJ6UWyTqiJwMzONbN1ZrbBzK7s4fXTzewFM2s3s4vCjEXCs6ehFdCEMyLZKrREYGZx4GbgPGAhcJmZLey22VvAJ4FfhxWHhK8jEWgoCZHsFGZn8SJgg7tvAjCzu4ELgDUdG7j75uC1ZIhxSMh2q0YgktXCbBqaBmxNW68MygbMzJaY2UozW1lVVZWR4CRzaoJEoDGFRLJTVnQWu/tSd69w94ry8vKow5E0T6+v5q+v7wKUCESyVZhNQ9uAGWnr04MyySEfv/U5AEYWxSkujEccjYgMRpg1ghXAfDObY2ZFwKXAshDfT4aYu3cup09SLyLZJbRE4O7twBXAg8DrwD3uvtrMrjWz8wHM7F1mVglcDPzUzFaHFY9kXtW+ls7lCaOUCESyVahDTLj7cmB5t7Kr05ZXkGoykiy0taapc3msagQiWSsrOotleKqsaexcLoxZhJGIyMFQIpBB21nbNR2lH2A7ERnelAhk0Oqa2zqX0zuORSS7KBHIoNU1tXcul47QiOYi2UrfXhmU2sY2ttY0Mn1cCecfO5X/edqcqEMSkUFSIpBBOfbahwA4etoYvn7ugoijEZGDoaYhGbBEsqs/oKxY1xIi2U6JQAZsY1V957ISgUj2UyKQAVu9vbZzWTcLiWQ/JQIZsL2NXbeNNrUlIoxERDJBiUAGrLG168e/WYlAJOspEciANbZ2PT9QVlwYYSQikgnq6ZMBa2xNUDaigK+9/3A+cPSUqMMRkYOkRCAD1tSaoKQozuWnzo46FBHJADUNyYA1tiYYWaTZyERyhRKBDFhjazslRapMiuQKJQIZMNUIRHKLEoEMmBKBSG5RIpABa1IiEMkpSgQyYI1t7YxUH4FIzlAikAHruH1URHKDEoEMWENLgpGFSgQiuUKJQAYkmXSa2tRHIJJLlAik355aX8Xcq5YDGmNIJJeox0/6tLO2mbZEku8/9EZn2anzJkQYkYhkkhKB9Ki+pZ1tNU1s2d3AFb9+kdZEcr/XF04ZHVFkIpJpSgTSqbG1nSffqGLKmBK+cd8rrN25D4AFh5Rx1hGTaGhJ8OnT51IYM8ws4mhFJFOUCAQAd+fy255nxeaa/crHjizkexcew3EzxkYTmIiETokgD7k7r26r5e26FnbWNvGnl3fQ0NrO6u11/OPJszhiymgWzRnPvEmjog5VRIaAEkEWq2tuY2dtM3VNbexraadqXwtb9zRSXd9Kc1uC5rYEdc1t7Gtuxx1qGlvZ29hGWyJJS3tXm/+8SaOYNraEi0+czrc/tJCiAt1MJpJPlAiGuea2BDtrm9le20RlTRObqxvYvLuBN6sbWf/2PtqTvt/2MYPxpUUUF8YpLowzuriAcSOLiBnMLS9lfGkRBTHjiCmjmT+pjOLCGHPLRxGPqc1fJF8pEQwD7s6O2mZeqdzL2p37WL+rntd31PF2bTMNrftPDl8QM2aMH8mciaWccXg5R0wZzeiSQsqKCxg/sohp40oojOuKXkT6T4kgQu7Og6vf5rrla9i6pwlIXdFPHzeSBYeUsfiwSUwYVcTk0cVMGVPM1LElTNcPvYhkWKiJwMzOBW4C4sDP3P173V4fAdwJnAjsBi5x981hxjQcNLUmeGTt29z82EZe31HH4ZPLuPaCIzly6hjmTRrFmBI9tSsiQye0RGBmceBm4H1AJbDCzJa5+5q0zT4F1Lj7PDO7FLgBuCSsmDLN3Uk6tCeTNLcl2VHbRCJos/eg6X5HbTNvVtdTWdPE9r3NbNndwMaqepIOh5aX8v2Lj+X846bqKl9EIhNmjWARsMHdNwGY2d3ABUB6IrgAuCZYvhf4sZmZu+/fA5oB96zYytKnNpF0B4ekO07w1wn+7V+W+k1P/dgn3UkknWTSSbjTnvB3dNQeyOjiAqaOLWHWhFLOO3oKx88cy+nzy9VJKyKRCzMRTAO2pq1XAif1to27t5tZLTABqE7fyMyWAEsAZs6cOahgxo4s5PDJZWAQM8MA22/ZgnUwjFgMwFLrwXYxM+IxoyDW9bcgHiMeMwrjxtSxJRTEYnQ8dGvAxLIRzJ1YytiRRYOKW0QkbFnRWezuS4GlABUVFYOqLZxz5CGcc+QhGY1LRCQXhNkwvQ2YkbY+PSjrcRszKwDGkOo0FhGRIRJmIlgBzDezOWZWBFwKLOu2zTLg8mD5IuDRMPoHRESkd6E1DQVt/lcAD5K6ffQ2d19tZtcCK919GXAr8Asz2wDsIZUsRERkCIXaR+Duy4Hl3cquTltuBi4OMwYRETkw3bwuIpLnlAhERPKcEoGISJ5TIhARyXOWbXdrmlkVsGWQu0+k21PLeUDnnB90zvnhYM55lruX9/RC1iWCg2FmK929Iuo4hpLOOT/onPNDWOespiERkTynRCAikufyLREsjTqACOic84POOT+Ecs551UcgIiLvlG81AhER6UaJQEQkz+VNIjCzc81snZltMLMro44nDGZ2m5ntMrPX0srGm9nDZrY++DsuyhgzycxmmNljZrbGzFab2ZeC8lw+52Ize97MXg7O+btB+Rwzey74fP8mGPo9p5hZ3MxeNLM/B+s5fc5mttnMXjWzl8xsZVAWymc7LxKBmcWBm4HzgIXAZWa2MNqoQvFz4NxuZVcCj7j7fOCRYD1XtANfdfeFwMnA54P/r7l8zi3Ame5+LHAccK6ZnQzcAPzQ3ecBNcCnogsxNF8CXk9bz4dzPsPdj0t7diCUz3ZeJAJgEbDB3Te5eytwN3BBxDFlnLs/SWpeh3QXAHcEy3cA/zCUMYXJ3Xe4+wvB8j5SPxLTyO1zdnevD1YLg38OnAncG5Tn1DkDmNl04IPAz4J1I8fPuRehfLbzJRFMA7amrVcGZflgsrvvCJZ3ApOjDCYsZjYbOB54jhw/56CJ5CVgF/AwsBHY6+7twSa5+Pn+T+DrQDJYn0Dun7MDD5nZKjNbEpSF8tnOisnrJTPc3c0s5+4XNrNRwH3Al929LnWxmJKL5+zuCeA4MxsL/B5YEG1E4TKzDwG73H2VmS2OOJyhdJq7bzOzScDDZrY2/cVMfrbzpUawDZiRtj49KMsHb5vZFIDg766I48koMysklQR+5e6/C4pz+pw7uPte4DHgFGCsmXVc2OXa5/vdwPlmtplUs+6ZwE3k9jnj7tuCv7tIJfxFhPTZzpdEsAKYH9xlUERqbuRlEcc0VJYBlwfLlwN/jDCWjAraiW8FXnf3H6S9lMvnXB7UBDCzEuB9pPpGHgMuCjbLqXN293919+nuPpvUd/dRd/8YOXzOZlZqZmUdy8A5wGuE9NnOmyeLzewDpNoZ48Bt7n5dtBFlnpndBSwmNVTt28B3gD8A9wAzSQ3f/RF3796hnJXM7DTgKeBVutqOryLVT5Cr53wMqU7COKkLuXvc/Vozm0vqank88CLwcXdviS7ScARNQ19z9w/l8jkH5/b7YLUA+LW7X2dmEwjhs503iUBERHqWL01DIiLSCyUCEZE8p0QgIpLnlAhERPKcEoGISJ7Tk8WSs4Jb7R4JVg8BEkBVsL4oGHeqt30rgE+4+xcH8H6bgX3B+wA8OZD9+3H8encflanjiXTQ7aOSF8zsGqDe3f9vWllB2lg1mXiPzUCFu1dn6pjdjq9EIKFQ05DkFTP7uZndYmbPATea2SIzeyYY5/7vZnZ4sN3itHHvrwnmenjczDaZ2YCu8oP9bgrGlX/NzBYF5ePN7A9m9oqZPRs8LIaZjTKz24Ox6F8xswvTjnVdMBfBs2aWU4PpSXSUCCQfTQdOdfevAGuB97j78cDVwPW97LMAeD+p8V6+E4xx1JPHgh/8l8zsn9PKR7r7ccDngNuCsu8CL7r7MaSeiL4zKP82UOvuRwevPRqUlwLPBnMRPAl8ekBnLdIL9RFIPvptMIInwBjgDjObT2rY395+4O8Phi9oMbNdpIb/rexhuzN6aRq6C1JzRpjZ6GC8oNOAC4PyR81sgpmNBs4mNaYOwWs1wWIr8OdgeRWpcYZEDppqBJKPGtKW/w14zN2PAj4MFPeyT/oYNgkGfhHVvTNuMJ1zbd7VqTeYGER6pEQg+W4MXcMXfzLE97kEOgfKq3X3WlID5n0sKF8MVLt7HanJZj7fsWOm5qUV6Y0SgeS7G4H/Y2Yvkpkr7PQ+gjvTypuD97iFrrl1rwFONLNXgO/RNbzwvwPjgo7ll4EzMhCXSK90+6hIyMzscVJDJ6+MOhaRnqhGICKS51QjEBHJc6oRiIjkOSUCEZE8p0QgIpLnlAhERPKcEoGISJ77/1NON3CXSKjTAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = np.linspace(0, 50, num=470)\n",
    "\n",
    "plt.plot(x, accu_list[30:], label='Score')\n",
    "y2 = accu_list[30:]\n",
    "\n",
    "# plt.title('Score')\n",
    "plt.xlabel('Train Epoch')\n",
    "plt.ylabel('Score')\n",
    "plt.savefig('train-3000-0.1-accu.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "y1 = loss_list[600:]\n",
    "# y2 = accu_list[30:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAupElEQVR4nO3deXxU9bn48c8zS/aEkBAStrAjoAhqZFFUUKHu2mottYt6tajVqtXan7e3t9rea29vF6je2mtp63bdN1q0uKCsFRHCjiD7viVhyULWyTy/P+YkTEJCEshkMjPP+/XKa875nmWerx7mOd+zfL+iqhhjjIldrnAHYIwxJrwsERhjTIyzRGCMMTHOEoExxsQ4SwTGGBPjPOEOoK26deum/fr1C3cYxhgTUZYvX16kqllNLYu4RNCvXz/y8/PDHYYxxkQUEdnZ3DK7NGSMMTEuphJBRXVtuEMwxphOJ2YSweKtRVz067m8s2IP9ja1McYcFzOJICM5jj4ZSTz0xmq++eclbCkoC3dIxhjTKcRMIhiak8bbd1/AL786gg37S7nyyYX89sONVNbY5SJjTGyLmUQA4HIJt4zJ5ZOHL+HakT35w7wtTJq+gHlfFoQ7NGOMCZuYSgR1uqXEM+3mUbz6vbHEuV3c/vwy7nlpOfuLK8IdmjHGdLiYTAR1xg3M5P0HLuaRr5zB3C8LuPx3C3ju0+3U+u1msjEmdsR0IgCI87i4d+Ig5vzwEvL6ZfDzd9dz0zOL2XSwNNyhGWNMh4j5RFAnNzOJ528/n99/YxQ7io5x9VOLmD5nE1U+u5lsjIlulgiCiAg3nNOLjx+6hKtG9ODJTzZzzVP/ZPnOI+EOzRhjQsYSQRMyU+J5cso5PHtbHseqfNz0zGIen/UFx6p84Q7NGGPanSWCk7h0aDYfPXQJ3x3blxc+28Hk6QuZv9EeNTXGRBdLBC1Iiffw8+vP4q27x5EY5+a255bxw9dXceRYdbhDM8aYdmGJoJXO65vBP+4fz/2XDebd1fuYNH0hH35xINxhGWPMabNE0AbxHjcPTRrCrPvGk50Wz13/t5z7X13JYWsdGGMimCWCUzC8Zxp/u/dCHp40hPfX7Wfy9AV8sG5/uMMyxphTYongFHndLn5w2WDe/cF4crokcPdLK7jvlRUcKqsKd2jGGNMmlghO09CcNGZ+/0J+NHkIH35xgMnTFzJ7rbUOjDGRwxJBO/C6Xdx36WDe+8FF9ExP5Psvr+Del1dQZK0DY0wEsETQjs7ISWXm9y/gka+cwZz1B5k8fSHvrdlnI6IZYzq1kCUCETlDRFYF/ZWIyION1pkgIsVB6/wsVPF0FI870Inde/ePp0/XRO57ZSXff3kFhaXWOjDGdE6eUO1YVTcCowBExA3sBWY2seoiVb0mVHGEy5DsVN6+5wL+vGg70+dsYsm2Bfz8+rO49uweiEi4wzPGmHoddWnoMmCrqu7soO/rFDxuF/dMGMg/7h9PbmYy97+6kntestaBMaZz6ahEMAV4tZll40RktYi8LyJnNrWCiEwVkXwRyS8sLAxdlCEyODuVt+8ex6NXDmXuxgImTV/A31fttXsHxphOQUL9YyQiccA+4ExVPdhoWRrgV9UyEbkKeFJVB59sf3l5eZqfnx+6gENsS0EpP3pzDat2H2Xy8Gz+86tn0T01IdxhGWOinIgsV9W8ppZ1RIvgSmBF4yQAoKolqlrmTM8GvCLSrQNiCptB3QP3Dv71yqHM31TI5OkLrXVgjAmrjkgE36SZy0IikiPOnVMRGe3Ec6gDYgort0u465KBzL7/Ivp3S+aB11Yx9f+WU1BSGe7QjDExKKSJQESSgUnAO0Fld4vI3c7sTcA6EVkNPAVM0Rg6NR7UPYW37r6Af7tqGAs3FTJp+kJmrtxjrQNjTIcK+T2C9hbp9wias7WwjEfeXM2KXUe5fFg2v/zqWXRPs3sHxpj2Ee57BKYVBmal8ObdF/DTq4exaHMhl09bwDsrrHVgjAk9SwSdiNsl3HnRAN5/4CIGZ6fy0BurufOFfA7avQNjTAhZIuiEBmSl8MZd4/jp1cP455YiJk1bwNvLrXVgjAkNSwSdVF3r4IMHL2ZIdioPv7maO17I50CxtQ6MMe3LEkEn179bMq/fNY6fXTOcxVuLmDR9AW/m77bWgTGm3VgiiABul/Av4/vz/gMXMzQnlUfeWsO/PL/MWgfGmHZhiSCC9O+WzOtTx/HYtcP5bNshJk1fwBvWOjDGnCZLBBHG5RJuv7A/HzxwMcN6pPHjt9Zw23PL2F9cEe7QjDERyhJBhOrXLZnXvjeWx68dztLth5k8bSEvf74Tv99aB8aYtrFEEMFcLuG2C/vzwYMXcVavLvzbzHVMmbGELQVl4Q7NGBNBWpUIRCRZRFzO9BARuU5EvKENzbRW38xkXvneGH5909lsPFjKVU8u4qlPNlPt84c7NGNMBGhti2AhkCAivYCPgO8Az4cqKNN2IsLNeX34+KFLmHxmNtPmbOKa/1nE8p1Hwh2aMaaTa20iEFUtB74G/FFVvw40OZqYCa+s1Hj+cMu5/PXWPEorfdz0zGIe+/s6yqp84Q7NGNNJtToRiMg44FvAP5wyd2hCMu3hsmHZzHnoEm4d148Xl+xk0rQFfLz+hLGBjDGm1YngQeBfgZmq+oWIDADmhSwq0y5S4j08ft2ZvH3PBaQleLnzxXzufXkFBaX2Ipox5rg2j0fg3DROUdWS0IR0ctE6HkGoVfv8/GnBVv5n7hbivS5+/JUzuGVMX9wuCXdoxpgOcNrjEYjIKyKS5ow4tg5YLyKPtGeQJrTiPC5+cNlg3n/wIs7u3YV///sX3PD0p6zefTTcoRljwqy1l4aGOy2AG4D3gf4Enhw6KRHZISJrRWSViJxwGi8BT4nIFhFZIyLntiV403YDs1J46Y4xPPXNczhQUskNf/yUn/5tLcXlNeEOzRgTJq1NBF7nvYEbgFmqWgO09prSRFUd1UyT5EpgsPM3FfjfVu7TnAYR4bqRPfnk4Uu47YJ+vPL5Li793Xwb88CYGNXaRPAnYAeQDCwUkb5Ae9wjuB54UQOWAOki0qMd9mtaIS3By2PXnsm7PxhPbmYSD7+5mm/MWMKmg6XhDs0Y04FalQhU9SlV7aWqVzk/2juBia3ZFPhIRJaLyNQmlvcCdgfN73HKGhCRqSKSLyL5hYWFrQnZtMGZPbvw9t0X8KuvjWCT82byL2dvoLTSLhcZEwtae7O4i4hMq/sxFpHfEWgdtGS8qp5L4BLQvSJy8akEqaozVDVPVfOysrJOZRemBS6XMGV0LnMfnsCN5/ZmxsJtXPq7wCA41pGdMdGttZeGngVKgZudvxLguZY2UtW9zmcBMBMY3WiVvUCfoPneTpkJk4zkOP77prP5+70X0rtrIo+8tYYb/vipdVVhTBRrbSIYqKqPqeo25+/nwICTbeB0VJdaNw1MJvDoabBZwHedp4fGAsWqur+NdTAhMLJPOm/ffQHTvzGSgyWV3Pi/i/nh66tsVDRjopCnletViMh4Vf0ngIhcCLQ0Eko2MFNE6r7nFVX9QETuBlDVZ4DZwFXAFqAcuL3tVTCh4nIJXz2nN5OH5/C/87cyY9E2Plh3gHsnDuTOiwaQ4LVeRoyJBq16s1hERgIvAl2coiPAraq6JoSxNcneLA6fXYfK+eXsDXzwxQF6d03kx1cM5ZoRPXDZ28nGdHqn/Waxqq5W1ZHA2cDZqnoOcGk7xmgiQG5mEs985zxeuXMMKfEe7n91JV/946d8tvVQuEMzxpyGNvc1VL+hyC5VzW3neFpkLYLOodav/G3lXn730Ub2FVdy6dDuPHrlUIZkp4Y7NGNME067RdDcfk9jWxPh3C7hxvN6M/dHE3j0yqEs23GYK36/kP/31hq7oWxMhDmdRGAPlxsSvG7uvmQgCx+ZyO0X9mfmyr1M+O08fvPhlxRX2AtpxkSCk14aEpFSmv7BFyBRVVv71FG7sUtDndvuw+X89qON/H3VPtISPEy9eAC3XdiflPgOP1SMMUFOdmnolO8RhIslgsiwfl8J0+Zs4uMNB8lIjuPuSwbwnbH9SIyzR06NCQdLBCZsVu0+yrQ5m1i4qZCs1HjumziIKaP7EO+xhGBMR7JEYMJu2Y7D/PbDjXy+/TA9uiRw18UDmDI6115KM6aDWCIwnYKqsnjrIZ78eDNLdxymW0ocd4wfwLfH5pKa4A13eMZENUsEptNZuv0wf5i3hYWbCklL8HD7hf25/cJ+pCfFhTs0Y6KSJQLTaa3efZSn523ho/UHSY5z8+2xfbn9wv7kdEkId2jGRBVLBKbT23iglD/O38K7q/fhEuHakT25Y3x/zurVpeWNjTEtskRgIsauQ+U8t3g7byzbzbHqWsYNyOTOi/oz8Yzu1rmdMafBEoGJOMUVNby2dBfPL97B/uJKBmQl8y8X9udr5/YiKc5eTjOmrSwRmIhVU+tn9tr9/GXRdtbuLSY13sON5/Xm22NzGdTdOrgzprUsEZiIp6os33mEl5bsZPbaA1TX+hk7IIPvjO3H5DOz8bpPp9ssY6KfJQITVYrKqngjfzevfL6LPUcqyEqNZ8r5ffj6eX3IzUwKd3jGdEphSQQi0ofAqGbZBDqum6GqTzZaZwLwd2C7U/SOqv7iZPu1RGDq1PqVBZsKeGnJLuZtLEAVxvTP4Ot5fbjyrBySraM7Y+qFKxH0AHqo6gpnEPvlwA2quj5onQnAj1T1mtbu1xKBacq+oxXMXLmXN/N3s+NQOclxbq4a0YOv5/Xh/H5dccbONiZmnSwRhOyUSVX3A/ud6VIR2QD0AtafdENjTkHP9ETunTiI708YSP7OI7yVv4f31uzjzeV76JuZxLVn9+TakT05I8duMBvTWIfcIxCRfsBC4CxVLQkqnwC8DewB9hFoHXzRxPZTgakAubm55+3cuTPkMZvIV17t4/21B5i5ci+LtxbhVxjcPYVrR/bkmrN7MCArJdwhGtNhwnqzWERSgAXAE6r6TqNlaYBfVctE5CrgSVUdfLL92aUhcyqKyqp4f90B3l29j2U7DqMKZ/ZM4+qzezB5eA6DultSMNEtbIlARLzAe8CHqjqtFevvAPJUtai5dSwRmNN1oLiSf6zdz7ur97Fq91EABnRLZtLwbCYNz+ac3K647S1mE2XCdbNYgBeAw6r6YDPr5AAHVVVFZDTwFtBXTxKUJQLTnvYdreDjDQeZs/4gn209hM+vZCbHcdmw7lw2LJsLBmZaF9kmKoQrEYwHFgFrAb9T/BMgF0BVnxGR+4B7AB9QATykqotPtl9LBCZUSiprmL+xkDnrDzL/ywJKq3y4XcK5uelcPDiLi4ZkMaJXF2stmIhkL5QZ00bVPj8rdh1h4aZCFm0uYu3eYgDSk7xcOKgbFw/uxtgBmeRmJNmjqSYiWCIw5jQdKqvin1uKWLS5iEWbCzlYUgVAdlo8o/tnMrp/BqP7ZTC4e4r1kmo6JUsExrQjVWVLQRmfbz/M0u2H+Xz7ofrEkJ7kJa9vBufkpjOqTzojenchze4xmE4gLC+UGROtRITB2akMzk7l22P7oqrsPlzB59sPsXT7YfJ3HuHjDQfr1x+YlczIPoHEMLJ3OmfkpJLgdYexBsY0ZC0CY0LgaHk1a/YUs3r3UVbvKWbV7qMUlQVaDW6XMKBbMsN6pDG0RyrDeqQxvEca3VPj7X6DCRlrERjTwdKT4rh4SBYXD8kCApeT9hdXsnr3UdbvL2HD/lKW7zzCrNX76rfpmuRlYFZK4K97MgOzUhiQlUKfrol4rJttE0KWCIzpACJCz/REeqYncuWIHvXlxRU1fLm/hA37S9h4sJSthcf45MuDvJ5fXb+O1y30y0xmQFYyuRlJ9MlIok/XJPpkJNK7a5JdZjKnzRKBMWHUJdHLmAGZjBmQ2aD8aHk1WwuPsa2wjK2Fx9haWMaWgjLmbyykyudvsG5Wajy9uwaSQk5aPNlpCeR0SSAnLYFs5y/OYy0K0zxLBMZ0QulJcZzXN47z+nZtUO73K0VlVew+Us6eIxXsPlzO7sMV7D5Szpo9R/mouPKERAGQmRxHVmo8mSlxZCTHk5kcF/hLiScjOY7MlMB8RnIcqQlee2kuxlgiMCaCuFxC97QEuqclcF7fE5erKsUVNRwoqeRAcSUHSyo5UFzFgZJKisqqOHysmrV7jnLoWDWllb5mvyc5zk1aopfUBA9pCc5norfBdGqCh5R4D0lxHhK9bhLjXCR6PSTGuUmKc5PgdZPodVtrJAJYIjAmiogI6UlxpCfFMTQn7aTrVvlqOXKspj5BHDpWxeFjNZRW1lBS4Qt8VtZQWumjsKyKbUXHKKmooaTSR62/9U8belziJIrAX4InkBw8bsHrdhHnduF1C55G04FlgXU8jaa9biHOE1jH4xLczp+I4BbBJYH/Fm5XYNrlElytWCZCYDpoWV3bSCQwXfdglyAEP+Qlcnyd+nlnnfrVpOF2QtPb1BXUbVv3NJnXLcR72v+ekCUCY2JUvMdNThc3OV0S2rSdqlJRU0tJhY+yqhoqqv1U1NQG/qp9VNTUUl5dS0V1LZV10zVB09W1+PxKTa2fal9g25LKwHRdeY3PT3Wt4vMHpmtqleraEy95xZq7LxnIo1cObff9WiIwxrSJiJAUF7gkBG1LIqdDVan1a31S8NUGEkRNrZ9av+LXuj/wO+uqcuIyv1KrTSzz02C9umWB7wYlsM3x+UBMEJjGWafh8hPLCNpGteE+6tcJ3nfdd6KM7J0ekv+2lgiMMRFBRPC4BY8bErFHZtuT3cUxxpgYZ4nAGGNiXMT1NSQihcCpjl7fDWh2GMwoZXWODVbn2HA6de6rqllNLYi4RHA6RCS/uU6XopXVOTZYnWNDqOpsl4aMMSbGWSIwxpgYF2uJYEa4AwgDq3NssDrHhpDUOabuERhjjDlRrLUIjDHGNGKJwBhjYlzMJAIRuUJENorIFhF5NNzxhIKIPCsiBSKyLqgsQ0TmiMhm57PryfYRaUSkj4jME5H1IvKFiDzglEdtvUUkQUSWishqp84/d8r7i8jnzjH+uojEhTvW9iQibhFZKSLvOfPRXt8dIrJWRFaJSL5TFpLjOiYSgYi4gaeBK4HhwDdFZHh4owqJ54ErGpU9CnyiqoOBT5z5aOIDHlbV4cBY4F7n/20017sKuFRVRwKjgCtEZCzw38B0VR0EHAHuCF+IIfEAsCFoPtrrCzBRVUcFvTsQkuM6JhIBMBrYoqrbVLUaeA24PswxtTtVXQgcblR8PfCCM/0CcENHxhRqqrpfVVc406UEfih6EcX11oAyZ9br/ClwKfCWUx5VdRaR3sDVwF+ceSGK63sSITmuYyUR9AJ2B83vccpiQbaq7nemDwDZ4QwmlESkH3AO8DlRXm/nMskqoACYA2wFjqpq3bBj0XaM/x74MVA3KEEm0V1fCCT3j0RkuYhMdcpCclxbN9QxRFVVRKLyeWERSQHeBh5U1RIJGjoqGuutqrXAKBFJB2YC7T9aSSchItcABaq6XEQmhDmcjjReVfeKSHdgjoh8GbywPY/rWGkR7AX6BM33dspiwUER6QHgfBaEOZ52JyJeAkngZVV9xymO+noDqOpRYB4wDkgXkbqTu2g6xi8ErhORHQQu614KPEn01hcAVd3rfBYQSPajCdFxHSuJYBkw2HnKIA6YAswKc0wdZRZwqzN9K/D3MMbS7pxrxX8FNqjqtKBFUVtvEclyWgKISCIwicC9kXnATc5qUVNnVf1XVe2tqv0I/Nudq6rfIkrrCyAiySKSWjcNTAbWEaLjOmbeLBaRqwhcZ3QDz6rqE+GNqP2JyKvABAJd1R4EHgP+BrwB5BLovvtmVW18Qzliich4YBGwluPXj39C4D5BVNZbRM4mcKPQTeBk7g1V/YWIDCBwxpwBrAS+rapV4Yu0/TmXhn6kqtdEc32dus10Zj3AK6r6hIhkEoLjOmYSgTHGmKbFyqUhY4wxzbBEYIwxMc4SgTHGxLiIe4+gW7du2q9fv3CHYYwxEWX58uVFzY1ZHHGJoF+/fuTn54c7DGOMiSgisrO5ZXZpyBhjYlzEtQiMMaenptZPTa0f1UBnNqrqfAIKijZY5nfKqC8Df/02zrrNbEeDMme74HXbuF1r1O3P51eaejy+rvsRaVDmfDYobb3Af402bnMKT+737prIgKyUtm/YAksExkQAVaW61k+8x93scp9fcYvwm482snBTIS4Rqn1+KmpqqaippbI68Onz27tDkeruSwby6JXt362UJQJjIsArS3fxmw838s//dyk/fms1I3unc9clA+uXP/zGav62ai+3XtCP5z7dwYBuyfTNTCLO4yLR6yYxzk2C1x2Y9rrxelwIgTNhQY6fEYsElYPL5ZwjB5W7GqwT2LbBds46jfcXvB0Iria2Oz5/4nZI68/Y3S7B7Tp+9l/n+Fm4nlCmjdaTNjYOTqUt0dbvyE5LOIVvaZklAmNCoLKmlvLqWjKSTxw0q9rnx+uWE36kAPYcKScjOQ6Py4XXLfgVXAKz1+7naHkNL362g9lrDzB77QG+M64vNT7lywMlzFy1F1V47tMdALx+1ziyUuNDXU0TJSwRGHOKav3KY7PWcdWIHtzxfD5/uTWPCwd1A+B7L+azaHMR2//rKvwKq3Yf5by+XSkqqyLvPz/ml18dwZgBGby/dj99MpL4dEsRZVU+Zq89wNgBGWzYX8plQ7szf1MhbpdQWBroQufXH2ys//7hP/uwfrpHlwSuG9WTPy3YxtCcVEsCpk0sERhzirYUlPHSkl28tGQXAH9etK0+ESzaXATAws1FLN95hKc+2cwr3xtDcXkNAD+ZubbZ/S7ZFuhD7J2VDXtV7prk5Uh5DcN6pHHjub2oqVXiPS4SvG4uH9adtEQvI3unM6xHWrvX1UQ3SwQmaqkqv/94Mzed15s56w9y47m9+WJfMaP7Z+Bxn/zJaVXl3TX7mTw8u/467stLdjEgK5lLhmQxc+VeXl+2u8E2a/cU88yCrfhq/fVltz67tH76D3O3nFZ9br+wP9PmbCI3I5E7LxrQ5DpXjehxWt9hYlPE9T6al5en9kKZacmizYV8569LG5TFeVxU+/z8+zXD+XJ/CT+9ejhdkryoKtuLjtU/lvenBVuZsXAbh45VMzQnlW1Fx8hIiuNASSUA9186iKda+FEXOX4TMt7josrnb7DcJXDtyJ7cdfFA3luzj/kbC+mS6OWzbYcAGNm7C4lxbipr/MS5XXjcwuPXncnk6Qt5+pZzufps+8E3bSMiy1U1r8lllghMNLr+6U9Zvftok8u6pcRTVFbFlPP7UFBaxbAeqTw9byv/uH88SXEeJv52fov7v3xYdz7eEBgcasr5ffh4QwFFZVX84ZZzGNk7nazUePYcKad7WgIJHjdet1Bd66emVknyBh4Bdbka3iyurKmlqsaPuCDB48bjLA9er6zKR0q8NeRN250sEdgRZaJSdaMz8GBFZYEbr685l3bmfhn4QZ8+Z1P9j3uwhyYNYdqcTVw7sicJHheDs1O4c/wAnv10O1U+P/dOHET+jsO8snQXV57VA7fzwz2oe2qD/cR73JzsNzzBG3jE82QsCZhQsKPKRJ1qn5+Sipo2b9dUElj86KX0TE/k22P7khTX8Ic6+Dp9Xr8M8vplnFrAxoSZJQITVZ6et4XffLix5RVPIictof5+QM/0RIAm3wcwJlqENBGIyBXAkwTGVv2Lqv6qmfVuBN4CzldVuwFg2mxH0TFueuaz+ss+rfXSHWOo8tVyVq8uHCiupHfXRDxuFyt2HiE1wc6TTGwI2ZEuIm7gaWASsAdYJiKzVHV9o/VSgQcIDDZuzCl5demuNicBgPP7d63vvyf49f2JQ7u3W2zGdHah7IZ6NLBFVbepajXwGnB9E+v9B/DfQGUIYzFR6liVj9eX7eJPC7e1absx/TO4Y3z/ZjtxMyaWhLLt2wsIfuNmDzAmeAURORfoo6r/EJFHmtuRiEwFpgLk5uaGIFQTqc587MOWV2rCzXl9uPG83u0cjTGRKWwD04iIC5gGPNzSuqo6Q1XzVDUvK6vJkdZMjCmtrOGbM5ac8vZ2/d+Y40KZCPYCfYLmeztldVKBs4D5IrIDGAvMEpEmX3gwJti8jYX1b+EG+/DBi/nzd1s+hFITvKEIy5iIFMrTomXAYBHpTyABTAFuqVuoqsVAt7p5EZkP/MieGjItKa/2sWrX0QZl/TKT+Ott5zMwK4We6S332W4tAmOOC1mLQFV9wH3Ah8AG4A1V/UJEfiEi14Xqe030e+TNNTz76fYGZZcNy2ag01dQaoKX528//6T7SLMWgTH1QnpapKqzgdmNyn7WzLoTQhmLiR6fbi06oaxxJ2yXDDn5vSRrERhznP1rMBEnrlEX0msfn3zCNf+mRv8KlmKJwJh6YXtqyJi2OlpezbP/3M6R8ur6sszkuDZ1xHbBwEwAvC2MR2BMLLHTIhMx3sjfzS9nfwnAA5cN5rpRPevvC7TWtJtH0d2GcTSmAUsEJmJsOlhGt5R45j8yoU2tgNemjuW7zy6l2ucnMc59wjgAxsQ6ax+biPD4rC94a/keBndPaXOf/GMHZJKVEmgFJHjtkDemMWsRmE5v6fbDPL94x2nt47WpY/l0S5H1LWRME+z0yHRqB4oruflPn9XPn2r/QH0ykpgy2vqpMqYp1iIwndorn+8E4FdfG8HX8/rUDwNpjGk/lghMp6SqfP/lFby/7gBXnJlzSmfzf701j5paDUF0xkQXSwSmU/piXwnvrzvAVSNymHbzqFPax2XDsts3KGOilN0jMJ3Ogk2FfO2Pi3EJ/OcNIxoMGG+MaX/WIjCdSrXPz30vryArNZ6pFw+wQeON6QCWCEyn8srnOymt8vH7KaPs0o4xHcQSgQk7v1/56z+3Myg7hcffXQ/ABQO7tbCVMaa9tDoRiEgikKuqG0MYj4lB24rKeGL2hvr55247n8Q4uy9gTEdp1c1iEbkWWAV84MyPEpFZIYzLxJCSSl+D+YlDu4cpEmNiU2ufGnocGA0cBVDVVUD/kERkYsrynUe4/bll9fPDeqSFMRpjYlNrLw3VqGpxo8E+7E0dc9rufGEZxRU1APzlu3mc17drmCMyJva0NhF8ISK3AG4RGQzcDywOXVgmVpRX19ZPD8lOpas9LmpMh2vtpaEfAGcCVcArQDHwYIhiMjGkyuevn06OtxvExoRDiy0CEXED/1DVicC/hT4kE6uS2zjOgDGmfbTYIlDVWsAvIl06IB4Tw+I91uOJMeHQ2lOwMmCtiMwBjtUVqur9IYnKRL19Ryv44/wtDcoaPYxgjOkgrU0E7zh/xrSLH76+is+3Hw53GMYYWpkIVPUFEYkDhjhFG1W1JnRhmWhXGXST2BgTXq1KBCIyAXgB2AEI0EdEblXVhSGLzEQ1uwhkTOfR2ktDvwMm1/UzJCJDgFeB80IVmIludjvAmM6jtYnAG9zZnKpuEhFviGIyMSA4D/zw8iGMG5gZtliMiXWtTQT5IvIX4CVn/ltAfmhCMrEg+Amhq0bkMDg7NYzRGBPbWvvg9j3AegJdS9zvTN/T0kYicoWIbBSRLSLyaBPLHxKR9SKyRkQ+EZG+bQneRK7gFoF1OW1MeLW2ReABnlTVaVD/tnH8yTZw1nkamATsAZaJyCxVXR+02kogT1XLReQe4NfAN9pYBxOBgu8RJMfZG8XGhFNrWwSfAIlB84nAxy1sMxrYoqrbVLUaeA24PngFVZ2nquXO7BKgdyvjMRFOgtoE1iIwJrxamwgSVLWsbsaZTmphm17A7qD5PU5Zc+4A3m9qgYhMFZF8EckvLCxsZcimM6vxH3+PwLqWMCa8Wvsv8JiInFs3IyJ5QEV7BSEi3wbygN80tVxVZ6hqnqrmZWVltdfXmjCqCOp+2rqWMCa8Wntx9kHgTRHZ58z3oOVr+XuBPkHzvZ2yBkTkcgK9ml6iqlWtjMdEuIqa2pZXMsZ0iJO2CETkfBHJUdVlwFDgdaCGwNjF21vY9zJgsIj0d7qnmAI0GOdYRM4B/gRcp6oFp1gHE4GCWwTGmPBq6dLQn4BqZ3oc8BMCTwIdAWacbENV9QH3AR8CG4A3VPULEfmFiFznrPYbIIVAa2OViMxqZncmylgiMKbzaOnSkFtV67qI/AYwQ1XfBt4WkVUt7VxVZwOzG5X9LGj68raFayLdhv0l/PD1VZRW+cIdijHG0VKLwC0idcniMmBu0DJ7+Nu02dPztvDlgVIArhvZkw8fvDjMERljWvoxfxVYICJFBJ4SWgQgIoMIjFtsTJsEvzx2+fBszsixriWMCbeTJgJVfUJEPiHwlNBHqqrOIheBAe2NaZU9R8r58VtrGgxGM2lYdhgjMsbUafHyjqouaaJsU2jCMdFo16FyPvnyIIu3Hqove/qWc+2NYmM6CbvOb0JqybZDTJnR8FzioUlDuPrsHmGKyBjTmCUCEzKqyvp9JfXzk4Zn85ubziYl3g47YzoT+xdpQmL34XIu+vU8eqUH+iqM97i4ZUwu6UlxYY7MGNOYJQITEnO/DLwovvdoBefkpvPGXePwuq1zOWM6I0sEpl1VVNfyP3M38+mWovqyCwZmWhIwphOzRGDaRUFpJRXVtfx50TZeWrILgCnn9+Gxa88kwWtJwJjOzBKBaRcPv7GaRZsDrYARvbpwRk4qP71muD0iakwEsERgTpuqsmRb4B2BJ756Ft88PxeXy8YYMCZSWCIwp62gtIqaWuXn153Jt8b0DXc4xpg2skRgTsuG/SXMWh0Yr8j6DTImMlkiMKflyicX1U8PtURgTESyxznMKSsur6mfjve47GUxYyKUJQJzypbuON6TaH2/tMaYiGOJwJySsiofi7cef2lsaA+7LGRMpLJ7BOaUXPXkInYdLqdLopdffnUE5/frGu6QjDGnyBKBabMdRcfYdbgcAJdgXUobE+Hs0pBps/X7j3ctfSTohrExJjJZIjBtVlhaVT899eIBYYzEGNMe7NKQabOC0krcLmH9L75CnPUqakzEs0Rg2qywtIpuKXHEe6xDOWOigZ3OmTYrKK2ie2pCuMMwxrQTSwSmzQpLq8hKjQ93GMaYdmKXhkyListrKKmsYfeRcu56cTmlVT7O7p0e7rCMMe3EEoGpd6zKx9Lth0lL9LBubwkrdh0hKc7Dm/m78fkb9iHxrTG5YYrSGNPeLBFEGb9fqfL5qfLVBj5rAtOVNUFlvlqqavxU+mopr65l04FSqmv9LNl2mO1Fx07Y53UjezJ2QCZHK6qZPDwbVRicbV1KGBMtQpoIROQK4EnADfxFVX/VaHk88CJwHnAI+Iaq7ghlTJ1Zla+W4vIajlbUcPhYNUfLqymrqqW82sexRp9lVT4KSqrYX1JBRXWt84Pvp7rW3+bvTY5zkxzvITXBwx9uOYekODc5aYkMzk6htNJHRrL1KmpMNAtZIhARN/A0MAnYAywTkVmquj5otTuAI6o6SESmAP8NfCNUMXUWfr9ysLSSbYXHWLX7KB9vOMjOQ+UcPlZ90u1EIDnOQ3K8m+Q4D91S4zk3tysp8R7iPW7ivS7iPS4SvG7iPa5AmcfllLtJ8J5YFu9x0T01Hk8z7wNYEjAm+oWyRTAa2KKq2wBE5DXgeiA4EVwPPO5MvwX8QUREtf07Nd5aWMbcDQUoiiooOJ9a34WyatPLNLCwvtwfNK0EVlCgsiZwZu7zK37VwKdfqfLVctQ50z9aXs2R8hpqg665j+yTzhVn5ZCTlkBGchxdEr1kJMeRnuQlNd5LkvPDn+B1IWJjARtj2lcoE0EvYHfQ/B5gTHPrqKpPRIqBTKAoeCURmQpMBcjNPbWblBv2l/DE7A2ntO3xOEAAl4gzHSgQZ1ndmbjH5cLtEtwuwSUQ53GTnuhlcPcUuibH0TXJS48uifTvlswZOal0S7FHMY0x4RMRN4tVdQYwAyAvL++UWgtfOTOHtY9PRkTqf7iFwA86jeYDy4PWs7NwY0wUC2Ui2Av0CZrv7ZQ1tc4eEfEAXQjcNG53XrcLr/WLY4wxJwjlL+MyYLCI9BeROGAKMKvROrOAW53pm4C5obg/YIwxpnkhaxE41/zvAz4k8Pjos6r6hYj8AshX1VnAX4H/E5EtwGECycIYY0wHkkg7AReRQmDnKW7ejUY3omOA1Tk2WJ1jw+nUua+qZjW1IOISwekQkXxVzQt3HB3J6hwbrM6xIVR1trunxhgT4ywRGGNMjIu1RDAj3AGEgdU5NlidY0NI6hxT9wiMMcacKNZaBMYYYxqxRGCMMTEuZhKBiFwhIhtFZIuIPBrueEJBRJ4VkQIRWRdUliEic0Rks/PZNZwxtjcR6SMi80RkvYh8ISIPOOVRW28RSRCRpSKy2qnzz53y/iLyuXOMv+680R81RMQtIitF5D1nPtrru0NE1orIKhHJd8pCclzHRCIIGhvhSmA48E0RGR7eqELieeCKRmWPAp+o6mDgE2c+mviAh1V1ODAWuNf5fxvN9a4CLlXVkcAo4AoRGUtgPI/pqjoIOEJgvI9o8gAQ3IVwtNcXYKKqjgp6dyAkx3VMJAKCxkZQ1WqgbmyEqKKqCwl01RHseuAFZ/oF4IaOjCnUVHW/qq5wpksJ/FD0IorrrQFlzqzX+VPgUgLjekCU1VlEegNXA39x5oUoru9JhOS4jpVE0NTYCL3CFEtHy1bV/c70ASA7nMGEkoj0A84BPifK6+1cJlkFFABzgK3AUVX1OatE2zH+e+DHQN1YrJlEd30hkNw/EpHlzpgsEKLjOiLGIzDtQ1VVRKLyeWERSQHeBh5U1ZLgMSSisd6qWguMEpF0YCYwNLwRhY6IXAMUqOpyEZkQ5nA60nhV3Ssi3YE5IvJl8ML2PK5jpUXQmrERotVBEekB4HwWhDmediciXgJJ4GVVfccpjvp6A6jqUWAeMA5Id8b1gOg6xi8ErhORHQQu614KPEn01hcAVd3rfBYQSPajCdFxHSuJoDVjI0Sr4DEfbgX+HsZY2p1zrfivwAZVnRa0KGrrLSJZTksAEUkEJhG4NzKPwLgeEEV1VtV/VdXeqtqPwL/duar6LaK0vgAikiwiqXXTwGRgHSE6rmPmzWIRuYrAdca6sRGeCG9E7U9EXgUmEOiq9iDwGPA34A0gl0D33TerauMbyhFLRMYDi4C1HL9+/BMC9wmist4icjaBG4VuAidzb6jqL0RkAIEz5gxgJfBtVa0KX6Ttz7k09CNVvSaa6+vUbaYz6wFeUdUnRCSTEBzXMZMIjDHGNC1WLg0ZY4xphiUCY4yJcZYIjDEmxlkiMMaYGGeJwBhjYpy9WWyilvOo3SfObA5QCxQ686Odfqea2zYP+K6q3t+G79sBlDrfA7CwLdu3Yv9lqprSXvszpo49Pmpigog8DpSp6m+DyjxBfdW0x3fsAPJUtai99tlo/5YITEjYpSETU0TkeRF5RkQ+B34tIqNF5DOnn/vFInKGs96EoH7vH3fGepgvIttEpE1n+c52Tzr9yq8TkdFOeYaI/E1E1ojIEudFMUQkRUSec/qiXyMiNwbt6wlnHIIlIhJVHemZ8LFEYGJRb+ACVX0I+BK4SFXPAX4G/LKZbYYCXyHQ38tjTv9GTZnn/OCvEpEfBpUnqeoo4PvAs07Zz4GVqno2gbehX3TK/x0oVtURzrK5TnkysMQZh2Ah8L021dqYZtg9AhOL3nR67wToArwgIoMJdPvb3A/8P5zuC6pEpIBA9797mlhvYjOXhl6FwJgRIpLm9BU0HrjRKZ8rIpkikgZcTqBPHZxlR5zJauA9Z3o5gT6GjDlt1iIwsehY0PR/APNU9SzgWiChmW2C+7Cppe0nUY1vxp3KzbkaPX5T71RiMKZJlghMrOvC8e6Lbwvh93wD6jvJK1bVYgKd5X3LKZ8AFKlqCYGBZu6t27C9xqU1pjmWCEys+zXwXyKykvY5ww6+R/BiUHml8x3PcHxs3ceB80RkDfArjncv/J9AV+fG8mpgYjvEZUyz7PFRY0JMROYT6Do5P9yxGNMUaxEYY0yMsxaBMcbEOGsRGGNMjLNEYIwxMc4SgTHGxDhLBMYYE+MsERhjTIz7/7s0tQKcD6glAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x1 = np.linspace(0, 50, num=2400)\n",
    "x2 = np.linspace(0, 50, num=470)\n",
    "\n",
    "plt.figure()\n",
    "plt.subplot(211)\n",
    "plt.plot(x1, y1)\n",
    "plt.ylabel('Loss')\n",
    "\n",
    "plt.subplot(212)\n",
    "plt.plot(x2, y2)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Train Epoch')\n",
    "\n",
    "plt.savefig('e2n_experiment.png')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Valid Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Valid Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label(flag: str) -> int:\n",
    "    return {\n",
    "        '-': 0,\n",
    "        'snmpgetattack': 1,\n",
    "        'warez': 2,\n",
    "        'portsweep': 3,\n",
    "        'nmap': 4,\n",
    "        'warezclient': 5,\n",
    "        'ipsweep': 6,\n",
    "        'dict': 7,\n",
    "        'neptune': 8,\n",
    "        'smurf': 9,\n",
    "        'pod': 10,\n",
    "        'snmpguess': 11,\n",
    "        'teardrop': 12,\n",
    "        'satan': 13,\n",
    "        'httptunnel-e': 14,\n",
    "        'ignore': 15,\n",
    "        'mscan': 16,\n",
    "        'guest': 17,\n",
    "        'rootkit': 18,\n",
    "        'back': 19,\n",
    "        'apache2': 20,\n",
    "        'processtable': 21,\n",
    "        'mailbomb': 22,\n",
    "        'smurfttl': 23,\n",
    "        'saint': 24\n",
    "    }.get(flag[:-1])\n",
    "\n",
    "\n",
    "def line_split(line: str) -> tuple:\n",
    "    src_ip, dst_ip, port, time = line.split('\\t')\n",
    "    return int(src_ip), int(dst_ip), int(port), int(time)\n",
    "\n",
    "# Data File Path\n",
    "\n",
    "train_set_path = 'data/train'\n",
    "valid_set_path = 'data/valid_query'\n",
    "test_set_path = 'data/test_query'\n",
    "\n",
    "train_set_list = os.listdir(train_set_path)\n",
    "valid_set_list = os.listdir(valid_set_path)\n",
    "test_set_path = os.listdir(test_set_path)\n",
    "\n",
    "file_index = 0\n",
    "\n",
    "# Features\n",
    "\n",
    "src_in_degree = collections.OrderedDict()\n",
    "src_out_degree = collections.OrderedDict()\n",
    "dst_in_degree = collections.OrderedDict()\n",
    "dst_out_degree = collections.OrderedDict()\n",
    "\n",
    "port_num = collections.OrderedDict()\n",
    "port_list = collections.OrderedDict()\n",
    "time_len = collections.OrderedDict()\n",
    "connect_num = collections.OrderedDict()\n",
    "label_list = collections.OrderedDict()\n",
    "\n",
    "# Tool Parameters\n",
    "\n",
    "nodes_origin = []\n",
    "edges_origin = {}\n",
    "\n",
    "edges_trans_src = []\n",
    "edges_trans_dst = []\n",
    "\n",
    "node_in_degree = {}\n",
    "node_out_degree = {}\n",
    "\n",
    "node_in_edges = {}\n",
    "node_out_edges = {}\n",
    "\n",
    "edge_src_list = {}\n",
    "edge_dst_list = {}\n",
    "\n",
    "# Count Parameters\n",
    "\n",
    "edge_index = 0\n",
    "\n",
    "print('Init variables done.')\n",
    "\n",
    "for file in valid_set_list:\n",
    "    file_index += 1\n",
    "    if file_index == 114:\n",
    "        continue\n",
    "    print('Loading File {}/{}...'.format(file_index, len(train_set_list)), end='\\r')\n",
    "    \n",
    "    file_path = valid_set_path + '/' + file\n",
    "    data = open(file_path)\n",
    "    \n",
    "    for line in data:\n",
    "        src_ip, dst_ip, port, time = line_split(line)\n",
    "\n",
    "        # Repeat Connections\n",
    "        if (src_ip, dst_ip) in edges_origin.keys():\n",
    "            edge_index_temp = edges_origin[(src_ip, dst_ip)]\n",
    "\n",
    "            # Assign Features\n",
    "            time_len[edge_index_temp] += time\n",
    "            connect_num[edge_index_temp] += 1\n",
    "            if port in port_list[edge_index_temp]:\n",
    "                pass\n",
    "            else:\n",
    "                port_num[edge_index_temp] += 1\n",
    "                port_list[edge_index_temp].append(port)\n",
    "\n",
    "        # New Connections     \n",
    "        else:\n",
    "            # Record Origin Node and Edge\n",
    "            nodes_origin.append(src_ip)\n",
    "            nodes_origin.append(dst_ip)\n",
    "            edges_origin[(src_ip, dst_ip)] = edge_index\n",
    "\n",
    "            # Update Node Parameters\n",
    "            if src_ip in node_out_degree.keys():\n",
    "                node_out_degree[src_ip] += 1\n",
    "            else:\n",
    "                node_out_degree[src_ip] = 1\n",
    "            if dst_ip in node_in_degree.keys():\n",
    "                node_in_degree[dst_ip] += 1\n",
    "            else:\n",
    "                node_in_degree[dst_ip] = 1\n",
    "\n",
    "            # Store Nodes' In-Out Edges\n",
    "            if src_ip in node_out_edges.keys():\n",
    "                node_out_edges[src_ip].append(edge_index)\n",
    "            else:\n",
    "                node_out_edges[src_ip] = [edge_index]\n",
    "            if dst_ip in node_in_edges.keys():\n",
    "                node_in_edges[dst_ip].append(edge_index)\n",
    "            else:\n",
    "                node_in_edges[dst_ip] = [edge_index]\n",
    "\n",
    "            # Update Edge Parameters\n",
    "            if edge_index in edge_src_list.keys():\n",
    "                edge_src_list[edge_index].append(src_ip)\n",
    "            else:\n",
    "                edge_src_list[edge_index] = [src_ip]\n",
    "            if edge_index in edge_dst_list.keys():\n",
    "                edge_dst_list[edge_index].append(dst_ip)\n",
    "            else:\n",
    "                edge_dst_list[edge_index] = [dst_ip]\n",
    "\n",
    "            # Assign Features\n",
    "            port_num[edge_index] = 1\n",
    "            port_list[edge_index] = [port]\n",
    "            time_len[edge_index] = time\n",
    "            connect_num[edge_index] = 1\n",
    "\n",
    "            edge_index += 1\n",
    "            \n",
    "print('Data Loading Done')\n",
    "\n",
    "# Remove Repeats\n",
    "nodes_origin = list(set(nodes_origin))\n",
    "for i in range(len(edge_src_list)):\n",
    "    edge_src_list[i] = list(set(edge_src_list[i]))\n",
    "for i in range(len(edge_dst_list)):\n",
    "    edge_dst_list[i] = list(set(edge_dst_list[i]))\n",
    "    \n",
    "# Create Transform Graph\n",
    "for node in nodes_origin:\n",
    "    if not node in node_in_edges.keys() or not node in node_out_edges.keys():\n",
    "        continue\n",
    "    if not node_in_edges[node] or not node_out_edges[node]:\n",
    "        continue\n",
    "    for src_edge in node_in_edges[node]:\n",
    "        for dst_edge in node_out_edges[node]:\n",
    "            edges_trans_src.append(src_edge)\n",
    "            edges_trans_dst.append(dst_edge)\n",
    "            \n",
    "print('Data clean up done.')\n",
    "    \n",
    "# Create Features\n",
    "for edge in edge_src_list.keys():\n",
    "    if edge_src_list[edge][0] not in node_in_degree.keys():\n",
    "        src_in_degree[edge] = 0\n",
    "    else:\n",
    "        src_in_degree[edge] = node_in_degree[edge_src_list[edge][0]]\n",
    "    if edge_src_list[edge][0] not in node_out_degree.keys():\n",
    "        src_out_degree[edge] = 0\n",
    "    else:\n",
    "        src_out_degree[edge] = node_out_degree[edge_src_list[edge][0]]\n",
    "    \n",
    "for edge in edge_dst_list.keys():\n",
    "    if edge_dst_list[edge][0] not in node_in_degree.keys():\n",
    "        dst_in_degree[edge] = 0\n",
    "    else:\n",
    "        dst_in_degree[edge] = node_in_degree[edge_dst_list[edge][0]]\n",
    "    if edge_dst_list[edge][0] not in node_out_degree.keys():\n",
    "        dst_out_degree[edge] = 0\n",
    "    else:\n",
    "        dst_out_degree[edge] = node_out_degree[edge_dst_list[edge][0]]\n",
    "    \n",
    "print('Feature create done.')\n",
    "    \n",
    "src_in_degree_norm = list(src_in_degree.values()) / np.linalg.norm(list(src_in_degree.values()))\n",
    "src_out_degree_norm = list(src_out_degree.values()) / np.linalg.norm(list(src_out_degree.values()))\n",
    "dst_in_degree_norm = list(dst_in_degree.values()) / np.linalg.norm(list(dst_in_degree.values()))\n",
    "dst_out_degree_norm = list(dst_out_degree.values()) / np.linalg.norm(list(dst_out_degree.values()))\n",
    "\n",
    "port_num_norm = list(port_num.values()) / np.linalg.norm(list(port_num.values()))\n",
    "time_len_norm = list(time_len.values()) / np.linalg.norm(list(time_len.values()))\n",
    "connect_num_norm = list(connect_num.values()) / np.linalg.norm(list(connect_num.values()))\n",
    "\n",
    "print('Normalize done.')\n",
    "\n",
    "feature_matrix = np.empty([1, 7])\n",
    "for i in range(len(edges_origin)):\n",
    "    feature_matrix = np.vstack((feature_matrix, [src_in_degree_norm[i], src_out_degree_norm[i], dst_in_degree_norm[i], dst_out_degree_norm[i], \\\n",
    "                                                 port_num_norm[i], time_len_norm[i], connect_num_norm[i]]))\n",
    "feature_matrix = np.delete(feature_matrix, obj=0, axis=0)    \n",
    "feature_matrix = feature_matrix.astype(np.float32)\n",
    "\n",
    "print('Data process done.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GCN(nn.Module):\n",
    "    def __init__(self, in_feats, hidden_size, num_classes):\n",
    "        super(GCN, self).__init__()\n",
    "        self.conv1 = GraphConv(in_feats, hidden_size)\n",
    "        self.conv2 = GraphConv(hidden_size, num_classes)\n",
    "\n",
    "    def forward(self, g, inputs):\n",
    "        h = self.conv1(g, inputs)\n",
    "        h = torch.relu(h)\n",
    "        h = self.conv2(g, h)\n",
    "        return h"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = dgl.DGLGraph()\n",
    "\n",
    "# Create Nodes and Edges\n",
    "\n",
    "g.add_nodes(len(edges_origin))\n",
    "g.add_edges(edges_trans_src, edges_trans_dst)\n",
    "\n",
    "# Assign Features\n",
    "\n",
    "g.ndata['src_in_degree'] = torch.tensor(src_in_degree_norm)\n",
    "g.ndata['src_out_degree'] = torch.tensor(src_out_degree_norm)\n",
    "g.ndata['dst_in_degree'] = torch.tensor(dst_in_degree_norm)\n",
    "g.ndata['dst_out_degree'] = torch.tensor(dst_out_degree_norm)\n",
    "g.ndata['port_num'] = torch.tensor(port_num_norm)\n",
    "g.ndata['time_len'] = torch.tensor(time_len_norm)\n",
    "g.ndata['connect_num'] = torch.tensor(connect_num_norm)\n",
    "\n",
    "g = dgl.add_self_loop(g)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Valid Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 | Loss: 11.9851 | Accu: 0.0016\n",
      "Epoch 100 | Loss: 11.9416 | Accu: 0.0097\n",
      "Epoch 200 | Loss: 11.7770 | Accu: 0.1974\n",
      "Epoch 300 | Loss: 11.4916 | Accu: 0.4552\n",
      "Epoch 400 | Loss: 11.0992 | Accu: 0.4699\n"
     ]
    }
   ],
   "source": [
    "net = GCN(7, 16, 25)\n",
    "\n",
    "inputs = torch.tensor(feature_matrix)\n",
    "labels = torch.tensor(list(label_list.values()), dtype=torch.float32)\n",
    "\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr=0.1)\n",
    "loss_func = nn.MSELoss()\n",
    "\n",
    "# Recoder\n",
    "loss_list = []\n",
    "accu_list = []\n",
    "\n",
    "for epoch in range(500):\n",
    "    outputs = net(g, inputs)\n",
    "    \n",
    "    total_number = 0\n",
    "    correct_number = 0\n",
    "    \n",
    "    outputs_max = np.argmax(outputs.detach().numpy(), axis=1)\n",
    "    labels_max = np.argmax(labels.detach().numpy(), axis=1)\n",
    "    \n",
    "    for i in range(len(outputs_max)):\n",
    "        if outputs_max[i] == labels_max[i]:\n",
    "            correct_number += 1\n",
    "        total_number += 1\n",
    "    \n",
    "    loss = loss_func(outputs, labels)\n",
    "    \n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    loss_list.append(loss.item()/g.number_of_nodes())\n",
    "    accu_list.append(correct_number / total_number)\n",
    "    \n",
    "    if epoch % 100 == 0:\n",
    "        print('Epoch {} | Loss: {:.4f} | Accu: {:.4f}'.format(epoch, loss.item()/g.number_of_nodes(), correct_number / total_number))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}