GAT(cora).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "GAT(cora).ipynb",
      "provenance": [],
      "mount_file_id": "15ATmoJHoFKNCGPJGumcdFUvLUGOCJR2M",
      "authorship_tag": "ABX9TyNd7GRmZm2e9jTOlFArKFQY",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mamoonmasud/graph_community/blob/main/GAT(cora).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HgfRQ7XKYxWi"
      },
      "source": [
        "import torch"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5QD2djm6Yy9x"
      },
      "source": [
        "#Checking if cuda is available. If return False, change runtime to GPU\n",
        "print(torch.cuda.is_available())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ejET8LdpY3Us",
        "outputId": "a9bf8510-608a-48e6-c17a-66f7871ee650"
      },
      "source": [
        "print(torch.version.cuda)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "11.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qk_iEi34ZGvW",
        "outputId": "31b12d9e-44be-40bc-b761-fb8837af2999"
      },
      "source": [
        "#Checking Python Version\n",
        "! python --version"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Python 3.7.12\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mHf3OIxMZYuF",
        "outputId": "8afa0396-b90b-42b6-cf10-8511521b83ce"
      },
      "source": [
        "# Installing DGL\n",
        "! pip install dgl-cu111 -f https://data.dgl.ai/wheels/repo.html "
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in links: https://data.dgl.ai/wheels/repo.html\n",
            "Collecting dgl-cu111\n",
            "  Downloading https://data.dgl.ai/wheels/dgl_cu111-0.7.2-cp37-cp37m-manylinux1_x86_64.whl (165.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 165.0 MB 52 kB/s \n",
            "\u001b[?25hRequirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from dgl-cu111) (1.4.1)\n",
            "Requirement already satisfied: networkx>=2.1 in /usr/local/lib/python3.7/dist-packages (from dgl-cu111) (2.6.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from dgl-cu111) (2.23.0)\n",
            "Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.7/dist-packages (from dgl-cu111) (1.19.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->dgl-cu111) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->dgl-cu111) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->dgl-cu111) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->dgl-cu111) (2021.10.8)\n",
            "Installing collected packages: dgl-cu111\n",
            "Successfully installed dgl-cu111-0.7.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VeQwSStNzPCK"
      },
      "source": [
        ""
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2OzXVYfndvZP"
      },
      "source": [
        "from google.colab import files"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H-E1uJeYZ0r5"
      },
      "source": [
        "# Doing all the imports:\n",
        "import dgl\n",
        "import dgl.function as fn\n",
        "import torch as th\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from dgl import DGLGraph\n",
        "from scipy import sparse as sp\n",
        "from math import log\n",
        "import pandas as pd\n",
        "from sklearn import metrics as skmetrics\n",
        "import pickle\n",
        "import numpy as np\n",
        "\n",
        "import itertools"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QpR5PsmktzCc"
      },
      "source": [
        "#from dgl.nn.pytorch.conv import GATConv as GAT"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RWCs4nFmspN9"
      },
      "source": [
        ""
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x_uXoCpDy_xC"
      },
      "source": [
        "def upload_files():\n",
        "  from google.colab import files\n",
        "  uploaded = files.upload()\n",
        "  for k, v in uploaded.items():\n",
        "    open(k, 'wb').write(v)\n",
        "  return list(uploaded.keys())"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgZG8gewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwoKICAgICAgbGV0IHBlcmNlbnREb25lID0gZmlsZURhdGEuYnl0ZUxlbmd0aCA9PT0gMCA/CiAgICAgICAgICAxMDAgOgogICAgICAgICAgTWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCk7CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPSBgJHtwZXJjZW50RG9uZX0lIGRvbmVgOwoKICAgIH0gd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCk7CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 157
        },
        "id": "-nlF-C5n0SNh",
        "outputId": "53de13a6-9d4b-4294-b158-8902431a5285"
      },
      "source": [
        "upload_files()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-1f11d472-c9b6-4a12-b5ac-673aaf4880a5\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-1f11d472-c9b6-4a12-b5ac-673aaf4880a5\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving GAT.py to GAT.py\n",
            "Saving performance.py to performance.py\n",
            "Saving Sparsemax.py to Sparsemax.py\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['GAT.py', 'performance.py', 'Sparsemax.py']"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DofZPMoDz6a-"
      },
      "source": [
        "import GAT\n",
        "import Sparsemax\n",
        "import performance as pf\n"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7JgH7hAsOwQt"
      },
      "source": [
        ""
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xA9iRvQqtTZk"
      },
      "source": [
        "# Graph Neural Network Definition\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "08EnIPQSaX9G"
      },
      "source": [
        "class GATLayer(nn.Module):\n",
        "    def __init__(self, g, in_dim, out_dim):\n",
        "        super(GATLayer, self).__init__()\n",
        "        self.g = g\n",
        "        # equation (1)\n",
        "        self.fc = nn.Linear(in_dim, out_dim, bias=False)\n",
        "        # equation (2)\n",
        "        self.attn_fc = nn.Linear(2 * out_dim, 1, bias=False)\n",
        "\n",
        "    def edge_attention(self, edges):\n",
        "        # edge UDF for equation (2)\n",
        "        z2 = th.cat([edges.src['z'], edges.dst['z']], dim=1)\n",
        "        a = self.attn_fc(z2)\n",
        "        return {'e': F.leaky_relu(a)}\n",
        "\n",
        "    def message_func(self, edges):\n",
        "        # message UDF for equation (3) & (4)\n",
        "        return {'z': edges.src['z'], 'e': edges.data['e']}\n",
        "\n",
        "    def reduce_func(self, nodes):\n",
        "        # reduce UDF for equation (3) & (4)\n",
        "        # equation (3)\n",
        "        alpha = F.softmax(nodes.mailbox['e'], dim=1)\n",
        "        # equation (4)\n",
        "        h = th.sum(alpha * nodes.mailbox['z'], dim=1)\n",
        "        return {'h': h}\n",
        "\n",
        "    def forward(self, h):\n",
        "        # equation (1)\n",
        "        z = self.fc(h)\n",
        "        self.g.ndata['z'] = z\n",
        "        # equation (2)\n",
        "        self.g.apply_edges(self.edge_attention)\n",
        "        # equation (3) & (4)\n",
        "        self.g.update_all(self.message_func, self.reduce_func)\n",
        "        return self.g.ndata.pop('h')\n"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "90qOw_TXbNTf"
      },
      "source": [
        "class MultiHeadGATLayer(nn.Module):\n",
        "    def __init__(self, g, in_dim, out_dim, num_heads, merge='cat'):\n",
        "        super(MultiHeadGATLayer, self).__init__()\n",
        "        self.heads = nn.ModuleList()\n",
        "        for i in range(num_heads):\n",
        "            self.heads.append(GATLayer(g, in_dim, out_dim))\n",
        "        self.merge = merge\n",
        "\n",
        "    def forward(self, h):\n",
        "        head_outs = [attn_head(h) for attn_head in self.heads]\n",
        "        if self.merge == 'cat':\n",
        "            # concat on the output feature dimension (dim=1)\n",
        "            return th.cat(head_outs, dim=1)\n",
        "        else:\n",
        "            # merge using average\n",
        "            return th.mean(torch.stack(head_outs))"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zXJk-TtibQhN"
      },
      "source": [
        "class Net(nn.Module):\n",
        "    def __init__(self, g, in_dim, hidden_dim, out_dim, num_heads):\n",
        "        super(Net, self).__init__()\n",
        "        self.layer1 = MultiHeadGATLayer(g, in_dim, hidden_dim, num_heads)\n",
        "        # Be aware that the input dimension is hidden_dim*num_heads since\n",
        "        # multiple head outputs are concatenated together. Also, only\n",
        "        # one attention head in the output layer.\n",
        "        self.layer2 = MultiHeadGATLayer(g, hidden_dim * num_heads, out_dim, 1)\n",
        "\n",
        "    def forward(self, h):\n",
        "        h = self.layer1(h)\n",
        "        h = F.elu(h)\n",
        "        h = self.layer2(h)\n",
        "        h = F.log_softmax(h, 1)\n",
        "        return h"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bx2sFf240dIQ"
      },
      "source": [
        "# Data Loading"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nHyZ_u4ybSje",
        "outputId": "c7718785-98e6-4994-c78c-6b5dce69311f"
      },
      "source": [
        "from dgl.data import citation_graph as citegrh\n",
        "import networkx as nx\n",
        "\n",
        "data = citegrh.load_cora()\n",
        "\n",
        "# features = th.FloatTensor(data.features)\n",
        "# labels = th.LongTensor(data.labels)\n",
        "# mask = th.ByteTensor(data.train_mask)\n",
        "#g1 = data.graph\n",
        "\n",
        "##\n",
        "g = data[0]\n",
        "num_class = data.num_classes\n",
        "\n",
        "features = g.ndata['feat']\n",
        "\n",
        "train_mask = g.ndata['train_mask']\n",
        "val_mask = g.ndata['val_mask']\n",
        "test_mask = g.ndata['test_mask']\n",
        "labels = g.ndata['label']\n",
        "\n",
        "# add self loop\n",
        "#g.remove_edges_from(nx.selfloop_edges(g))\n",
        "g  = dgl.remove_self_loop(g)\n",
        "#g = DGLGraph(g)\n",
        "g.add_edges(g.nodes(), g.nodes())\n",
        "\n",
        "comb_mask = torch.zeros(len(train_mask), dtype=torch.bool)\n",
        "\n",
        "for i in range(len(comb_mask)):\n",
        "    if test_mask[i] or train_mask[i] or val_mask[i]:\n",
        "        comb_mask[i] = True\n"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  NumNodes: 2708\n",
            "  NumEdges: 10556\n",
            "  NumFeats: 1433\n",
            "  NumClasses: 7\n",
            "  NumTrainingSamples: 140\n",
            "  NumValidationSamples: 500\n",
            "  NumTestSamples: 1000\n",
            "Done loading data from cached files.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sntihQrF1Bpi",
        "outputId": "3bf20952-a841-455b-9de4-18f0954a97f1"
      },
      "source": [
        "print('We have %d nodes.' % g.number_of_nodes())\n",
        "print('We have %d edges.' % g.number_of_edges())"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "We have 2708 nodes.\n",
            "We have 13264 edges.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hSXAbdMR0ncM"
      },
      "source": [
        "# Selecting Training Set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m0KdxXbFtizM"
      },
      "source": [
        "percentage_train = 0.1\n",
        "percentage_val = 0.1\n",
        "\n",
        "with open(\"drive/MyDrive/CSCE689/data/cora_permutation3.pickle\",\"rb\") as f:\n",
        "    perm1 = pickle.load(f)\n",
        "mask_train = np.zeros(g.number_of_nodes())\n",
        "mask_val = np.zeros(g.number_of_nodes())\n",
        "\n",
        "i_train = int(percentage_train*g.number_of_nodes())\n",
        "i_val = i_train + int(percentage_val*g.number_of_nodes())\n",
        "mask_train[perm1[range(0,i_train)]] = 1\n",
        "mask_val[perm1[range(i_train,i_val)]] = 1\n",
        "mask_train = th.BoolTensor(mask_train)\n",
        "mask_val = th.BoolTensor(mask_val)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0KC_oeecIWZo"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Oyi6Z9od0prq"
      },
      "source": [
        "loss_function = pf.perm_inv_loss(labels)\n",
        "import copy"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M0HYmM23IVg2",
        "outputId": "4ad9c9c1-0803-4ac0-b7c6-9ec07d1664b1"
      },
      "source": [
        "import time\n",
        "\n",
        "net = GAT.GAT_Net_fast(g=g, in_feats=features.shape[1], hidden_size=100, hidden_layers=2, out_feats=len(np.unique(labels)),\n",
        "dropout=0.2, batchnorm=False, num_heads=1, residual = False)\n",
        "#print(net)\n",
        "\n",
        "optimizer = th.optim.Adam(net.parameters(), lr=1e-2, weight_decay=1e-2)\n",
        "net.train() # Set to training mode (use dropout)\n",
        "\n",
        "dur = []\n",
        "loss_ev = []\n",
        "current_best = 0 #arbitrarily high\n",
        "current_best_epoch = 0\n",
        "current_best_params = None\n",
        "no_improvement_for = 0\n",
        "\n",
        "for epoch in range(10000):\n",
        "    if epoch >=3:\n",
        "        t0 = time.time()\n",
        "\n",
        "    # Compute loss for test nodes (only for validation, not used by optimizer)\n",
        "    net.eval()\n",
        "    prediction = net(features)\n",
        "    train_rand=pf.rand_score(labels[mask_train].numpy(),np.argmax(prediction[mask_train].detach().numpy(), axis=1))\n",
        "    validation_rand=pf.rand_score(labels[mask_val].numpy(),np.argmax(prediction[mask_val].detach().numpy(), axis=1))\n",
        "    if train_rand>current_best:\n",
        "        current_best = train_rand\n",
        "        current_best_epoch = epoch\n",
        "        current_best_params = copy.deepcopy(net.state_dict())\n",
        "        no_improvement_for = 0\n",
        "    else: no_improvement_for += 1\n",
        "    \n",
        "    if no_improvement_for>100:\n",
        "        break\n",
        "    \n",
        "    net.train()\n",
        "\n",
        "    # Compute loss for train nodes\n",
        "    logits = net(features)\n",
        "\n",
        "    #loss = loss_function.approximate_loss(logits,mask_train,nclasses=7)\n",
        "    loss = F.nll_loss(logits[mask_train], labels[mask_train])\n",
        "    loss_ev.append(loss.detach().item())\n",
        "    #print(np.unique(np.argmax(logits[mask_train].detach().numpy(),1)))\n",
        "    \n",
        "    \n",
        "    optimizer.zero_grad()\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "\n",
        "    if epoch >=3:\n",
        "        dur.append(time.time() - t0)\n",
        "        print(f\"Epoch {epoch:05d} | Loss {loss.item():.4f} | Train.Rand {train_rand:.4f} | Valid.Rand {validation_rand:.4f} | Time(s) {np.mean(dur):.4f}\")\n",
        "    else:\n",
        "        print(f\"Epoch {epoch:05d} | Loss {loss.item():.4f} | Train.Rand {train_rand:.4f} | Valid.Rand {validation_rand:.4f} | Time(s) unknown\")\n",
        "        \n",
        "net.load_state_dict(current_best_params)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00000 | Loss 3.6732 | Train.Rand 0.0824 | Valid.Rand 0.0905 | Time(s) unknown\n",
            "Epoch 00001 | Loss 1.1571 | Train.Rand 0.4782 | Valid.Rand 0.3784 | Time(s) unknown\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00002 | Loss 0.7583 | Train.Rand 0.6778 | Valid.Rand 0.4908 | Time(s) unknown\n",
            "Epoch 00003 | Loss 0.6842 | Train.Rand 0.7634 | Valid.Rand 0.5421 | Time(s) 0.1978\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00004 | Loss 0.5568 | Train.Rand 0.7906 | Valid.Rand 0.5945 | Time(s) 0.1877\n",
            "Epoch 00005 | Loss 0.4891 | Train.Rand 0.8048 | Valid.Rand 0.6274 | Time(s) 0.1910\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00006 | Loss 0.3706 | Train.Rand 0.8365 | Valid.Rand 0.6043 | Time(s) 0.1894\n",
            "Epoch 00007 | Loss 0.3590 | Train.Rand 0.8858 | Valid.Rand 0.5944 | Time(s) 0.1865\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00008 | Loss 0.3649 | Train.Rand 0.8807 | Valid.Rand 0.6192 | Time(s) 0.1845\n",
            "Epoch 00009 | Loss 0.3458 | Train.Rand 0.8760 | Valid.Rand 0.6425 | Time(s) 0.1850\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00010 | Loss 0.2835 | Train.Rand 0.8784 | Valid.Rand 0.6259 | Time(s) 0.1843\n",
            "Epoch 00011 | Loss 0.3403 | Train.Rand 0.8991 | Valid.Rand 0.6208 | Time(s) 0.1838\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00012 | Loss 0.2939 | Train.Rand 0.8837 | Valid.Rand 0.6243 | Time(s) 0.1833\n",
            "Epoch 00013 | Loss 0.3051 | Train.Rand 0.8781 | Valid.Rand 0.6131 | Time(s) 0.1832\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00014 | Loss 0.2600 | Train.Rand 0.8945 | Valid.Rand 0.5655 | Time(s) 0.1830\n",
            "Epoch 00015 | Loss 0.2212 | Train.Rand 0.8936 | Valid.Rand 0.5618 | Time(s) 0.1824\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00016 | Loss 0.2314 | Train.Rand 0.9219 | Valid.Rand 0.5685 | Time(s) 0.1820\n",
            "Epoch 00017 | Loss 0.2024 | Train.Rand 0.9323 | Valid.Rand 0.5848 | Time(s) 0.1821\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00018 | Loss 0.1798 | Train.Rand 0.9429 | Valid.Rand 0.5819 | Time(s) 0.1819\n",
            "Epoch 00019 | Loss 0.1701 | Train.Rand 0.9408 | Valid.Rand 0.6049 | Time(s) 0.1816\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00020 | Loss 0.1866 | Train.Rand 0.9392 | Valid.Rand 0.6030 | Time(s) 0.1814\n",
            "Epoch 00021 | Loss 0.2486 | Train.Rand 0.9280 | Valid.Rand 0.6073 | Time(s) 0.1812\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00022 | Loss 0.1585 | Train.Rand 0.9419 | Valid.Rand 0.6159 | Time(s) 0.1809\n",
            "Epoch 00023 | Loss 0.1753 | Train.Rand 0.9350 | Valid.Rand 0.6198 | Time(s) 0.1806\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00024 | Loss 0.1833 | Train.Rand 0.9350 | Valid.Rand 0.6236 | Time(s) 0.1806\n",
            "Epoch 00025 | Loss 0.1696 | Train.Rand 0.9564 | Valid.Rand 0.6523 | Time(s) 0.1805\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00026 | Loss 0.1495 | Train.Rand 0.9392 | Valid.Rand 0.6531 | Time(s) 0.1805\n",
            "Epoch 00027 | Loss 0.1962 | Train.Rand 0.9170 | Valid.Rand 0.6532 | Time(s) 0.1804\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00028 | Loss 0.1402 | Train.Rand 0.9281 | Valid.Rand 0.6330 | Time(s) 0.1802\n",
            "Epoch 00029 | Loss 0.1591 | Train.Rand 0.9441 | Valid.Rand 0.6332 | Time(s) 0.1803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00030 | Loss 0.1333 | Train.Rand 0.9403 | Valid.Rand 0.6309 | Time(s) 0.1802\n",
            "Epoch 00031 | Loss 0.1417 | Train.Rand 0.9585 | Valid.Rand 0.6072 | Time(s) 0.1803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00032 | Loss 0.1322 | Train.Rand 0.9681 | Valid.Rand 0.6073 | Time(s) 0.1802\n",
            "Epoch 00033 | Loss 0.1291 | Train.Rand 0.9500 | Valid.Rand 0.5983 | Time(s) 0.1802\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00034 | Loss 0.1215 | Train.Rand 0.9500 | Valid.Rand 0.6061 | Time(s) 0.1802\n",
            "Epoch 00035 | Loss 0.1095 | Train.Rand 0.9520 | Valid.Rand 0.6118 | Time(s) 0.1802\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00036 | Loss 0.1506 | Train.Rand 0.9520 | Valid.Rand 0.6114 | Time(s) 0.1802\n",
            "Epoch 00037 | Loss 0.1439 | Train.Rand 0.9769 | Valid.Rand 0.6298 | Time(s) 0.1803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00038 | Loss 0.1634 | Train.Rand 0.9700 | Valid.Rand 0.6472 | Time(s) 0.1802\n",
            "Epoch 00039 | Loss 0.1446 | Train.Rand 0.9813 | Valid.Rand 0.6467 | Time(s) 0.1802\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00040 | Loss 0.1538 | Train.Rand 0.9813 | Valid.Rand 0.6527 | Time(s) 0.1801\n",
            "Epoch 00041 | Loss 0.1136 | Train.Rand 0.9813 | Valid.Rand 0.6536 | Time(s) 0.1801\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00042 | Loss 0.1404 | Train.Rand 0.9610 | Valid.Rand 0.6540 | Time(s) 0.1801\n",
            "Epoch 00043 | Loss 0.1289 | Train.Rand 0.9610 | Valid.Rand 0.6384 | Time(s) 0.1805\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00044 | Loss 0.1663 | Train.Rand 0.9666 | Valid.Rand 0.6509 | Time(s) 0.1804\n",
            "Epoch 00045 | Loss 0.1422 | Train.Rand 0.9825 | Valid.Rand 0.6504 | Time(s) 0.1804\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00046 | Loss 0.1061 | Train.Rand 0.9553 | Valid.Rand 0.6455 | Time(s) 0.1804\n",
            "Epoch 00047 | Loss 0.1441 | Train.Rand 0.9377 | Valid.Rand 0.6263 | Time(s) 0.1802\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00048 | Loss 0.1306 | Train.Rand 0.9544 | Valid.Rand 0.6049 | Time(s) 0.1803\n",
            "Epoch 00049 | Loss 0.1206 | Train.Rand 0.9570 | Valid.Rand 0.5932 | Time(s) 0.1803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00050 | Loss 0.1398 | Train.Rand 0.9658 | Valid.Rand 0.5984 | Time(s) 0.1803\n",
            "Epoch 00051 | Loss 0.1325 | Train.Rand 0.9588 | Valid.Rand 0.6451 | Time(s) 0.1804\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00052 | Loss 0.1513 | Train.Rand 0.9697 | Valid.Rand 0.6296 | Time(s) 0.1804\n",
            "Epoch 00053 | Loss 0.1247 | Train.Rand 0.9813 | Valid.Rand 0.5918 | Time(s) 0.1804\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00054 | Loss 0.1420 | Train.Rand 0.9704 | Valid.Rand 0.5923 | Time(s) 0.1804\n",
            "Epoch 00055 | Loss 0.0930 | Train.Rand 0.9613 | Valid.Rand 0.5901 | Time(s) 0.1804\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00056 | Loss 0.1705 | Train.Rand 0.9613 | Valid.Rand 0.5900 | Time(s) 0.1803\n",
            "Epoch 00057 | Loss 0.0841 | Train.Rand 0.9700 | Valid.Rand 0.5954 | Time(s) 0.1803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00058 | Loss 0.1225 | Train.Rand 0.9586 | Valid.Rand 0.5840 | Time(s) 0.1803\n",
            "Epoch 00059 | Loss 0.0978 | Train.Rand 0.9771 | Valid.Rand 0.5568 | Time(s) 0.1803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00060 | Loss 0.1098 | Train.Rand 0.9771 | Valid.Rand 0.5570 | Time(s) 0.1803\n",
            "Epoch 00061 | Loss 0.1182 | Train.Rand 0.9771 | Valid.Rand 0.5647 | Time(s) 0.1802\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00062 | Loss 0.0727 | Train.Rand 0.9883 | Valid.Rand 0.5896 | Time(s) 0.1804\n",
            "Epoch 00063 | Loss 0.0897 | Train.Rand 0.9883 | Valid.Rand 0.5970 | Time(s) 0.1804\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00064 | Loss 0.1047 | Train.Rand 0.9837 | Valid.Rand 0.6057 | Time(s) 0.1803\n",
            "Epoch 00065 | Loss 0.1228 | Train.Rand 0.9818 | Valid.Rand 0.6167 | Time(s) 0.1803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00066 | Loss 0.0971 | Train.Rand 0.9588 | Valid.Rand 0.5983 | Time(s) 0.1804\n",
            "Epoch 00067 | Loss 0.0956 | Train.Rand 0.9702 | Valid.Rand 0.5974 | Time(s) 0.1803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00068 | Loss 0.0567 | Train.Rand 0.9702 | Valid.Rand 0.6005 | Time(s) 0.1803\n",
            "Epoch 00069 | Loss 0.0807 | Train.Rand 0.9813 | Valid.Rand 0.6032 | Time(s) 0.1803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00070 | Loss 0.1227 | Train.Rand 0.9813 | Valid.Rand 0.5895 | Time(s) 0.1803\n",
            "Epoch 00071 | Loss 0.0980 | Train.Rand 0.9634 | Valid.Rand 0.5847 | Time(s) 0.1804\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00072 | Loss 0.0875 | Train.Rand 0.9409 | Valid.Rand 0.5729 | Time(s) 0.1803\n",
            "Epoch 00073 | Loss 0.0851 | Train.Rand 0.9659 | Valid.Rand 0.5750 | Time(s) 0.1804\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00074 | Loss 0.0723 | Train.Rand 0.9659 | Valid.Rand 0.5678 | Time(s) 0.1804\n",
            "Epoch 00075 | Loss 0.0802 | Train.Rand 1.0000 | Valid.Rand 0.5734 | Time(s) 0.1804\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00076 | Loss 0.0904 | Train.Rand 0.9929 | Valid.Rand 0.5893 | Time(s) 0.1804\n",
            "Epoch 00077 | Loss 0.1071 | Train.Rand 0.9625 | Valid.Rand 0.5997 | Time(s) 0.1804\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00078 | Loss 0.0923 | Train.Rand 0.9628 | Valid.Rand 0.5967 | Time(s) 0.1803\n",
            "Epoch 00079 | Loss 0.0985 | Train.Rand 0.9628 | Valid.Rand 0.5988 | Time(s) 0.1803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00080 | Loss 0.0823 | Train.Rand 0.9818 | Valid.Rand 0.6066 | Time(s) 0.1803\n",
            "Epoch 00081 | Loss 0.0592 | Train.Rand 0.9929 | Valid.Rand 0.6218 | Time(s) 0.1803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00082 | Loss 0.0905 | Train.Rand 0.9549 | Valid.Rand 0.6057 | Time(s) 0.1803\n",
            "Epoch 00083 | Loss 0.0746 | Train.Rand 0.9662 | Valid.Rand 0.5997 | Time(s) 0.1803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00084 | Loss 0.0927 | Train.Rand 0.9592 | Valid.Rand 0.5797 | Time(s) 0.1803\n",
            "Epoch 00085 | Loss 0.0689 | Train.Rand 0.9707 | Valid.Rand 0.5961 | Time(s) 0.1802\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00086 | Loss 0.0804 | Train.Rand 0.9929 | Valid.Rand 0.6135 | Time(s) 0.1803\n",
            "Epoch 00087 | Loss 0.0776 | Train.Rand 0.9863 | Valid.Rand 0.5896 | Time(s) 0.1803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00088 | Loss 0.0802 | Train.Rand 0.9929 | Valid.Rand 0.6000 | Time(s) 0.1803\n",
            "Epoch 00089 | Loss 0.1021 | Train.Rand 0.9588 | Valid.Rand 0.6163 | Time(s) 0.1803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00090 | Loss 0.1079 | Train.Rand 0.9929 | Valid.Rand 0.6008 | Time(s) 0.1803\n",
            "Epoch 00091 | Loss 0.0610 | Train.Rand 0.9813 | Valid.Rand 0.5873 | Time(s) 0.1804\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00092 | Loss 0.0932 | Train.Rand 1.0000 | Valid.Rand 0.5791 | Time(s) 0.1803\n",
            "Epoch 00093 | Loss 0.1035 | Train.Rand 0.9520 | Valid.Rand 0.5533 | Time(s) 0.1804\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00094 | Loss 0.0817 | Train.Rand 0.9660 | Valid.Rand 0.5466 | Time(s) 0.1805\n",
            "Epoch 00095 | Loss 0.1357 | Train.Rand 0.9651 | Valid.Rand 0.5623 | Time(s) 0.1805\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00096 | Loss 0.1356 | Train.Rand 0.9818 | Valid.Rand 0.5709 | Time(s) 0.1805\n",
            "Epoch 00097 | Loss 0.0943 | Train.Rand 0.9929 | Valid.Rand 0.5838 | Time(s) 0.1805\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00098 | Loss 0.0956 | Train.Rand 0.9929 | Valid.Rand 0.5783 | Time(s) 0.1805\n",
            "Epoch 00099 | Loss 0.1140 | Train.Rand 0.9667 | Valid.Rand 0.5638 | Time(s) 0.1806\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00100 | Loss 0.1220 | Train.Rand 0.9567 | Valid.Rand 0.5563 | Time(s) 0.1806\n",
            "Epoch 00101 | Loss 0.1026 | Train.Rand 0.9635 | Valid.Rand 0.5418 | Time(s) 0.1807\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00102 | Loss 0.1252 | Train.Rand 0.9193 | Valid.Rand 0.5274 | Time(s) 0.1807\n",
            "Epoch 00103 | Loss 0.1351 | Train.Rand 0.9418 | Valid.Rand 0.5184 | Time(s) 0.1807\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00104 | Loss 0.1197 | Train.Rand 0.9709 | Valid.Rand 0.5321 | Time(s) 0.1807\n",
            "Epoch 00105 | Loss 0.1373 | Train.Rand 0.9737 | Valid.Rand 0.5278 | Time(s) 0.1806\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00106 | Loss 0.0990 | Train.Rand 0.9623 | Valid.Rand 0.5217 | Time(s) 0.1806\n",
            "Epoch 00107 | Loss 0.1017 | Train.Rand 0.9508 | Valid.Rand 0.5399 | Time(s) 0.1806\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00108 | Loss 0.1090 | Train.Rand 0.9700 | Valid.Rand 0.5458 | Time(s) 0.1805\n",
            "Epoch 00109 | Loss 0.1685 | Train.Rand 0.9410 | Valid.Rand 0.5448 | Time(s) 0.1805\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00110 | Loss 0.1075 | Train.Rand 0.9772 | Valid.Rand 0.5495 | Time(s) 0.1806\n",
            "Epoch 00111 | Loss 0.1038 | Train.Rand 0.9588 | Valid.Rand 0.5270 | Time(s) 0.1805\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00112 | Loss 0.1449 | Train.Rand 0.9514 | Valid.Rand 0.5414 | Time(s) 0.1805\n",
            "Epoch 00113 | Loss 0.1054 | Train.Rand 0.9636 | Valid.Rand 0.5842 | Time(s) 0.1806\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00114 | Loss 0.1375 | Train.Rand 0.9769 | Valid.Rand 0.5966 | Time(s) 0.1806\n",
            "Epoch 00115 | Loss 0.1108 | Train.Rand 0.9699 | Valid.Rand 0.6178 | Time(s) 0.1806\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00116 | Loss 0.0851 | Train.Rand 0.9700 | Valid.Rand 0.6299 | Time(s) 0.1806\n",
            "Epoch 00117 | Loss 0.1068 | Train.Rand 0.9699 | Valid.Rand 0.6126 | Time(s) 0.1806\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00118 | Loss 0.1027 | Train.Rand 0.9702 | Valid.Rand 0.6046 | Time(s) 0.1807\n",
            "Epoch 00119 | Loss 0.0908 | Train.Rand 0.9772 | Valid.Rand 0.5708 | Time(s) 0.1807\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00120 | Loss 0.0639 | Train.Rand 0.9883 | Valid.Rand 0.5759 | Time(s) 0.1807\n",
            "Epoch 00121 | Loss 0.0732 | Train.Rand 0.9883 | Valid.Rand 0.5863 | Time(s) 0.1807\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00122 | Loss 0.1041 | Train.Rand 0.9883 | Valid.Rand 0.5887 | Time(s) 0.1807\n",
            "Epoch 00123 | Loss 0.0776 | Train.Rand 0.9816 | Valid.Rand 0.5793 | Time(s) 0.1807\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00124 | Loss 0.0687 | Train.Rand 0.9816 | Valid.Rand 0.5666 | Time(s) 0.1807\n",
            "Epoch 00125 | Loss 0.1102 | Train.Rand 0.9840 | Valid.Rand 0.5597 | Time(s) 0.1806\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00126 | Loss 0.1108 | Train.Rand 0.9929 | Valid.Rand 0.5620 | Time(s) 0.1806\n",
            "Epoch 00127 | Loss 0.0685 | Train.Rand 1.0000 | Valid.Rand 0.5708 | Time(s) 0.1807\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00128 | Loss 0.0892 | Train.Rand 1.0000 | Valid.Rand 0.5859 | Time(s) 0.1806\n",
            "Epoch 00129 | Loss 0.0718 | Train.Rand 1.0000 | Valid.Rand 0.5704 | Time(s) 0.1806\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00130 | Loss 0.0677 | Train.Rand 0.9884 | Valid.Rand 0.5763 | Time(s) 0.1806\n",
            "Epoch 00131 | Loss 0.0918 | Train.Rand 0.9795 | Valid.Rand 0.5946 | Time(s) 0.1806\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00132 | Loss 0.0820 | Train.Rand 0.9883 | Valid.Rand 0.5644 | Time(s) 0.1806\n",
            "Epoch 00133 | Loss 0.0894 | Train.Rand 0.9697 | Valid.Rand 0.5695 | Time(s) 0.1806\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00134 | Loss 0.1086 | Train.Rand 0.9653 | Valid.Rand 0.5561 | Time(s) 0.1806\n",
            "Epoch 00135 | Loss 0.0943 | Train.Rand 0.9769 | Valid.Rand 0.5566 | Time(s) 0.1805\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00136 | Loss 0.1047 | Train.Rand 0.9929 | Valid.Rand 0.5575 | Time(s) 0.1805\n",
            "Epoch 00137 | Loss 0.0685 | Train.Rand 0.9756 | Valid.Rand 0.5514 | Time(s) 0.1805\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00138 | Loss 0.0976 | Train.Rand 0.9922 | Valid.Rand 0.5753 | Time(s) 0.1805\n",
            "Epoch 00139 | Loss 0.0610 | Train.Rand 0.9922 | Valid.Rand 0.6044 | Time(s) 0.1805\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00140 | Loss 0.0793 | Train.Rand 0.9744 | Valid.Rand 0.5804 | Time(s) 0.1805\n",
            "Epoch 00141 | Loss 0.1232 | Train.Rand 0.9851 | Valid.Rand 0.5520 | Time(s) 0.1804\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00142 | Loss 0.0936 | Train.Rand 0.9911 | Valid.Rand 0.5573 | Time(s) 0.1805\n",
            "Epoch 00143 | Loss 0.0897 | Train.Rand 0.9929 | Valid.Rand 0.5759 | Time(s) 0.1804\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00144 | Loss 0.1286 | Train.Rand 0.9853 | Valid.Rand 0.5619 | Time(s) 0.1804\n",
            "Epoch 00145 | Loss 0.1002 | Train.Rand 0.9737 | Valid.Rand 0.5996 | Time(s) 0.1804\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00146 | Loss 0.0968 | Train.Rand 0.9572 | Valid.Rand 0.6072 | Time(s) 0.1803\n",
            "Epoch 00147 | Loss 0.0869 | Train.Rand 0.9577 | Valid.Rand 0.5856 | Time(s) 0.1803\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00148 | Loss 0.1605 | Train.Rand 0.9771 | Valid.Rand 0.5483 | Time(s) 0.1803\n",
            "Epoch 00149 | Loss 0.0602 | Train.Rand 0.9806 | Valid.Rand 0.5604 | Time(s) 0.1802\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00150 | Loss 0.0900 | Train.Rand 0.9670 | Valid.Rand 0.5581 | Time(s) 0.1802\n",
            "Epoch 00151 | Loss 0.0951 | Train.Rand 0.9778 | Valid.Rand 0.5621 | Time(s) 0.1802\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00152 | Loss 0.1213 | Train.Rand 0.9548 | Valid.Rand 0.5895 | Time(s) 0.1802\n",
            "Epoch 00153 | Loss 0.0569 | Train.Rand 0.9813 | Valid.Rand 0.5804 | Time(s) 0.1801\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00154 | Loss 0.0819 | Train.Rand 0.9813 | Valid.Rand 0.5552 | Time(s) 0.1801\n",
            "Epoch 00155 | Loss 0.0807 | Train.Rand 0.9911 | Valid.Rand 0.5716 | Time(s) 0.1801\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00156 | Loss 0.1196 | Train.Rand 0.9929 | Valid.Rand 0.5984 | Time(s) 0.1801\n",
            "Epoch 00157 | Loss 0.0894 | Train.Rand 0.9700 | Valid.Rand 0.6014 | Time(s) 0.1801\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00158 | Loss 0.0763 | Train.Rand 0.9813 | Valid.Rand 0.5905 | Time(s) 0.1801\n",
            "Epoch 00159 | Loss 0.0972 | Train.Rand 0.9929 | Valid.Rand 0.5528 | Time(s) 0.1801\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00160 | Loss 0.1015 | Train.Rand 0.9851 | Valid.Rand 0.5336 | Time(s) 0.1800\n",
            "Epoch 00161 | Loss 0.0982 | Train.Rand 0.9813 | Valid.Rand 0.5526 | Time(s) 0.1801\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00162 | Loss 0.0731 | Train.Rand 0.9922 | Valid.Rand 0.5610 | Time(s) 0.1800\n",
            "Epoch 00163 | Loss 0.0528 | Train.Rand 1.0000 | Valid.Rand 0.5744 | Time(s) 0.1800\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00164 | Loss 0.0593 | Train.Rand 0.9926 | Valid.Rand 0.5610 | Time(s) 0.1800\n",
            "Epoch 00165 | Loss 0.0929 | Train.Rand 0.9941 | Valid.Rand 0.5628 | Time(s) 0.1800\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00166 | Loss 0.0898 | Train.Rand 1.0000 | Valid.Rand 0.5657 | Time(s) 0.1799\n",
            "Epoch 00167 | Loss 0.0704 | Train.Rand 0.9926 | Valid.Rand 0.5720 | Time(s) 0.1799\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00168 | Loss 0.0659 | Train.Rand 0.9926 | Valid.Rand 0.5731 | Time(s) 0.1799\n",
            "Epoch 00169 | Loss 0.0811 | Train.Rand 0.9777 | Valid.Rand 0.5762 | Time(s) 0.1799\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00170 | Loss 0.0844 | Train.Rand 0.9745 | Valid.Rand 0.5806 | Time(s) 0.1799\n",
            "Epoch 00171 | Loss 0.0694 | Train.Rand 0.9819 | Valid.Rand 0.5899 | Time(s) 0.1799\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00172 | Loss 0.1094 | Train.Rand 0.9884 | Valid.Rand 0.5842 | Time(s) 0.1799\n",
            "Epoch 00173 | Loss 0.0647 | Train.Rand 1.0000 | Valid.Rand 0.5818 | Time(s) 0.1799\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n",
            "/usr/local/lib/python3.7/dist-packages/torch/autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00174 | Loss 0.0766 | Train.Rand 0.9922 | Valid.Rand 0.6024 | Time(s) 0.1799\n",
            "Epoch 00175 | Loss 0.1038 | Train.Rand 0.9813 | Valid.Rand 0.5890 | Time(s) 0.1799\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zvF64MJDIYmW",
        "outputId": "facaeebb-5324-4b33-f43c-a28d0c3e0726"
      },
      "source": [
        "net.load_state_dict(current_best_params)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bFpZlJOeOKlS",
        "outputId": "967e755d-2a9a-4875-b941-0fc862ff4d46"
      },
      "source": [
        "np.unique(np.argmax(logits[mask_train].detach().numpy(),1))"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 1, 2, 3, 4, 5, 6])"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NqUOV82Cb3dq"
      },
      "source": [
        ""
      ],
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jpuKzkwMQL0_",
        "outputId": "a92daa14-5f27-4bf3-cda4-81d1a4dcf16f"
      },
      "source": [
        "# Visualise predictions\n",
        "net.eval() # Set net to evaluation mode (deactivates dropout)\n",
        "final_prediction = net(features).detach()\n",
        "a = np.transpose(np.vstack([final_prediction[comb_mask].numpy().argmax(axis=1),labels[comb_mask].numpy()]))\n",
        "a[a[:,0].argsort()][np.random.choice(range(a.shape[0]),size=10)]\n",
        "\n",
        "# As can be seen, the graph net predicts correct labels"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[2, 2],\n",
              "       [2, 2],\n",
              "       [2, 2],\n",
              "       [3, 3],\n",
              "       [0, 0],\n",
              "       [2, 2],\n",
              "       [0, 0],\n",
              "       [3, 3],\n",
              "       [3, 3],\n",
              "       [4, 4]])"
            ]
          },
          "metadata": {},
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a9c8JUmkeATZ"
      },
      "source": [
        ""
      ],
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JM_hDALScdxW"
      },
      "source": [
        ""
      ],
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2wmHeyTrSf1H"
      },
      "source": [
        "## Evaluation\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C46ZjsYgZ2tu"
      },
      "source": [
        "# Performnace evaluation functions\n",
        "\n",
        "def variation_of_information_score(labels, preds):\n",
        "    def mi(x, y):\n",
        "        contingency = skmetrics.cluster.contingency_matrix(x, y, sparse=True)\n",
        "        # print(contingency.todense())\n",
        "        nzx, nzy, nz_val = sp.find(contingency)\n",
        "        contingency_sum = contingency.sum()\n",
        "\n",
        "        pi = np.ravel(contingency.sum(axis=1))\n",
        "        pj = np.ravel(contingency.sum(axis=0))\n",
        "        # print(nz_val)\n",
        "        log_contingency_nm = np.log(nz_val)\n",
        "        # print(log_contingency_nm)\n",
        "        contingency_nm = nz_val / contingency_sum\n",
        "        # print(contingency_nm)\n",
        "\n",
        "        # Don't need to calculate the full outer product, just for non-zeroes\n",
        "        outer = pi.take(nzx).astype(np.int64, copy=False) * pj.take(nzy).astype(\n",
        "            np.int64, copy=False\n",
        "        )\n",
        "        # print(outer)\n",
        "        log_outer = -np.log(outer) + log(pi.sum()) + log(pj.sum())\n",
        "        # print(log_outer)\n",
        "        mi = (\n",
        "            contingency_nm * (log_contingency_nm - log(contingency_sum))\n",
        "            + contingency_nm * log_outer\n",
        "        )\n",
        "        # print(mi)\n",
        "        return mi.sum()\n",
        "\n",
        "    return mi(labels, labels) + mi(preds, preds) - 2 * mi(labels, preds)\n",
        "\n",
        "\n",
        "def mutual_info_score(labels, preds):\n",
        "    return skmetrics.adjusted_mutual_info_score(labels, preds, average_method=\"arithmetic\")\n",
        "\n",
        "def compute_performance(labels, logits, splits):\n",
        "    logits = logits.detach().numpy()\n",
        "    preds = np.argmax(logits, axis=1)\n",
        "    labels = labels.numpy()\n",
        "    pred_sets = {\n",
        "        \"All \": preds,\n",
        "        \"Train\": preds[mask_train],\n",
        "        \"Val\": preds[mask_val],\n",
        "        \"Test\": preds[test_mask],\n",
        "    }\n",
        "    label_sets = {\n",
        "        \"All \": labels,\n",
        "        \"Train\": labels[mask_train],\n",
        "        \"Val\": labels[mask_val],\n",
        "        \"Test\": labels[test_mask],\n",
        "    }\n",
        "    eval_functions = {\n",
        "        \"Rand-Index\": rand_score,\n",
        "        \"Mutual Information\": mutual_info_score,\n",
        "        \"Variation of Information\": variation_of_information_score,\n",
        "    }\n",
        "    scores = {\n",
        "        subset: {\n",
        "            name: func(label_sets[subset], pred_sets[subset])\n",
        "            for name, func in eval_functions.items()\n",
        "        }\n",
        "        for subset in pred_sets.keys()\n",
        "    }\n",
        "    return scores\n",
        "\n",
        "def performance_as_df(labels, logits, splits):\n",
        "    scores = compute_performance(labels, logits, splits)\n",
        "    return pd.DataFrame(scores)\n",
        "\n",
        "def rand_score(labels, preds):\n",
        "    return skmetrics.adjusted_rand_score(labels, preds)"
      ],
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        },
        "id": "nUltH0-xRaNk",
        "outputId": "06498f13-61f8-415d-934e-be19526e89bc"
      },
      "source": [
        "net.eval() # Set net to evaluation mode (deactivates dropout)\n",
        "final_prediction = net(features).detach()\n",
        "performance_as_df(labels,final_prediction,comb_mask)"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>All</th>\n",
              "      <th>Train</th>\n",
              "      <th>Val</th>\n",
              "      <th>Test</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Rand-Index</th>\n",
              "      <td>0.627315</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.573450</td>\n",
              "      <td>0.611194</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Mutual Information</th>\n",
              "      <td>0.607157</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.579221</td>\n",
              "      <td>0.592505</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Variation of Information</th>\n",
              "      <td>1.436844</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.465711</td>\n",
              "      <td>1.472488</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                              All   Train       Val      Test\n",
              "Rand-Index                0.627315    1.0  0.573450  0.611194\n",
              "Mutual Information        0.607157    1.0  0.579221  0.592505\n",
              "Variation of Information  1.436844    0.0  1.465711  1.472488"
            ]
          },
          "metadata": {},
          "execution_count": 64
        }
      ]
    }
  ]
}