GAT(GitHub).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "GAT(GitHub).ipynb",
      "provenance": [],
      "mount_file_id": "1E9fXsmxZWNwCEElJpilDHXh6nQxMo1uQ",
      "authorship_tag": "ABX9TyN8pR3WRpy9GsNn4pS/XltB",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mamoonmasud/graph_community/blob/main/GAT(GitHub).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HgfRQ7XKYxWi"
      },
      "source": [
        "import torch"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5QD2djm6Yy9x",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7cc28d42-95e7-4806-8327-88837b455d6d"
      },
      "source": [
        "#Checking if cuda is available. If return False, change runtime to GPU\n",
        "print(torch.cuda.is_available())"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "True\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ejET8LdpY3Us",
        "outputId": "54900dce-e40d-4672-9590-d6ea8374015b"
      },
      "source": [
        "print(torch.version.cuda)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "11.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qk_iEi34ZGvW",
        "outputId": "7cce97f5-861d-4e7b-ce0b-978622c36378"
      },
      "source": [
        "#Checking Python Version\n",
        "! python --version"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Python 3.7.12\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mHf3OIxMZYuF",
        "outputId": "4efb4d9c-7563-4911-ce52-20f42fcce8dd"
      },
      "source": [
        "# Installing DGL\n",
        "! pip install dgl-cu111 -f https://data.dgl.ai/wheels/repo.html "
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in links: https://data.dgl.ai/wheels/repo.html\n",
            "Collecting dgl-cu111\n",
            "  Downloading https://data.dgl.ai/wheels/dgl_cu111-0.7.2-cp37-cp37m-manylinux1_x86_64.whl (165.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 165.0 MB 50 kB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.7/dist-packages (from dgl-cu111) (1.19.5)\n",
            "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from dgl-cu111) (1.4.1)\n",
            "Requirement already satisfied: networkx>=2.1 in /usr/local/lib/python3.7/dist-packages (from dgl-cu111) (2.6.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from dgl-cu111) (2.23.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->dgl-cu111) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->dgl-cu111) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->dgl-cu111) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->dgl-cu111) (2.10)\n",
            "Installing collected packages: dgl-cu111\n",
            "Successfully installed dgl-cu111-0.7.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VeQwSStNzPCK"
      },
      "source": [
        ""
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2OzXVYfndvZP"
      },
      "source": [
        "from google.colab import files"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H-E1uJeYZ0r5"
      },
      "source": [
        "# Doing all the imports:\n",
        "import dgl\n",
        "import dgl.function as fn\n",
        "import torch as th\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from dgl import DGLGraph\n",
        "from dgl.data import DGLDataset\n",
        "from scipy import sparse as sp\n",
        "from math import log\n",
        "import pandas as pd\n",
        "from sklearn import metrics as skmetrics\n",
        "import pickle\n",
        "import numpy as np\n",
        "import os\n",
        "import itertools"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QpR5PsmktzCc"
      },
      "source": [
        "#from dgl.nn.pytorch.conv import GATConv as GAT"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RWCs4nFmspN9"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x_uXoCpDy_xC"
      },
      "source": [
        "def upload_files():\n",
        "  from google.colab import files\n",
        "  uploaded = files.upload()\n",
        "  for k, v in uploaded.items():\n",
        "    open(k, 'wb').write(v)\n",
        "  return list(uploaded.keys())"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgZG8gewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwoKICAgICAgbGV0IHBlcmNlbnREb25lID0gZmlsZURhdGEuYnl0ZUxlbmd0aCA9PT0gMCA/CiAgICAgICAgICAxMDAgOgogICAgICAgICAgTWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCk7CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPSBgJHtwZXJjZW50RG9uZX0lIGRvbmVgOwoKICAgIH0gd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCk7CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 106
        },
        "id": "-nlF-C5n0SNh",
        "outputId": "c8a72af5-bb59-49bf-c405-a3996539492e"
      },
      "source": [
        "upload_files()"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-95ab9856-04c5-4105-aa7e-8fce03ad7ee0\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-95ab9856-04c5-4105-aa7e-8fce03ad7ee0\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving GAT.py to GAT.py\n",
            "Saving performance.py to performance.py\n",
            "Saving Sparsemax.py to Sparsemax.py\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['GAT.py', 'performance.py', 'Sparsemax.py']"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DofZPMoDz6a-"
      },
      "source": [
        "import GAT\n",
        "import Sparsemax\n",
        "import performance as pf\n"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7JgH7hAsOwQt"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xA9iRvQqtTZk"
      },
      "source": [
        "# Graph Neural Network Definition\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "08EnIPQSaX9G"
      },
      "source": [
        "class GATLayer(nn.Module):\n",
        "    def __init__(self, g, in_dim, out_dim):\n",
        "        super(GATLayer, self).__init__()\n",
        "        self.g = g\n",
        "        # equation (1)\n",
        "        self.fc = nn.Linear(in_dim, out_dim, bias=False)\n",
        "        # equation (2)\n",
        "        self.attn_fc = nn.Linear(2 * out_dim, 1, bias=False)\n",
        "\n",
        "    def edge_attention(self, edges):\n",
        "        # edge UDF for equation (2)\n",
        "        z2 = th.cat([edges.src['z'], edges.dst['z']], dim=1)\n",
        "        a = self.attn_fc(z2)\n",
        "        return {'e': F.leaky_relu(a)}\n",
        "\n",
        "    def message_func(self, edges):\n",
        "        # message UDF for equation (3) & (4)\n",
        "        return {'z': edges.src['z'], 'e': edges.data['e']}\n",
        "\n",
        "    def reduce_func(self, nodes):\n",
        "        # reduce UDF for equation (3) & (4)\n",
        "        # equation (3)\n",
        "        alpha = F.softmax(nodes.mailbox['e'], dim=1)\n",
        "        # equation (4)\n",
        "        h = th.sum(alpha * nodes.mailbox['z'], dim=1)\n",
        "        return {'h': h}\n",
        "\n",
        "    def forward(self, h):\n",
        "        # equation (1)\n",
        "        z = self.fc(h)\n",
        "        self.g.ndata['z'] = z\n",
        "        # equation (2)\n",
        "        self.g.apply_edges(self.edge_attention)\n",
        "        # equation (3) & (4)\n",
        "        self.g.update_all(self.message_func, self.reduce_func)\n",
        "        return self.g.ndata.pop('h')\n"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "90qOw_TXbNTf"
      },
      "source": [
        "class MultiHeadGATLayer(nn.Module):\n",
        "    def __init__(self, g, in_dim, out_dim, num_heads, merge='cat'):\n",
        "        super(MultiHeadGATLayer, self).__init__()\n",
        "        self.heads = nn.ModuleList()\n",
        "        for i in range(num_heads):\n",
        "            self.heads.append(GATLayer(g, in_dim, out_dim))\n",
        "        self.merge = merge\n",
        "\n",
        "    def forward(self, h):\n",
        "        head_outs = [attn_head(h) for attn_head in self.heads]\n",
        "        if self.merge == 'cat':\n",
        "            # concat on the output feature dimension (dim=1)\n",
        "            return th.cat(head_outs, dim=1)\n",
        "        else:\n",
        "            # merge using average\n",
        "            return th.mean(torch.stack(head_outs))"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zXJk-TtibQhN"
      },
      "source": [
        "class Net(nn.Module):\n",
        "    def __init__(self, g, in_dim, hidden_dim, out_dim, num_heads):\n",
        "        super(Net, self).__init__()\n",
        "        self.layer1 = MultiHeadGATLayer(g, in_dim, hidden_dim, num_heads)\n",
        "        # Be aware that the input dimension is hidden_dim*num_heads since\n",
        "        # multiple head outputs are concatenated together. Also, only\n",
        "        # one attention head in the output layer.\n",
        "        self.layer2 = MultiHeadGATLayer(g, hidden_dim * num_heads, out_dim, 1)\n",
        "\n",
        "    def forward(self, h):\n",
        "        h = self.layer1(h)\n",
        "        h = F.elu(h)\n",
        "        h = self.layer2(h)\n",
        "        h = F.log_softmax(h, 1)\n",
        "        return h"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bx2sFf240dIQ"
      },
      "source": [
        "# Data Loading"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Generating a dgl graph from the given csv files of github dataset at: http://snap.stanford.edu/data/github-social.html\n",
        "\n",
        "class GitHubDataset(DGLDataset):\n",
        "    def __init__(self):\n",
        "        super().__init__(name='github_dataset')\n",
        "\n",
        "    def process(self):\n",
        "        nodes_data = pd.read_csv('drive/MyDrive/CSCE689/data_git/papers.csv')\n",
        "        edges_data = pd.read_csv('drive/MyDrive/CSCE689/data_git/git_edges.csv')\n",
        "        node_target = pd.read_csv('drive/MyDrive/CSCE689/data_git/musae_git_target.csv')\n",
        "        node_features = torch.from_numpy(nodes_data.to_numpy())\n",
        "        node_labels = torch.from_numpy(node_target[\"ml_target\"].astype('category').cat.codes.to_numpy())\n",
        "        #edge_features = torch.from_numpy(edges_data['Weight'].to_numpy())\n",
        "        edges_src = torch.from_numpy(edges_data['source'].to_numpy())\n",
        "        edges_dst = torch.from_numpy(edges_data['target'].to_numpy())\n",
        "\n",
        "        self.graph = dgl.graph((edges_src, edges_dst), num_nodes=nodes_data.shape[0])\n",
        "        self.graph.ndata['feat'] = node_features\n",
        "        self.graph.ndata['label'] = node_labels\n",
        "        #self.graph.edata['weight'] = edge_features\n",
        "\n",
        "        # If your dataset is a node classification dataset, you will need to assign\n",
        "        # masks indicating whether a node belongs to training, validation, and test set.\n",
        "        n_nodes = nodes_data.shape[0]\n",
        "        n_train = int(n_nodes * 0.6)\n",
        "        n_val = int(n_nodes * 0.2)\n",
        "        train_mask = torch.zeros(n_nodes, dtype=torch.bool)\n",
        "        val_mask = torch.zeros(n_nodes, dtype=torch.bool)\n",
        "        test_mask = torch.zeros(n_nodes, dtype=torch.bool)\n",
        "        train_mask[:n_train] = True\n",
        "        val_mask[n_train:n_train + n_val] = True\n",
        "        test_mask[n_train + n_val:] = True\n",
        "        self.graph.ndata['train_mask'] = train_mask\n",
        "        self.graph.ndata['val_mask'] = val_mask\n",
        "        self.graph.ndata['test_mask'] = test_mask\n",
        "\n",
        "    def __getitem__(self, i):\n",
        "        return self.graph\n",
        "\n",
        "    def __len__(self):\n",
        "        return 1\n",
        "\n",
        "dataset = GitHubDataset()\n",
        "graph = dataset[0]\n"
      ],
      "metadata": {
        "id": "c6E6i5J4FHsL"
      },
      "execution_count": 107,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nHyZ_u4ybSje"
      },
      "source": [
        "num_class = 2\n",
        "\n",
        "features = graph.ndata['feat']\n",
        "\n",
        "train_mask = graph.ndata['train_mask']\n",
        "val_mask = graph.ndata['val_mask']\n",
        "test_mask = graph.ndata['test_mask']\n",
        "labels = graph.ndata['label']\n",
        "\n",
        "# add self loop\n",
        "#g.remove_edges_from(nx.selfloop_edges(g))\n",
        "graph = dgl.remove_self_loop(graph)\n",
        "#g = DGLGraph(g)\n",
        "graph.add_edges(graph.nodes(), graph.nodes())\n",
        "\n",
        "comb_mask = torch.zeros(len(train_mask), dtype=torch.bool)\n",
        "\n",
        "for i in range(len(comb_mask)):\n",
        "    if test_mask[i] or train_mask[i] or val_mask[i]:\n",
        "        comb_mask[i] = True\n"
      ],
      "execution_count": 108,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "features = features.float()"
      ],
      "metadata": {
        "id": "61AvgXVWH5HF"
      },
      "execution_count": 111,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "labels = labels.long()"
      ],
      "metadata": {
        "id": "EPKjFoZdK4r2"
      },
      "execution_count": 125,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "lAbrWfrbNujh"
      },
      "execution_count": 126,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sntihQrF1Bpi",
        "outputId": "b57bea4f-0a78-43fc-cdcd-4efad7d918b4"
      },
      "source": [
        "print('We have %d nodes.' % graph.number_of_nodes())\n",
        "print('We have %d edges.' % graph.number_of_edges())"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "We have 37700 nodes.\n",
            "We have 326703 edges.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hSXAbdMR0ncM"
      },
      "source": [
        "# Selecting Training Set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m0KdxXbFtizM"
      },
      "source": [
        "percentage_train = 0.6\n",
        "percentage_val = 0.2\n",
        "\n",
        "with open(\"drive/MyDrive/CSCE689/data_git/git_permutation4.pickle\",\"rb\") as f:\n",
        "    perm1 = pickle.load(f)\n",
        "mask_train = np.zeros(graph.number_of_nodes())\n",
        "mask_val = np.zeros(graph.number_of_nodes())\n",
        "\n",
        "i_train = int(percentage_train*graph.number_of_nodes())\n",
        "i_val = i_train + int(percentage_val*graph.number_of_nodes())\n",
        "mask_train[perm1[range(0,i_train)]] = 1\n",
        "mask_val[perm1[range(i_train,i_val)]] = 1\n",
        "mask_train = th.BoolTensor(mask_train)\n",
        "mask_val = th.BoolTensor(mask_val)"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "g= graph"
      ],
      "metadata": {
        "id": "_Xr7_mmTGitL"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0KC_oeecIWZo"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Oyi6Z9od0prq"
      },
      "source": [
        "loss_function = pf.perm_inv_loss(labels)\n",
        "import copy"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M0HYmM23IVg2",
        "outputId": "cb7163fa-0304-4f62-f2b6-25d14bd6401a"
      },
      "source": [
        "import time\n",
        "\n",
        "net = GAT.GAT_Net_fast(g=g, in_feats=features.shape[1], hidden_size=100, hidden_layers=2, out_feats=len(np.unique(labels)),\n",
        "dropout=0.2, batchnorm=False, num_heads=1, residual = False)\n",
        "#print(net)\n",
        "\n",
        "optimizer = th.optim.Adam(net.parameters(), lr=1e-2, weight_decay=1e-2)\n",
        "net.train() # Set to training mode (use dropout)\n",
        "\n",
        "dur = []\n",
        "loss_ev = []\n",
        "current_best = 0 #arbitrarily high\n",
        "current_best_epoch = 0\n",
        "current_best_params = None\n",
        "no_improvement_for = 0\n",
        "\n",
        "for epoch in range(1000):\n",
        "    if epoch >=3:\n",
        "        t0 = time.time()\n",
        "\n",
        "    # Compute loss for test nodes (only for validation, not used by optimizer)\n",
        "    net.eval()\n",
        "    \n",
        "    prediction = net(features)\n",
        "    train_rand = pf.rand_score(labels[mask_train].numpy(),np.argmax(prediction[mask_train].detach().numpy(), axis=1))\n",
        "    validation_rand = pf.rand_score(labels[mask_val].numpy(),np.argmax(prediction[mask_val].detach().numpy(), axis=1))\n",
        "    if train_rand>current_best:\n",
        "        current_best = train_rand\n",
        "        current_best_epoch = epoch\n",
        "        current_best_params = copy.deepcopy(net.state_dict())\n",
        "        no_improvement_for = 0\n",
        "    else: no_improvement_for += 1\n",
        "    \n",
        "    if no_improvement_for>50:\n",
        "        break\n",
        "    \n",
        "    net.train()\n",
        "\n",
        "    # Compute loss for train nodes\n",
        "    logits = net(features)\n",
        "\n",
        "    #loss = loss_function.approximate_loss(logits,mask_train,nclasses=7)\n",
        "    loss = F.nll_loss(logits[mask_train], labels[mask_train])\n",
        "    loss_ev.append(loss.detach().item())\n",
        "    #print(np.unique(np.argmax(logits[mask_train].detach().numpy(),1)))\n",
        "    \n",
        "    \n",
        "    optimizer.zero_grad()\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "\n",
        "    if epoch >=3:\n",
        "        dur.append(time.time() - t0)\n",
        "        print(f\"Epoch {epoch:05d} | Loss {loss.item():.4f} | Train.Rand {train_rand:.4f} | Valid.Rand {validation_rand:.4f} | Time(s) {np.mean(dur):.4f}\")\n",
        "    else:\n",
        "        print(f\"Epoch {epoch:05d} | Loss {loss.item():.4f} | Train.Rand {train_rand:.4f} | Valid.Rand {validation_rand:.4f} | Time(s) unknown\")\n",
        "        \n",
        "net.load_state_dict(current_best_params)"
      ],
      "execution_count": 127,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 00000 | Loss 2.1574 | Train.Rand 0.0244 | Valid.Rand 0.0301 | Time(s) unknown\n",
            "Epoch 00001 | Loss 1.5081 | Train.Rand 0.0306 | Valid.Rand 0.0359 | Time(s) unknown\n",
            "Epoch 00002 | Loss 1.0722 | Train.Rand 0.0035 | Valid.Rand 0.0170 | Time(s) unknown\n",
            "Epoch 00003 | Loss 1.2732 | Train.Rand -0.0004 | Valid.Rand 0.0001 | Time(s) 1.6988\n",
            "Epoch 00004 | Loss 1.3794 | Train.Rand 0.0002 | Valid.Rand -0.0006 | Time(s) 1.7021\n",
            "Epoch 00005 | Loss 1.1344 | Train.Rand 0.0093 | Valid.Rand 0.0103 | Time(s) 1.7004\n",
            "Epoch 00006 | Loss 0.9873 | Train.Rand 0.0116 | Valid.Rand 0.0170 | Time(s) 1.7034\n",
            "Epoch 00007 | Loss 0.9455 | Train.Rand 0.0073 | Valid.Rand 0.0117 | Time(s) 1.7049\n",
            "Epoch 00008 | Loss 0.8582 | Train.Rand 0.0050 | Valid.Rand 0.0100 | Time(s) 1.7022\n",
            "Epoch 00009 | Loss 0.7704 | Train.Rand 0.0137 | Valid.Rand 0.0098 | Time(s) 1.7009\n",
            "Epoch 00010 | Loss 0.8068 | Train.Rand 0.0082 | Valid.Rand 0.0027 | Time(s) 1.6989\n",
            "Epoch 00011 | Loss 0.8074 | Train.Rand -0.0009 | Valid.Rand -0.0033 | Time(s) 1.6947\n",
            "Epoch 00012 | Loss 0.7892 | Train.Rand 0.0135 | Valid.Rand 0.0049 | Time(s) 1.6955\n",
            "Epoch 00013 | Loss 0.7816 | Train.Rand 0.0239 | Valid.Rand 0.0194 | Time(s) 1.7008\n",
            "Epoch 00014 | Loss 0.7428 | Train.Rand 0.0191 | Valid.Rand 0.0221 | Time(s) 1.7044\n",
            "Epoch 00015 | Loss 0.7586 | Train.Rand 0.0195 | Valid.Rand 0.0179 | Time(s) 1.7081\n",
            "Epoch 00016 | Loss 0.7241 | Train.Rand 0.0171 | Valid.Rand 0.0173 | Time(s) 1.7102\n",
            "Epoch 00017 | Loss 0.6999 | Train.Rand 0.0157 | Valid.Rand 0.0194 | Time(s) 1.7156\n",
            "Epoch 00018 | Loss 0.6965 | Train.Rand 0.0205 | Valid.Rand 0.0239 | Time(s) 1.7147\n",
            "Epoch 00019 | Loss 0.7067 | Train.Rand 0.0263 | Valid.Rand 0.0255 | Time(s) 1.7145\n",
            "Epoch 00020 | Loss 0.7017 | Train.Rand 0.0274 | Valid.Rand 0.0305 | Time(s) 1.7136\n",
            "Epoch 00021 | Loss 0.6634 | Train.Rand 0.0316 | Valid.Rand 0.0348 | Time(s) 1.7101\n",
            "Epoch 00022 | Loss 0.6564 | Train.Rand 0.0494 | Valid.Rand 0.0575 | Time(s) 1.7075\n",
            "Epoch 00023 | Loss 0.6766 | Train.Rand 0.0595 | Valid.Rand 0.0582 | Time(s) 1.7068\n",
            "Epoch 00024 | Loss 0.6506 | Train.Rand 0.0611 | Valid.Rand 0.0592 | Time(s) 1.7057\n",
            "Epoch 00025 | Loss 0.6491 | Train.Rand 0.0467 | Valid.Rand 0.0387 | Time(s) 1.7051\n",
            "Epoch 00026 | Loss 0.6431 | Train.Rand 0.0456 | Valid.Rand 0.0393 | Time(s) 1.7029\n",
            "Epoch 00027 | Loss 0.6482 | Train.Rand 0.0595 | Valid.Rand 0.0486 | Time(s) 1.7042\n",
            "Epoch 00028 | Loss 0.6280 | Train.Rand 0.0833 | Valid.Rand 0.0763 | Time(s) 1.7029\n",
            "Epoch 00029 | Loss 0.6272 | Train.Rand 0.0811 | Valid.Rand 0.0766 | Time(s) 1.7009\n",
            "Epoch 00030 | Loss 0.6315 | Train.Rand 0.0736 | Valid.Rand 0.0672 | Time(s) 1.7004\n",
            "Epoch 00031 | Loss 0.6228 | Train.Rand 0.0665 | Valid.Rand 0.0545 | Time(s) 1.7031\n",
            "Epoch 00032 | Loss 0.6419 | Train.Rand 0.0634 | Valid.Rand 0.0561 | Time(s) 1.7050\n",
            "Epoch 00033 | Loss 0.6175 | Train.Rand 0.0583 | Valid.Rand 0.0526 | Time(s) 1.7050\n",
            "Epoch 00034 | Loss 0.6273 | Train.Rand 0.0531 | Valid.Rand 0.0500 | Time(s) 1.7044\n",
            "Epoch 00035 | Loss 0.6200 | Train.Rand 0.0521 | Valid.Rand 0.0471 | Time(s) 1.7033\n",
            "Epoch 00036 | Loss 0.6171 | Train.Rand 0.0575 | Valid.Rand 0.0511 | Time(s) 1.7027\n",
            "Epoch 00037 | Loss 0.6151 | Train.Rand 0.0561 | Valid.Rand 0.0443 | Time(s) 1.7029\n",
            "Epoch 00038 | Loss 0.6170 | Train.Rand 0.0580 | Valid.Rand 0.0485 | Time(s) 1.7036\n",
            "Epoch 00039 | Loss 0.6323 | Train.Rand 0.0581 | Valid.Rand 0.0518 | Time(s) 1.7019\n",
            "Epoch 00040 | Loss 0.6131 | Train.Rand 0.0510 | Valid.Rand 0.0391 | Time(s) 1.7006\n",
            "Epoch 00041 | Loss 0.6004 | Train.Rand 0.0411 | Valid.Rand 0.0393 | Time(s) 1.6994\n",
            "Epoch 00042 | Loss 0.6125 | Train.Rand 0.0335 | Valid.Rand 0.0330 | Time(s) 1.7003\n",
            "Epoch 00043 | Loss 0.9316 | Train.Rand 0.0347 | Valid.Rand 0.0352 | Time(s) 1.7004\n",
            "Epoch 00044 | Loss 0.6045 | Train.Rand 0.0382 | Valid.Rand 0.0379 | Time(s) 1.6997\n",
            "Epoch 00045 | Loss 0.6049 | Train.Rand 0.0284 | Valid.Rand 0.0238 | Time(s) 1.6990\n",
            "Epoch 00046 | Loss 0.6158 | Train.Rand 0.0255 | Valid.Rand 0.0198 | Time(s) 1.7012\n",
            "Epoch 00047 | Loss 0.5847 | Train.Rand 0.0249 | Valid.Rand 0.0192 | Time(s) 1.7029\n",
            "Epoch 00048 | Loss 0.5888 | Train.Rand 0.0298 | Valid.Rand 0.0285 | Time(s) 1.7035\n",
            "Epoch 00049 | Loss 0.5911 | Train.Rand 0.0341 | Valid.Rand 0.0290 | Time(s) 1.7051\n",
            "Epoch 00050 | Loss 0.6073 | Train.Rand 0.0304 | Valid.Rand 0.0275 | Time(s) 1.7068\n",
            "Epoch 00051 | Loss 0.5952 | Train.Rand 0.0238 | Valid.Rand 0.0227 | Time(s) 1.7083\n",
            "Epoch 00052 | Loss 0.5887 | Train.Rand 0.0148 | Valid.Rand 0.0131 | Time(s) 1.7084\n",
            "Epoch 00053 | Loss 0.5880 | Train.Rand 0.0204 | Valid.Rand 0.0193 | Time(s) 1.7076\n",
            "Epoch 00054 | Loss 0.5863 | Train.Rand 0.0196 | Valid.Rand 0.0159 | Time(s) 1.7075\n",
            "Epoch 00055 | Loss 0.5812 | Train.Rand 0.0139 | Valid.Rand 0.0119 | Time(s) 1.7071\n",
            "Epoch 00056 | Loss 0.5857 | Train.Rand 0.0238 | Valid.Rand 0.0303 | Time(s) 1.7055\n",
            "Epoch 00057 | Loss 0.5938 | Train.Rand 0.0279 | Valid.Rand 0.0210 | Time(s) 1.7046\n",
            "Epoch 00058 | Loss 0.5763 | Train.Rand 0.0273 | Valid.Rand 0.0300 | Time(s) 1.7063\n",
            "Epoch 00059 | Loss 0.5851 | Train.Rand 0.0191 | Valid.Rand 0.0160 | Time(s) 1.7067\n",
            "Epoch 00060 | Loss 0.5849 | Train.Rand 0.0190 | Valid.Rand 0.0175 | Time(s) 1.7064\n",
            "Epoch 00061 | Loss 0.5859 | Train.Rand 0.0259 | Valid.Rand 0.0310 | Time(s) 1.7054\n",
            "Epoch 00062 | Loss 0.5800 | Train.Rand 0.0287 | Valid.Rand 0.0282 | Time(s) 1.7045\n",
            "Epoch 00063 | Loss 0.5841 | Train.Rand 0.0355 | Valid.Rand 0.0358 | Time(s) 1.7036\n",
            "Epoch 00064 | Loss 0.5810 | Train.Rand 0.0470 | Valid.Rand 0.0469 | Time(s) 1.7025\n",
            "Epoch 00065 | Loss 0.5729 | Train.Rand 0.0467 | Valid.Rand 0.0484 | Time(s) 1.7018\n",
            "Epoch 00066 | Loss 0.5803 | Train.Rand 0.0430 | Valid.Rand 0.0506 | Time(s) 1.7015\n",
            "Epoch 00067 | Loss 0.5660 | Train.Rand 0.0269 | Valid.Rand 0.0305 | Time(s) 1.7006\n",
            "Epoch 00068 | Loss 0.5830 | Train.Rand 0.0223 | Valid.Rand 0.0332 | Time(s) 1.6995\n",
            "Epoch 00069 | Loss 0.5775 | Train.Rand 0.0228 | Valid.Rand 0.0312 | Time(s) 1.6997\n",
            "Epoch 00070 | Loss 0.5706 | Train.Rand 0.0243 | Valid.Rand 0.0356 | Time(s) 1.6990\n",
            "Epoch 00071 | Loss 0.5714 | Train.Rand 0.0280 | Valid.Rand 0.0398 | Time(s) 1.6987\n",
            "Epoch 00072 | Loss 0.5885 | Train.Rand 0.0490 | Valid.Rand 0.0536 | Time(s) 1.6984\n",
            "Epoch 00073 | Loss 0.5660 | Train.Rand 0.0550 | Valid.Rand 0.0570 | Time(s) 1.6991\n",
            "Epoch 00074 | Loss 0.5874 | Train.Rand 0.0341 | Valid.Rand 0.0404 | Time(s) 1.6987\n",
            "Epoch 00075 | Loss 0.5759 | Train.Rand 0.0295 | Valid.Rand 0.0354 | Time(s) 1.6981\n",
            "Epoch 00076 | Loss 0.5756 | Train.Rand 0.0314 | Valid.Rand 0.0348 | Time(s) 1.6974\n",
            "Epoch 00077 | Loss 0.5794 | Train.Rand 0.0425 | Valid.Rand 0.0453 | Time(s) 1.6963\n",
            "Epoch 00078 | Loss 0.5715 | Train.Rand 0.0625 | Valid.Rand 0.0671 | Time(s) 1.6965\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {},
          "execution_count": 127
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "labels."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yaQjMYevHEaR",
        "outputId": "2dbd727b-a32c-42db-d166-bc4cc1c0b377"
      },
      "execution_count": 124,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.int8"
            ]
          },
          "metadata": {},
          "execution_count": 124
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "id": "l-vcPWhxsKAT",
        "outputId": "c2eabb94-4909-438e-c0e7-d63388595f99"
      },
      "source": [
        "import seaborn as sns\n",
        "sns.lineplot(x=range(len(loss_ev)), y=loss_ev)\n"
      ],
      "execution_count": 128,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f7f482097d0>"
            ]
          },
          "metadata": {},
          "execution_count": 128
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3zcVZ3/8ddnZjKTTO7XXpL0fqNQ2kKQQgELCBa8oMii6CK6aNfryspvV9l1va3uqvvTVX4qbAVkdV1c5SYgeEPYchEkLS30SltK2zRtM73kfp85vz9mJk3aXKbpJJOZvp+PRx7NzJzMfJrLOyfn9jXnHCIikv48qS5ARESSQ4EuIpIhFOgiIhlCgS4ikiEU6CIiGcKXqhcuKytzM2bMSNXLi4ikpbVr1x5yzpUP9ljKAn3GjBnU1tam6uVFRNKSme0e6rERh1zMrNrMnjKzzWa2ycw+M0ibD5jZK2b2qpk9b2aLT7VoERE5OYn00HuBW51z68wsH1hrZr93zm3u12YX8Gbn3FEzuwpYDZw/BvWKiMgQRgx059x+YH/s/RYz2wJUApv7tXm+34e8AFQluU4RERnBSa1yMbMZwFLgxWGa3Qw8McTHrzKzWjOrDYVCJ/PSIiIygoQD3czygAeAW5xzzUO0uZRooH9usMedc6udczXOuZry8kEnaUVEZJQSWuViZllEw/xnzrkHh2hzNnAXcJVz7nDyShQRkUQkssrFgLuBLc657wzRZhrwIHCjc+615JYoIiKJSKSHvhy4EXjVzNbH7vsHYBqAc+5O4ItAKfDDaP7T65yrSX65sO1AC49uqOfDy2dQmhcYi5cQEUlLiaxyeRawEdp8BPhIsooazuuhVr7/1A6uXjRFgS4i0k/aneWSG4j+Dmrv7k1xJSIiE0saBroXgLbucIorERGZWNIu0IP+aA+9rUs9dBGR/tIu0PMCCnQRkcGkXaAH/bEhFwW6iMgAaRfo8UlRjaGLiAyUdoEe8HnwekyrXEREjpN2gW5mBP1e2rrUQxcR6S/tAh2iE6MaQxcRGSgtAz3o99KuMXQRkQHSMtDzAj5a1UMXERkgLQM96PdpUlRE5DhpGei5AS+tmhQVERkgTQNdPXQRkeOlZaAH/T4tWxQROU5aBnqu36tliyIix0nkEnTVZvaUmW02s01m9plB2piZ3W5mO8zsFTM7Z2zKjcoN+OjoCROOuLF8GRGRtJJID70XuNU5txBYBnzSzBYe1+YqYG7sbRVwR1KrPE78THSNo4uIHDNioDvn9jvn1sXebwG2AJXHNbsG+ImLegEoMrMpSa825thVizSOLiISd1Jj6GY2A1gKvHjcQ5XA3n636zgx9DGzVWZWa2a1oVDo5CrtJ1cXuRAROUHCgW5mecADwC3OuebRvJhzbrVzrsY5V1NeXj6apwD6n4muHrqISFxCgW5mWUTD/GfOuQcHabIPqO53uyp235jou2qRxtBFRPokssrFgLuBLc657wzR7BHgg7HVLsuAJufc/iTWOUCwbwxdgS4iEudLoM1y4EbgVTNbH7vvH4BpAM65O4HHgauBHUA78OHkl3pMbmzIRdv/RUSOGTHQnXPPAjZCGwd8MllFjaRvlYsmRUVE+qTpTtFooOsIXRGRY9Iy0IN9G4s05CIiEpeWgZ7l9eD3ebTKRUSkn7QMdNABXSIix0vfQA/4aNcqFxGRPukb6H5dV1REpL/0DfSAV5OiIiL9pHGg+zQpKiLST9oGelCToiIiA6RtoOcGdF1REZH+0jfQ/T4dziUi0k/aBnow4FUPXUSkn7QN9Dy/j+5whO7eSKpLERGZENI20HUmuojIQGkb6HmxA7ratBZdRARI7IpF95hZg5ltHOLxQjN71Mw2mNkmMxvTi1vEBf06E11EpL9Eeuj3AiuHefyTwGbn3GJgBfBtM/OfemnDyw3Er1qkQBcRgQQC3Tm3BjgyXBMgP3bt0bxY2zFP2fhFLrT9X0QkKpFrio7k+0QvEl0P5APvdc6N+dKT+GXo1EMXEYlKxqToW4H1wFRgCfB9MysYrKGZrTKzWjOrDYVCp/SiuVrlIiIyQDIC/cPAgy5qB7ALWDBYQ+fcaudcjXOupry8/JReNNcfW+WizUUiIkByAn0PcDmAmU0C5gOvJ+F5hxVfh64DukREokYcQzez+4iuXikzszrgS0AWgHPuTuCfgXvN7FXAgM855w6NWcUxwSytQxcR6W/EQHfO3TDC4/XAlUmrKEEejxH0e7UOXUQkJm13ikJ0c5EuciEiEpXWgZ6nExdFRPqkdaAH/T5NioqIxKR1oOfpuqIiIn3SOtCDAa+2/ouIxKR1oOf6fdr6LyISk96BHvDSrklREREgzQNdk6IiIsekdaDnBry0dffinEt1KSIiKZfmge4j4qBLF4oWEUnzQPfrTHQRkbj0DvT4meiaGBURSfNAj5+Jrs1FIiLpHeg6E11E5Ji0DvS8gM5EFxGJS+tAD/rVQxcRiRsx0M3sHjNrMLONw7RZYWbrzWyTmf1vckscWp6GXERE+iTSQ78XWDnUg2ZWBPwQeKdz7kzgL5JT2siCsUlRHdAlIpJAoDvn1gBHhmnyfuBB59yeWPuGJNU2oviyRa1DFxFJzhj6PKDYzJ42s7Vm9sGhGprZKjOrNbPaUCh0yi8c8Hnweox2LVsUEUlKoPuAc4G3AW8F/snM5g3W0Dm32jlX45yrKS8vP+UXNoteKFqXoRMRiYbxqaoDDjvn2oA2M1sDLAZeS8JzjyhXJy6KiADJ6aH/CrjIzHxmFgTOB7Yk4XkTkqurFomIAAn00M3sPmAFUGZmdcCXgCwA59ydzrktZvYb4BUgAtzlnBtyiWOy5QZ01SIREUgg0J1zNyTQ5t+Af0tKRScp1+/TpKiICGm+UxRiF7nQpKiISPoHetDv02mLIiJkQKDnBnzqoYuIkAmB7vdq2aKICJkQ6AEfHT1hwhFdKFpETm8ZEOjxA7rUSxeR01vaB3pBdhYAzZ0KdBE5vaV9oBcFo4He2N6d4kpERFIrAwLdD0Bje0+KKxERSa0MCPR4D12BLiKnt/QP9JxYD71DQy4icnpL/0BXD11EBMiAQM/O8pKd5dGkqIic9tI+0AGKg36OqocuIqe5jAj0wpwsDbmIyGkvIwK9OOjXkIuInPZGDHQzu8fMGsxs2KsQmdl5ZtZrZtclr7zEFAWzaOxQD11ETm+J9NDvBVYO18DMvMA3gd8loaaTVqQeuojIyIHunFsDHBmh2aeBB4CGZBR1soqC0TF053Tiooicvk55DN3MKoF3A3ck0HaVmdWaWW0oFDrVl+5TlJNFb8TR1q0LXYjI6SsZk6LfBT7nnIuM1NA5t9o5V+OcqykvL0/CS0cVx85zOdqmYRcROX35kvAcNcDPzQygDLjazHqdcw8n4bkTUhjbLdrU0UP1CG1/8NQOunojfPaKeWNfmIjIODrlHrpzbqZzboZzbgZwP/CJ8Qxz6NdDH2FitDccYfWa1/nV+n3jUZaIyLgasYduZvcBK4AyM6sDvgRkATjn7hzT6hKU6HkuL+9tpKmjh/buXsIRh9dj41GeiMi4GDHQnXM3JPpkzrkPnVI1o5ToRS6e3BJdhNMTdhxo7qSyKGfMaxMRGS8ZsVO07wjdEXrof9x6kFx/9Bqke4+0j3ldIiLjKSMC3e/zkOv3DrtbdO+Rdl472Mq151T13RYRySQZEegQ3S063KToU9uiwy03XjAdM9h7tGO8ShMRGRcZFOhZNA0z5PLklgZmleUyb1I+kwuyqTuqHrqIZJaMCvSheuhtXb38aedhLl1QAUB1cZC6I+qhi0hmyZxAz/EPOYb+3I5DdIcjXB4L9KqSHPaqhy4iGSZzAj049EUuntrWQH7AR82MEiDaQz/Q3ElXr85+EZHMkWGB3k0kMvDEReccT25p4OJ5Zfh90f9udUkQ56C+sTMVpYqIjImMCfTioJ+Ig5au3gH3b6pvpqGli8sWTOq7r6o4uqFISxdFJJNkTKAX5sQO6Dpu2OXJLQ2YwYr5x053rC4JAlCnpYsikkEyJtDjB3Q1dgxc6bKhrpF5FfmU5QX67ptckE2W1zQxKiIZJWMCPX6ey9Hjeug7Q63MmZQ34D6vx5halKMhFxHJKBkU6PHzXI710Dt7wuw90s7s8rwT2lcXB7VbVEQySgYF+olH6O4+3E7Ewezy3BPaV5fkUKceuohkkIwJ9PikaP9A39HQCjBoD72qOMjhtm7ajlsVIyKSrjIm0LO8HvIDvgHb/3eGooE+a5Aeenzp4r5GDbuISGYYMdDN7B4zazCzjUM8/gEze8XMXjWz581scfLLTExhMIumftv/d4ZaqSzKIeg/8Toe8aWLmhgVkUyRSA/9XmDlMI/vAt7snFsE/DOwOgl1jUpx0D9gUnRnqHXQ3jlEJ0VBgS4imWPEQHfOrQGODPP48865o7GbLwBVSartpEVPXIz20CMRx86GtkHHzwHK8vzkZHm10kVEMkayx9BvBp4Y6kEzW2VmtWZWGwqFkvzS0aWL8SGXA82ddPSEmV0xeKCbGVXFWosuIpkjaYFuZpcSDfTPDdXGObfaOVfjnKspLy8fqtmoFeUcOxM9PiE62JLFuKriHG3/F5GMkZRAN7OzgbuAa5xzh5PxnKNRHJsUjQ63RAN9zhBDLhCdGNX2fxHJFKcc6GY2DXgQuNE599qplzR6hUE/zkFzZw87Q23kB3yU5weGbF9dHKSls3fYS9eJiKSLE9fzHcfM7gNWAGVmVgd8CcgCcM7dCXwRKAV+aGYAvc65mrEqeDjF/XaL7gy1Mqsij1hNg6ouiR2je7SdwmDhuNQoIjJWRgx059wNIzz+EeAjSavoFBw7oKubnaFWls8pG7Z9Vb+li2dVKtBFJL1lzE5RgMKc6AFddUc7ONjcNeSSxbi+tegaRxeRDJBRgR4fclm3J7osfqRALwxmkZ/t00oXEckIGRXo8SN01+6OBvqciqGXLMbNLMvtO8RLRCSdZVSgx09c3FTfjNdjTCsZOdAXVRbyal3TCReXFhFJNxkV6F6PUZDtIxxxTC8J4veN/N9bUl1ES1dv30YkEZF0lVGBDlCcGx12mTXC+Hnc0mlFAKzf2zhmNYmIjIeMC/Si2LDL7ATGzwFmleWRH/Ap0EUk7WVeoMcmRofb8t+fx2OcXV2oQBeRtJeBgR7voScW6BAdR996oIXOnvBYlSUiMuYyL9DjQy5liQf64qoiwhHHxn1NY1WWiMiYG3Hrf7pZedYUPB6jMNZTT8SSfhOjNTNKxqo0EZExlXGBfsHsUi6YXXpSH1ORn01lUQ4vaxxdRNJYxg25jNaS6iI2KNBFJI0p0GMWVxdSd7SDQ61dqS5FRGRUFOgxS6qLAVi/R710EUlPCvSYsyoL8HqMDXUKdBFJTyMGupndY2YNZrZxiMfNzG43sx1m9oqZnZP8Msde0O9j3qR8bTASkbSVSA/9XmDlMI9fBcyNva0C7jj1slJjSXUR6/c26uRFEUlLIwa6c24NcGSYJtcAP3FRLwBFZjYlWQWOp6XVRbR09rLrcFuqSxEROWnJGEOvBPb2u10Xu+8EZrbKzGrNrDYUCiXhpZNrcXVsg5EmRkUkDY3rpKhzbrVzrsY5V1NeXj6eL52QORV55Gf7eHj9Pg27iEjaSUag7wOq+92uit2Xdrwe4+/eOp9nth/iR8+8nupyREROSjIC/RHgg7HVLsuAJufc/iQ8b0rcuGw6Vy+azLd+u421u4ebOhARmVgSWbZ4H/AnYL6Z1ZnZzWb2MTP7WKzJ48DrwA7gR8AnxqzacWBmfOM9Z1NVnMOn/vtljrZ1p7okEZGEmHOpGSuuqalxtbW1KXntRGzc18S1P3yei+aWcdcHa/B4LNUliYhgZmudczWDPaadokM4q7KQf3r7GfxxawP/U7t35A8QEUkxBfow/nLZdBZOKeDnf96T6lIkTXT2hGlo7kx1GXKaUqAPw8y49pxKNtQ1sTPUmupyJA3c+b87ufr2Z0jVUKac3hToI3jH4ql4DH71clquxJRx9nqojUOt3bR29aa6FDkNKdBHMKkgm+Vzynho/T71umREDS2dsX91rr6MPwV6At69tJK9RzpYt+doqkuRCS4e5CEFuqSAAj0Bbz1zMjlZXh7SsIuMINQcDXL10CUVFOgJyA34uPLMSTz2yn66eyOpLkcmqPbuXlpiY+da6SKpoEBP0LuWVtLY3sPT2xpSXYpMUA3Nx3rlIV2bVlJAgZ6gi+eUUZrr5+H1GnaRwfUfZgk1K9Bl/CnQE+TzenjH4qn8YUsDTR09qS5HJqD4Cpdcv1c9dEkJBfpJuPacSrp7IzzxatoeJiljKD7kcubUwgHDLyLjRYF+EhZVFjKrLFfDLjKogy2d+L0e5k7K6+uti4wnBfpJMDPetbSSF3cdob6xI9XlyAQTau6iPD9ARX42R9t7tCJKxp0C/SS9a0klzsEjG+pTXYpMMA0tsUAvCABwSOPoMs4U6CdpWmmQc6YV8bA2GclxGlo6mVQQoDwvELutQJfxlVCgm9lKM9tmZjvM7PODPD7NzJ4ys5fN7BUzuzr5pU4c715aydYDLWzZ35zqUmQCaWjpoiI/u6+Hru3/Mt58IzUwMy/wA+AKoA54ycwecc5t7tfsC8AvnHN3mNlCopelmzEG9U4Ibzt7Kl95dDMPr9/HGVMK+u7feqCZf3l8K62dPfRGHL1hR362j09dNoeL55ansGIZa509YRrbe6jID1CeH++ha2JUxlciPfQ3ATucc68757qBnwPXHNfGAfFkKwQyeoC5JNfPm+eV88j6eiKR6AmMB5o6+dA9L7FxXxO5AR+luX6mFuWwr7GDG+/+Mx/9SS27D7eluHIZK/HeeEVBgLK8AGbqocv4G7GHDlQC/a/BVgecf1ybLwO/M7NPA7nAWwZ7IjNbBawCmDZt2snWOqG8a2klT25t4MVdR1hUVchf3fsSLZ09/PJjF7Jw6rFee2dPmHue28X3/7iDK76zhvefP40V88s5d3ox+dlZKfwfSDLFx8sr8rPJ8nooCfo1hi7jLpFAT8QNwL3OuW+b2QXAT83sLOfcgHVbzrnVwGqIXiQ6Sa+dEm85YxK5fi8PrKtj9ZqdbDvYwl031QwIc4DsLC+fWDGH95xTxbd+s42fvrCbe59/A4/BGVMKuGbJVD568SzMdBHqdBaKDa/Ex8/L8wPaXCTjLpFA3wdU97tdFbuvv5uBlQDOuT+ZWTZQBmTsSVY5fi8rz5rC/WvrAPjau87i0vkVQ7afVJDNt69fzFevOZOX9zTy5zeO8Oz2EP/y+FZysrzceMGMcapcxkL/HjpEA13b/2W8JTKG/hIw18xmmpkfeB/wyHFt9gCXA5jZGUA2EEpmoRPRdedWAbDqkln85bLpCX1MbsDHRXPL+OwV87j/Yxdy2YIKvvLoZl58/fBYlipj7GBzJ16PUZrrB2KBriN0ZZyNGOjOuV7gU8BvgS1EV7NsMrOvmtk7Y81uBT5qZhuA+4APudPgem0XzC7lmb+/lNuuWjCqj/d4jO++bwnTSoN84mfr2Ndv9+kbh9q47cFX+dGa15NVroyhhuYuyvL8eDzRobOK/GxCrV26bKGMq4TG0J1zjxNditj/vi/2e38zsDy5paWH6pLgKX18QXYWP/pgDe/6/nOs+kkt375+MXc9s4uHXt5HxDmcA5/X+PDymUmqWMZCfA16XEV+gJ6wo7G9h+JYr11krGmn6AQwuzyP792whM37m1n53Wd4dEM9H7pwBn/6/OVcuXASX3l0M7/SgWATWkNLF5NiE6JAv7XoGkeX8ZOsVS5yii5bMIlvXns2Ow+1cvPymVQURHt7t9+wlJvu+TO3/mIDRcHo+neZeEItnSypLuq7XZF/bLfo/Mn5qSpLTjPqoU8g159XzW1XndEX5hBd9vijm2qYOymfj//XWtbuPpLCCmUwPeEIh1q7+0Ic0G5RSQkFehooyM7iP//qPMrzA9yw+kV++sJuTbZNIPFTFSv6DbnEfylrt6iMJwV6mqjIz+bhTyznwjml/NPDG/nsLzbQ3t07ZPu2rl6e3X6IrQea6ewJj2Olp5/4BqJJ/SZFc/1ecrK8GkOXcaUx9DRSnOvnnpvO4wdP7eA7f3iNTfVN3HThDKYUZjOpIJuioJ8Xdh7miY0HWLM91HeBBTOoLg5yVmUBX37nmQNWY8ipa2g5sYduZlQUBBToMq4U6GnG4zE+fflclk4r5pb/Wc8/PrTxhDZTC7P5wPnTWDG/guaOHnY0tLIz1MofthzkQFMn961aRsDnTUH1mSk+Tn78L8qK/EDfkQAi40GBnqYumlvGC7ddRqi1iwNNnRxo6uRQaxeLqopYXFU46Nkwj7+6n0/8bB1feGgj37rubJ0fkyQHm7swg7K8gevNy/MDbD3QkqKq5HSkQE9jPq+HKYU5TCnMSaj91Yum8DeXz+X2J7dzxpQC/uqioTcrPb2tgd2H27lx2fS+3Y8yuFBLJ6W5fnzegVNSFfnZPPPaoRRVJacjBfpp5pbL57LtQDNf+/Vm5lTkcclx69oPtXbxlUc382jsmqnP7jjEv793CXkBfasMpaG5a9B5ifL8AC1dvXR0h8nxa4hLxp5+Sk8zHo/xneuX8J47nuevf7qWZbNKOGdaMedOL6a+qZOv/XozbV29/O1b5pGf7ePrj2/hPT98nrtuqjnlYw4yVUNL14AJ0bjyfpuLppXqcydjT4F+GsoN+LjnQ+fx//64ndo3jvLUtmMHY54zrYhvvuds5k6K7m6cOymPT/5sHdf84Dk+v3IB5QUBcv0+gn4vUwqzKc07McgAwhGH9zQZqmlo6eSMKSfuBu3bLdraqUCXcaFAP01NLcrhX689G4Cm9h5e3nuUzp4wVyycPCCIL55bzsOfXM5H/rOWv3/glROeZ3JBNmdOLeDMqQV0hx07GlrY3tDK3iPtLJ9Txr+8e9Ep9+w7usPsa2xnTsXE20IfjjhCLUMPuQC60IWMGwW6UBjMYsUwF+eYVZ7HE7dczK5DbbR3h2nvCtPa1cveI+1sqm9iU30zT21rwOsxZpXlcdbUQi6dX8Eva/dy5b+v4dYr5/Hh5TNPusfunON3mw/y1Uc3s6+xg4+vmM2tV8w7YfIxlQ63dRFxDDiYKy4e8lqLLuNFgS4JCfi8LJhcMOTjnT1hfB4bELarLpnFFx7eyNd+vYVHNtSzfE4Zfq8Hv8+Dx4z9TR3sPtzOniPtHGrtYkl1EZfMLeeSeeVkZ3n48iObeGpbiAWT83n30krueHonG/Y2cvsNSymLDfV0dId5dschmjt6OGd6MTNKg6Nejrkz1MoPntrBYxv2c+WZk/inty9kUsHwm7Dive/yQXroJbl+PLpYtIwjBbokRXbWias4phblcPdNNTz2yn6+8cRW7n5mF93hY5eZzc/2Mb00yMIpBRTkZPHSG0f4+uNb+PrjW4Do9vkvvO0MbrpwBlleDxfMjh578Pbbn+Wjl8zixdcPs2Z7iM6eY89Zmuvn3OnFLJicT3l+oO+tKOinIDuLghwfAZ8X5xzt3WFaOnvZ39TBPc+9wa9fqcfv83DFwkn8bvNBnt4W4rNXzOODF0wf8q+C0CC7ROO8HqMsLzDsAV3xM3m0J0CSIaFAN7OVwPcAL3CXc+4bg7S5Hvgy4IANzrn3J7FOSVNmxjsWT+Udi6cC0QDrDkcIRxxB/4nffvWNHTyzPUTd0Q4+cP50Jhce6/leX1PNwikFfPxna/nnxzYzpTCb99ZUc8XCyVQUBFi7+yi1bxyldvcRfr/lIEOdXxbweeiNOMKRYw1y/V5WXTKbj1w8k7K8AG8cauOLj2ziq49t5v61dfz1m2ex8qzJA3bYdnSHeW5HdJ15/5MW+6soCJzQQ49EHOv2HOVX6+t5/NX9nF1VyB1/ee6gvxRFToaNdGqfmXmB14ArgDqi1xi9IXaVonibucAvgMucc0fNrMI5N+wFomtqalxtbe2p1i+nofbuXuobO5ldnjtkz7Y3HOFIezehli5CLV00dfTQ3NFDc2cvzR09+LxGfnYW+dk+CnOyWD677IQrCznneGLjAb7xxFb2HGmnJNfPdedWcdGcMn6z6QCPrq+npauXBZPzefTTF5E1SC/+wz/+Mxvqmrh4bhkRFw3z9Xsb2dfYQcDn4fxZpTyzPcRFc8pYfWNNwuvVj7R18x9rdvLo+nrOm1nCe2uqWTarVJvATgNmttY5VzPoYwkE+gXAl51zb43dvg3AOfev/dp8C3jNOXdXokUp0CVdRCKOZ3Yc4r9f3M0ftjQQjjiyszxcfdYUrj+vmjfNKBkySP/rhd38x5qdeMxibzC9NJd3LJ7CFQsnkxfwcf/aOv7u/g1cOLuUuz543rCh3tTRw93PvM7dz+6ivSfMRXPK2LC3kebOXqpLcrh2aRXnzyzhrKpCCrKzkvY56AlH2FzfTHVJkBJdUi+lTjXQrwNWOuc+Ert9I3C+c+5T/do8TLQXv5zosMyXnXO/GeS5VgGrAKZNm3bu7t27R/c/EkmRg82drN/byAWzS5MamA+uq+PWX25g2cxSPnXZHOqORieL6452cLi1m6Pt3Rxt6+ZQazfd4QhvWzSFW94yl7mT8unsCfPbTQf4Re1enttxuO85Z5XnsmByPtk+Lz5vdMI64POQH/CRl+0jPzuL0lw/cyrymFYS7JsncM5xuK2bXYfaWLf7KM/vPMxLbxyhvTtMXsDHx1fM5uaLZg4YIopEHK81tNDZE8EAT+wvp+5whK7eMF29EXAwpyKPquKcE/6y6glHaO8OU5DtS/l8wjPbQ6x5LcSqS2b3LT3tb+O+Jjbvb+YtZ0xKyS+38Qj0x4Ae4HqgClgDLHLONQ71vOqhiwz00Mt13PqLDcSH9r0eY2pRNmV5AUqCfoqCfkrz/FyzZCpnTi0c9Dka27t5pa6JV+oa2VDXxM5QKz3hCL1hR0/Y0dUTprW794T5Bb/Xw6zyXHxeY/ehdlq6jp21P6cijwtnl3Lu9GIe3bCfP2w5yNTCbD575Xy8HvjfbSHWbD/EkbbuhP6f+dk+Fk4poKo4yIHmDvYcaae+sZNwxOH3eSjPCw6yEFgAAAgeSURBVDCpIMCM0lyWzSrlgtmlfXsZmtp7WLvnCGt3H6U8L8B1NdUDjqUIRxwPrKvjzqd3YgYLJhcwf3I+8ybls7i6cNhzjzbVN/GNJ7byzPbovEhZnp9/+4vFXBpb0tvdG+H2J7fzw6d3EHHg8xgr5lfwnnMquWB2dLgr/ldYU0cP9Y0d7GvspL6xg/K8AMtml1JZlNi5S8MZjyGXO4EXnXM/jt1+Evi8c+6loZ5XgS5yos31zTS2d1NdEmRKYfaYrLmPRBztPWFaO3s50NzJjoZWtje0sP1gK70Rx8zSIDPKcplRmsvCqQUnLN38087DfP3xzWzc1wxEVxa9eV45F80tozjox+GIRDvk+H3Rvwr8Pg+RiGPbwRY21zezeX8z9Y0dTC3KYVpJkGklQQpzsgi1dhFq7uJgSyfbDrRwqDX6S6K6JIdglo9tB6OnV3o9RjjiyM/28YHzp/Ph5TPYVN/EN5/YxraDLSyqLGRyYTbbDrSw50h7X+1TCrM5Z1oxi6oKyfJ66AlH6O6NsL2hlcdeqacwJ4tPXTqHZbNK+T+/3MDWAy186MIZvGtpJZ9/4BW2HmjhunOr+MD50/jNxgM89PK+k9pnUF2Sw7KZpbxzyVQunju66wOfaqD7iA6nXA7sIzop+n7n3KZ+bVYSnSi9yczKgJeBJc65w4M9JyjQRdJZfF6hJOjnzKkFYzIZ65xje0Mrz+84xPM7D9PVG6FmejE1M0pYUl3EtoMt/GjN6zyxcX+0JgfTS4P83Vvn87ZFU/qGbtq6etl2sIX1exp5eW8j63YfZV9jx4DXysnyctOFM/j4itkU5kSH0jp7wnzzN1v58XNvAFCWF+Bfr13EFQsn9X1cOOJ4bschtje04pzDOQg7R17AR2VxDpVFOUwuzGbf0Q5eeP0wL7x+mBd3HeHm5TP59OVzR/V5OaVAjz3B1cB3iY6P3+Oc+7qZfRWodc49YtHP3LeBlUAY+Lpz7ufDPacCXUSSYffhNn7+0l4qi3K4vqYav2/kv2qa2ntwRId4/F4PXo8NOXb/9LYGnt1+iE9cOicpY+aRSHTp7miXqZ5yoI8FBbqIyMkbLtAnzqEYIiJyShToIiIZQoEuIpIhFOgiIhlCgS4ikiEU6CIiGUKBLiKSIRToIiIZImUbi8wsBIz2uMUy4FASy0km1TY6E7k2mNj1qbbRSdfapjvnBj0IJmWBfirMrHaonVKpptpGZyLXBhO7PtU2OplYm4ZcREQyhAJdRCRDpGugr051AcNQbaMzkWuDiV2fahudjKstLcfQRUTkROnaQxcRkeMo0EVEMkTaBbqZrTSzbWa2w8w+n+Ja7jGzBjPb2O++EjP7vZltj/1bnKLaqs3sKTPbbGabzOwzE6U+M8s2sz+b2YZYbV+J3T/TzF6MfW3/x8zG/5Lqx2r0mtnLsQugT5jazOwNM3vVzNabWW3svpR/TWN1FJnZ/Wa21cy2mNkFE6E2M5sf+3zF35rN7JaJUFusvr+N/RxsNLP7Yj8fo/p+S6tANzMv8APgKmAhcIOZLUxhSfcSvexef58HnnTOzQWejN1OhV7gVufcQmAZ8MnY52oi1NcFXOacWwwsAVaa2TLgm8C/O+fmAEeBm1NQW9xngC39bk+k2i51zi3pt055InxNAb4H/MY5twBYTPTzl/LanHPbYp+vJcC5QDvw0ESozcwqgb8BapxzZxG9zOf7GO33W/TCpunxBlwA/Lbf7duA21Jc0wxgY7/b24ApsfenANtS/XmL1fIr4IqJVh8QBNYB5xPdGecb7Gs9zjVVEf0Bvwx4DLAJVNsbQNlx96X8awoUAruILbSYSLUdV8+VwHMTpTagEtgLlAC+2PfbW0f7/ZZWPXSO/efj6mL3TSSTnHP7Y+8fACYN13g8mNkMYCnwIhOkvtiQxnqgAfg9sBNodM71xpqk8mv7XeDvgUjsdikTpzYH/M7M1prZqth9E+FrOhMIAT+ODVXdZWa5E6S2/t4H3Bd7P+W1Oef2Af8X2APsB5qAtYzy+y3dAj2tuOiv15SuCzWzPOAB4BbnXHP/x1JZn3Mu7KJ/AlcBbwIWpKKO45nZ24EG59zaVNcyhIucc+cQHXb8pJld0v/BFH5NfcA5wB3OuaVAG8cNYaT65yE2Dv1O4JfHP5aq2mLj9tcQ/YU4FcjlxGHchKVboO8DqvvdrordN5EcNLMpALF/G1JViJllEQ3znznnHpxo9QE45xqBp4j+WVlkZr7YQ6n62i4H3mlmbwA/Jzrs8r0JUlu8R4dzroHoOPCbmBhf0zqgzjn3Yuz2/UQDfiLUFncVsM45dzB2eyLU9hZgl3Mu5JzrAR4k+j04qu+3dAv0l4C5sRlgP9E/nx5JcU3HewS4Kfb+TUTHrsedmRlwN7DFOfedfg+lvD4zKzezotj7OUTH9rcQDfbrUlmbc+4251yVc24G0e+vPzrnPjARajOzXDPLj79PdDx4IxPga+qcOwDsNbP5sbsuBzZPhNr6uYFjwy0wMWrbAywzs2DsZzb+eRvd91sqJyhGOYlwNfAa0THXf0xxLfcRHffqIdpDuZnoeOuTwHbgD0BJimq7iOifkK8A62NvV0+E+oCzgZdjtW0Evhi7fxbwZ2AH0T+LAyn++q4AHpsotcVq2BB72xT//p8IX9NYHUuA2tjX9WGgeALVlgscBgr73TdRavsKsDX2s/BTIDDa7zdt/RcRyRDpNuQiIiJDUKCLiGQIBbqISIZQoIuIZAgFuohIhlCgi4hkCAW6iEiG+P8nf3PBV7ZnbwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zvF64MJDIYmW",
        "outputId": "9e27c0cc-6268-4bbf-8f29-b377af11bb90"
      },
      "source": [
        "net.load_state_dict(current_best_params)"
      ],
      "execution_count": 129,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {},
          "execution_count": 129
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bFpZlJOeOKlS",
        "outputId": "4ac1c440-c9a1-46cc-8bd8-8b0e194a5565"
      },
      "source": [
        "np.unique(np.argmax(logits[mask_train].detach().numpy(),1))"
      ],
      "execution_count": 130,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 1])"
            ]
          },
          "metadata": {},
          "execution_count": 130
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NqUOV82Cb3dq"
      },
      "source": [
        ""
      ],
      "execution_count": 130,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jpuKzkwMQL0_",
        "outputId": "0eff80b5-92a4-4a03-8203-319f47d4c16a"
      },
      "source": [
        "# Visualise predictions\n",
        "net.eval() # Set net to evaluation mode (deactivates dropout)\n",
        "final_prediction = net(features).detach()\n",
        "a = np.transpose(np.vstack([final_prediction[comb_mask].numpy().argmax(axis=1),labels[comb_mask].numpy()]))\n",
        "a[a[:,0].argsort()][np.random.choice(range(a.shape[0]),size=10)]\n",
        "\n",
        "# As can be seen, the graph net predicts correct labels"
      ],
      "execution_count": 131,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0, 0],\n",
              "       [0, 0],\n",
              "       [0, 1],\n",
              "       [0, 1],\n",
              "       [0, 0],\n",
              "       [1, 0],\n",
              "       [0, 0],\n",
              "       [0, 0],\n",
              "       [0, 0],\n",
              "       [1, 1]])"
            ]
          },
          "metadata": {},
          "execution_count": 131
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a9c8JUmkeATZ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "ccff1cb9-e36b-456e-bcc3-e19334f72fd7"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "\n",
        "dataset = dgl.data.CoraGraphDataset()\n",
        "g = dataset[0]\n",
        "options = {\n",
        "    'node_color': 'black',\n",
        "    'node_size': 20,\n",
        "    'width': 1,\n",
        "}\n",
        "G = dgl.to_networkx(g)\n",
        "plt.figure(figsize=[15,7])\n",
        "nx.draw(G, **options)\n"
      ],
      "execution_count": 132,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading /root/.dgl/cora_v2.zip from https://data.dgl.ai/dataset/cora_v2.zip...\n",
            "Extracting file to /root/.dgl/cora_v2\n",
            "Finished data loading and preprocessing.\n",
            "  NumNodes: 2708\n",
            "  NumEdges: 10556\n",
            "  NumFeats: 1433\n",
            "  NumClasses: 7\n",
            "  NumTrainingSamples: 140\n",
            "  NumValidationSamples: 500\n",
            "  NumTestSamples: 1000\n",
            "Done saving data into cached files.\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-132-4be5fa17607c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0mG\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdgl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_networkx\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfigsize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m15\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m7\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0mnx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mG\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'nx' is not defined"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1080x504 with 0 Axes>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JM_hDALScdxW"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2wmHeyTrSf1H"
      },
      "source": [
        "## Evaluation\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C46ZjsYgZ2tu"
      },
      "source": [
        "# Performnace evaluation functions\n",
        "\n",
        "def variation_of_information_score(labels, preds):\n",
        "    def mi(x, y):\n",
        "        contingency = skmetrics.cluster.contingency_matrix(x, y, sparse=True)\n",
        "        # print(contingency.todense())\n",
        "        nzx, nzy, nz_val = sp.find(contingency)\n",
        "        contingency_sum = contingency.sum()\n",
        "\n",
        "        pi = np.ravel(contingency.sum(axis=1))\n",
        "        pj = np.ravel(contingency.sum(axis=0))\n",
        "        # print(nz_val)\n",
        "        log_contingency_nm = np.log(nz_val)\n",
        "        # print(log_contingency_nm)\n",
        "        contingency_nm = nz_val / contingency_sum\n",
        "        # print(contingency_nm)\n",
        "\n",
        "        # Don't need to calculate the full outer product, just for non-zeroes\n",
        "        outer = pi.take(nzx).astype(np.int64, copy=False) * pj.take(nzy).astype(\n",
        "            np.int64, copy=False\n",
        "        )\n",
        "        # print(outer)\n",
        "        log_outer = -np.log(outer) + log(pi.sum()) + log(pj.sum())\n",
        "        # print(log_outer)\n",
        "        mi = (\n",
        "            contingency_nm * (log_contingency_nm - log(contingency_sum))\n",
        "            + contingency_nm * log_outer\n",
        "        )\n",
        "        # print(mi)\n",
        "        return mi.sum()\n",
        "\n",
        "    return mi(labels, labels) + mi(preds, preds) - 2 * mi(labels, preds)\n",
        "\n",
        "\n",
        "def mutual_info_score(labels, preds):\n",
        "    return skmetrics.adjusted_mutual_info_score(labels, preds, average_method=\"arithmetic\")\n",
        "\n",
        "def compute_performance(labels, logits, splits):\n",
        "    logits = logits.detach().numpy()\n",
        "    preds = np.argmax(logits, axis=1)\n",
        "    labels = labels.numpy()\n",
        "    pred_sets = {\n",
        "        \"All \": preds,\n",
        "        \"Train\": preds[mask_train],\n",
        "        \"Val\": preds[mask_val],\n",
        "        \"Test\": preds[test_mask],\n",
        "    }\n",
        "    label_sets = {\n",
        "        \"All \": labels,\n",
        "        \"Train\": labels[mask_train],\n",
        "        \"Val\": labels[mask_val],\n",
        "        \"Test\": labels[test_mask],\n",
        "    }\n",
        "    eval_functions = {\n",
        "        \"Rand-Index\": rand_score,\n",
        "        \"Mutual Information\": mutual_info_score,\n",
        "        \"Variation of Information\": variation_of_information_score,\n",
        "    }\n",
        "    scores = {\n",
        "        subset: {\n",
        "            name: func(label_sets[subset], pred_sets[subset])\n",
        "            for name, func in eval_functions.items()\n",
        "        }\n",
        "        for subset in pred_sets.keys()\n",
        "    }\n",
        "    return scores\n",
        "\n",
        "def performance_as_df(labels, logits, splits):\n",
        "    scores = compute_performance(labels, logits, splits)\n",
        "    return pd.DataFrame(scores)\n",
        "\n",
        "def rand_score(labels, preds):\n",
        "    return skmetrics.adjusted_rand_score(labels, preds)"
      ],
      "execution_count": 133,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        },
        "id": "nUltH0-xRaNk",
        "outputId": "60a84198-eb88-46ea-e09e-9c90ade83aaf"
      },
      "source": [
        "net.eval() # Set net to evaluation mode (deactivates dropout)\n",
        "final_prediction = net(features).detach()\n",
        "performance_as_df(labels,final_prediction,comb_mask)"
      ],
      "execution_count": 134,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>All</th>\n",
              "      <th>Train</th>\n",
              "      <th>Val</th>\n",
              "      <th>Test</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Rand-Index</th>\n",
              "      <td>0.081777</td>\n",
              "      <td>0.083345</td>\n",
              "      <td>0.076261</td>\n",
              "      <td>0.073702</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Mutual Information</th>\n",
              "      <td>0.022555</td>\n",
              "      <td>0.022489</td>\n",
              "      <td>0.021113</td>\n",
              "      <td>0.018171</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Variation of Information</th>\n",
              "      <td>0.908057</td>\n",
              "      <td>0.905504</td>\n",
              "      <td>0.917646</td>\n",
              "      <td>1.075580</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                              All      Train       Val      Test\n",
              "Rand-Index                0.081777  0.083345  0.076261  0.073702\n",
              "Mutual Information        0.022555  0.022489  0.021113  0.018171\n",
              "Variation of Information  0.908057  0.905504  0.917646  1.075580"
            ]
          },
          "metadata": {},
          "execution_count": 134
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XVNusA8avfGU"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}