GNN/05_Aggregation_PNA+LAF.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Ramsi-K/3D-Vision-Playground/blob/main/GNN/05_Aggregation_PNA%2BLAF.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WCth0QB57I3b"
      },
      "source": [
        "# Tutorial5: Aggregation\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9lsYUPgM7I3d"
      },
      "source": [
        "In this tutorial we will override the aggregation method of the GIN convolution module of Pytorch Geometric implementing the following methods:\n",
        "\n",
        "- Principal Neighborhood Aggregation (PNA)\n",
        "- Learning Aggregation Functions (LAF)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "a6RTq8KF7I3e",
        "outputId": "0cceb43f-348b-4ecc-a4c3-7bb4c306a350",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.2.1+cu121\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.9/10.9 MB\u001b[0m \u001b[31m34.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.0/5.0 MB\u001b[0m \u001b[31m33.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for torch_geometric (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "import torch\n",
        "os.environ['TORCH'] = torch.__version__\n",
        "print(torch.__version__)\n",
        "\n",
        "!pip install -q torch-scatter -f https://data.pyg.org/whl/torch-${TORCH}.html\n",
        "!pip install -q torch-sparse -f https://data.pyg.org/whl/torch-${TORCH}.html\n",
        "!pip install -q git+https://github.com/pyg-team/pytorch_geometric.git"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "ujQV96dI7I3f",
        "outputId": "893e7999-ff22-4955-ec5b-046761084eef",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch._C.Generator at 0x7cd59017d710>"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "import torch\n",
        "torch.manual_seed(42)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZgdBek8D7I3f"
      },
      "source": [
        "### Message Passing Class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "BnpidYuc7I3f"
      },
      "outputs": [],
      "source": [
        "from torch_geometric.nn import MessagePassing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V4Tx2P9B7I3g"
      },
      "source": [
        "We are interested in the <span style='color:Blue'>aggregate</span> method, or, if you are using a sparse adjacency matrix, in the <span style='color:Blue'>message_and_aggregate</span> method. Convolutional classes in PyG extend MessagePassing, we construct our custom convoutional class extending GINConv."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rShgdUKY7I3g"
      },
      "source": [
        "Scatter operation in <span style='color:Blue'>aggregate</span>:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e4_lCh_y7I3g"
      },
      "source": [
        "<img src=\"https://raw.githubusercontent.com/rusty1s/pytorch_scatter/master/docs/source/_figures/add.svg?sanitize=true\" width=\"500\">"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "vFIOjG_H7I3g"
      },
      "outputs": [],
      "source": [
        "from torch.nn import Parameter, Module, Sigmoid\n",
        "import torch\n",
        "import torch_scatter\n",
        "import torch.nn.functional as F\n",
        "\n",
        "class AbstractLAFLayer(Module):\n",
        "    def __init__(self, **kwargs):\n",
        "        super(AbstractLAFLayer, self).__init__()\n",
        "        assert 'units' in kwargs or 'weights' in kwargs\n",
        "        if 'device' in kwargs.keys():\n",
        "            self.device = kwargs['device']\n",
        "        else:\n",
        "            self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "        self.ngpus = torch.cuda.device_count()\n",
        "\n",
        "        if 'kernel_initializer' in kwargs.keys():\n",
        "            assert kwargs['kernel_initializer'] in [\n",
        "                'random_normal',\n",
        "                'glorot_normal',\n",
        "                'he_normal',\n",
        "                'random_uniform',\n",
        "                'glorot_uniform',\n",
        "                'he_uniform']\n",
        "            self.kernel_initializer = kwargs['kernel_initializer']\n",
        "        else:\n",
        "            self.kernel_initializer = 'random_normal'\n",
        "\n",
        "        if 'weights' in kwargs.keys():\n",
        "            self.weights = Parameter(kwargs['weights'].to(self.device), \\\n",
        "                                     requires_grad=True)\n",
        "            self.units = self.weights.shape[1]\n",
        "        else:\n",
        "            self.units = kwargs['units']\n",
        "            params = torch.empty(12, self.units, device=self.device)\n",
        "            if self.kernel_initializer == 'random_normal':\n",
        "                torch.nn.init.normal_(params)\n",
        "            elif self.kernel_initializer == 'glorot_normal':\n",
        "                torch.nn.init.xavier_normal_(params)\n",
        "            elif self.kernel_initializer == 'he_normal':\n",
        "                torch.nn.init.kaiming_normal_(params)\n",
        "            elif self.kernel_initializer == 'random_uniform':\n",
        "                torch.nn.init.uniform_(params)\n",
        "            elif self.kernel_initializer == 'glorot_uniform':\n",
        "                torch.nn.init.xavier_uniform_(params)\n",
        "            elif self.kernel_initializer == 'he_uniform':\n",
        "                torch.nn.init.kaiming_uniform_(params)\n",
        "            self.weights = Parameter(params, \\\n",
        "                                     requires_grad=True)\n",
        "        e = torch.tensor([1,-1,1,-1], dtype=torch.float32, device=self.device)\n",
        "        self.e = Parameter(e, requires_grad=False)\n",
        "        num_idx = torch.tensor([1,1,0,0], dtype=torch.float32, device=self.device).\\\n",
        "                                view(1,1,-1,1)\n",
        "        self.num_idx = Parameter(num_idx, requires_grad=False)\n",
        "        den_idx = torch.tensor([0,0,1,1], dtype=torch.float32, device=self.device).\\\n",
        "                                view(1,1,-1,1)\n",
        "        self.den_idx = Parameter(den_idx, requires_grad=False)\n",
        "\n",
        "\n",
        "class LAFLayer(AbstractLAFLayer):\n",
        "    def __init__(self, eps=1e-7, **kwargs):\n",
        "        super(LAFLayer, self).__init__(**kwargs)\n",
        "        self.eps = eps\n",
        "\n",
        "    def forward(self, data, index, dim=0, **kwargs):\n",
        "        eps = self.eps\n",
        "        sup = 1.0 - eps\n",
        "        e = self.e\n",
        "\n",
        "        x = torch.clamp(data, eps, sup)\n",
        "        x = torch.unsqueeze(x, -1)\n",
        "        e = e.view(1,1,-1)\n",
        "\n",
        "        exps = (1. - e)/2. + x*e\n",
        "        exps = torch.unsqueeze(exps, -1)\n",
        "        exps = torch.pow(exps, torch.relu(self.weights[0:4]))\n",
        "\n",
        "        scatter = torch_scatter.scatter_add(exps, index.view(-1), dim=dim)\n",
        "        scatter = torch.clamp(scatter, eps)\n",
        "\n",
        "        sqrt = torch.pow(scatter, torch.relu(self.weights[4:8]))\n",
        "        alpha_beta = self.weights[8:12].view(1,1,4,-1)\n",
        "        terms = sqrt * alpha_beta\n",
        "\n",
        "        num = torch.sum(terms * self.num_idx, dim=2)\n",
        "        den = torch.sum(terms * self.den_idx, dim=2)\n",
        "\n",
        "        multiplier = 2.0*torch.clamp(torch.sign(den), min=0.0) - 1.0\n",
        "\n",
        "        den = torch.where((den < eps) & (den > -eps), multiplier*eps, den)\n",
        "\n",
        "        res = num / den\n",
        "        return res\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "OO-tlFFB7I3h"
      },
      "outputs": [],
      "source": [
        "from torch_geometric.nn import GINConv\n",
        "from torch.nn import Linear"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1Bop6OWd7I3h"
      },
      "source": [
        "### LAF Aggregation Module"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MdoCNi4g7I3h"
      },
      "source": [
        "<img src=\"https://github.com/AntonioLonga/PytorchGeometricTutorial/blob/main/Tutorial5/laf.png?raw=1\" width=\"500\">"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "jMMHzIIY7I3h"
      },
      "outputs": [],
      "source": [
        "class GINLAFConv(GINConv):\n",
        "    def __init__(self, nn, units=1, node_dim=32, **kwargs):\n",
        "        super(GINLAFConv, self).__init__(nn, **kwargs)\n",
        "        self.laf = LAFLayer(units=units, kernel_initializer='random_uniform')\n",
        "        self.mlp = torch.nn.Linear(node_dim*units, node_dim)\n",
        "        self.dim = node_dim\n",
        "        self.units = units\n",
        "\n",
        "    def aggregate(self, inputs, index):\n",
        "        x = torch.sigmoid(inputs)\n",
        "        x = self.laf(x, index)\n",
        "        x = x.view((-1, self.dim * self.units))\n",
        "        x = self.mlp(x)\n",
        "        return x\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WIyeTk-d7I3h"
      },
      "source": [
        "### PNA Aggregation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "acsWHl2w7I3h"
      },
      "source": [
        "<img src=\"https://github.com/AntonioLonga/PytorchGeometricTutorial/blob/main/Tutorial5/pna.png?raw=1\" width=\"800\">"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "e9Y6mnKv7I3h"
      },
      "outputs": [],
      "source": [
        "class GINPNAConv(GINConv):\n",
        "    def __init__(self, nn, node_dim=32, **kwargs):\n",
        "        super(GINPNAConv, self).__init__(nn, **kwargs)\n",
        "        self.mlp = torch.nn.Linear(node_dim*12, node_dim)\n",
        "        self.delta = 2.5749\n",
        "\n",
        "    def aggregate(self, inputs, index):\n",
        "        sums = torch_scatter.scatter_add(inputs, index, dim=0)\n",
        "        maxs = torch_scatter.scatter_max(inputs, index, dim=0)[0]\n",
        "        means = torch_scatter.scatter_mean(inputs, index, dim=0)\n",
        "        var = torch.relu(torch_scatter.scatter_mean(inputs ** 2, index, dim=0) - means ** 2)\n",
        "\n",
        "        aggrs = [sums, maxs, means, var]\n",
        "        c_idx = index.bincount().float().view(-1, 1)\n",
        "        l_idx = torch.log(c_idx + 1.)\n",
        "\n",
        "        amplification_scaler = [c_idx / self.delta * a for a in aggrs]\n",
        "        attenuation_scaler = [self.delta / c_idx * a for a in aggrs]\n",
        "        combinations = torch.cat(aggrs+ amplification_scaler+ attenuation_scaler, dim=1)\n",
        "        x = self.mlp(combinations)\n",
        "\n",
        "        return x\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aZH5QSXN7I3h"
      },
      "source": [
        "### Test the new classes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "mkjNe9lg7I3h"
      },
      "outputs": [],
      "source": [
        "from torch_geometric.nn import MessagePassing, SAGEConv, GINConv, global_add_pool\n",
        "import torch_scatter\n",
        "import torch.nn.functional as F\n",
        "from torch.nn import Sequential, Linear, ReLU\n",
        "from torch_geometric.datasets import TUDataset\n",
        "from torch_geometric.data import DataLoader\n",
        "import os.path as osp\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "GJ3p6-ML7I3h",
        "outputId": "670f079f-2afd-40a9-9734-d03528d84ce0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading https://www.chrsmrrs.com/graphkerneldatasets/MUTAG.zip\n",
            "Processing...\n",
            "Done!\n",
            "/usr/local/lib/python3.10/dist-packages/torch_geometric/deprecation.py:26: UserWarning: 'data.DataLoader' is deprecated, use 'loader.DataLoader' instead\n",
            "  warnings.warn(out)\n"
          ]
        }
      ],
      "source": [
        "path = osp.join('./', 'data', 'TU')\n",
        "dataset = TUDataset(path, name='MUTAG').shuffle()\n",
        "test_dataset = dataset[:len(dataset) // 10]\n",
        "train_dataset = dataset[len(dataset) // 10:]\n",
        "test_loader = DataLoader(test_dataset, batch_size=128)\n",
        "train_loader = DataLoader(train_dataset, batch_size=128)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "BSlIUB7u7I3i"
      },
      "outputs": [],
      "source": [
        "class LAFNet(torch.nn.Module):\n",
        "    def __init__(self):\n",
        "        super(LAFNet, self).__init__()\n",
        "\n",
        "        num_features = dataset.num_features\n",
        "        dim = 32\n",
        "        units = 3\n",
        "\n",
        "        nn1 = Sequential(Linear(num_features, dim), ReLU(), Linear(dim, dim))\n",
        "        self.conv1 = GINLAFConv(nn1, units=units, node_dim=num_features)\n",
        "        self.bn1 = torch.nn.BatchNorm1d(dim)\n",
        "\n",
        "        nn2 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim))\n",
        "        self.conv2 = GINLAFConv(nn2, units=units, node_dim=dim)\n",
        "        self.bn2 = torch.nn.BatchNorm1d(dim)\n",
        "\n",
        "        nn3 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim))\n",
        "        self.conv3 = GINLAFConv(nn3, units=units, node_dim=dim)\n",
        "        self.bn3 = torch.nn.BatchNorm1d(dim)\n",
        "\n",
        "        nn4 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim))\n",
        "        self.conv4 = GINLAFConv(nn4, units=units, node_dim=dim)\n",
        "        self.bn4 = torch.nn.BatchNorm1d(dim)\n",
        "\n",
        "        nn5 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim))\n",
        "        self.conv5 = GINLAFConv(nn5, units=units, node_dim=dim)\n",
        "        self.bn5 = torch.nn.BatchNorm1d(dim)\n",
        "\n",
        "        self.fc1 = Linear(dim, dim)\n",
        "        self.fc2 = Linear(dim, dataset.num_classes)\n",
        "\n",
        "    def forward(self, x, edge_index, batch):\n",
        "        x = F.relu(self.conv1(x, edge_index))\n",
        "        x = self.bn1(x)\n",
        "        x = F.relu(self.conv2(x, edge_index))\n",
        "        x = self.bn2(x)\n",
        "        x = F.relu(self.conv3(x, edge_index))\n",
        "        x = self.bn3(x)\n",
        "        x = F.relu(self.conv4(x, edge_index))\n",
        "        x = self.bn4(x)\n",
        "        x = F.relu(self.conv5(x, edge_index))\n",
        "        x = self.bn5(x)\n",
        "        x = global_add_pool(x, batch)\n",
        "        x = F.relu(self.fc1(x))\n",
        "        x = F.dropout(x, p=0.5, training=self.training)\n",
        "        x = self.fc2(x)\n",
        "        return F.log_softmax(x, dim=-1)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "rt4872m87I3i"
      },
      "outputs": [],
      "source": [
        "class PNANet(torch.nn.Module):\n",
        "    def __init__(self):\n",
        "        super(PNANet, self).__init__()\n",
        "\n",
        "        num_features = dataset.num_features\n",
        "        dim = 32\n",
        "\n",
        "        nn1 = Sequential(Linear(num_features, dim), ReLU(), Linear(dim, dim))\n",
        "        self.conv1 = GINPNAConv(nn1, node_dim=num_features)\n",
        "        self.bn1 = torch.nn.BatchNorm1d(dim)\n",
        "\n",
        "        nn2 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim))\n",
        "        self.conv2 = GINPNAConv(nn2, node_dim=dim)\n",
        "        self.bn2 = torch.nn.BatchNorm1d(dim)\n",
        "\n",
        "        nn3 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim))\n",
        "        self.conv3 = GINPNAConv(nn3, node_dim=dim)\n",
        "        self.bn3 = torch.nn.BatchNorm1d(dim)\n",
        "\n",
        "        nn4 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim))\n",
        "        self.conv4 = GINPNAConv(nn4, node_dim=dim)\n",
        "        self.bn4 = torch.nn.BatchNorm1d(dim)\n",
        "\n",
        "        nn5 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim))\n",
        "        self.conv5 = GINPNAConv(nn5, node_dim=dim)\n",
        "        self.bn5 = torch.nn.BatchNorm1d(dim)\n",
        "\n",
        "        self.fc1 = Linear(dim, dim)\n",
        "        self.fc2 = Linear(dim, dataset.num_classes)\n",
        "\n",
        "    def forward(self, x, edge_index, batch):\n",
        "        x = F.relu(self.conv1(x, edge_index))\n",
        "        x = self.bn1(x)\n",
        "        x = F.relu(self.conv2(x, edge_index))\n",
        "        x = self.bn2(x)\n",
        "        x = F.relu(self.conv3(x, edge_index))\n",
        "        x = self.bn3(x)\n",
        "        x = F.relu(self.conv4(x, edge_index))\n",
        "        x = self.bn4(x)\n",
        "        x = F.relu(self.conv5(x, edge_index))\n",
        "        x = self.bn5(x)\n",
        "        x = global_add_pool(x, batch)\n",
        "        x = F.relu(self.fc1(x))\n",
        "        x = F.dropout(x, p=0.5, training=self.training)\n",
        "        x = self.fc2(x)\n",
        "        return F.log_softmax(x, dim=-1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "IsLsK_Nt7I3i"
      },
      "outputs": [],
      "source": [
        "class GINNet(torch.nn.Module):\n",
        "    def __init__(self):\n",
        "        super(GINNet, self).__init__()\n",
        "\n",
        "        num_features = dataset.num_features\n",
        "        dim = 32\n",
        "\n",
        "        nn1 = Sequential(Linear(num_features, dim), ReLU(), Linear(dim, dim))\n",
        "        self.conv1 = GINConv(nn1)\n",
        "        self.bn1 = torch.nn.BatchNorm1d(dim)\n",
        "\n",
        "        nn2 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim))\n",
        "        self.conv2 = GINConv(nn2)\n",
        "        self.bn2 = torch.nn.BatchNorm1d(dim)\n",
        "\n",
        "        nn3 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim))\n",
        "        self.conv3 = GINConv(nn3)\n",
        "        self.bn3 = torch.nn.BatchNorm1d(dim)\n",
        "\n",
        "        nn4 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim))\n",
        "        self.conv4 = GINConv(nn4)\n",
        "        self.bn4 = torch.nn.BatchNorm1d(dim)\n",
        "\n",
        "        nn5 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim))\n",
        "        self.conv5 = GINConv(nn5)\n",
        "        self.bn5 = torch.nn.BatchNorm1d(dim)\n",
        "\n",
        "        self.fc1 = Linear(dim, dim)\n",
        "        self.fc2 = Linear(dim, dataset.num_classes)\n",
        "\n",
        "    def forward(self, x, edge_index, batch):\n",
        "        x = F.relu(self.conv1(x, edge_index))\n",
        "        x = self.bn1(x)\n",
        "        x = F.relu(self.conv2(x, edge_index))\n",
        "        x = self.bn2(x)\n",
        "        x = F.relu(self.conv3(x, edge_index))\n",
        "        x = self.bn3(x)\n",
        "        x = F.relu(self.conv4(x, edge_index))\n",
        "        x = self.bn4(x)\n",
        "        x = F.relu(self.conv5(x, edge_index))\n",
        "        x = self.bn5(x)\n",
        "        x = global_add_pool(x, batch)\n",
        "        x = F.relu(self.fc1(x))\n",
        "        x = F.dropout(x, p=0.5, training=self.training)\n",
        "        x = self.fc2(x)\n",
        "        return F.log_softmax(x, dim=-1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "scrolled": false,
        "id": "qigvGSNX7I3i",
        "outputId": "2b4ab359-206b-4e56-8fe0-c57882d9bc77",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 001, Train Loss: 0.8650472, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 002, Train Loss: 0.7599045, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 003, Train Loss: 0.8972136, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 004, Train Loss: 0.6134784, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 005, Train Loss: 0.5924671, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 006, Train Loss: 0.5319859, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 007, Train Loss: 0.5401638, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 008, Train Loss: 0.5089008, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 009, Train Loss: 0.5440796, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 010, Train Loss: 0.5598108, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 011, Train Loss: 0.5183228, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 012, Train Loss: 0.4977737, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 013, Train Loss: 0.4780328, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 014, Train Loss: 0.5809572, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 015, Train Loss: 0.4476953, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 016, Train Loss: 0.4370331, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 017, Train Loss: 0.4296935, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 018, Train Loss: 0.4225018, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 019, Train Loss: 0.3562843, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 020, Train Loss: 0.3778081, Train Acc: 0.6764706, Test Acc: 0.6111111\n",
            "Epoch: 021, Train Loss: 0.3774645, Train Acc: 0.7235294, Test Acc: 0.8333333\n",
            "Epoch: 022, Train Loss: 0.3672958, Train Acc: 0.7588235, Test Acc: 0.6111111\n",
            "Epoch: 023, Train Loss: 0.3983777, Train Acc: 0.4823529, Test Acc: 0.4444444\n",
            "Epoch: 024, Train Loss: 0.3779517, Train Acc: 0.3352941, Test Acc: 0.3888889\n",
            "Epoch: 025, Train Loss: 0.3336472, Train Acc: 0.3294118, Test Acc: 0.3888889\n",
            "Epoch: 026, Train Loss: 0.3448226, Train Acc: 0.3294118, Test Acc: 0.3888889\n",
            "Epoch: 027, Train Loss: 0.3556136, Train Acc: 0.3294118, Test Acc: 0.3888889\n",
            "Epoch: 028, Train Loss: 0.3377177, Train Acc: 0.3294118, Test Acc: 0.3888889\n",
            "Epoch: 029, Train Loss: 0.3382654, Train Acc: 0.3294118, Test Acc: 0.3888889\n",
            "Epoch: 030, Train Loss: 0.3644957, Train Acc: 0.3352941, Test Acc: 0.3888889\n",
            "Epoch: 031, Train Loss: 0.3476817, Train Acc: 0.4117647, Test Acc: 0.4444444\n",
            "Epoch: 032, Train Loss: 0.3127073, Train Acc: 0.5470588, Test Acc: 0.5000000\n",
            "Epoch: 033, Train Loss: 0.3574253, Train Acc: 0.5352941, Test Acc: 0.5000000\n",
            "Epoch: 034, Train Loss: 0.3283083, Train Acc: 0.4117647, Test Acc: 0.4444444\n",
            "Epoch: 035, Train Loss: 0.3374432, Train Acc: 0.3352941, Test Acc: 0.3888889\n",
            "Epoch: 036, Train Loss: 0.3255821, Train Acc: 0.4117647, Test Acc: 0.4444444\n",
            "Epoch: 037, Train Loss: 0.2874793, Train Acc: 0.6352941, Test Acc: 0.5555556\n",
            "Epoch: 038, Train Loss: 0.3000287, Train Acc: 0.7117647, Test Acc: 0.6666667\n",
            "Epoch: 039, Train Loss: 0.2963587, Train Acc: 0.7117647, Test Acc: 0.5555556\n",
            "Epoch: 040, Train Loss: 0.2933287, Train Acc: 0.7764706, Test Acc: 0.6666667\n",
            "Epoch: 041, Train Loss: 0.2949790, Train Acc: 0.8058824, Test Acc: 0.7222222\n",
            "Epoch: 042, Train Loss: 0.3127043, Train Acc: 0.7470588, Test Acc: 0.6111111\n",
            "Epoch: 043, Train Loss: 0.3181303, Train Acc: 0.4647059, Test Acc: 0.4444444\n",
            "Epoch: 044, Train Loss: 0.2890976, Train Acc: 0.7470588, Test Acc: 0.6666667\n",
            "Epoch: 045, Train Loss: 0.3041718, Train Acc: 0.5647059, Test Acc: 0.5555556\n",
            "Epoch: 046, Train Loss: 0.2898609, Train Acc: 0.3294118, Test Acc: 0.3888889\n",
            "Epoch: 047, Train Loss: 0.2808521, Train Acc: 0.3294118, Test Acc: 0.3888889\n",
            "Epoch: 048, Train Loss: 0.3032045, Train Acc: 0.3352941, Test Acc: 0.3888889\n",
            "Epoch: 049, Train Loss: 0.3091900, Train Acc: 0.5705882, Test Acc: 0.5555556\n",
            "Epoch: 050, Train Loss: 0.2742498, Train Acc: 0.8470588, Test Acc: 0.7222222\n",
            "Epoch: 051, Train Loss: 0.2873945, Train Acc: 0.8352941, Test Acc: 0.7222222\n",
            "Epoch: 052, Train Loss: 0.2941733, Train Acc: 0.8764706, Test Acc: 0.8333333\n",
            "Epoch: 053, Train Loss: 0.2895809, Train Acc: 0.7411765, Test Acc: 0.7777778\n",
            "Epoch: 054, Train Loss: 0.2811435, Train Acc: 0.7294118, Test Acc: 0.7777778\n",
            "Epoch: 055, Train Loss: 0.2994587, Train Acc: 0.7411765, Test Acc: 0.7777778\n",
            "Epoch: 056, Train Loss: 0.2950836, Train Acc: 0.7823529, Test Acc: 0.8333333\n",
            "Epoch: 057, Train Loss: 0.2753096, Train Acc: 0.8882353, Test Acc: 0.8333333\n",
            "Epoch: 058, Train Loss: 0.2822786, Train Acc: 0.8823529, Test Acc: 0.8888889\n",
            "Epoch: 059, Train Loss: 0.2544703, Train Acc: 0.7823529, Test Acc: 0.8888889\n",
            "Epoch: 060, Train Loss: 0.2656169, Train Acc: 0.7823529, Test Acc: 0.8888889\n",
            "Epoch: 061, Train Loss: 0.2973233, Train Acc: 0.8882353, Test Acc: 0.8888889\n",
            "Epoch: 062, Train Loss: 0.2539657, Train Acc: 0.5117647, Test Acc: 0.5000000\n",
            "Epoch: 063, Train Loss: 0.2550300, Train Acc: 0.3529412, Test Acc: 0.3888889\n",
            "Epoch: 064, Train Loss: 0.3024226, Train Acc: 0.6117647, Test Acc: 0.6111111\n",
            "Epoch: 065, Train Loss: 0.2856011, Train Acc: 0.7941176, Test Acc: 0.7222222\n",
            "Epoch: 066, Train Loss: 0.2656752, Train Acc: 0.7411765, Test Acc: 0.7222222\n",
            "Epoch: 067, Train Loss: 0.2777416, Train Acc: 0.7470588, Test Acc: 0.7777778\n",
            "Epoch: 068, Train Loss: 0.2712155, Train Acc: 0.7705882, Test Acc: 0.7777778\n",
            "Epoch: 069, Train Loss: 0.2870202, Train Acc: 0.8588235, Test Acc: 0.8888889\n",
            "Epoch: 070, Train Loss: 0.2590990, Train Acc: 0.8294118, Test Acc: 0.8333333\n",
            "Epoch: 071, Train Loss: 0.2660991, Train Acc: 0.4352941, Test Acc: 0.3888889\n",
            "Epoch: 072, Train Loss: 0.2775441, Train Acc: 0.3411765, Test Acc: 0.3888889\n",
            "Epoch: 073, Train Loss: 0.2535625, Train Acc: 0.3352941, Test Acc: 0.3888889\n",
            "Epoch: 074, Train Loss: 0.2914483, Train Acc: 0.7117647, Test Acc: 0.6666667\n",
            "Epoch: 075, Train Loss: 0.2360902, Train Acc: 0.8470588, Test Acc: 0.7222222\n",
            "Epoch: 076, Train Loss: 0.2623871, Train Acc: 0.8352941, Test Acc: 0.7222222\n",
            "Epoch: 077, Train Loss: 0.2580584, Train Acc: 0.8411765, Test Acc: 0.7777778\n",
            "Epoch: 078, Train Loss: 0.2489168, Train Acc: 0.7352941, Test Acc: 0.6666667\n",
            "Epoch: 079, Train Loss: 0.2664888, Train Acc: 0.4588235, Test Acc: 0.4444444\n",
            "Epoch: 080, Train Loss: 0.2479216, Train Acc: 0.4647059, Test Acc: 0.5000000\n",
            "Epoch: 081, Train Loss: 0.2458562, Train Acc: 0.7529412, Test Acc: 0.7222222\n",
            "Epoch: 082, Train Loss: 0.2254077, Train Acc: 0.9000000, Test Acc: 0.8333333\n",
            "Epoch: 083, Train Loss: 0.2321653, Train Acc: 0.8941176, Test Acc: 0.8333333\n",
            "Epoch: 084, Train Loss: 0.2566983, Train Acc: 0.8470588, Test Acc: 0.8333333\n",
            "Epoch: 085, Train Loss: 0.2191187, Train Acc: 0.7882353, Test Acc: 0.7777778\n",
            "Epoch: 086, Train Loss: 0.2376892, Train Acc: 0.8470588, Test Acc: 0.8333333\n",
            "Epoch: 087, Train Loss: 0.2322022, Train Acc: 0.8882353, Test Acc: 1.0000000\n",
            "Epoch: 088, Train Loss: 0.2647700, Train Acc: 0.9058824, Test Acc: 0.9444444\n",
            "Epoch: 089, Train Loss: 0.2188623, Train Acc: 0.8000000, Test Acc: 0.8333333\n",
            "Epoch: 090, Train Loss: 0.2154218, Train Acc: 0.8000000, Test Acc: 0.8333333\n",
            "Epoch: 091, Train Loss: 0.2320241, Train Acc: 0.8823529, Test Acc: 0.9444444\n",
            "Epoch: 092, Train Loss: 0.2433650, Train Acc: 0.9117647, Test Acc: 0.8333333\n",
            "Epoch: 093, Train Loss: 0.2431234, Train Acc: 0.8000000, Test Acc: 0.7777778\n",
            "Epoch: 094, Train Loss: 0.2307503, Train Acc: 0.8117647, Test Acc: 0.7777778\n",
            "Epoch: 095, Train Loss: 0.2057205, Train Acc: 0.8823529, Test Acc: 0.7777778\n",
            "Epoch: 096, Train Loss: 0.2414358, Train Acc: 0.9235294, Test Acc: 0.8333333\n",
            "Epoch: 097, Train Loss: 0.2332241, Train Acc: 0.8352941, Test Acc: 0.8333333\n",
            "Epoch: 098, Train Loss: 0.2291197, Train Acc: 0.7529412, Test Acc: 0.7222222\n",
            "Epoch: 099, Train Loss: 0.2266680, Train Acc: 0.8647059, Test Acc: 0.8888889\n",
            "Epoch: 100, Train Loss: 0.2406777, Train Acc: 0.8705882, Test Acc: 0.8333333\n"
          ]
        }
      ],
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "net = \"LAF\"\n",
        "if net == \"LAF\":\n",
        "    model = LAFNet().to(device)\n",
        "elif net == \"PNA\":\n",
        "    model = PNANet().to(device)\n",
        "elif net == \"GIN\":\n",
        "    GINNet().to(device)\n",
        "\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
        "\n",
        "\n",
        "def train(epoch):\n",
        "    model.train()\n",
        "\n",
        "    if epoch == 51:\n",
        "        for param_group in optimizer.param_groups:\n",
        "            param_group['lr'] = 0.5 * param_group['lr']\n",
        "\n",
        "    loss_all = 0\n",
        "    for data in train_loader:\n",
        "        data = data.to(device)\n",
        "        optimizer.zero_grad()\n",
        "        output = model(data.x, data.edge_index, data.batch)\n",
        "        loss = F.nll_loss(output, data.y)\n",
        "        loss.backward()\n",
        "        loss_all += loss.item() * data.num_graphs\n",
        "        optimizer.step()\n",
        "    return loss_all / len(train_dataset)\n",
        "\n",
        "\n",
        "def test(loader):\n",
        "    model.eval()\n",
        "\n",
        "    correct = 0\n",
        "    for data in loader:\n",
        "        data = data.to(device)\n",
        "        output = model(data.x, data.edge_index, data.batch)\n",
        "        pred = output.max(dim=1)[1]\n",
        "        correct += pred.eq(data.y).sum().item()\n",
        "    return correct / len(loader.dataset)\n",
        "\n",
        "\n",
        "for epoch in range(1, 101):\n",
        "    train_loss = train(epoch)\n",
        "    train_acc = test(train_loader)\n",
        "    test_acc = test(test_loader)\n",
        "    print('Epoch: {:03d}, Train Loss: {:.7f}, '\n",
        "          'Train Acc: {:.7f}, Test Acc: {:.7f}'.format(epoch, train_loss,\n",
        "                                                       train_acc, test_acc))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "Uuys6oUR7I3i",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5ea24740-12b5-4956-eceb-5b4355502fd1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 001, Train Loss: 1.3497391, Train Acc: 0.3294118, Test Acc: 0.3888889\n",
            "Epoch: 002, Train Loss: 0.8683372, Train Acc: 0.3294118, Test Acc: 0.3888889\n",
            "Epoch: 003, Train Loss: 0.7337412, Train Acc: 0.3294118, Test Acc: 0.3888889\n",
            "Epoch: 004, Train Loss: 0.7280867, Train Acc: 0.3294118, Test Acc: 0.3888889\n",
            "Epoch: 005, Train Loss: 0.8280224, Train Acc: 0.3294118, Test Acc: 0.3888889\n",
            "Epoch: 006, Train Loss: 0.7722415, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 007, Train Loss: 0.6033175, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 008, Train Loss: 0.4331027, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 009, Train Loss: 0.4725457, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 010, Train Loss: 0.4262507, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 011, Train Loss: 0.3770855, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 012, Train Loss: 0.3984250, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 013, Train Loss: 0.4109223, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 014, Train Loss: 0.3501784, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 015, Train Loss: 0.2860184, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 016, Train Loss: 0.2797274, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 017, Train Loss: 0.2462570, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 018, Train Loss: 0.2397365, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 019, Train Loss: 0.2560576, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 020, Train Loss: 0.2310867, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 021, Train Loss: 0.2210936, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 022, Train Loss: 0.2002977, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 023, Train Loss: 0.1741187, Train Acc: 0.6705882, Test Acc: 0.6111111\n",
            "Epoch: 024, Train Loss: 0.1753883, Train Acc: 0.6764706, Test Acc: 0.6111111\n",
            "Epoch: 025, Train Loss: 0.1354335, Train Acc: 0.6764706, Test Acc: 0.6111111\n",
            "Epoch: 026, Train Loss: 0.1207889, Train Acc: 0.7000000, Test Acc: 0.6666667\n",
            "Epoch: 027, Train Loss: 0.1275360, Train Acc: 0.6941176, Test Acc: 0.6666667\n",
            "Epoch: 028, Train Loss: 0.1121714, Train Acc: 0.7000000, Test Acc: 0.7222222\n",
            "Epoch: 029, Train Loss: 0.1149847, Train Acc: 0.7058824, Test Acc: 0.7222222\n",
            "Epoch: 030, Train Loss: 0.0870609, Train Acc: 0.7176471, Test Acc: 0.7777778\n",
            "Epoch: 031, Train Loss: 0.1304918, Train Acc: 0.7058824, Test Acc: 0.8333333\n",
            "Epoch: 032, Train Loss: 0.1471310, Train Acc: 0.7647059, Test Acc: 0.8333333\n",
            "Epoch: 033, Train Loss: 0.0883307, Train Acc: 0.7705882, Test Acc: 0.8888889\n",
            "Epoch: 034, Train Loss: 0.0901929, Train Acc: 0.8000000, Test Acc: 0.9444444\n",
            "Epoch: 035, Train Loss: 0.0956914, Train Acc: 0.8294118, Test Acc: 1.0000000\n",
            "Epoch: 036, Train Loss: 0.0967218, Train Acc: 0.8882353, Test Acc: 0.9444444\n",
            "Epoch: 037, Train Loss: 0.0677803, Train Acc: 0.8823529, Test Acc: 0.8333333\n",
            "Epoch: 038, Train Loss: 0.0725969, Train Acc: 0.8941176, Test Acc: 0.8888889\n",
            "Epoch: 039, Train Loss: 0.0461554, Train Acc: 0.8705882, Test Acc: 0.9444444\n",
            "Epoch: 040, Train Loss: 0.0685510, Train Acc: 0.8705882, Test Acc: 1.0000000\n",
            "Epoch: 041, Train Loss: 0.0716990, Train Acc: 0.8882353, Test Acc: 0.9444444\n",
            "Epoch: 042, Train Loss: 0.0841498, Train Acc: 0.9058824, Test Acc: 0.9444444\n",
            "Epoch: 043, Train Loss: 0.0575552, Train Acc: 0.9529412, Test Acc: 0.8888889\n",
            "Epoch: 044, Train Loss: 0.0478302, Train Acc: 0.9823529, Test Acc: 0.8888889\n",
            "Epoch: 045, Train Loss: 0.0516917, Train Acc: 0.9470588, Test Acc: 0.8888889\n",
            "Epoch: 046, Train Loss: 0.0376829, Train Acc: 0.8764706, Test Acc: 0.8888889\n",
            "Epoch: 047, Train Loss: 0.0944155, Train Acc: 0.9294118, Test Acc: 0.8333333\n",
            "Epoch: 048, Train Loss: 0.0604479, Train Acc: 0.9529412, Test Acc: 0.7222222\n",
            "Epoch: 049, Train Loss: 0.0656420, Train Acc: 0.9823529, Test Acc: 0.8333333\n",
            "Epoch: 050, Train Loss: 0.0540702, Train Acc: 0.9764706, Test Acc: 0.7777778\n",
            "Epoch: 051, Train Loss: 0.0452024, Train Acc: 0.9764706, Test Acc: 0.8333333\n",
            "Epoch: 052, Train Loss: 0.0328488, Train Acc: 0.9823529, Test Acc: 0.8888889\n",
            "Epoch: 053, Train Loss: 0.0451938, Train Acc: 0.9823529, Test Acc: 0.8333333\n",
            "Epoch: 054, Train Loss: 0.0271671, Train Acc: 0.9764706, Test Acc: 0.8333333\n",
            "Epoch: 055, Train Loss: 0.0361339, Train Acc: 0.9882353, Test Acc: 0.7777778\n",
            "Epoch: 056, Train Loss: 0.0204628, Train Acc: 0.9764706, Test Acc: 0.7777778\n",
            "Epoch: 057, Train Loss: 0.0207347, Train Acc: 0.9764706, Test Acc: 0.7777778\n",
            "Epoch: 058, Train Loss: 0.0256155, Train Acc: 0.9823529, Test Acc: 0.7777778\n",
            "Epoch: 059, Train Loss: 0.0280545, Train Acc: 0.9823529, Test Acc: 0.7777778\n",
            "Epoch: 060, Train Loss: 0.0223794, Train Acc: 0.9882353, Test Acc: 0.8333333\n",
            "Epoch: 061, Train Loss: 0.0108289, Train Acc: 0.9705882, Test Acc: 0.8333333\n",
            "Epoch: 062, Train Loss: 0.0316626, Train Acc: 0.9764706, Test Acc: 0.8333333\n",
            "Epoch: 063, Train Loss: 0.0226496, Train Acc: 0.9764706, Test Acc: 0.8888889\n",
            "Epoch: 064, Train Loss: 0.0213496, Train Acc: 0.9941176, Test Acc: 0.8888889\n",
            "Epoch: 065, Train Loss: 0.0199417, Train Acc: 0.9941176, Test Acc: 0.8888889\n",
            "Epoch: 066, Train Loss: 0.0139472, Train Acc: 0.9941176, Test Acc: 0.8888889\n",
            "Epoch: 067, Train Loss: 0.0271445, Train Acc: 0.9941176, Test Acc: 0.8888889\n",
            "Epoch: 068, Train Loss: 0.0193949, Train Acc: 0.9941176, Test Acc: 0.8888889\n",
            "Epoch: 069, Train Loss: 0.0261998, Train Acc: 0.9941176, Test Acc: 0.8888889\n",
            "Epoch: 070, Train Loss: 0.0180488, Train Acc: 1.0000000, Test Acc: 0.8888889\n",
            "Epoch: 071, Train Loss: 0.0225507, Train Acc: 1.0000000, Test Acc: 0.8888889\n",
            "Epoch: 072, Train Loss: 0.0219721, Train Acc: 1.0000000, Test Acc: 0.8888889\n",
            "Epoch: 073, Train Loss: 0.0118929, Train Acc: 1.0000000, Test Acc: 0.8888889\n",
            "Epoch: 074, Train Loss: 0.0093817, Train Acc: 1.0000000, Test Acc: 0.8888889\n",
            "Epoch: 075, Train Loss: 0.0119331, Train Acc: 1.0000000, Test Acc: 0.8888889\n",
            "Epoch: 076, Train Loss: 0.0210117, Train Acc: 0.9941176, Test Acc: 0.8333333\n",
            "Epoch: 077, Train Loss: 0.0173290, Train Acc: 0.9941176, Test Acc: 0.8333333\n",
            "Epoch: 078, Train Loss: 0.0112130, Train Acc: 0.9941176, Test Acc: 0.8333333\n",
            "Epoch: 079, Train Loss: 0.0147933, Train Acc: 0.9941176, Test Acc: 0.8888889\n",
            "Epoch: 080, Train Loss: 0.0163544, Train Acc: 0.9941176, Test Acc: 0.8888889\n",
            "Epoch: 081, Train Loss: 0.0078255, Train Acc: 1.0000000, Test Acc: 0.8888889\n",
            "Epoch: 082, Train Loss: 0.0079931, Train Acc: 1.0000000, Test Acc: 0.8888889\n",
            "Epoch: 083, Train Loss: 0.0051889, Train Acc: 1.0000000, Test Acc: 0.8888889\n",
            "Epoch: 084, Train Loss: 0.0112106, Train Acc: 1.0000000, Test Acc: 0.8888889\n",
            "Epoch: 085, Train Loss: 0.0122358, Train Acc: 1.0000000, Test Acc: 0.8888889\n",
            "Epoch: 086, Train Loss: 0.0115622, Train Acc: 1.0000000, Test Acc: 0.8888889\n",
            "Epoch: 087, Train Loss: 0.0100610, Train Acc: 1.0000000, Test Acc: 0.8888889\n",
            "Epoch: 088, Train Loss: 0.0129200, Train Acc: 1.0000000, Test Acc: 0.8888889\n",
            "Epoch: 089, Train Loss: 0.0084742, Train Acc: 0.9941176, Test Acc: 0.8888889\n",
            "Epoch: 090, Train Loss: 0.0141422, Train Acc: 0.9941176, Test Acc: 0.8888889\n",
            "Epoch: 091, Train Loss: 0.0049024, Train Acc: 0.9941176, Test Acc: 0.8888889\n",
            "Epoch: 092, Train Loss: 0.0083834, Train Acc: 0.9941176, Test Acc: 0.8888889\n",
            "Epoch: 093, Train Loss: 0.0130252, Train Acc: 0.9941176, Test Acc: 0.8888889\n",
            "Epoch: 094, Train Loss: 0.0084299, Train Acc: 0.9941176, Test Acc: 0.8888889\n",
            "Epoch: 095, Train Loss: 0.0107380, Train Acc: 0.9941176, Test Acc: 0.8888889\n",
            "Epoch: 096, Train Loss: 0.0113044, Train Acc: 1.0000000, Test Acc: 0.8888889\n",
            "Epoch: 097, Train Loss: 0.0128534, Train Acc: 1.0000000, Test Acc: 0.8888889\n",
            "Epoch: 098, Train Loss: 0.0117190, Train Acc: 0.9941176, Test Acc: 0.8888889\n",
            "Epoch: 099, Train Loss: 0.0093422, Train Acc: 0.9941176, Test Acc: 0.8333333\n",
            "Epoch: 100, Train Loss: 0.0071716, Train Acc: 0.9941176, Test Acc: 0.8333333\n"
          ]
        }
      ],
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "net = \"PNA\"\n",
        "if net == \"LAF\":\n",
        "    model = LAFNet().to(device)\n",
        "elif net == \"PNA\":\n",
        "    model = PNANet().to(device)\n",
        "elif net == \"GIN\":\n",
        "    GINNet().to(device)\n",
        "\n",
        "optimizer = torch.optim.AdamW(model.parameters(), lr=0.001)\n",
        "\n",
        "\n",
        "def train(epoch):\n",
        "    model.train()\n",
        "\n",
        "    if epoch == 51:\n",
        "        for param_group in optimizer.param_groups:\n",
        "            param_group['lr'] = 0.5 * param_group['lr']\n",
        "\n",
        "    loss_all = 0\n",
        "    for data in train_loader:\n",
        "        data = data.to(device)\n",
        "        optimizer.zero_grad()\n",
        "        output = model(data.x, data.edge_index, data.batch)\n",
        "        loss = F.nll_loss(output, data.y)\n",
        "        loss.backward()\n",
        "        loss_all += loss.item() * data.num_graphs\n",
        "        optimizer.step()\n",
        "    return loss_all / len(train_dataset)\n",
        "\n",
        "\n",
        "def test(loader):\n",
        "    model.eval()\n",
        "\n",
        "    correct = 0\n",
        "    for data in loader:\n",
        "        data = data.to(device)\n",
        "        output = model(data.x, data.edge_index, data.batch)\n",
        "        pred = output.max(dim=1)[1]\n",
        "        correct += pred.eq(data.y).sum().item()\n",
        "    return correct / len(loader.dataset)\n",
        "\n",
        "\n",
        "for epoch in range(1, 101):\n",
        "    train_loss = train(epoch)\n",
        "    train_acc = test(train_loader)\n",
        "    test_acc = test(test_loader)\n",
        "    print('Epoch: {:03d}, Train Loss: {:.7f}, '\n",
        "          'Train Acc: {:.7f}, Test Acc: {:.7f}'.format(epoch, train_loss,\n",
        "                                                       train_acc, test_acc))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "net = \"GIN\"\n",
        "if net == \"LAF\":\n",
        "    model = LAFNet().to(device)\n",
        "elif net == \"PNA\":\n",
        "    model = PNANet().to(device)\n",
        "elif net == \"GIN\":\n",
        "    GINNet().to(device)\n",
        "\n",
        "optimizer = torch.optim.AdamW(model.parameters(), lr=0.001)\n",
        "\n",
        "\n",
        "def train(epoch):\n",
        "    model.train()\n",
        "\n",
        "    if epoch == 51:\n",
        "        for param_group in optimizer.param_groups:\n",
        "            param_group['lr'] = 0.5 * param_group['lr']\n",
        "\n",
        "    loss_all = 0\n",
        "    for data in train_loader:\n",
        "        data = data.to(device)\n",
        "        optimizer.zero_grad()\n",
        "        output = model(data.x, data.edge_index, data.batch)\n",
        "        loss = F.nll_loss(output, data.y)\n",
        "        loss.backward()\n",
        "        loss_all += loss.item() * data.num_graphs\n",
        "        optimizer.step()\n",
        "    return loss_all / len(train_dataset)\n",
        "\n",
        "\n",
        "def test(loader):\n",
        "    model.eval()\n",
        "\n",
        "    correct = 0\n",
        "    for data in loader:\n",
        "        data = data.to(device)\n",
        "        output = model(data.x, data.edge_index, data.batch)\n",
        "        pred = output.max(dim=1)[1]\n",
        "        correct += pred.eq(data.y).sum().item()\n",
        "    return correct / len(loader.dataset)\n",
        "\n",
        "\n",
        "for epoch in range(1, 101):\n",
        "    train_loss = train(epoch)\n",
        "    train_acc = test(train_loader)\n",
        "    test_acc = test(test_loader)\n",
        "    print('Epoch: {:03d}, Train Loss: {:.7f}, '\n",
        "          'Train Acc: {:.7f}, Test Acc: {:.7f}'.format(epoch, train_loss,\n",
        "                                                       train_acc, test_acc))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5r84Po-xYVOQ",
        "outputId": "ffae629d-6793-4026-bf4f-237af7b99230"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 001, Train Loss: 0.1543812, Train Acc: 0.8705882, Test Acc: 0.9444444\n",
            "Epoch: 002, Train Loss: 0.3757242, Train Acc: 0.8882353, Test Acc: 0.8333333\n",
            "Epoch: 003, Train Loss: 0.2250766, Train Acc: 0.9058824, Test Acc: 0.7777778\n",
            "Epoch: 004, Train Loss: 0.1418931, Train Acc: 0.9000000, Test Acc: 0.8333333\n",
            "Epoch: 005, Train Loss: 0.1487077, Train Acc: 0.9235294, Test Acc: 0.8333333\n",
            "Epoch: 006, Train Loss: 0.0935999, Train Acc: 0.9058824, Test Acc: 0.8333333\n",
            "Epoch: 007, Train Loss: 0.1021548, Train Acc: 0.9176471, Test Acc: 0.7777778\n",
            "Epoch: 008, Train Loss: 0.1058675, Train Acc: 0.9470588, Test Acc: 0.8888889\n",
            "Epoch: 009, Train Loss: 0.0800030, Train Acc: 0.9352941, Test Acc: 0.8333333\n",
            "Epoch: 010, Train Loss: 0.1170794, Train Acc: 0.9176471, Test Acc: 0.7777778\n",
            "Epoch: 011, Train Loss: 0.0797732, Train Acc: 0.9117647, Test Acc: 0.8333333\n",
            "Epoch: 012, Train Loss: 0.0924640, Train Acc: 0.9058824, Test Acc: 0.7777778\n",
            "Epoch: 013, Train Loss: 0.0513738, Train Acc: 0.9294118, Test Acc: 0.7777778\n",
            "Epoch: 014, Train Loss: 0.0614671, Train Acc: 0.9294118, Test Acc: 0.7777778\n",
            "Epoch: 015, Train Loss: 0.0479347, Train Acc: 0.9176471, Test Acc: 0.7777778\n",
            "Epoch: 016, Train Loss: 0.0584519, Train Acc: 0.9000000, Test Acc: 0.8333333\n",
            "Epoch: 017, Train Loss: 0.0549918, Train Acc: 0.9176471, Test Acc: 0.8333333\n",
            "Epoch: 018, Train Loss: 0.0534451, Train Acc: 0.9529412, Test Acc: 0.8333333\n",
            "Epoch: 019, Train Loss: 0.0280595, Train Acc: 0.9882353, Test Acc: 0.8333333\n",
            "Epoch: 020, Train Loss: 0.0717960, Train Acc: 0.9764706, Test Acc: 0.8333333\n",
            "Epoch: 021, Train Loss: 0.0321016, Train Acc: 0.9705882, Test Acc: 0.8333333\n",
            "Epoch: 022, Train Loss: 0.0288013, Train Acc: 0.9647059, Test Acc: 0.8333333\n",
            "Epoch: 023, Train Loss: 0.0189139, Train Acc: 0.9647059, Test Acc: 0.8333333\n",
            "Epoch: 024, Train Loss: 0.0130764, Train Acc: 0.9823529, Test Acc: 0.8333333\n",
            "Epoch: 025, Train Loss: 0.0202296, Train Acc: 0.9764706, Test Acc: 0.8333333\n",
            "Epoch: 026, Train Loss: 0.0147458, Train Acc: 0.9882353, Test Acc: 0.8333333\n",
            "Epoch: 027, Train Loss: 0.0105133, Train Acc: 0.9941176, Test Acc: 0.8333333\n",
            "Epoch: 028, Train Loss: 0.0075279, Train Acc: 0.9941176, Test Acc: 0.8333333\n",
            "Epoch: 029, Train Loss: 0.0095293, Train Acc: 0.9882353, Test Acc: 0.8333333\n",
            "Epoch: 030, Train Loss: 0.0126280, Train Acc: 0.9882353, Test Acc: 0.8333333\n",
            "Epoch: 031, Train Loss: 0.0079384, Train Acc: 0.9941176, Test Acc: 0.8333333\n",
            "Epoch: 032, Train Loss: 0.0120276, Train Acc: 0.9823529, Test Acc: 0.8333333\n",
            "Epoch: 033, Train Loss: 0.0072766, Train Acc: 0.9882353, Test Acc: 0.8333333\n",
            "Epoch: 034, Train Loss: 0.0100288, Train Acc: 0.9823529, Test Acc: 0.8333333\n",
            "Epoch: 035, Train Loss: 0.0159473, Train Acc: 0.9882353, Test Acc: 0.8333333\n",
            "Epoch: 036, Train Loss: 0.0072844, Train Acc: 0.9823529, Test Acc: 0.8333333\n",
            "Epoch: 037, Train Loss: 0.0046067, Train Acc: 0.9823529, Test Acc: 0.8333333\n",
            "Epoch: 038, Train Loss: 0.0093492, Train Acc: 0.9941176, Test Acc: 0.8333333\n",
            "Epoch: 039, Train Loss: 0.0052369, Train Acc: 0.9882353, Test Acc: 0.8333333\n",
            "Epoch: 040, Train Loss: 0.0047525, Train Acc: 0.9882353, Test Acc: 0.8333333\n",
            "Epoch: 041, Train Loss: 0.0087503, Train Acc: 0.9882353, Test Acc: 0.8333333\n",
            "Epoch: 042, Train Loss: 0.0021545, Train Acc: 0.9823529, Test Acc: 0.7777778\n",
            "Epoch: 043, Train Loss: 0.0039228, Train Acc: 0.9764706, Test Acc: 0.7777778\n",
            "Epoch: 044, Train Loss: 0.0042726, Train Acc: 0.9823529, Test Acc: 0.7777778\n",
            "Epoch: 045, Train Loss: 0.0037024, Train Acc: 0.9705882, Test Acc: 0.7777778\n",
            "Epoch: 046, Train Loss: 0.0060803, Train Acc: 0.9823529, Test Acc: 0.7777778\n",
            "Epoch: 047, Train Loss: 0.0034611, Train Acc: 0.9823529, Test Acc: 0.7777778\n",
            "Epoch: 048, Train Loss: 0.0056211, Train Acc: 0.9764706, Test Acc: 0.7777778\n",
            "Epoch: 049, Train Loss: 0.0049050, Train Acc: 0.9764706, Test Acc: 0.7777778\n",
            "Epoch: 050, Train Loss: 0.0074722, Train Acc: 0.9823529, Test Acc: 0.7777778\n",
            "Epoch: 051, Train Loss: 0.0025539, Train Acc: 0.9882353, Test Acc: 0.7777778\n",
            "Epoch: 052, Train Loss: 0.0039450, Train Acc: 0.9941176, Test Acc: 0.8333333\n",
            "Epoch: 053, Train Loss: 0.0036104, Train Acc: 1.0000000, Test Acc: 0.8333333\n",
            "Epoch: 054, Train Loss: 0.0044570, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 055, Train Loss: 0.0029137, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 056, Train Loss: 0.0021807, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 057, Train Loss: 0.0012837, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 058, Train Loss: 0.0013967, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 059, Train Loss: 0.0015003, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 060, Train Loss: 0.0011881, Train Acc: 0.9941176, Test Acc: 0.7777778\n",
            "Epoch: 061, Train Loss: 0.0018032, Train Acc: 0.9941176, Test Acc: 0.7777778\n",
            "Epoch: 062, Train Loss: 0.0018816, Train Acc: 0.9941176, Test Acc: 0.7777778\n",
            "Epoch: 063, Train Loss: 0.0029092, Train Acc: 0.9941176, Test Acc: 0.7777778\n",
            "Epoch: 064, Train Loss: 0.0028035, Train Acc: 0.9941176, Test Acc: 0.7777778\n",
            "Epoch: 065, Train Loss: 0.0023646, Train Acc: 0.9941176, Test Acc: 0.7777778\n",
            "Epoch: 066, Train Loss: 0.0030593, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 067, Train Loss: 0.0006509, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 068, Train Loss: 0.0047645, Train Acc: 0.9941176, Test Acc: 0.7777778\n",
            "Epoch: 069, Train Loss: 0.0028100, Train Acc: 0.9941176, Test Acc: 0.7777778\n",
            "Epoch: 070, Train Loss: 0.0061960, Train Acc: 0.9941176, Test Acc: 0.7777778\n",
            "Epoch: 071, Train Loss: 0.0015897, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 072, Train Loss: 0.0023837, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 073, Train Loss: 0.0006545, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 074, Train Loss: 0.0017835, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 075, Train Loss: 0.0035376, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 076, Train Loss: 0.0021876, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 077, Train Loss: 0.0032244, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 078, Train Loss: 0.0022429, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 079, Train Loss: 0.0013306, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 080, Train Loss: 0.0005241, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 081, Train Loss: 0.0015545, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 082, Train Loss: 0.0010169, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 083, Train Loss: 0.0015336, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 084, Train Loss: 0.0006424, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 085, Train Loss: 0.0006514, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 086, Train Loss: 0.0020132, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 087, Train Loss: 0.0022427, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 088, Train Loss: 0.0016486, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 089, Train Loss: 0.0011926, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 090, Train Loss: 0.0032095, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 091, Train Loss: 0.0018466, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 092, Train Loss: 0.0019558, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 093, Train Loss: 0.0009050, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 094, Train Loss: 0.0023666, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 095, Train Loss: 0.0009373, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 096, Train Loss: 0.0010030, Train Acc: 0.9941176, Test Acc: 0.7777778\n",
            "Epoch: 097, Train Loss: 0.0025636, Train Acc: 0.9941176, Test Acc: 0.7777778\n",
            "Epoch: 098, Train Loss: 0.0046557, Train Acc: 1.0000000, Test Acc: 0.7777778\n",
            "Epoch: 099, Train Loss: 0.0027510, Train Acc: 0.9823529, Test Acc: 0.8333333\n",
            "Epoch: 100, Train Loss: 0.0120788, Train Acc: 1.0000000, Test Acc: 0.8333333\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.12"
    },
    "colab": {
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}