reproduce.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a7081c50-ed9c-4be4-970d-6b382e5c14d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from data import Dataset\n",
    "from torch_geometric.data import DataLoader\n",
    "\n",
    "from evaluate import evaluate_node, evaluate_clustering, run_similarity_search"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee4dc29f-79ac-4f76-9885-fb1902476d6c",
   "metadata": {},
   "source": [
    "## Evaluation of embeddings\n",
    "Due to memory limitation of uploading files, we attached only two embeddings from our model.  \n",
    "They are embeddings from node classification task for **Amazon computers** dataset and **Coauthor CS** dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "acb5cc43-95a6-4557-ba11-ae344b10bf20",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate node classification results\n",
      "** Val: 90.0800 (0.7563) | Test: 89.8100 (0.3403) **\n"
     ]
    }
   ],
   "source": [
    "# Computers dataset\n",
    "computers_dataset = Dataset(root=\"data\", dataset=\"computers\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_computers_node.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "evaluate_node(embeddings, computers_dataset, \"computers\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1239dc1c-0d18-4814-98d4-ea91d1d7ce08",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate node classification results\n",
      "** Val: 93.1506 (0.4517) | Test: 93.3022 (0.2037) **\n"
     ]
    }
   ],
   "source": [
    "# CS dataset\n",
    "cs_dataset = Dataset(root=\"data\", dataset=\"cs\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_cs_node.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "evaluate_node(embeddings, cs_dataset, \"cs\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4afeb3f-b0d0-4856-93b9-a904e47a4b51",
   "metadata": {},
   "source": [
    "### Evaluation for Node Classification\n",
    "This is for reproducing **Table 2** in the paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f6d8a955-d876-451e-9375-95b405f3cc34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate node classification results\n",
      "** Val: 78.3041 (0.9529) | Test: 77.5517 (0.5158) **\n"
     ]
    }
   ],
   "source": [
    "# WikiCS dataset\n",
    "wikics_dataset = Dataset(root=\"data\", dataset=\"wikics\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_wikics_node.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "evaluate_node(embeddings, wikics_dataset, \"wikics\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "76bb3ed4-296d-47c5-9c7e-a2d56623dc37",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate node classification results\n",
      "** Val: 90.1127 (0.7972) | Test: 89.7959 (0.3182) **\n"
     ]
    }
   ],
   "source": [
    "# Computers dataset\n",
    "computers_dataset = Dataset(root=\"data\", dataset=\"computers\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_computers_node.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "evaluate_node(embeddings, computers_dataset, \"computers\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "767c085d-a7be-40f1-9b96-10550b8d4893",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate node classification results\n",
      "** Val: 93.0980 (0.9691) | Test: 93.1634 (0.3363) **\n"
     ]
    }
   ],
   "source": [
    "# Photo dataset\n",
    "photo_dataset = Dataset(root=\"data\", dataset=\"photo\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_photo_node.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "evaluate_node(embeddings, photo_dataset, \"photo\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a7cc13f6-cb9a-4700-9ab6-a2c1f8371218",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate node classification results\n",
      "** Val: 93.1833 (0.4807) | Test: 93.3073 (0.2143) **\n"
     ]
    }
   ],
   "source": [
    "# CS dataset\n",
    "cs_dataset = Dataset(root=\"data\", dataset=\"cs\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_cs_node.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "evaluate_node(embeddings, cs_dataset, \"cs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "65132bea-7aee-42dc-b48c-c9a7e9323f3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate node classification results\n",
      "** Val: 95.5697 (0.3342) | Test: 95.6067 (0.0860) **\n"
     ]
    }
   ],
   "source": [
    "# Physics dataset\n",
    "physics_dataset = Dataset(root=\"data\", dataset=\"physics\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_physics_node.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "evaluate_node(embeddings, physics_dataset, \"physics\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "988c85ab-61b0-4a6b-a94d-d52cceb29ab0",
   "metadata": {},
   "source": [
    "### Clustering\n",
    "This is for reproducing **Table 3** in the paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8db513cb-fa82-4e10-bb61-e2b964ce1696",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate clustering results\n",
      "** Clustering NMI: 0.4115 | homogeneity score: 0.4288 **\n"
     ]
    }
   ],
   "source": [
    "# WikiCS dataset\n",
    "wikics_dataset = Dataset(root=\"data\", dataset=\"wikics\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_wikics_clustering.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "evaluate_clustering(embeddings, wikics_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "36920051-08e4-4710-a008-85c5c153d526",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate clustering results\n",
      "** Clustering NMI: 0.5442 | homogeneity score: 0.5936 **\n"
     ]
    }
   ],
   "source": [
    "# Computers dataset\n",
    "computers_dataset = Dataset(root=\"data\", dataset=\"computers\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_computers_clustering.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "evaluate_clustering(embeddings, computers_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "927f50e1-33d4-4079-b061-f7924df50940",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate clustering results\n",
      "** Clustering NMI: 0.6647 | homogeneity score: 0.6807 **\n"
     ]
    }
   ],
   "source": [
    "# Photo dataset\n",
    "photo_dataset = Dataset(root=\"data\", dataset=\"photo\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_photo_clustering.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "evaluate_clustering(embeddings, photo_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5ca2e67c-aa56-40c0-8b15-4d0e254cdf2f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate clustering results\n",
      "** Clustering NMI: 0.7778 | homogeneity score: 0.8088 **\n"
     ]
    }
   ],
   "source": [
    "# CS dataset\n",
    "cs_dataset = Dataset(root=\"data\", dataset=\"cs\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_cs_clustering.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "evaluate_clustering(embeddings, cs_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6eacfd6a-d82f-4243-89b6-2f820ac9c1cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate clustering results\n",
      "** Clustering NMI: 0.7288 | homogeneity score: 0.7353 **\n"
     ]
    }
   ],
   "source": [
    "# Physics dataset\n",
    "physics_dataset = Dataset(root=\"data\", dataset=\"physics\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_physics_clustering.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "evaluate_clustering(embeddings, physics_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fde2dd0e-223d-4fe1-b015-3c0f0a392f39",
   "metadata": {},
   "source": [
    "### Similarity Search\n",
    "This is for reproducing **Table 4** in the paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f6e890bb-0301-488d-ba51-f3969c1e3cee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate similarity search results\n",
      "** sim@5 : 0.781 | sim@10 : 0.7657 **\n"
     ]
    }
   ],
   "source": [
    "# WikiCS dataset\n",
    "wikics_dataset = Dataset(root=\"data\", dataset=\"wikics\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_wikics_similarity.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "run_similarity_search(embeddings, wikics_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "87ff7fd5-f211-41c2-955c-a2a1f33dd05b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate similarity search results\n",
      "** sim@5 : 0.8944 | sim@10 : 0.8867 **\n"
     ]
    }
   ],
   "source": [
    "# Computers dataset\n",
    "computers_dataset = Dataset(root=\"data\", dataset=\"computers\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_computers_similarity.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "run_similarity_search(embeddings, computers_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "144e6869-5651-462a-8c48-cf0a9cb76218",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate similarity search results\n",
      "** sim@5 : 0.9226 | sim@10 : 0.9158 **\n"
     ]
    }
   ],
   "source": [
    "# Photo dataset\n",
    "photo_dataset = Dataset(root=\"data\", dataset=\"photo\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_photo_similarity.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "run_similarity_search(embeddings, photo_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "594e6ae2-594a-453b-85d8-4abb1711c443",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate similarity search results\n",
      "** sim@5 : 0.918 | sim@10 : 0.9141 **\n"
     ]
    }
   ],
   "source": [
    "# CS dataset\n",
    "cs_dataset = Dataset(root=\"data\", dataset=\"cs\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_cs_similarity.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "run_similarity_search(embeddings, cs_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4bfc2842-b105-4a76-88bc-4af5b4bb6e6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate similarity search results\n",
      "** sim@5 : 0.9529 | sim@10 : 0.9491 **\n"
     ]
    }
   ],
   "source": [
    "# Physics dataset\n",
    "physics_dataset = Dataset(root=\"data\", dataset=\"physics\")[0]\n",
    "embedding_path = \"./embeddings/embeddings_physics_similarity.pt\"\n",
    "embeddings = torch.load(embedding_path)\n",
    "\n",
    "run_similarity_search(embeddings, physics_dataset)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}