eval_face_detection_ensemble.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/system/apps/userenv/kschweig/quam/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import glob\n",
    "import copy\n",
    "from tqdm import tqdm\n",
    "from typing import List\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision as tv\n",
    "from torch.utils.data import Subset, Dataset, DataLoader\n",
    "\n",
    "from source.constants import RESULTS_PATH\n",
    "from source.data.face_detection import get_fair_face, get_utk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "method_seeds = [42, 142, 242, 342, 442]\n",
    "dseed = 42\n",
    "\n",
    "model = [\"resnet18\", \"resnet34\", \"resnet50\"][0]\n",
    "\n",
    "target = 3 # 0, 1, 2, 3\n",
    "\n",
    "device = \"cuda:7\"\n",
    "batch_size = 2048 # 1024 ~ 10GB VRAM / 2048 ~ 15GB VRAM for resnet18 ~ 1GB more for resnet34, 1024 ~ 17GB for resnet50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "ff_train_ds, ff_test_ds = get_fair_face(target=target, binarize=True, augment=False)\n",
    "utk_test_ds = get_utk(target=target, binarize=True)\n",
    "\n",
    "run_path = os.path.join(RESULTS_PATH, f\"fairface_target{target}_{model}_mseed{method_seeds[0]}_dseed{dseed}\")\n",
    "fair_inds = torch.load(os.path.join(run_path, \"fair_inds.pt\"))\n",
    "val_inds = torch.load(os.path.join(run_path, \"val_inds.pt\"))\n",
    "\n",
    "fair_ds = Subset(ff_train_ds, indices=fair_inds)\n",
    "val_ds = Subset(ff_train_ds, indices=val_inds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def evaluate(networks: List, ds: Dataset):\n",
    "    probits = list()\n",
    "    for network in tqdm(networks):\n",
    "        network.eval()\n",
    "        probits_ = list()\n",
    "        for x, _, _ in DataLoader(ds, batch_size = batch_size, shuffle=False, drop_last=False):\n",
    "            x = x.to(device)\n",
    "\n",
    "            probits_.append(torch.softmax(network.forward(x), dim=1).cpu())\n",
    "        probits_ = torch.concat(probits_, dim=0)\n",
    "        probits.append(probits_)\n",
    "    return torch.stack(probits, dim=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [05:04<00:00, 30.43s/it]\n",
      "100%|██████████| 10/10 [04:42<00:00, 28.28s/it]\n",
      "100%|██████████| 10/10 [04:16<00:00, 25.66s/it]\n",
      "100%|██████████| 10/10 [13:08<00:00, 78.82s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluated method seed 42\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [03:49<00:00, 22.99s/it]\n",
      "100%|██████████| 10/10 [04:29<00:00, 26.99s/it]\n",
      "100%|██████████| 10/10 [04:15<00:00, 25.55s/it]\n",
      "100%|██████████| 10/10 [11:41<00:00, 70.13s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluated method seed 142\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [03:47<00:00, 22.78s/it]\n",
      "100%|██████████| 10/10 [04:21<00:00, 26.10s/it]\n",
      "100%|██████████| 10/10 [03:52<00:00, 23.21s/it]\n",
      "100%|██████████| 10/10 [12:21<00:00, 74.12s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluated method seed 242\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [04:27<00:00, 26.71s/it]\n",
      "100%|██████████| 10/10 [04:09<00:00, 24.97s/it]\n",
      "100%|██████████| 10/10 [04:33<00:00, 27.36s/it]\n",
      "100%|██████████| 10/10 [14:19<00:00, 85.92s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluated method seed 342\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [05:13<00:00, 31.34s/it]\n",
      "100%|██████████| 10/10 [04:19<00:00, 25.94s/it]\n",
      "100%|██████████| 10/10 [04:20<00:00, 26.01s/it]\n",
      "100%|██████████| 10/10 [14:36<00:00, 87.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluated method seed 442\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for mseed in method_seeds:\n",
    "\n",
    "    path = os.path.join(RESULTS_PATH, f\"fairface_target{target}_{model}_mseed{mseed}_dseed{dseed}\")\n",
    "\n",
    "    # load networks\n",
    "    networks = list()\n",
    "    model_files = glob.glob(os.path.join(path, \"models\", \"*.pt\"))\n",
    "    for model_file in sorted(model_files):\n",
    "        if model == \"resnet18\":\n",
    "            network = tv.models.resnet18(weights=None) \n",
    "            network.fc = nn.Linear(in_features=512, out_features=2)\n",
    "        elif model == \"resnet34\":\n",
    "            network = tv.models.resnet34(weights=None) \n",
    "            network.fc = nn.Linear(in_features=512, out_features=2)\n",
    "        elif model == \"resnet50\":\n",
    "            network = tv.models.resnet50(weights=None) \n",
    "            network.fc = nn.Linear(in_features=2048, out_features=2)\n",
    "\n",
    "        network.load_state_dict(torch.load(model_file, map_location=device))\n",
    "        network.to(device)\n",
    "        networks.append(copy.deepcopy(network))\n",
    "\n",
    "    # evaluate\n",
    "    torch.save(evaluate(networks, fair_ds), os.path.join(path, f\"fair_probits_t{target}.pt\"))\n",
    "    torch.save(evaluate(networks, val_ds), os.path.join(path, f\"val_probits_t{target}.pt\"))\n",
    "    torch.save(evaluate(networks, ff_test_ds), os.path.join(path, f\"ff_test_probits_t{target}.pt\"))\n",
    "    torch.save(evaluate(networks, utk_test_ds), os.path.join(path, f\"utk_test_probits_t{target}.pt\"))\n",
    "    print(\"Evaluated method seed\", mseed)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "quam",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}