interpretabnet_synthetic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Installing Packages and Dependencies\n",
    "\n",
    "```\n",
    "conda create -n interpretabnet python=3.10\n",
    "conda activate interpretabnet\n",
    "```\n",
    "\n",
    "```\n",
    "pip install -r requirements.txt\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data generation function import\n",
    "from syn_data_generation import generate_data\n",
    "from matplotlib import pyplot as plt\n",
    "from pytorch_tabnet.tab_model import TabNetClassifier\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import accuracy_score, average_precision_score, roc_auc_score\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Processing - Synthetic Data from L2X (https://arxiv.org/abs/1802.07814)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Synthetic data type\n",
    "idx = 2\n",
    "data_sets = ['Syn1','Syn2','Syn3','Syn4','Syn5','Syn6']\n",
    "data_type = data_sets[idx]\n",
    "\n",
    "# Data output can be either binary (Y) or Probability (Prob)\n",
    "data_out_sets = ['Y','Prob']\n",
    "data_out = data_out_sets[0]\n",
    "\n",
    "# Number of Training and Testing samples\n",
    "train_N = 10000\n",
    "test_N = 10000\n",
    "\n",
    "# Seeds (different seeds for training and testing)\n",
    "train_seed = 0\n",
    "test_seed = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_data(data_type, data_out):\n",
    "\n",
    "    x_train, y_train, g_train = generate_data(n = train_N, data_type = data_type, seed = train_seed, out = data_out)\n",
    "    x_test,  y_test,  g_test  = generate_data(n = test_N,  data_type = data_type, seed = test_seed,  out = data_out)\n",
    "\n",
    "    return x_train, y_train, g_train, x_test, y_test, g_test\n",
    "\n",
    "x_train, y_train, g_train, x_test, y_test, g_test = create_data(data_type, data_out)\n",
    "\n",
    "# binary\n",
    "y_train = y_train[:, 0]\n",
    "y_test = y_test[:, 0]\n",
    "cat_idxs = []\n",
    "cat_dims = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tuning Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device used cuda: 0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0  | loss: 20.96395| train_auc: 0.85258 | valid_auc: 0.84815 |  0:00:13s\n",
      "epoch 1  | loss: 20.98631| train_auc: 0.88572 | valid_auc: 0.88042 |  0:00:27s\n",
      "epoch 2  | loss: 20.85886| train_auc: 0.89681 | valid_auc: 0.88959 |  0:00:41s\n",
      "epoch 3  | loss: 21.04312| train_auc: 0.89998 | valid_auc: 0.89502 |  0:00:54s\n",
      "epoch 4  | loss: 20.82679| train_auc: 0.9063  | valid_auc: 0.89823 |  0:01:08s\n",
      "epoch 5  | loss: 20.80269| train_auc: 0.90816 | valid_auc: 0.89915 |  0:01:21s\n",
      "epoch 6  | loss: 20.82927| train_auc: 0.90968 | valid_auc: 0.8987  |  0:01:35s\n",
      "epoch 7  | loss: 20.96574| train_auc: 0.91086 | valid_auc: 0.89971 |  0:01:49s\n",
      "epoch 8  | loss: 20.83868| train_auc: 0.91103 | valid_auc: 0.89865 |  0:02:03s\n",
      "epoch 9  | loss: 20.87111| train_auc: 0.91326 | valid_auc: 0.89962 |  0:02:16s\n",
      "Stop training because you reached max_epochs = 10 with best_epoch = 7 and best_valid_auc = 0.89971\n",
      "Best weights from best epoch are automatically used!\n",
      "Optimum Hyperparameters Training [16, 1.5, 0.001, 0.025, 4750]\n",
      "Device used cuda: 0\n",
      "epoch 0  | loss: 20.9347 | train_auc: 0.84976 | valid_auc: 0.84861 |  0:00:13s\n",
      "epoch 1  | loss: 20.97865| train_auc: 0.88965 | valid_auc: 0.87964 |  0:00:26s\n",
      "epoch 2  | loss: 20.86204| train_auc: 0.90127 | valid_auc: 0.88938 |  0:00:40s\n",
      "epoch 3  | loss: 21.03821| train_auc: 0.90698 | valid_auc: 0.89231 |  0:00:54s\n",
      "epoch 4  | loss: 20.82166| train_auc: 0.91061 | valid_auc: 0.89318 |  0:01:07s\n",
      "epoch 5  | loss: 20.80083| train_auc: 0.91645 | valid_auc: 0.89365 |  0:01:21s\n",
      "epoch 6  | loss: 20.83021| train_auc: 0.91629 | valid_auc: 0.89337 |  0:01:34s\n",
      "epoch 7  | loss: 20.96517| train_auc: 0.91951 | valid_auc: 0.89177 |  0:01:48s\n",
      "epoch 8  | loss: 20.83566| train_auc: 0.91965 | valid_auc: 0.89285 |  0:02:01s\n",
      "epoch 9  | loss: 20.87787| train_auc: 0.92038 | valid_auc: 0.89488 |  0:02:15s\n",
      "Stop training because you reached max_epochs = 10 with best_epoch = 9 and best_valid_auc = 0.89488\n",
      "Best weights from best epoch are automatically used!\n",
      "Device used cuda: 0\n",
      "epoch 0  | loss: 21.12103| train_auc: 0.83359 | valid_auc: 0.82453 |  0:00:13s\n",
      "epoch 1  | loss: 21.00465| train_auc: 0.88518 | valid_auc: 0.87044 |  0:00:27s\n",
      "epoch 2  | loss: 20.84959| train_auc: 0.90026 | valid_auc: 0.88275 |  0:00:40s\n",
      "epoch 3  | loss: 21.02187| train_auc: 0.91121 | valid_auc: 0.89108 |  0:00:54s\n",
      "epoch 4  | loss: 20.80234| train_auc: 0.91823 | valid_auc: 0.8907  |  0:01:07s\n",
      "epoch 5  | loss: 20.77364| train_auc: 0.91858 | valid_auc: 0.89191 |  0:01:21s\n",
      "epoch 6  | loss: 20.80656| train_auc: 0.92128 | valid_auc: 0.89062 |  0:01:35s\n",
      "epoch 7  | loss: 20.95012| train_auc: 0.92481 | valid_auc: 0.89246 |  0:01:48s\n",
      "epoch 8  | loss: 20.81311| train_auc: 0.92503 | valid_auc: 0.88881 |  0:02:02s\n",
      "epoch 9  | loss: 20.85029| train_auc: 0.92643 | valid_auc: 0.89173 |  0:02:15s\n",
      "Stop training because you reached max_epochs = 10 with best_epoch = 7 and best_valid_auc = 0.89246\n",
      "Best weights from best epoch are automatically used!\n",
      "Device used cuda: 0\n",
      "epoch 0  | loss: 20.96395| train_accuracy: 0.7778  | valid_accuracy: 0.777   |  0:00:13s\n",
      "epoch 1  | loss: 20.98631| train_accuracy: 0.7982  | valid_accuracy: 0.7963  |  0:00:27s\n",
      "epoch 2  | loss: 20.85886| train_accuracy: 0.8105  | valid_accuracy: 0.809   |  0:00:41s\n",
      "epoch 3  | loss: 21.04312| train_accuracy: 0.805   | valid_accuracy: 0.8084  |  0:00:54s\n",
      "epoch 4  | loss: 20.82679| train_accuracy: 0.819   | valid_accuracy: 0.8169  |  0:01:08s\n",
      "epoch 5  | loss: 20.80269| train_accuracy: 0.8234  | valid_accuracy: 0.8166  |  0:01:22s\n",
      "epoch 6  | loss: 20.82927| train_accuracy: 0.8238  | valid_accuracy: 0.8174  |  0:01:35s\n",
      "epoch 7  | loss: 20.96574| train_accuracy: 0.8252  | valid_accuracy: 0.8186  |  0:01:49s\n",
      "epoch 8  | loss: 20.83868| train_accuracy: 0.8261  | valid_accuracy: 0.819   |  0:02:03s\n",
      "epoch 9  | loss: 20.87111| train_accuracy: 0.8288  | valid_accuracy: 0.8192  |  0:02:16s\n",
      "Stop training because you reached max_epochs = 10 with best_epoch = 9 and best_valid_accuracy = 0.8192\n",
      "Best weights from best epoch are automatically used!\n",
      "Optimum Hyperparameters Training [16, 4, 1.0, 0.001, 0.025, 4750]\n",
      "Device used cuda: 0\n",
      "epoch 0  | loss: 20.98835| train_accuracy: 0.7704  | valid_accuracy: 0.7684  |  0:00:13s\n",
      "epoch 1  | loss: 21.0101 | train_accuracy: 0.8007  | valid_accuracy: 0.7981  |  0:00:27s\n",
      "epoch 2  | loss: 20.87442| train_accuracy: 0.8093  | valid_accuracy: 0.8061  |  0:00:41s\n",
      "epoch 3  | loss: 21.06081| train_accuracy: 0.8017  | valid_accuracy: 0.797   |  0:00:54s\n",
      "epoch 4  | loss: 20.83836| train_accuracy: 0.8136  | valid_accuracy: 0.8145  |  0:01:08s\n",
      "epoch 5  | loss: 20.80866| train_accuracy: 0.8189  | valid_accuracy: 0.817   |  0:01:21s\n",
      "epoch 6  | loss: 20.83393| train_accuracy: 0.8203  | valid_accuracy: 0.817   |  0:01:35s\n",
      "epoch 7  | loss: 20.96635| train_accuracy: 0.8198  | valid_accuracy: 0.8178  |  0:01:49s\n",
      "epoch 8  | loss: 20.83388| train_accuracy: 0.8232  | valid_accuracy: 0.8198  |  0:02:02s\n",
      "epoch 9  | loss: 20.87549| train_accuracy: 0.8209  | valid_accuracy: 0.8183  |  0:02:16s\n",
      "Stop training because you reached max_epochs = 10 with best_epoch = 8 and best_valid_accuracy = 0.8198\n",
      "Best weights from best epoch are automatically used!\n",
      "Optimum Hyperparameters Training [16, 4, 1.2, 0.001, 0.025, 4750]\n",
      "Device used cuda: 0\n",
      "epoch 0  | loss: 21.04364| train_accuracy: 0.7511  | valid_accuracy: 0.7488  |  0:00:13s\n",
      "epoch 1  | loss: 21.04552| train_accuracy: 0.7878  | valid_accuracy: 0.7844  |  0:00:26s\n",
      "epoch 2  | loss: 20.9061 | train_accuracy: 0.7956  | valid_accuracy: 0.7918  |  0:00:39s\n",
      "epoch 3  | loss: 21.06141| train_accuracy: 0.7909  | valid_accuracy: 0.7869  |  0:00:53s\n",
      "epoch 4  | loss: 20.82357| train_accuracy: 0.8069  | valid_accuracy: 0.7962  |  0:01:07s\n",
      "epoch 5  | loss: 20.78485| train_accuracy: 0.8059  | valid_accuracy: 0.8001  |  0:01:20s\n",
      "epoch 6  | loss: 20.79817| train_accuracy: 0.8047  | valid_accuracy: 0.8008  |  0:01:34s\n",
      "epoch 7  | loss: 20.91017| train_accuracy: 0.809   | valid_accuracy: 0.8046  |  0:01:48s\n",
      "epoch 8  | loss: 20.76268| train_accuracy: 0.8089  | valid_accuracy: 0.7995  |  0:02:01s\n",
      "epoch 9  | loss: 20.8063 | train_accuracy: 0.8116  | valid_accuracy: 0.8043  |  0:02:14s\n",
      "Stop training because you reached max_epochs = 10 with best_epoch = 7 and best_valid_accuracy = 0.8046\n",
      "Best weights from best epoch are automatically used!\n",
      "Device used cuda: 0\n",
      "epoch 0  | loss: 21.14716| train_accuracy: 0.7172  | valid_accuracy: 0.7187  |  0:00:13s\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[8], line 60\u001b[0m\n\u001b[1;32m     46\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m gams \u001b[38;5;129;01min\u001b[39;00m gammas:\n\u001b[1;32m     47\u001b[0m     clf \u001b[38;5;241m=\u001b[39m TabNetClassifier(\n\u001b[1;32m     48\u001b[0m         n_d\u001b[38;5;241m=\u001b[39mopt_ndna,\n\u001b[1;32m     49\u001b[0m         n_a\u001b[38;5;241m=\u001b[39mopt_ndna,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     57\u001b[0m         mask_type \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msoftmax\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m     58\u001b[0m     )\n\u001b[0;32m---> 60\u001b[0m     \u001b[43mclf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     61\u001b[0m \u001b[43m        \u001b[49m\u001b[43mX_train\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mx_train\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_train\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43my_train\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     62\u001b[0m \u001b[43m        \u001b[49m\u001b[43meval_set\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx_train\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_train\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43mx_test\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_test\u001b[49m\u001b[43m)\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     63\u001b[0m \u001b[43m        \u001b[49m\u001b[43meval_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mtrain\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mvalid\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m256\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     64\u001b[0m \u001b[43m        \u001b[49m\u001b[43mvirtual_batch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m256\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     65\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmax_epochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m10\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43meval_metric\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43maccuracy\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[1;32m     66\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     68\u001b[0m     y_pred \u001b[38;5;241m=\u001b[39m clf\u001b[38;5;241m.\u001b[39mpredict_proba(x_test)\n\u001b[1;32m     69\u001b[0m     test_acc \u001b[38;5;241m=\u001b[39m roc_auc_score(y_score\u001b[38;5;241m=\u001b[39my_pred[:,\u001b[38;5;241m1\u001b[39m], y_true\u001b[38;5;241m=\u001b[39my_test)\n",
      "File \u001b[0;32m/voyager/projects/jacobyhsi/test/InterpreTabNet/pytorch_tabnet/abstract_model.py:272\u001b[0m, in \u001b[0;36mTabModel.fit\u001b[0;34m(self, X_train, y_train, eval_set, eval_name, eval_metric, loss_fn, weights, max_epochs, patience, batch_size, virtual_batch_size, num_workers, drop_last, callbacks, pin_memory, from_unsupervised)\u001b[0m\n\u001b[1;32m    270\u001b[0m \u001b[38;5;66;03m# Apply predict epoch to all eval sets\u001b[39;00m\n\u001b[1;32m    271\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m eval_name, valid_dataloader \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(eval_names, valid_dataloaders):\n\u001b[0;32m--> 272\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_predict_epoch\u001b[49m\u001b[43m(\u001b[49m\u001b[43meval_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalid_dataloader\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    274\u001b[0m \u001b[38;5;66;03m# Call method on_epoch_end for all callbacks\u001b[39;00m\n\u001b[1;32m    275\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_callback_container\u001b[38;5;241m.\u001b[39mon_epoch_end(\n\u001b[1;32m    276\u001b[0m     epoch_idx, logs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhistory\u001b[38;5;241m.\u001b[39mepoch_metrics\n\u001b[1;32m    277\u001b[0m )\n",
      "File \u001b[0;32m/voyager/projects/jacobyhsi/test/InterpreTabNet/pytorch_tabnet/abstract_model.py:571\u001b[0m, in \u001b[0;36mTabModel._predict_epoch\u001b[0;34m(self, name, loader)\u001b[0m\n\u001b[1;32m    569\u001b[0m \u001b[38;5;66;03m# Main loop\u001b[39;00m\n\u001b[1;32m    570\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m batch_idx, (X, y) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(loader):\n\u001b[0;32m--> 571\u001b[0m     scores \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_predict_batch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    572\u001b[0m     list_y_true\u001b[38;5;241m.\u001b[39mappend(y)\n\u001b[1;32m    573\u001b[0m     list_y_score\u001b[38;5;241m.\u001b[39mappend(scores)\n",
      "File \u001b[0;32m/voyager/projects/jacobyhsi/test/InterpreTabNet/pytorch_tabnet/abstract_model.py:601\u001b[0m, in \u001b[0;36mTabModel._predict_batch\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    598\u001b[0m X \u001b[38;5;241m=\u001b[39m X\u001b[38;5;241m.\u001b[39mto(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice)\u001b[38;5;241m.\u001b[39mfloat()\n\u001b[1;32m    600\u001b[0m \u001b[38;5;66;03m# compute model output\u001b[39;00m\n\u001b[0;32m--> 601\u001b[0m scores, _ \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnetwork\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    603\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(scores, \u001b[38;5;28mlist\u001b[39m):\n\u001b[1;32m    604\u001b[0m     scores \u001b[38;5;241m=\u001b[39m [x\u001b[38;5;241m.\u001b[39mcpu()\u001b[38;5;241m.\u001b[39mdetach()\u001b[38;5;241m.\u001b[39mnumpy() \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m scores]\n",
      "File \u001b[0;32m/voyager/projects/jacobyhsi/.conda/envs/interpretabnet/lib/python3.10/site-packages/torch/nn/modules/module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1530\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1532\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/voyager/projects/jacobyhsi/.conda/envs/interpretabnet/lib/python3.10/site-packages/torch/nn/modules/module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1539\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1540\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1541\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1544\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m/voyager/projects/jacobyhsi/test/InterpreTabNet/pytorch_tabnet/tab_network.py:673\u001b[0m, in \u001b[0;36mTabNet.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    671\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, x):\n\u001b[1;32m    672\u001b[0m     x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membedder(x)\n\u001b[0;32m--> 673\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtabnet\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/voyager/projects/jacobyhsi/.conda/envs/interpretabnet/lib/python3.10/site-packages/torch/nn/modules/module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1530\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1532\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/voyager/projects/jacobyhsi/.conda/envs/interpretabnet/lib/python3.10/site-packages/torch/nn/modules/module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1539\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1540\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1541\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1544\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m/voyager/projects/jacobyhsi/test/InterpreTabNet/pytorch_tabnet/tab_network.py:543\u001b[0m, in \u001b[0;36mTabNetNoEmbeddings.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    541\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, x):\n\u001b[1;32m    542\u001b[0m     res \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[0;32m--> 543\u001b[0m     steps_output, M_loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencoder\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    544\u001b[0m     res \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39msum(torch\u001b[38;5;241m.\u001b[39mstack(steps_output, dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m), dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m)\n\u001b[1;32m    546\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_multi_task:\n\u001b[1;32m    547\u001b[0m         \u001b[38;5;66;03m# Result will be in list format\u001b[39;00m\n",
      "File \u001b[0;32m/voyager/projects/jacobyhsi/.conda/envs/interpretabnet/lib/python3.10/site-packages/torch/nn/modules/module.py:1532\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1530\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1532\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/voyager/projects/jacobyhsi/.conda/envs/interpretabnet/lib/python3.10/site-packages/torch/nn/modules/module.py:1541\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1536\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1537\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1539\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1540\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1541\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1543\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1544\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m/voyager/projects/jacobyhsi/test/InterpreTabNet/pytorch_tabnet/tab_network.py:233\u001b[0m, in \u001b[0;36mTabNetEncoder.forward\u001b[0;34m(self, x, prior)\u001b[0m\n\u001b[1;32m    231\u001b[0m     steps_output\u001b[38;5;241m.\u001b[39mappend(d)\n\u001b[1;32m    232\u001b[0m     \u001b[38;5;66;03m# update attention\u001b[39;00m\n\u001b[0;32m--> 233\u001b[0m     att \u001b[38;5;241m=\u001b[39m \u001b[43mout\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mn_d\u001b[49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m    234\u001b[0m M_loss \u001b[38;5;241m/\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mn_steps\n\u001b[1;32m    235\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m steps_output, M_loss\n",
      "File \u001b[0;32m/voyager/projects/jacobyhsi/.conda/envs/interpretabnet/lib/python3.10/site-packages/torch/fx/traceback.py:67\u001b[0m, in \u001b[0;36mformat_stack\u001b[0;34m()\u001b[0m\n\u001b[1;32m     64\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m [current_meta\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstack_trace\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m)]\n\u001b[1;32m     65\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     66\u001b[0m     \u001b[38;5;66;03m# fallback to traceback.format_stack()\u001b[39;00m\n\u001b[0;32m---> 67\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtraceback\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mformat_list\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtraceback\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mextract_stack\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/voyager/projects/jacobyhsi/.conda/envs/interpretabnet/lib/python3.10/traceback.py:39\u001b[0m, in \u001b[0;36mformat_list\u001b[0;34m(extracted_list)\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mformat_list\u001b[39m(extracted_list):\n\u001b[1;32m     28\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Format a list of tuples or FrameSummary objects for printing.\u001b[39;00m\n\u001b[1;32m     29\u001b[0m \n\u001b[1;32m     30\u001b[0m \u001b[38;5;124;03m    Given a list of tuples or FrameSummary objects as returned by\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     37\u001b[0m \u001b[38;5;124;03m    whose source text line is not None.\u001b[39;00m\n\u001b[1;32m     38\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m---> 39\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mStackSummary\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_list\u001b[49m\u001b[43m(\u001b[49m\u001b[43mextracted_list\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mformat()\n",
      "File \u001b[0;32m/voyager/projects/jacobyhsi/.conda/envs/interpretabnet/lib/python3.10/traceback.py:399\u001b[0m, in \u001b[0;36mStackSummary.from_list\u001b[0;34m(klass, a_list)\u001b[0m\n\u001b[1;32m    397\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m frame \u001b[38;5;129;01min\u001b[39;00m a_list:\n\u001b[1;32m    398\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(frame, FrameSummary):\n\u001b[0;32m--> 399\u001b[0m         result\u001b[38;5;241m.\u001b[39mappend(frame)\n\u001b[1;32m    400\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    401\u001b[0m         filename, lineno, name, line \u001b[38;5;241m=\u001b[39m frame\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "nd_na = [16, 32, 128]\n",
    "gammas = [1.0, 1.2, 1.5, 2.0]\n",
    "lambda_sparses = [0.001, 0.01, 0.1, 0.3]\n",
    "learn_r = [0.005, 0.01, 0.02, 0.025]\n",
    "\n",
    "opt_ndna = 32\n",
    "opt_gamma = 1.5\n",
    "opt_lambda = 0.001\n",
    "opt_lr = 0.025\n",
    "\n",
    "opt_nsteps = 4\n",
    "opt_reg_m = 4750\n",
    "\n",
    "ndna_test_acc = 0\n",
    "for ndna in nd_na:\n",
    "    clf = TabNetClassifier(\n",
    "        n_d=ndna,\n",
    "        n_a=ndna,\n",
    "        n_steps=opt_nsteps,\n",
    "        gamma=gammas[0],\n",
    "        lambda_sparse=lambda_sparses[0],\n",
    "        cat_idxs=cat_idxs,\n",
    "        cat_dims=cat_dims,\n",
    "        optimizer_params=dict(lr=learn_r[0]),\n",
    "        reg_m=opt_reg_m,\n",
    "        mask_type = 'softmax'\n",
    "    )\n",
    "\n",
    "    clf.fit(\n",
    "        X_train=x_train, y_train=y_train,\n",
    "        eval_set=[(x_train, y_train), (x_test, y_test)],\n",
    "        eval_name=['train', 'valid'], batch_size=256,\n",
    "        virtual_batch_size=256,\n",
    "        max_epochs=10, eval_metric=['auc']\n",
    "    )\n",
    "\n",
    "    y_pred = clf.predict_proba(x_test)\n",
    "    test_acc = roc_auc_score(y_score=y_pred[:,1], y_true=y_test)\n",
    "\n",
    "    if test_acc > ndna_test_acc:\n",
    "        opt_ndna = ndna\n",
    "        ndna_test_acc = test_acc\n",
    "        print(\"Optimum Hyperparameters Training\", [opt_ndna, opt_gamma, opt_lambda, opt_lr, opt_reg_m])\n",
    "\n",
    "gams_test_acc = 0\n",
    "for gams in gammas:\n",
    "    clf = TabNetClassifier(\n",
    "        n_d=opt_ndna,\n",
    "        n_a=opt_ndna,\n",
    "        n_steps=opt_nsteps,\n",
    "        gamma=gams,\n",
    "        lambda_sparse=lambda_sparses[0],\n",
    "        cat_idxs=cat_idxs,\n",
    "        cat_dims=cat_dims,\n",
    "        optimizer_params=dict(lr=learn_r[0]),\n",
    "        reg_m=opt_reg_m,\n",
    "        mask_type = 'softmax'\n",
    "    )\n",
    "\n",
    "    clf.fit(\n",
    "        X_train=x_train, y_train=y_train,\n",
    "        eval_set=[(x_train, y_train), (x_test, y_test)],\n",
    "        eval_name=['train', 'valid'], batch_size=256,\n",
    "        virtual_batch_size=256,\n",
    "        max_epochs=10, eval_metric=['accuracy']\n",
    "    )\n",
    "\n",
    "    y_pred = clf.predict_proba(x_test)\n",
    "    test_acc = roc_auc_score(y_score=y_pred[:,1], y_true=y_test)\n",
    "\n",
    "    if test_acc > gams_test_acc:\n",
    "        opt_gamma = gams\n",
    "        gams_test_acc = test_acc\n",
    "        print(\"Optimum Hyperparameters Training\", [opt_ndna, opt_nsteps, opt_gamma, opt_lambda, opt_lr, opt_reg_m])\n",
    "\n",
    "lamb_test_acc = 0\n",
    "for lambs in lambda_sparses:\n",
    "    clf = TabNetClassifier(\n",
    "        n_d=opt_ndna,\n",
    "        n_a=opt_ndna,\n",
    "        n_steps=opt_nsteps,\n",
    "        gamma=opt_gamma,\n",
    "        lambda_sparse=lambs,\n",
    "        cat_idxs=cat_idxs,\n",
    "        cat_dims=cat_dims,\n",
    "        optimizer_params=dict(lr=learn_r[0]),\n",
    "        reg_m=opt_reg_m,\n",
    "        mask_type = 'softmax'\n",
    "    )\n",
    "\n",
    "    clf.fit(\n",
    "        X_train=x_train, y_train=y_train,\n",
    "        eval_set=[(x_train, y_train), (x_test, y_test)],\n",
    "        eval_name=['train', 'valid'], batch_size=256,\n",
    "        virtual_batch_size=256,\n",
    "        max_epochs=10, eval_metric=['accuracy']\n",
    "    )\n",
    "\n",
    "    y_pred = clf.predict_proba(x_test)\n",
    "    test_acc = roc_auc_score(y_score=y_pred[:,1], y_true=y_test)\n",
    "    if test_acc > lamb_test_acc:\n",
    "        opt_lambda = lambs\n",
    "        lamb_test_acc = test_acc\n",
    "        print(\"Optimum Hyperparameters Training\", [opt_ndna, opt_nsteps, opt_gamma, opt_lambda, opt_lr, opt_reg_m])\n",
    "\n",
    "lr_test_accuracy = 0\n",
    "for lr in learn_r:\n",
    "    clf = TabNetClassifier(\n",
    "        n_d=opt_ndna,\n",
    "        n_a=opt_ndna,\n",
    "        n_steps=opt_nsteps,\n",
    "        gamma=opt_gamma,\n",
    "        lambda_sparse=opt_lambda,\n",
    "        cat_idxs=cat_idxs,\n",
    "        cat_dims=cat_dims,\n",
    "        optimizer_params=dict(lr=lr),\n",
    "        reg_m=opt_reg_m,\n",
    "        mask_type = 'softmax'\n",
    "    )\n",
    "\n",
    "    clf.fit(\n",
    "        X_train=x_train, y_train=y_train,\n",
    "        eval_set=[(x_train, y_train), (x_test, y_test)],\n",
    "        eval_name=['train', 'valid'], batch_size=256,\n",
    "        virtual_batch_size=256,\n",
    "        max_epochs=10, eval_metric=['accuracy']\n",
    "    )\n",
    "\n",
    "    y_pred = clf.predict_proba(x_test)\n",
    "    test_acc = roc_auc_score(y_score=y_pred[:,1], y_true=y_test)\n",
    "\n",
    "    if test_acc > lr_test_accuracy:\n",
    "        opt_lr = lr\n",
    "        lr_test_accuracy = test_acc\n",
    "        print(\"Optimum Hyperparameters Training\", [opt_ndna, opt_nsteps, opt_gamma, opt_lambda, opt_lr, opt_reg_m])\n",
    "\n",
    "print(\"Optimum Hyperparameters\", [opt_ndna, opt_nsteps, opt_gamma, opt_lambda, opt_lr, opt_reg_m])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optimized Hyperparameters\n",
    "opt_ndna = 16\n",
    "opt_nsteps = 4\n",
    "opt_gamma = 1.2\n",
    "opt_lambda = 0.001\n",
    "opt_lr = 0.025"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optimized Run with tuned $r_M = 4750$ for Syn3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device used cuda: 0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0  | loss: 21.11836| train_auc: 0.84381 | valid_auc: 0.84349 |  0:00:05s\n",
      "epoch 1  | loss: 20.8597 | train_auc: 0.87899 | valid_auc: 0.87941 |  0:00:10s\n",
      "epoch 2  | loss: 21.01706| train_auc: 0.89204 | valid_auc: 0.88811 |  0:00:15s\n",
      "epoch 3  | loss: 20.92565| train_auc: 0.8967  | valid_auc: 0.89135 |  0:00:20s\n",
      "epoch 4  | loss: 20.91524| train_auc: 0.90092 | valid_auc: 0.89249 |  0:00:25s\n",
      "epoch 5  | loss: 20.8521 | train_auc: 0.90385 | valid_auc: 0.89471 |  0:00:31s\n",
      "epoch 6  | loss: 20.79781| train_auc: 0.90564 | valid_auc: 0.89469 |  0:00:36s\n",
      "epoch 7  | loss: 20.84079| train_auc: 0.9049  | valid_auc: 0.89714 |  0:00:41s\n",
      "epoch 8  | loss: 20.70139| train_auc: 0.90345 | valid_auc: 0.89659 |  0:00:46s\n",
      "epoch 9  | loss: 20.80239| train_auc: 0.90518 | valid_auc: 0.89802 |  0:00:51s\n",
      "epoch 10 | loss: 20.80586| train_auc: 0.90446 | valid_auc: 0.89739 |  0:00:57s\n",
      "epoch 11 | loss: 20.82054| train_auc: 0.90452 | valid_auc: 0.89727 |  0:01:02s\n",
      "epoch 12 | loss: 20.73434| train_auc: 0.90453 | valid_auc: 0.89542 |  0:01:07s\n",
      "epoch 13 | loss: 20.78792| train_auc: 0.89835 | valid_auc: 0.89372 |  0:01:12s\n",
      "epoch 14 | loss: 20.58917| train_auc: 0.89824 | valid_auc: 0.89417 |  0:01:17s\n",
      "epoch 15 | loss: 20.59828| train_auc: 0.89678 | valid_auc: 0.89118 |  0:01:22s\n",
      "epoch 16 | loss: 20.66063| train_auc: 0.88747 | valid_auc: 0.88799 |  0:01:28s\n",
      "epoch 17 | loss: 20.57157| train_auc: 0.88667 | valid_auc: 0.88735 |  0:01:33s\n",
      "epoch 18 | loss: 20.72286| train_auc: 0.89356 | valid_auc: 0.89204 |  0:01:38s\n",
      "epoch 19 | loss: 20.57277| train_auc: 0.89541 | valid_auc: 0.89052 |  0:01:43s\n",
      "\n",
      "Early stopping occurred at epoch 19 with best_epoch = 9 and best_valid_auc = 0.89802\n",
      "Best weights from best epoch are automatically used!\n",
      "FINAL ROC AUC SCORE FOR Syn3 : 0.8980195982371406\n"
     ]
    }
   ],
   "source": [
    "# rm_lst = [10, 100, 1000, 10000, 100000]\n",
    "rm_lst = [4750]\n",
    "for rm in rm_lst:\n",
    "    opt_ndna = 16\n",
    "    opt_nsteps = 4\n",
    "    opt_gamma = 1.2\n",
    "    opt_lambda = 0.001\n",
    "    opt_lr = 0.025\n",
    "    opt_reg_m = rm\n",
    "\n",
    "    clf = TabNetClassifier(\n",
    "        n_d=opt_ndna,\n",
    "        n_a=opt_ndna,\n",
    "        n_steps=opt_nsteps,\n",
    "        gamma=opt_gamma,\n",
    "        lambda_sparse=opt_lambda,\n",
    "        cat_idxs=cat_idxs,\n",
    "        cat_dims=cat_dims,\n",
    "        optimizer_params=dict(lr=opt_lr),\n",
    "        mask_type = 'softmax',\n",
    "        reg_m=opt_reg_m\n",
    "    )\n",
    "    # max epoch 50\n",
    "    clf.fit(\n",
    "        X_train=x_train, y_train=y_train,\n",
    "        eval_set=[(x_train, y_train), (x_test, y_test)],\n",
    "        eval_name=['train', 'valid'],\n",
    "        max_epochs=50, eval_metric=['auc']\n",
    "    )\n",
    "\n",
    "    y_pred = clf.predict_proba(x_test)\n",
    "    test_acc = roc_auc_score(y_score=y_pred[:,1], y_true=y_test)\n",
    "    print(f\"FINAL ROC AUC SCORE FOR {data_type} : {test_acc}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Full results of synthetic experiments can be found in Appendix D of our paper: https://arxiv.org/pdf/2406.00426"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "interpretabnet",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}