Exp2_MNIST_sample_all_weights.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 310
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4511,
     "status": "ok",
     "timestamp": 1588324677970,
     "user": {
      "displayName": "Marius Hobbhahn",
      "photoUrl": "",
      "userId": "09428085039491522481"
     },
     "user_tz": -120
    },
    "id": "Py4m-IYBiUvO",
    "outputId": "841cca17-1299-4a06-c3ab-686a99002dad"
   },
   "outputs": [],
   "source": [
    "#!nvidia-smi\n",
    "#using a GeForce GTX1080 Ti for reproducibility for all timing experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3nOUiVSuk9CZ"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torch import nn, optim, autograd\n",
    "from torch.nn import functional as F\n",
    "from torch.distributions.multivariate_normal import MultivariateNormal\n",
    "from torch.autograd import Variable\n",
    "import numpy as np\n",
    "from sklearn.utils import shuffle as skshuffle\n",
    "from math import *\n",
    "from backpack import backpack, extend\n",
    "from backpack.extensions import KFAC, DiagHessian, DiagGGNMC\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import scipy\n",
    "from tqdm import tqdm, trange\n",
    "import pytest\n",
    "from DirLPA_utils import * \n",
    "import time\n",
    "from torch.distributions.normal import Normal\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "s = 123\n",
    "np.random.seed(s)\n",
    "torch.manual_seed(s)\n",
    "torch.cuda.manual_seed(s)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DXo1iPmgk9Co"
   },
   "outputs": [],
   "source": [
    "def LPADirNN(num_classes=10, num_LL=256):\n",
    "    \n",
    "    features = torch.nn.Sequential(\n",
    "        torch.nn.Conv2d(1, 32, 5),\n",
    "        torch.nn.ReLU(),\n",
    "        torch.nn.MaxPool2d(2,2),\n",
    "        torch.nn.Conv2d(32, 64, 5),\n",
    "        torch.nn.ReLU(),\n",
    "        torch.nn.MaxPool2d(2,2),\n",
    "        torch.nn.Flatten(),\n",
    "        torch.nn.Linear(4 * 4 * 64, num_LL), #changed from 500\n",
    "        torch.nn.Linear(num_LL, num_classes)  #changed from 500\n",
    "    )\n",
    "    return(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "R15L1O-Gk9Cz"
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE_TRAIN_MNIST = 32#128\n",
    "BATCH_SIZE_TEST_MNIST = 32#128\n",
    "MAX_ITER_MNIST = 6\n",
    "LR_TRAIN_MNIST = 10e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4TP39zZjk9C7"
   },
   "outputs": [],
   "source": [
    "MNIST_transform = torchvision.transforms.ToTensor()\n",
    "\n",
    "MNIST_train = torchvision.datasets.MNIST(\n",
    "        '~/data/mnist',\n",
    "        train=True,\n",
    "        download=True,\n",
    "        transform=MNIST_transform)\n",
    "\n",
    "mnist_train_loader = torch.utils.data.dataloader.DataLoader(\n",
    "    MNIST_train,\n",
    "    batch_size=BATCH_SIZE_TRAIN_MNIST,\n",
    "    shuffle=True\n",
    ")\n",
    "\n",
    "\n",
    "MNIST_test = torchvision.datasets.MNIST(\n",
    "        '~/data/mnist',\n",
    "        train=False,\n",
    "        download=True,\n",
    "        transform=MNIST_transform)\n",
    "\n",
    "mnist_test_loader = torch.utils.data.dataloader.DataLoader(\n",
    "    MNIST_test,\n",
    "    batch_size=BATCH_SIZE_TEST_MNIST,\n",
    "    shuffle=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "O6KhNJJ3k9DC"
   },
   "outputs": [],
   "source": [
    "mnist_model = LPADirNN(num_LL=256).cuda()\n",
    "loss_function = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "mnist_train_optimizer = torch.optim.Adam(mnist_model.parameters(), lr=1e-3, weight_decay=5e-4)\n",
    "MNIST_PATH = \"pretrained_weights/MNIST_pretrained_10_classes.pth\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZXIV3Ms6k9DK"
   },
   "outputs": [],
   "source": [
    "#Training routine\n",
    "\n",
    "def train(model, train_loader, optimizer, max_iter, path, verbose=True):\n",
    "    max_len = len(train_loader)\n",
    "\n",
    "    for iter in range(max_iter):\n",
    "        for batch_idx, (x, y) in enumerate(train_loader):\n",
    "            \n",
    "            x, y = x.cuda(), y.cuda()\n",
    "            \n",
    "            output = model(x)\n",
    "\n",
    "            accuracy = get_accuracy(output, y)\n",
    "\n",
    "            loss = loss_function(output, y)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            if verbose and batch_idx % 50 == 0:\n",
    "                print(\n",
    "                    \"Iteration {}; {}/{} \\t\".format(iter, batch_idx, max_len) +\n",
    "                    \"Minibatch Loss %.3f  \" % (loss) +\n",
    "                    \"Accuracy %.0f\" % (accuracy * 100) + \"%\"\n",
    "                )\n",
    "\n",
    "    print(\"saving model at: {}\".format(path))\n",
    "    torch.save(mnist_model.state_dict(), path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "H8neqsXnk9DR"
   },
   "outputs": [],
   "source": [
    "#train(mnist_model, mnist_train_loader, mnist_train_optimizer, MAX_ITER_MNIST, MNIST_PATH, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 200
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11701,
     "status": "ok",
     "timestamp": 1588324685272,
     "user": {
      "displayName": "Marius Hobbhahn",
      "photoUrl": "",
      "userId": "09428085039491522481"
     },
     "user_tz": -120
    },
    "id": "F8AhVaAkk9DX",
    "outputId": "966e26e9-84f2-4f0d-b234-8bbbb8d175a8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading model from: pretrained_weights/MNIST_pretrained_10_classes.pth\n",
      "Batch 0/313 \tAccuracy 100%\n",
      "Batch 10/313 \tAccuracy 97%\n",
      "Batch 20/313 \tAccuracy 94%\n",
      "Batch 30/313 \tAccuracy 97%\n",
      "Batch 40/313 \tAccuracy 97%\n",
      "Batch 50/313 \tAccuracy 97%\n",
      "Batch 60/313 \tAccuracy 100%\n",
      "Batch 70/313 \tAccuracy 100%\n",
      "Batch 80/313 \tAccuracy 100%\n",
      "Batch 90/313 \tAccuracy 97%\n",
      "Batch 100/313 \tAccuracy 100%\n",
      "Batch 110/313 \tAccuracy 94%\n",
      "Batch 120/313 \tAccuracy 97%\n",
      "Batch 130/313 \tAccuracy 97%\n",
      "Batch 140/313 \tAccuracy 100%\n",
      "Batch 150/313 \tAccuracy 97%\n",
      "Batch 160/313 \tAccuracy 100%\n",
      "Batch 170/313 \tAccuracy 100%\n",
      "Batch 180/313 \tAccuracy 100%\n",
      "Batch 190/313 \tAccuracy 97%\n",
      "Batch 200/313 \tAccuracy 100%\n",
      "Batch 210/313 \tAccuracy 100%\n",
      "Batch 220/313 \tAccuracy 100%\n",
      "Batch 230/313 \tAccuracy 100%\n",
      "Batch 240/313 \tAccuracy 100%\n",
      "Batch 250/313 \tAccuracy 100%\n",
      "Batch 260/313 \tAccuracy 100%\n",
      "Batch 270/313 \tAccuracy 100%\n",
      "Batch 280/313 \tAccuracy 100%\n",
      "Batch 290/313 \tAccuracy 100%\n",
      "Batch 300/313 \tAccuracy 100%\n",
      "Batch 310/313 \tAccuracy 100%\n",
      "overall test accuracy on MNIST: 99.00 %\n"
     ]
    }
   ],
   "source": [
    "#predict in distribution\n",
    "MNIST_PATH = \"pretrained_weights/MNIST_pretrained_10_classes.pth\"\n",
    "\n",
    "mnist_model = LPADirNN(num_LL=256).cuda()\n",
    "print(\"loading model from: {}\".format(MNIST_PATH))\n",
    "mnist_model.load_state_dict(torch.load(MNIST_PATH))\n",
    "mnist_model.eval()\n",
    "\n",
    "acc = []\n",
    "\n",
    "max_len = len(mnist_test_loader)\n",
    "for batch_idx, (x, y) in enumerate(mnist_test_loader):\n",
    "\n",
    "    x, y = x.cuda(), y.cuda()\n",
    "    output = mnist_model(x)\n",
    "\n",
    "    accuracy = get_accuracy(output, y)\n",
    "    if batch_idx % 10 == 0:\n",
    "        print(\n",
    "            \"Batch {}/{} \\t\".format(batch_idx, max_len) + \n",
    "            \"Accuracy %.0f\" % (accuracy * 100) + \"%\"\n",
    "        )\n",
    "    acc.append(accuracy)\n",
    "\n",
    "avg_acc = np.mean(acc)\n",
    "print('overall test accuracy on MNIST: {:.02f} %'.format(avg_acc * 100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XegYEGgyk9De"
   },
   "outputs": [],
   "source": [
    "# get all weight dists\n",
    "def get_Hessian_NN(model, train_loader, prec0, device='cpu', verbose=True):\n",
    "    lossfunc = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "    extend(lossfunc, debug=False)\n",
    "    extend(model, debug=False)\n",
    "\n",
    "    Hessian_diag = []\n",
    "    for param in model.parameters():\n",
    "        ps = param.size()\n",
    "        print(\"parameter size: \", ps)\n",
    "        Hessian_diag.append(torch.zeros(ps, device=device))\n",
    "\n",
    "    var0 = 1/prec0\n",
    "    max_len = len(train_loader)\n",
    "\n",
    "    with backpack(DiagHessian()):\n",
    "\n",
    "        for batch_idx, (x, y) in enumerate(train_loader):\n",
    "\n",
    "            if device == 'cuda':\n",
    "                x, y = x.float().cuda(), y.long().cuda()\n",
    "\n",
    "            model.zero_grad()\n",
    "            lossfunc(model(x), y).backward()\n",
    "\n",
    "            with torch.no_grad():\n",
    "                # Hessian of weight\n",
    "                for idx, param in enumerate(model.parameters()):\n",
    "\n",
    "                    H_ = param.diag_h\n",
    "                    #add prior here\n",
    "                    H_ += var0 * torch.ones(H_.size(), device=device)\n",
    "\n",
    "                    rho = 1-1/(batch_idx+1)\n",
    "\n",
    "                    Hessian_diag[idx] = rho*Hessian_diag[idx] + (1-rho)* H_\n",
    "            \n",
    "            if verbose and batch_idx%20==0:\n",
    "                print(\"Batch: {}/{}\".format(batch_idx, max_len))\n",
    "    \n",
    "    return(Hessian_diag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vHYyizO3k9Dl"
   },
   "outputs": [],
   "source": [
    "# sample from weights\n",
    "def get_param_dists(model, Hessian):\n",
    "    param_dists = []\n",
    "    for idx, param in enumerate(model.parameters()):\n",
    "        ps = param.size()\n",
    "        mu = param.view(-1)\n",
    "        Sigma = Hessian[idx].view(-1)\n",
    "        print(\"mu size: \", mu.size())\n",
    "        print(\"Sigma size: \", Sigma.size())\n",
    "        dist = Normal(mu, Sigma)\n",
    "        param_dists.append(dist)\n",
    "        \n",
    "    return(param_dists)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kZ_b15Phk9Dx"
   },
   "source": [
    "## Load additional data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "p5Dp07GZk9Dz"
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE_TEST_FMNIST = 32#128\n",
    "BATCH_SIZE_TEST_KMNIST = 32#128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "j1Enw3h1k9D5"
   },
   "outputs": [],
   "source": [
    "FMNIST_test = torchvision.datasets.FashionMNIST(\n",
    "        '~/data/fmnist', train=False, download=True,\n",
    "        transform=MNIST_transform)   #torchvision.transforms.ToTensor())\n",
    "\n",
    "FMNIST_test_loader = torch.utils.data.DataLoader(\n",
    "    FMNIST_test,\n",
    "    batch_size=BATCH_SIZE_TEST_FMNIST, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bktersHdk9D_"
   },
   "outputs": [],
   "source": [
    "KMNIST_test = torchvision.datasets.KMNIST(\n",
    "        '~/data/kmnist', train=False, download=True,\n",
    "        transform=MNIST_transform)\n",
    "\n",
    "KMNIST_test_loader = torch.utils.data.DataLoader(\n",
    "    KMNIST_test,\n",
    "    batch_size=BATCH_SIZE_TEST_KMNIST, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OLmD7N_7k9EF"
   },
   "outputs": [],
   "source": [
    "\"\"\"Load notMNIST\"\"\"\n",
    "\n",
    "import os\n",
    "import numpy as np\n",
    "import torch\n",
    "from PIL import Image\n",
    "from torch.utils.data.dataset import Dataset\n",
    "from matplotlib.pyplot import imread\n",
    "from torch import Tensor\n",
    "\n",
    "\"\"\"\n",
    "Loads the train/test set. \n",
    "Every image in the dataset is 28x28 pixels and the labels are numbered from 0-9\n",
    "for A-J respectively.\n",
    "Set root to point to the Train/Test folders.\n",
    "\"\"\"\n",
    "\n",
    "# Creating a sub class of torch.utils.data.dataset.Dataset\n",
    "class notMNIST(Dataset):\n",
    "\n",
    "    # The init method is called when this class will be instantiated\n",
    "    def __init__(self, root, transform):\n",
    "        \n",
    "        #super(notMNIST, self).__init__(root, transform=transform)\n",
    "\n",
    "        self.transform = transform\n",
    "        \n",
    "        Images, Y = [], []\n",
    "        folders = os.listdir(root)\n",
    "\n",
    "        for folder in folders:\n",
    "            folder_path = os.path.join(root, folder)\n",
    "            for ims in os.listdir(folder_path):\n",
    "                try:\n",
    "                    img_path = os.path.join(folder_path, ims)\n",
    "                    Images.append(np.array(imread(img_path)))\n",
    "                    Y.append(ord(folder) - 65)  # Folders are A-J so labels will be 0-9\n",
    "                except:\n",
    "                    # Some images in the dataset are damaged\n",
    "                    print(\"File {}/{} is broken\".format(folder, ims))\n",
    "        data = [(x, y) for x, y in zip(Images, Y)]\n",
    "        self.data = data\n",
    "        self.targets = torch.Tensor(Y)\n",
    "\n",
    "    # The number of items in the dataset\n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "\n",
    "    # The Dataloader is a generator that repeatedly calls the getitem method.\n",
    "    # getitem is supposed to return (X, Y) for the specified index.\n",
    "    def __getitem__(self, index):\n",
    "        img = self.data[index][0]\n",
    "\n",
    "        if self.transform is not None:\n",
    "            img = self.transform(img)\n",
    "            \n",
    "        # Input for Conv2D should be Channels x Height x Width\n",
    "        img_tensor = Tensor(img).view(1, 28, 28).float()\n",
    "        label = self.data[index][1]\n",
    "        return (img_tensor, label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 31761,
     "status": "ok",
     "timestamp": 1588324705378,
     "user": {
      "displayName": "Marius Hobbhahn",
      "photoUrl": "",
      "userId": "09428085039491522481"
     },
     "user_tz": -120
    },
    "id": "3NrXr7Yjk9EL",
    "outputId": "9dbce2e9-a31e-452b-d068-1b906be6ef53"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File F/Q3Jvc3NvdmVyIEJvbGRPYmxpcXVlLnR0Zg==.png is broken\n",
      "File A/RGVtb2NyYXRpY2FCb2xkT2xkc3R5bGUgQm9sZC50dGY=.png is broken\n"
     ]
    }
   ],
   "source": [
    "root = os.path.expanduser('~/data')\n",
    "\n",
    "# Instantiating the notMNIST dataset class we created\n",
    "notMNIST_test = notMNIST(root=os.path.join(root, 'notMNIST_small'),\n",
    "                               transform=MNIST_transform)\n",
    "\n",
    "# Creating a dataloader\n",
    "not_mnist_test_loader = torch.utils.data.dataloader.DataLoader(\n",
    "                            dataset=notMNIST_test,\n",
    "                            batch_size=BATCH_SIZE_TEST_KMNIST,\n",
    "                            shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cAtmj_7Pk9EW"
   },
   "outputs": [],
   "source": [
    "targets = MNIST_test.targets.numpy()\n",
    "targets_FMNIST = FMNIST_test.targets.numpy()\n",
    "targets_notMNIST = notMNIST_test.targets.numpy().astype(int)\n",
    "targets_KMNIST = KMNIST_test.targets.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4GI8Cq3kqINE"
   },
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def predict_MAP(model, test_loader, num_samples=1, cuda=False):\n",
    "    py = []\n",
    "\n",
    "    max_len = int(np.ceil(len(test_loader.dataset)/len(test_loader)))\n",
    "    for batch_idx, (x, y) in enumerate(test_loader):\n",
    "\n",
    "        if cuda:\n",
    "            x, y = x.cuda(), y.cuda()\n",
    "\n",
    "        py_ = 0\n",
    "        for _ in range(num_samples):\n",
    "            py_ += torch.softmax(model(x), 1)\n",
    "        py_ /= num_samples\n",
    "\n",
    "        py.append(py_)\n",
    "    return torch.cat(py, dim=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zBC4Su-uk9EW"
   },
   "source": [
    "# MAP estimate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cnDA049ck9Eb"
   },
   "outputs": [],
   "source": [
    "mnist_test_in_MAP = predict_MAP(mnist_model, mnist_test_loader, cuda=True).cpu().numpy()\n",
    "mnist_test_out_fmnist_MAP = predict_MAP(mnist_model, FMNIST_test_loader, cuda=True).cpu().numpy()\n",
    "mnist_test_out_notMNIST_MAP = predict_MAP(mnist_model, not_mnist_test_loader, cuda=True).cpu().numpy()\n",
    "mnist_test_out_KMNIST_MAP = predict_MAP(mnist_model, KMNIST_test_loader, cuda=True).cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VuHpPvHik9Ef"
   },
   "outputs": [],
   "source": [
    "acc_in_MAP, prob_correct_in_MAP, ent_in_MAP, MMC_in_MAP = get_in_dist_values(mnist_test_in_MAP, targets)\n",
    "acc_out_FMNIST_MAP, prob_correct_out_FMNIST_MAP, ent_out_FMNIST_MAP, MMC_out_FMNIST_MAP, auroc_out_FMNIST_MAP = get_out_dist_values(mnist_test_in_MAP, mnist_test_out_fmnist_MAP, targets_FMNIST)\n",
    "acc_out_notMNIST_MAP, prob_correct_out_notMNIST_MAP, ent_out_notMNIST_MAP, MMC_out_notMNIST_MAP, auroc_out_notMNIST_MAP = get_out_dist_values(mnist_test_in_MAP, mnist_test_out_notMNIST_MAP, targets_notMNIST)\n",
    "acc_out_KMNIST_MAP, prob_correct_out_KMNIST_MAP, ent_out_KMNIST_MAP, MMC_out_KMNIST_MAP, auroc_out_KMNIST_MAP = get_out_dist_values(mnist_test_in_MAP, mnist_test_out_KMNIST_MAP, targets_KMNIST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 90
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 33872,
     "status": "ok",
     "timestamp": 1588324707518,
     "user": {
      "displayName": "Marius Hobbhahn",
      "photoUrl": "",
      "userId": "09428085039491522481"
     },
     "user_tz": -120
    },
    "id": "LF_LO_FNk9El",
    "outputId": "53488ea8-fdbf-4ee9-d5a3-79c7586f3ab3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[In, MAP, mnist] Accuracy: 0.990; average entropy: 0.035;     MMC: 0.989; Prob @ correct: 0.100\n",
      "[Out-MAP, KFAC, FMNIST] Accuracy: 0.108; Average entropy: 1.390;    MMC: 0.514; AUROC: 0.993; Prob @ correct: 0.100\n",
      "[Out-MAP, KFAC, notMNIST] Accuracy: 0.108; Average entropy: 0.808;    MMC: 0.709; AUROC: 0.956; Prob @ correct: 0.100\n",
      "[Out-MAP, KFAC, KMNIST] Accuracy: 0.083; Average entropy: 0.861;    MMC: 0.688; AUROC: 0.973; Prob @ correct: 0.100\n"
     ]
    }
   ],
   "source": [
    "print_in_dist_values(acc_in_MAP, prob_correct_in_MAP, ent_in_MAP, MMC_in_MAP, 'mnist', 'MAP')\n",
    "print_out_dist_values(acc_out_FMNIST_MAP, prob_correct_out_FMNIST_MAP, ent_out_FMNIST_MAP, MMC_out_FMNIST_MAP, auroc_out_FMNIST_MAP, 'FMNIST', 'MAP')\n",
    "print_out_dist_values(acc_out_notMNIST_MAP, prob_correct_out_notMNIST_MAP, ent_out_notMNIST_MAP, MMC_out_notMNIST_MAP, auroc_out_notMNIST_MAP, 'notMNIST', 'MAP')\n",
    "print_out_dist_values(acc_out_KMNIST_MAP, prob_correct_out_KMNIST_MAP, ent_out_KMNIST_MAP, MMC_out_KMNIST_MAP, auroc_out_KMNIST_MAP, 'KMNIST', 'MAP')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 33868,
     "status": "ok",
     "timestamp": 1588324707520,
     "user": {
      "displayName": "Marius Hobbhahn",
      "photoUrl": "",
      "userId": "09428085039491522481"
     },
     "user_tz": -120
    },
    "id": "sHBdSW8Kk9Et",
    "outputId": "7c39d3d5-1281-4c8c-f775-e760a824ee66"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000, 10)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "print(np.shape(mnist_test_in_MAP))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 163
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 33862,
     "status": "ok",
     "timestamp": 1588324707521,
     "user": {
      "displayName": "Marius Hobbhahn",
      "photoUrl": "",
      "userId": "09428085039491522481"
     },
     "user_tz": -120
    },
    "id": "dFv6EyWlk9Ez",
    "outputId": "8710e6ee-bbf4-4c6a-e05d-ddd618ad9af0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.992 with std 0.000\n",
      "MMC in: 0.990 with std 0.000\n",
      "MMC out fmnist: 0.552 with std 0.000\n",
      "MMC out notmnist: 0.709 with std 0.000\n",
      "MMC out kmnist: 0.679 with std 0.000\n",
      "AUROC out fmnist: 0.991 with std 0.000\n",
      "AUROC out notmnist: 0.960 with std 0.000\n",
      "AUROC out kmnist: 0.979 with std 0.000\n"
     ]
    }
   ],
   "source": [
    "#MAP estimate\n",
    "#seeds are 123,124,125,126,127\n",
    "acc_in = [0.992]\n",
    "mmc_in = [0.990]\n",
    "mmc_out_fmnist = [0.552]\n",
    "mmc_out_notmnist = [0.709]\n",
    "mmc_out_kmnist = [0.679]\n",
    "\n",
    "auroc_out_fmnist = [0.991]\n",
    "auroc_out_notmnist = [0.960]\n",
    "auroc_out_kmnist = [0.979]\n",
    "\n",
    "print(\"accuracy: {:.03f} with std {:.03f}\".format(np.mean(acc_in), np.std(acc_in)))\n",
    "\n",
    "print(\"MMC in: {:.03f} with std {:.03f}\".format(np.mean(mmc_in), np.std(mmc_in)))\n",
    "print(\"MMC out fmnist: {:.03f} with std {:.03f}\".format(np.mean(mmc_out_fmnist), np.std(mmc_out_fmnist)))\n",
    "print(\"MMC out notmnist: {:.03f} with std {:.03f}\".format(np.mean(mmc_out_notmnist), np.std(mmc_out_notmnist)))\n",
    "print(\"MMC out kmnist: {:.03f} with std {:.03f}\".format(np.mean(mmc_out_kmnist), np.std(mmc_out_kmnist)))\n",
    "\n",
    "print(\"AUROC out fmnist: {:.03f} with std {:.03f}\".format(np.mean(auroc_out_fmnist), np.std(auroc_out_fmnist)))\n",
    "print(\"AUROC out notmnist: {:.03f} with std {:.03f}\".format(np.mean(auroc_out_notmnist), np.std(auroc_out_notmnist)))\n",
    "print(\"AUROC out kmnist: {:.03f} with std {:.03f}\".format(np.mean(auroc_out_kmnist), np.std(auroc_out_kmnist)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "BpLksQf2k9E4"
   },
   "source": [
    "# Diag Hessian sample all weights estimate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 602
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 65729,
     "status": "ok",
     "timestamp": 1588324739394,
     "user": {
      "displayName": "Marius Hobbhahn",
      "photoUrl": "",
      "userId": "09428085039491522481"
     },
     "user_tz": -120
    },
    "id": "USPP24d8k9E5",
    "outputId": "0ce1eba5-75d8-4708-82a2-4346801b250f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "parameter size:  torch.Size([32, 1, 5, 5])\n",
      "parameter size:  torch.Size([32])\n",
      "parameter size:  torch.Size([64, 32, 5, 5])\n",
      "parameter size:  torch.Size([64])\n",
      "parameter size:  torch.Size([256, 1024])\n",
      "parameter size:  torch.Size([256])\n",
      "parameter size:  torch.Size([10, 256])\n",
      "parameter size:  torch.Size([10])\n",
      "Batch: 0/1875\n",
      "Batch: 20/1875\n",
      "Batch: 40/1875\n",
      "Batch: 60/1875\n",
      "Batch: 80/1875\n",
      "Batch: 100/1875\n",
      "Batch: 120/1875\n",
      "Batch: 140/1875\n",
      "Batch: 160/1875\n",
      "Batch: 180/1875\n",
      "Batch: 200/1875\n",
      "Batch: 220/1875\n",
      "Batch: 240/1875\n",
      "Batch: 260/1875\n",
      "Batch: 280/1875\n",
      "Batch: 300/1875\n",
      "Batch: 320/1875\n",
      "Batch: 340/1875\n",
      "Batch: 360/1875\n",
      "Batch: 380/1875\n",
      "Batch: 400/1875\n",
      "Batch: 420/1875\n",
      "Batch: 440/1875\n",
      "Batch: 460/1875\n",
      "Batch: 480/1875\n",
      "Batch: 500/1875\n",
      "Batch: 520/1875\n",
      "Batch: 540/1875\n",
      "Batch: 560/1875\n",
      "Batch: 580/1875\n",
      "Batch: 600/1875\n",
      "Batch: 620/1875\n",
      "Batch: 640/1875\n",
      "Batch: 660/1875\n",
      "Batch: 680/1875\n",
      "Batch: 700/1875\n",
      "Batch: 720/1875\n",
      "Batch: 740/1875\n",
      "Batch: 760/1875\n",
      "Batch: 780/1875\n",
      "Batch: 800/1875\n",
      "Batch: 820/1875\n",
      "Batch: 840/1875\n",
      "Batch: 860/1875\n",
      "Batch: 880/1875\n",
      "Batch: 900/1875\n",
      "Batch: 920/1875\n",
      "Batch: 940/1875\n",
      "Batch: 960/1875\n",
      "Batch: 980/1875\n",
      "Batch: 1000/1875\n",
      "Batch: 1020/1875\n",
      "Batch: 1040/1875\n",
      "Batch: 1060/1875\n",
      "Batch: 1080/1875\n",
      "Batch: 1100/1875\n",
      "Batch: 1120/1875\n",
      "Batch: 1140/1875\n",
      "Batch: 1160/1875\n",
      "Batch: 1180/1875\n",
      "Batch: 1200/1875\n",
      "Batch: 1220/1875\n",
      "Batch: 1240/1875\n",
      "Batch: 1260/1875\n",
      "Batch: 1280/1875\n",
      "Batch: 1300/1875\n",
      "Batch: 1320/1875\n",
      "Batch: 1340/1875\n",
      "Batch: 1360/1875\n",
      "Batch: 1380/1875\n",
      "Batch: 1400/1875\n",
      "Batch: 1420/1875\n",
      "Batch: 1440/1875\n",
      "Batch: 1460/1875\n",
      "Batch: 1480/1875\n",
      "Batch: 1500/1875\n",
      "Batch: 1520/1875\n",
      "Batch: 1540/1875\n",
      "Batch: 1560/1875\n",
      "Batch: 1580/1875\n",
      "Batch: 1600/1875\n",
      "Batch: 1620/1875\n",
      "Batch: 1640/1875\n",
      "Batch: 1660/1875\n",
      "Batch: 1680/1875\n",
      "Batch: 1700/1875\n",
      "Batch: 1720/1875\n",
      "Batch: 1740/1875\n",
      "Batch: 1760/1875\n",
      "Batch: 1780/1875\n",
      "Batch: 1800/1875\n",
      "Batch: 1820/1875\n",
      "Batch: 1840/1875\n",
      "Batch: 1860/1875\n"
     ]
    }
   ],
   "source": [
    "HessianNN_mnist = get_Hessian_NN(mnist_model, mnist_train_loader, prec0=30, device='cuda', verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 310
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 65726,
     "status": "ok",
     "timestamp": 1588324739397,
     "user": {
      "displayName": "Marius Hobbhahn",
      "photoUrl": "",
      "userId": "09428085039491522481"
     },
     "user_tz": -120
    },
    "id": "RoCcle_Vk9E-",
    "outputId": "8261840b-d9e7-4b98-9fc8-8eeda641f862"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mu size:  torch.Size([800])\n",
      "Sigma size:  torch.Size([800])\n",
      "mu size:  torch.Size([32])\n",
      "Sigma size:  torch.Size([32])\n",
      "mu size:  torch.Size([51200])\n",
      "Sigma size:  torch.Size([51200])\n",
      "mu size:  torch.Size([64])\n",
      "Sigma size:  torch.Size([64])\n",
      "mu size:  torch.Size([262144])\n",
      "Sigma size:  torch.Size([262144])\n",
      "mu size:  torch.Size([256])\n",
      "Sigma size:  torch.Size([256])\n",
      "mu size:  torch.Size([2560])\n",
      "Sigma size:  torch.Size([2560])\n",
      "mu size:  torch.Size([10])\n",
      "Sigma size:  torch.Size([10])\n"
     ]
    }
   ],
   "source": [
    "mnist_param_dist = get_param_dists(mnist_model, HessianNN_mnist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pjqJd0llk9FD"
   },
   "outputs": [],
   "source": [
    "# predict with different samples\n",
    "def get_stacked_results(param_dists, data_loader, network, num_samples=100, verbose=False, timing=False):\n",
    "\n",
    "    results = []\n",
    "    if timing:\n",
    "        time_sum = 0\n",
    "\n",
    "    test_model = network(num_classes=10).cuda()\n",
    "    test_model.eval()\n",
    "    #new_state_dict = fmnist_model.state_dict()\n",
    "    new_state_dict = dict()\n",
    "    for s in range(num_samples):\n",
    "        #torch.manual_seed(s)\n",
    "        t0 = time.time()\n",
    "        for idx, (p_name, param) in enumerate(test_model.named_parameters()):\n",
    "            ps = param.size()\n",
    "            new_weights = param_dists[idx].sample().view(ps)\n",
    "            new_state_dict[p_name] = new_weights\n",
    "\n",
    "        #print(new_state_dict.keys())\n",
    "        test_model.load_state_dict(new_state_dict)\n",
    "        \n",
    "        inter_results = []\n",
    "        for batch_idx, (x, y) in enumerate(data_loader):\n",
    "            x = x.cuda()\n",
    "            y_pred_logits = test_model(x).detach()\n",
    "            #print(y_pred_logits)\n",
    "            y_pred = F.softmax(y_pred_logits, dim=1).detach()\n",
    "            inter_results.append(y_pred)\n",
    "            if verbose and batch_idx % 100 == 0:\n",
    "                print(\"s: {}; batch_idx: {}\".format(s, batch_idx))\n",
    "                \n",
    "        results.append(torch.cat(inter_results, dim=0))\n",
    "        t1 = time.time()\n",
    "        if timing:\n",
    "            time_sum += t1-t0\n",
    "    \n",
    "    stacked_results = torch.stack(results)\n",
    "    print(\"time sum: \", time_sum)\n",
    "    return(stacked_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 134780,
     "status": "ok",
     "timestamp": 1588324808460,
     "user": {
      "displayName": "Marius Hobbhahn",
      "photoUrl": "",
      "userId": "09428085039491522481"
     },
     "user_tz": -120
    },
    "id": "zjPkLM1sk9FH",
    "outputId": "eeeba1df-6e85-4639-ce89-9405ef0b6931"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time sum:  75.36639976501465\n"
     ]
    }
   ],
   "source": [
    "stacked_results_mnist = get_stacked_results(param_dists=mnist_param_dist, data_loader=mnist_test_loader\\\n",
    "                                            ,network=LPADirNN, num_samples=100, verbose=False, timing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 205244,
     "status": "ok",
     "timestamp": 1588324878931,
     "user": {
      "displayName": "Marius Hobbhahn",
      "photoUrl": "",
      "userId": "09428085039491522481"
     },
     "user_tz": -120
    },
    "id": "XiInw3qYk9FM",
    "outputId": "6de77487-8736-4089-9fcf-10ed028d8ee0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time sum:  76.33020257949829\n"
     ]
    }
   ],
   "source": [
    "stacked_results_fmnist = get_stacked_results(param_dists=mnist_param_dist, data_loader=FMNIST_test_loader\\\n",
    "                                            ,network=LPADirNN, num_samples=100, verbose=False, timing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 242866,
     "status": "ok",
     "timestamp": 1588324916558,
     "user": {
      "displayName": "Marius Hobbhahn",
      "photoUrl": "",
      "userId": "09428085039491522481"
     },
     "user_tz": -120
    },
    "id": "CKHT0-Wdk9FP",
    "outputId": "1674a08e-c9b4-4b36-8fe9-da92760ef024"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time sum:  42.24465465545654\n"
     ]
    }
   ],
   "source": [
    "stacked_results_notmnist = get_stacked_results(param_dists=mnist_param_dist, data_loader=not_mnist_test_loader\\\n",
    "                                            ,network=LPADirNN, num_samples=100, verbose=False, timing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 312178,
     "status": "ok",
     "timestamp": 1588324985877,
     "user": {
      "displayName": "Marius Hobbhahn",
      "photoUrl": "",
      "userId": "09428085039491522481"
     },
     "user_tz": -120
    },
    "id": "Ty6fZfY4k9FT",
    "outputId": "e53212e2-356c-4b90-dd8a-e3eb29b9aa33"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time sum:  75.95980525016785\n"
     ]
    }
   ],
   "source": [
    "stacked_results_kmnist = get_stacked_results(param_dists=mnist_param_dist, data_loader=KMNIST_test_loader\\\n",
    "                                            ,network=LPADirNN, num_samples=100, verbose=False, timing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "eVaq7u6hk9FZ"
   },
   "outputs": [],
   "source": [
    "results_mnist = torch.mean(stacked_results_mnist, dim=0).cpu().numpy()\n",
    "results_fmnist = torch.mean(stacked_results_fmnist, dim=0).cpu().numpy()\n",
    "results_notmnist = torch.mean(stacked_results_notmnist, dim=0).cpu().numpy()\n",
    "results_kmnist = torch.mean(stacked_results_kmnist, dim=0).cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tNaE6PZck9Fc"
   },
   "outputs": [],
   "source": [
    "acc_in_S, prob_correct_in_S, ent_in_S, MMC_in_S = get_in_dist_values(results_mnist, targets)\n",
    "acc_out_FMNIST_S, prob_correct_out_FMNIST_S, ent_out_FMNIST_S, MMC_out_FMNIST_S, auroc_out_FMNIST_S = get_out_dist_values(results_mnist, results_fmnist, targets_FMNIST)\n",
    "acc_out_notMNIST_S, prob_correct_out_notMNIST_S, ent_out_notMNIST_S, MMC_out_notMNIST_S, auroc_out_notMNIST_S = get_out_dist_values(results_mnist, results_notmnist, targets_notMNIST)\n",
    "acc_out_KMNIST_S, prob_correct_out_KMNIST_S, ent_out_KMNIST_S, MMC_out_KMNIST_S, auroc_out_KMNIST_S = get_out_dist_values(results_mnist, results_kmnist, targets_KMNIST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 90
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 312167,
     "status": "ok",
     "timestamp": 1588324985882,
     "user": {
      "displayName": "Marius Hobbhahn",
      "photoUrl": "",
      "userId": "09428085039491522481"
     },
     "user_tz": -120
    },
    "id": "kq1LPFQRk9Fh",
    "outputId": "5547d72f-b605-42e0-d07b-53ff393bfc77"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[In, Sampling full, mnist] Accuracy: 0.989; average entropy: 0.071;     MMC: 0.979; Prob @ correct: 0.100\n",
      "[Out-Sampling full, KFAC, FMNIST] Accuracy: 0.090; Average entropy: 1.486;    MMC: 0.473; AUROC: 0.991; Prob @ correct: 0.100\n",
      "[Out-Sampling full, KFAC, notMNIST] Accuracy: 0.124; Average entropy: 0.978;    MMC: 0.647; AUROC: 0.954; Prob @ correct: 0.100\n",
      "[Out-Sampling full, KFAC, KMNIST] Accuracy: 0.086; Average entropy: 0.997;    MMC: 0.633; AUROC: 0.966; Prob @ correct: 0.100\n"
     ]
    }
   ],
   "source": [
    "print_in_dist_values(acc_in_S, prob_correct_in_S, ent_in_S, MMC_in_S, 'mnist', 'Sampling full')\n",
    "print_out_dist_values(acc_out_FMNIST_S, prob_correct_out_FMNIST_S, ent_out_FMNIST_S, MMC_out_FMNIST_S, auroc_out_FMNIST_S, 'FMNIST', 'Sampling full')\n",
    "print_out_dist_values(acc_out_notMNIST_S, prob_correct_out_notMNIST_S, ent_out_notMNIST_S, MMC_out_notMNIST_S, auroc_out_notMNIST_S, 'notMNIST', 'Sampling full')\n",
    "print_out_dist_values(acc_out_KMNIST_S, prob_correct_out_KMNIST_S, ent_out_KMNIST_S, MMC_out_KMNIST_S, auroc_out_KMNIST_S, 'KMNIST', 'Sampling full')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 236
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 631,
     "status": "ok",
     "timestamp": 1588325049227,
     "user": {
      "displayName": "Marius Hobbhahn",
      "photoUrl": "",
      "userId": "09428085039491522481"
     },
     "user_tz": -120
    },
    "id": "kvHcfwYbwLyU",
    "outputId": "59b1266d-50f8-47ac-b8d2-ddc07eeb39db"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sampling time in: 76.279 with std 0.917\n",
      "Sampling time out fmnist: 76.233 with std 0.269\n",
      "Sampling time out notmnist: 42.591 with std 0.291\n",
      "Sampling time out kmnist: 75.975 with std 0.273\n",
      "accuracy: 0.989 with std 0.000\n",
      "MMC in: 0.979 with std 0.000\n",
      "MMC out fmnist: 0.465 with std 0.004\n",
      "MMC out notmnist: 0.645 with std 0.001\n",
      "MMC out kmnist: 0.632 with std 0.001\n",
      "AUROC out fmnist: 0.992 with std 0.000\n",
      "AUROC out notmnist: 0.955 with std 0.001\n",
      "AUROC out kmnist: 0.967 with std 0.001\n"
     ]
    }
   ],
   "source": [
    "#Diag Sampling with 100 samples when sampling all weights\n",
    "import numpy as np\n",
    "#seeds are 123,124,125,126,127\n",
    "time_sampling_all_in = [77.30279994010925, 75.88909602165222, 77.45202136039734, 75.38714742660522, 75.36639976501465]\n",
    "time_sampling_all_out_fmnist = [75.75277590751648, 76.20959711074829, 76.57610511779785, 76.29606461524963, 76.33020257949829]\n",
    "time_sampling_all_out_notmnist = [42.952558755874634, 42.86951684951782, 42.28128266334534, 42.60728096961975, 42.24465465545654]\n",
    "time_sampling_all_out_kmnist = [76.0160961151123, 75.4875876903534, 76.32327890396118, 76.08931422233582, 75.95980525016785]\n",
    "\n",
    "acc_in = [0.989, 0.989, 0.990, 0.989, 0.989]\n",
    "mmc_in = [0.979, 0.980, 0.980, 0.979, 0.979]\n",
    "mmc_out_fmnist = [0.463, 0.465, 0.461, 0.465, 0.473]\n",
    "mmc_out_notmnist = [0.644, 0.644, 0.645, 0.645, 0.647]\n",
    "mmc_out_kmnist = [0.632, 0.631, 0.635, 0.631, 0.633]\n",
    "\n",
    "auroc_out_fmnist = [0.992, 0.992, 0.992, 0.991, 0.991]\n",
    "auroc_out_notmnist = [0.956, 0.955, 0.955, 0.954, 0.954]\n",
    "auroc_out_kmnist = [0.968, 0.968, 0.966, 0.968, 0.966]\n",
    "\n",
    "print(\"Sampling time in: {:.03f} with std {:.03f}\".format(np.mean(time_sampling_all_in), np.std(time_sampling_all_in)))\n",
    "print(\"Sampling time out fmnist: {:.03f} with std {:.03f}\".format(np.mean(time_sampling_all_out_fmnist), np.std(time_sampling_all_out_fmnist)))\n",
    "print(\"Sampling time out notmnist: {:.03f} with std {:.03f}\".format(np.mean(time_sampling_all_out_notmnist), np.std(time_sampling_all_out_notmnist)))\n",
    "print(\"Sampling time out kmnist: {:.03f} with std {:.03f}\".format(np.mean(time_sampling_all_out_kmnist), np.std(time_sampling_all_out_kmnist)))\n",
    "\n",
    "print(\"accuracy: {:.03f} with std {:.03f}\".format(np.mean(acc_in), np.std(acc_in)))\n",
    "\n",
    "print(\"MMC in: {:.03f} with std {:.03f}\".format(np.mean(mmc_in), np.std(mmc_in)))\n",
    "print(\"MMC out fmnist: {:.03f} with std {:.03f}\".format(np.mean(mmc_out_fmnist), np.std(mmc_out_fmnist)))\n",
    "print(\"MMC out notmnist: {:.03f} with std {:.03f}\".format(np.mean(mmc_out_notmnist), np.std(mmc_out_notmnist)))\n",
    "print(\"MMC out kmnist: {:.03f} with std {:.03f}\".format(np.mean(mmc_out_kmnist), np.std(mmc_out_kmnist)))\n",
    "\n",
    "print(\"AUROC out fmnist: {:.03f} with std {:.03f}\".format(np.mean(auroc_out_fmnist), np.std(auroc_out_fmnist)))\n",
    "print(\"AUROC out notmnist: {:.03f} with std {:.03f}\".format(np.mean(auroc_out_notmnist), np.std(auroc_out_notmnist)))\n",
    "print(\"AUROC out kmnist: {:.03f} with std {:.03f}\".format(np.mean(auroc_out_kmnist), np.std(auroc_out_kmnist)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 236
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 312158,
     "status": "ok",
     "timestamp": 1588324985886,
     "user": {
      "displayName": "Marius Hobbhahn",
      "photoUrl": "",
      "userId": "09428085039491522481"
     },
     "user_tz": -120
    },
    "id": "howYBhkik9Fl",
    "outputId": "b7c8c1a0-9cf7-44b7-8649-6bfebc119f09"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sampling time in: 1131.086 with std 66.344\n",
      "Sampling time out fmnist: 1116.892 with std 65.585\n",
      "Sampling time out notmnist: 447.418 with std 28.362\n",
      "Sampling time out kmnist: 1100.796 with std 63.696\n",
      "accuracy: 0.992 with std 0.000\n",
      "MMC in: 0.989 with std 0.000\n",
      "MMC out fmnist: 0.540 with std 0.000\n",
      "MMC out notmnist: 0.701 with std 0.000\n",
      "MMC out kmnist: 0.672 with std 0.000\n",
      "AUROC out fmnist: 0.992 with std 0.000\n",
      "AUROC out notmnist: 0.961 with std 0.000\n",
      "AUROC out kmnist: 0.978 with std 0.000\n"
     ]
    }
   ],
   "source": [
    "#Diag Sampling with 1000 samples when sampling all weights\n",
    "import numpy as np\n",
    "#seeds are 123,124,125,126,127\n",
    "time_sampling_all_in = [1175.2953972816467, 1188.364794254303, 1121.3945801258087, 1164.1343219280243, 1006.2414374351501]\n",
    "time_sampling_all_out_fmnist = [1170.4062821865082, 1176.4168524742126, 1094.737497329712, 1143.832529783249, 999.0646016597748]\n",
    "time_sampling_all_out_notmnist = [478.3686921596527, 473.6282377243042, 452.92081594467163, 429.9301836490631, 402.24077248573303]\n",
    "time_sampling_all_out_kmnist = [1162.3914835453033, 1155.9036095142365, 1123.0117349624634, 1072.0317740440369, 990.6405141353607]\n",
    "\n",
    "acc_in = [0.992, 0.992, 0.992, 0.992, 0.992]\n",
    "mmc_in = [0.989, 0.989, 0.989, 0.989, 0.989]\n",
    "mmc_out_fmnist = [0.540, 0.540, 0.540, 0.539, 0.539]\n",
    "mmc_out_notmnist = [0.701, 0.701, 0.701, 0.702, 0.702]\n",
    "mmc_out_kmnist = [0.672, 0.672, 0.672, 0.672, 0.672]\n",
    "\n",
    "auroc_out_fmnist = [0.992, 0.992, 0.992, 0.992, 0.992]\n",
    "auroc_out_notmnist = [0.960, 0.961, 0.961, 0.961, 0.960]\n",
    "auroc_out_kmnist = [0.978, 0.978, 0.978, 0.978, 0.978]\n",
    "\n",
    "print(\"Sampling time in: {:.03f} with std {:.03f}\".format(np.mean(time_sampling_all_in), np.std(time_sampling_all_in)))\n",
    "print(\"Sampling time out fmnist: {:.03f} with std {:.03f}\".format(np.mean(time_sampling_all_out_fmnist), np.std(time_sampling_all_out_fmnist)))\n",
    "print(\"Sampling time out notmnist: {:.03f} with std {:.03f}\".format(np.mean(time_sampling_all_out_notmnist), np.std(time_sampling_all_out_notmnist)))\n",
    "print(\"Sampling time out kmnist: {:.03f} with std {:.03f}\".format(np.mean(time_sampling_all_out_kmnist), np.std(time_sampling_all_out_kmnist)))\n",
    "\n",
    "print(\"accuracy: {:.03f} with std {:.03f}\".format(np.mean(acc_in), np.std(acc_in)))\n",
    "\n",
    "print(\"MMC in: {:.03f} with std {:.03f}\".format(np.mean(mmc_in), np.std(mmc_in)))\n",
    "print(\"MMC out fmnist: {:.03f} with std {:.03f}\".format(np.mean(mmc_out_fmnist), np.std(mmc_out_fmnist)))\n",
    "print(\"MMC out notmnist: {:.03f} with std {:.03f}\".format(np.mean(mmc_out_notmnist), np.std(mmc_out_notmnist)))\n",
    "print(\"MMC out kmnist: {:.03f} with std {:.03f}\".format(np.mean(mmc_out_kmnist), np.std(mmc_out_kmnist)))\n",
    "\n",
    "print(\"AUROC out fmnist: {:.03f} with std {:.03f}\".format(np.mean(auroc_out_fmnist), np.std(auroc_out_fmnist)))\n",
    "print(\"AUROC out notmnist: {:.03f} with std {:.03f}\".format(np.mean(auroc_out_notmnist), np.std(auroc_out_notmnist)))\n",
    "print(\"AUROC out kmnist: {:.03f} with std {:.03f}\".format(np.mean(auroc_out_kmnist), np.std(auroc_out_kmnist)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nODVol1yk9Fp"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Exp2_MNIST_sample_all_weights.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}