make_histograms.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import eval_paths\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Q_to_subscript(s):\n",
    "    if 'GOODQ' in s:\n",
    "        match = re.search(r'\\d+', s)\n",
    "        name = r'GOOD\\textsubscript{' + match.group(0) + '}'\n",
    "        return name\n",
    "    else:\n",
    "        return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward(a):\n",
    "    a = np.deg2rad(a)\n",
    "    return np.rad2deg(np.log(np.abs(np.tan(a) + 1.0 / np.cos(a))))\n",
    "\n",
    "\n",
    "def inverse(a):\n",
    "    a = np.deg2rad(a)\n",
    "    return np.rad2deg(np.arctan(np.sinh(a)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_hist(confs, color):\n",
    "    ymax = len(confs)\n",
    "    y_change = 0.2\n",
    "    def forward(a):\n",
    "        return (a <= y_change*ymax) * a + (a > y_change*ymax) * (y_change*ymax + 0.1*(a-y_change*ymax))\n",
    "    def inverse(b):\n",
    "        return (b <= y_change*ymax)*b + (b > y_change*ymax) * y_change*ymax + 1/0.1*(b-y_change*ymax)\n",
    "    fig = plt.hist(confs, color=color, bins=50, align='mid', range=(0.1, 1.0))\n",
    "    plt.ylim(ymin=0, ymax=1.0*ymax)\n",
    "    plt.yscale('function', functions=(forward, inverse))\n",
    "    plt.xlim(xmin=0.09, xmax=1.02)\n",
    "    plt.xticks(np.arange(0.1, 1.01, 0.3))\n",
    "    if 5000 < ymax < 20000:\n",
    "        plt.yticks(np.concatenate([np.arange(0, ymax*y_change+1, 500), np.arange(4000, ymax-1000, 2000), [ymax]]))\n",
    "    elif 20000 < ymax < 25000:\n",
    "        plt.yticks(np.concatenate([np.arange(0, ymax*y_change+1, 1000), np.arange(8000, ymax-2000, 4000), [ymax]]))\n",
    "    elif 25000 < ymax < 40000:\n",
    "        plt.yticks(np.concatenate([np.arange(0, ymax*y_change+1, 1500), np.arange(12000, ymax-3000, 6000), [ymax]]))\n",
    "    elif 100 < ymax < 1000:\n",
    "        plt.yticks(np.concatenate([np.arange(0, ymax*y_change+1, 20), np.arange(100, ymax-30, 100), [ymax]]))\n",
    "    plt.axvline(confs.mean(), color='k', linestyle=\"-\", dashes=(1, 3), linewidth=2)\n",
    "    #plt.axvline(pred_in_confidences.mean(), color='w', linestyle=\"-\", dashes=(2, 6), linewidth=2)\n",
    "    plt.axhline(y_change*ymax, color='#DDDDDD', linestyle=\"--\", linewidth=1)\n",
    "    #dotted_line2 = lines.Line2D([], [], linewidth=5, linestyle=\"-\", dashes=(5, 4), color='black')\n",
    "    #save_path = model_folder + f'/sample_images/{short_name}{in_name}_in_pred_confidences_hist'\n",
    "    #plt.savefig(save_path, bbox_inches='tight')\n",
    "    #plt.show()\n",
    "    #plt.close()\n",
    "    return fig\n",
    "    #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_log_cifar10_filedest = 'evals/2020-10-14_16:34:38.log'\n",
    "in_name_c10 = 'CIFAR10'\n",
    "out_names_c10 = ['CIFAR100', 'SVHN', 'LSUN', 'Noise_Dataset_noise_uniform']\n",
    "model_list_c10 = ['OE', 'ACET', 'GOODQ40', 'GOODQ80', 'GOODQ90', 'GOODQ100']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_log_svhn_filedest = 'evals/2020-10-14_18:14:40.log'\n",
    "in_name_svhn = 'SVHN'\n",
    "out_names_svhn = ['CIFAR100', 'CIFAR10', 'LSUN', 'Noise_Dataset_noise_uniform']\n",
    "model_list_svhn = ['OE', 'ACET', 'GOODQ40', 'GOODQ80', 'GOODQ90', 'GOODQ100']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_log_mnist_filedest = 'evals/2020-10-15_15:13:11.log'\n",
    "in_name_mnist = 'MNIST'\n",
    "out_names_mnist = ['FashionMNIST', 'EMNIST', 'CIFAR10', 'Noise_Dataset_noise_uniform']\n",
    "model_list_mnist = ['Plain', 'OE', 'ACET', 'GOODQ40', 'GOODQ80', 'GOODQ90', 'GOODQ100']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_histograms(eval_log_filedest, in_name, out_names, model_list):\n",
    "    eval_log = open(eval_log_filedest, 'r')\n",
    "    model_rows = {}\n",
    "    for l in list(eval_log):\n",
    "        l = l[:-1]\n",
    "        if l[:len('Evaluation outputs saved in')] == 'Evaluation outputs saved in':\n",
    "            shortname = eval_paths.get_shortname(l) \n",
    "            if shortname not in model_list:\n",
    "                continue\n",
    "            values_folder = l.split(' ')[-1] + '/values/'\n",
    "            histsave_folder =  'evals/plots/histograms_final/'\n",
    "            os.makedirs(histsave_folder, exist_ok=True)\n",
    "            if shortname != 'Plain':\n",
    "                pred_in_confidences = np.loadtxt(values_folder + f'{shortname}_{in_name}_pred_in_confidences.txt')\n",
    "            else:\n",
    "                pred_in_confidences = np.loadtxt(values_folder + f'plain_{in_name}_pred_in_confidences.txt')\n",
    "            fig = plot_hist(pred_in_confidences, '#00CC44')\n",
    "            filename = f'{in_name}-{shortname}_{in_name}_pred_confidences_hist.png'\n",
    "            save_path = histsave_folder + filename\n",
    "            plt.savefig(save_path, bbox_inches='tight')\n",
    "            #plt.show()\n",
    "            plt.close()\n",
    "            print(rf'{Q_to_subscript(shortname)}')\n",
    "            print(rf' &  \\includegraphics[width=1.0\\linewidth]{{images/histograms_final/{filename}}}')\n",
    "            for n in out_names:\n",
    "                #print(n)\n",
    "                #pred_out_confidences = np.loadtxt(values_folder + f'{shortname}_{n}_pred_out_confidences.txt')\n",
    "                #fig = plot_hist(pred_out_confidences, '#4400DD')\n",
    "                #plt.show()\n",
    "                #plt.close()\n",
    "                if shortname != 'Plain':\n",
    "                    ub_out_confidences = np.loadtxt(values_folder + f'{shortname}{n}_ub_el_out_confidences.txt')\n",
    "                else:\n",
    "                    ub_out_confidences = np.loadtxt(values_folder + f'plain{n}_ub_el_out_confidences.txt')\n",
    "                fig = plot_hist(ub_out_confidences, '#4400DD')\n",
    "                filename = f'{in_name}-{shortname}_{n}_ub_confidences_hist.png'\n",
    "                save_path = histsave_folder + filename\n",
    "                plt.savefig(save_path, bbox_inches='tight')\n",
    "                #plt.show()\n",
    "                plt.close()\n",
    "                print(rf' &  \\includegraphics[width=1.0\\linewidth]{{images/histograms_final/{filename}}}')\n",
    "            print('\\\\\\\\')"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "make_histograms(eval_log_cifar10_filedest, in_name_c10, out_names_c10, model_list_c10)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "make_histograms(eval_log_svhn_filedest, in_name_svhn, out_names_svhn, model_list_svhn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plain\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-Plain_MNIST_pred_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-Plain_FashionMNIST_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-Plain_EMNIST_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-Plain_CIFAR10_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-Plain_Noise_Dataset_noise_uniform_ub_confidences_hist.png}\n",
      "\\\\\n",
      "OE\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-OE_MNIST_pred_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-OE_FashionMNIST_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-OE_EMNIST_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-OE_CIFAR10_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-OE_Noise_Dataset_noise_uniform_ub_confidences_hist.png}\n",
      "\\\\\n",
      "ACET\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-ACET_MNIST_pred_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-ACET_FashionMNIST_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-ACET_EMNIST_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-ACET_CIFAR10_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-ACET_Noise_Dataset_noise_uniform_ub_confidences_hist.png}\n",
      "\\\\\n",
      "GOOD\\textsubscript{40}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ40_MNIST_pred_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ40_FashionMNIST_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ40_EMNIST_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ40_CIFAR10_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ40_Noise_Dataset_noise_uniform_ub_confidences_hist.png}\n",
      "\\\\\n",
      "GOOD\\textsubscript{80}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ80_MNIST_pred_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ80_FashionMNIST_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ80_EMNIST_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ80_CIFAR10_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ80_Noise_Dataset_noise_uniform_ub_confidences_hist.png}\n",
      "\\\\\n",
      "GOOD\\textsubscript{90}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ90_MNIST_pred_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ90_FashionMNIST_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ90_EMNIST_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ90_CIFAR10_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ90_Noise_Dataset_noise_uniform_ub_confidences_hist.png}\n",
      "\\\\\n",
      "GOOD\\textsubscript{100}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ100_MNIST_pred_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ100_FashionMNIST_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ100_EMNIST_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ100_CIFAR10_ub_confidences_hist.png}\n",
      " &  \\includegraphics[width=1.0\\linewidth]{images/histograms_final/MNIST-GOODQ100_Noise_Dataset_noise_uniform_ub_confidences_hist.png}\n",
      "\\\\\n"
     ]
    }
   ],
   "source": [
    "make_histograms(eval_log_mnist_filedest, in_name_mnist, out_names_mnist, model_list_mnist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def latex_len5(s):\n",
    "    if len(s) < 5:\n",
    "        return '\\\\phantom{' + (5-len(s))*'0' + '}' + s\n",
    "    else:\n",
    "        return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_conf_table(eval_log_filedest, in_name, out_names, model_list):\n",
    "    eval_log = open(eval_log_filedest, 'r')\n",
    "    model_rows = {}\n",
    "    for l in list(eval_log):\n",
    "        l = l[:-1]\n",
    "        if l[:len('Evaluation outputs saved in')] == 'Evaluation outputs saved in':\n",
    "            shortname = eval_paths.get_shortname(l) \n",
    "            if shortname not in model_list:\n",
    "                continue\n",
    "            values_folder = l.split(' ')[-1] + '/values/'\n",
    "            if shortname != 'Plain':\n",
    "                pred_in_confidences = np.loadtxt(values_folder + f'{shortname}_{in_name}_pred_in_confidences.txt')\n",
    "            else:\n",
    "                pred_in_confidences = np.loadtxt(values_folder + f'plain_{in_name}_pred_in_confidences.txt')\n",
    "            model_rows[shortname] = rf'{Q_to_subscript(shortname)}'\n",
    "            model_rows[shortname] += f'\\n & {100*pred_in_confidences.mean():.1f}'\n",
    "            for n in out_names:\n",
    "                if shortname != 'Plain':\n",
    "                    ub_out_confidences = np.loadtxt(values_folder + f'{shortname}{n}_ub_el_out_confidences.txt')\n",
    "                    pred_out_confidences = np.loadtxt(values_folder + f'{shortname}{n}_pred_out_confidences.txt')\n",
    "                else:\n",
    "                    ub_out_confidences = np.loadtxt(values_folder + f'plain{n}_ub_el_out_confidences.txt')\n",
    "                    pred_out_confidences = np.loadtxt(values_folder + f'plain{n}_pred_out_confidences.txt')\n",
    "                model_rows[shortname] += f'\\n & ' + f'{100*pred_out_confidences.mean():.1f}' + '\\ / ' + latex_len5(f'{100*ub_out_confidences.mean():.1f}')\n",
    "            model_rows[shortname] += '\\\\\\\\'\n",
    "    for model in model_list:\n",
    "        print(model_rows[model])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_list_conf = ['Plain', 'CEDA', 'OE', 'ACET', 'GOODQ0', 'GOODQ20', 'GOODQ40', 'GOODQ60', 'GOODQ80', 'GOODQ90', 'GOODQ95', 'GOODQ100']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['FashionMNIST', 'EMNIST', 'CIFAR10', 'Noise_Dataset_noise_uniform']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out_names_mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plain\n",
      " & 99.7\n",
      " & 79.2\\ / 100.0\n",
      " & 91.5\\ / 100.0\n",
      " & 77.2\\ / 100.0\n",
      " & 79.6\\ / 100.0\\\\\n",
      "CEDA\n",
      " & 99.7\n",
      " & 22.0\\ / 100.0\n",
      " & 88.3\\ / 100.0\n",
      " & 10.0\\ / 100.0\n",
      " & 10.0\\ / 100.0\\\\\n",
      "OE\n",
      " & 99.7\n",
      " & 25.4\\ / 100.0\n",
      " & 87.9\\ / 100.0\n",
      " & 10.1\\ / 100.0\n",
      " & 10.0\\ / 100.0\\\\\n",
      "ACET\n",
      " & 99.6\n",
      " & 12.3\\ / 100.0\n",
      " & 75.0\\ / 100.0\n",
      " & 10.0\\ / 100.0\n",
      " & 10.0\\ / 100.0\\\\\n",
      "GOOD\\textsubscript{0}\n",
      " & 99.7\n",
      " & 20.6\\ / 100.0\n",
      " & 87.9\\ / 100.0\n",
      " & 10.0\\ / 100.0\n",
      " & 10.0\\ / 100.0\\\\\n",
      "GOOD\\textsubscript{20}\n",
      " & 99.5\n",
      " & 19.4\\ / \\phantom{0}93.0\n",
      " & 70.2\\ / 100.0\n",
      " & 10.0\\ / \\phantom{0}76.6\n",
      " & 10.0\\ / \\phantom{0}10.0\\\\\n",
      "GOOD\\textsubscript{40}\n",
      " & 99.3\n",
      " & 17.7\\ / \\phantom{0}76.8\n",
      " & 58.2\\ / 100.0\n",
      " & 10.0\\ / \\phantom{0}43.7\n",
      " & 10.0\\ / \\phantom{0}10.0\\\\\n",
      "GOOD\\textsubscript{60}\n",
      " & 99.2\n",
      " & 15.8\\ / \\phantom{0}66.0\n",
      " & 51.7\\ / 100.0\n",
      " & 10.0\\ / \\phantom{0}24.8\n",
      " & 10.0\\ / \\phantom{0}10.0\\\\\n",
      "GOOD\\textsubscript{80}\n",
      " & 99.0\n",
      " & 16.3\\ / \\phantom{0}55.1\n",
      " & 40.7\\ / \\phantom{0}98.6\n",
      " & 10.0\\ / \\phantom{0}15.7\n",
      " & 10.0\\ / \\phantom{0}10.0\\\\\n",
      "GOOD\\textsubscript{90}\n",
      " & 98.8\n",
      " & 14.2\\ / \\phantom{0}47.5\n",
      " & 38.3\\ / \\phantom{0}98.2\n",
      " & 10.0\\ / \\phantom{0}12.7\n",
      " & 10.0\\ / \\phantom{0}10.0\\\\\n",
      "GOOD\\textsubscript{95}\n",
      " & 98.7\n",
      " & 13.1\\ / \\phantom{0}42.6\n",
      " & 32.2\\ / \\phantom{0}98.2\n",
      " & 10.0\\ / \\phantom{0}11.6\n",
      " & 10.0\\ / \\phantom{0}10.0\\\\\n",
      "GOOD\\textsubscript{100}\n",
      " & 98.4\n",
      " & 10.8\\ / \\phantom{0}40.8\n",
      " & 27.1\\ / \\phantom{0}99.2\n",
      " & 10.0\\ / \\phantom{0}11.0\n",
      " & 10.0\\ / \\phantom{0}10.0\\\\\n"
     ]
    }
   ],
   "source": [
    "make_conf_table(eval_log_mnist_filedest, in_name_mnist, out_names_mnist, model_list_conf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}