CP_NA_Semi-synthetic_Plots.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import generation as gen\n",
    "import utils\n",
    "import files\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm.autonotebook import tqdm\n",
    "import datasets\n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import matplotlib as mpl\n",
    "from mpl_toolkits.axes_grid1.inset_locator import zoomed_inset_axes, mark_inset\n",
    "from matplotlib.backends.backend_pgf import FigureCanvasPgf\n",
    "mpl.backend_bases.register_backend('pdf', FigureCanvasPgf)\n",
    "import matplotlib.lines as mlines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "size=19\n",
    "mpl.rcParams.update({\n",
    "    \"pgf.texsystem\": \"pdflatex\",\n",
    "    'font.family': 'serif',\n",
    "    'font.serif': 'Times',\n",
    "    'text.usetex': True,\n",
    "    'pgf.rcfonts': False,\n",
    "    'font.size': size,\n",
    "    'axes.labelsize':size,\n",
    "    'axes.titlesize':size,\n",
    "    'figure.titlesize':size,\n",
    "    'xtick.labelsize':size,\n",
    "    'ytick.labelsize':size,\n",
    "    'legend.fontsize':size,\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets_names = ['meps_19', 'bio', 'concrete', 'bike']\n",
    "dataset_base_path = \"./data/cqr_datasets/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets_sizes_normal = {'meps_19': {'train': 1000, 'cal': 500, 'test_pattern': 100},\n",
    "                         'meps_20': {'train': 1000, 'cal': 500, 'test_pattern': 100},\n",
    "                         'meps_21': {'train': 1000, 'cal': 500, 'test_pattern': 100},\n",
    "                         'bio': {'train': 1000, 'cal': 500, 'test_pattern': 100},\n",
    "                         'concrete': {'train': 630, 'cal': 200, 'test_pattern': 100},\n",
    "                         'bike': {'train': 1000, 'cal': 500, 'test_pattern': 100}}\n",
    "\n",
    "datasets_sizes_small = {'meps_19': {'train': 500, 'cal': 250, 'test_pattern': 100},\n",
    "                        'meps_20': {'train': 500, 'cal': 250, 'test_pattern': 100},\n",
    "                        'meps_21': {'train': 500, 'cal': 250, 'test_pattern': 100},\n",
    "                        'bio': {'train': 500, 'cal': 250, 'test_pattern': 100},\n",
    "                        'concrete': {'train': 330, 'cal': 100, 'test_pattern': 100},\n",
    "                        'bike': {'train': 500, 'cal': 250, 'test_pattern': 100}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_rep = 100\n",
    "alpha = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prob_missing = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "imputation = 'iterative_ridge'\n",
    "mask = 'Yes'\n",
    "protection = 'No'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "methods = ['CQR', 'CQR_MDA']#QR_TrainCal\n",
    "basemodel = 'NNet'\n",
    "exacts = [False, True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_pipelines = []\n",
    "for method in methods: \n",
    "    if method == 'CQR_MDA':\n",
    "        name_temp = files.get_name_method(method, basemodel, mask=mask, protection=protection, exact=True)\n",
    "        if not name_temp in name_pipelines:\n",
    "            name_pipelines.append(name_temp)\n",
    "    name_temp = files.get_name_method(method, basemodel, mask=mask, protection=protection, exact=False)\n",
    "    if not name_temp in name_pipelines:\n",
    "        name_pipelines.append(name_temp)\n",
    "    \n",
    "current_pipeline = method+'_'+basemodel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if mask == 'Yes':\n",
    "    dict_methods = {'QR_TrainCal_NNet_Mask': 'QR',\n",
    "                    'CQR_NNet_Mask': 'CQR', \n",
    "                    'CQR_MDA_Exact_NNet_Mask': 'CQR-MDA-Exact',\n",
    "                    'CQR_MDA_Nested_NNet_Mask': 'CQR-MDA-Nested'}\n",
    "else:\n",
    "    dict_methods = {'QR_TrainCal_NNet': 'QR',\n",
    "                    'CQR_NNet': 'CQR', \n",
    "                    'CQR_MDA_Exact_NNet': 'CQR-MDA-Exact',\n",
    "                    'CQR_MDA_Nested_NNet': 'CQR-MDA-Nested'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sizes = ['small', 'normal']\n",
    "sizes = ['normal']\n",
    "        \n",
    "dict_cov = dict.fromkeys(datasets_names)\n",
    "dict_len = dict.fromkeys(datasets_names)\n",
    "dict_cov_patterns = dict.fromkeys(datasets_names)\n",
    "dict_len_patterns = dict.fromkeys(datasets_names)\n",
    "dict_cov_worst = dict.fromkeys(datasets_names)\n",
    "dict_len_worst = dict.fromkeys(datasets_names)\n",
    "dict_cov_best = dict.fromkeys(datasets_names)\n",
    "dict_len_best = dict.fromkeys(datasets_names)\n",
    "\n",
    "for dataset_name in datasets_names:\n",
    "\n",
    "    dict_cov[dataset_name] = dict.fromkeys(sizes)\n",
    "    dict_len[dataset_name] = dict.fromkeys(sizes)\n",
    "    dict_cov_patterns[dataset_name] = dict.fromkeys(sizes)\n",
    "    dict_len_patterns[dataset_name] = dict.fromkeys(sizes)\n",
    "    dict_cov_worst[dataset_name] = dict.fromkeys(sizes)\n",
    "    dict_len_worst[dataset_name] = dict.fromkeys(sizes)\n",
    "    dict_cov_best[dataset_name] = dict.fromkeys(sizes)\n",
    "    dict_len_best[dataset_name] = dict.fromkeys(sizes)\n",
    "\n",
    "    for size in sizes:\n",
    "\n",
    "        dict_cov[dataset_name][size] = dict.fromkeys(name_pipelines)\n",
    "        dict_len[dataset_name][size] = dict.fromkeys(name_pipelines)\n",
    "        dict_cov_patterns[dataset_name][size] = dict.fromkeys(name_pipelines)\n",
    "        dict_len_patterns[dataset_name][size] = dict.fromkeys(name_pipelines)\n",
    "        dict_cov_worst[dataset_name][size] = dict.fromkeys(name_pipelines)\n",
    "        dict_len_worst[dataset_name][size] = dict.fromkeys(name_pipelines)\n",
    "        dict_cov_best[dataset_name][size] = dict.fromkeys(name_pipelines)\n",
    "        dict_len_best[dataset_name][size] = dict.fromkeys(name_pipelines)\n",
    "\n",
    "        for pipeline in name_pipelines:\n",
    "            dict_cov_patterns[dataset_name][size][pipeline] = {}\n",
    "            dict_len_patterns[dataset_name][size][pipeline] = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset_name in datasets_names:\n",
    "    \n",
    "    print(dataset_name)\n",
    "    \n",
    "    df, target, var_missing = datasets.GetDataset(dataset_name, dataset_base_path)\n",
    "\n",
    "    d = df.shape[1]-1\n",
    "\n",
    "    params_missing = {}\n",
    "    params_missing['var_missing'] = var_missing\n",
    "    params_missing['prob_missing'] = prob_missing\n",
    "\n",
    "    for size in sizes:\n",
    "        \n",
    "        if size == 'normal':\n",
    "            train_size = datasets_sizes_normal[dataset_name]['train']\n",
    "            cal_size = datasets_sizes_normal[dataset_name]['cal']\n",
    "            \n",
    "            if dataset_name == 'concrete':\n",
    "                nb_sample_pattern = datasets_sizes_normal[dataset_name]['test_pattern']\n",
    "                params_test = {'iid':{'test_size': 200}, 'fixed_nb_sample_pattern':{'nb_sample_pattern': nb_sample_pattern}}\n",
    "            else:\n",
    "                nb_sample_pattern = datasets_sizes_normal[dataset_name]['test_pattern']\n",
    "                params_test = {'iid':{'test_size': 2000}, 'fixed_nb_sample_pattern':{'nb_sample_pattern': nb_sample_pattern}}\n",
    "            params_test = gen.process_test(params_test, d=d, params_missing=params_missing)\n",
    "\n",
    "            max_test_size = np.max(params_test['test_size'])\n",
    "            \n",
    "        elif size == 'small':\n",
    "            train_size = datasets_sizes_small[dataset_name]['train']\n",
    "            cal_size = datasets_sizes_small[dataset_name]['cal']\n",
    "            \n",
    "            if dataset_name == 'concrete':\n",
    "                nb_sample_pattern = datasets_sizes_small[dataset_name]['test_pattern']\n",
    "                params_test = {'iid':{'test_size': 200}, 'fixed_nb_sample_pattern':{'nb_sample_pattern': nb_sample_pattern}}\n",
    "            else:\n",
    "                nb_sample_pattern = datasets_sizes_small[dataset_name]['test_pattern']\n",
    "                params_test = {'iid':{'test_size': 2000}, 'fixed_nb_sample_pattern':{'nb_sample_pattern': nb_sample_pattern}}\n",
    "            params_test = gen.process_test(params_test, d=d, params_missing=params_missing)\n",
    "\n",
    "            max_test_size = np.max(params_test['test_size'])\n",
    "\n",
    "        name_method = []\n",
    "\n",
    "        for pipeline in name_pipelines:\n",
    "\n",
    "            name_method = np.append(name_method, '_'.join([imputation, pipeline]))\n",
    "\n",
    "            key = -1\n",
    "\n",
    "            data, results = utils.get_data_results(pipeline, train_size, cal_size, params_test, n_rep, imputation=imputation,\n",
    "                                                   dataset = dataset_name,\n",
    "                                                   params_missing=params_missing,\n",
    "                                                   parent_results='results', parent_data='data', extension='xz')\n",
    "\n",
    "            contains, lengths = utils.compute_PI_metrics(data, results, 'iid')\n",
    "\n",
    "            metrics = utils.compute_metrics_cond(n_rep, data, results, 'fixed_nb_sample_pattern', cond='Pattern')\n",
    "\n",
    "            dict_cov[dataset_name][size][pipeline] = np.mean(contains, axis=1)\n",
    "            dict_len[dataset_name][size][pipeline] = np.mean(lengths, axis=1)\n",
    "\n",
    "            for key_pattern in list(metrics.keys()):\n",
    "\n",
    "                dict_cov_patterns[dataset_name][size][pipeline][key_pattern] = metrics[key_pattern]['avg_cov']\n",
    "                dict_len_patterns[dataset_name][size][pipeline][key_pattern] = metrics[key_pattern]['avg_len']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset_name in datasets_names:\n",
    "    \n",
    "    for size in sizes:\n",
    "        \n",
    "        for pipeline in name_pipelines:\n",
    "\n",
    "            avg_cov = dict.fromkeys(dict_cov_patterns[dataset_name][size][pipeline].keys())\n",
    "\n",
    "            for key in list(avg_cov.keys()):\n",
    "                avg_cov[key] = np.mean(dict_cov_patterns[dataset_name][size][pipeline][key])\n",
    "\n",
    "            worst_index = np.argmin(list(avg_cov.values()))\n",
    "            worst_key = list(avg_cov.keys())[worst_index]\n",
    "            dict_cov_worst[dataset_name][size][pipeline] = dict_cov_patterns[dataset_name][size][pipeline][worst_key]\n",
    "            dict_len_worst[dataset_name][size][pipeline] = dict_len_patterns[dataset_name][size][pipeline][worst_key]\n",
    "\n",
    "            best_index = np.argmax(list(avg_cov.values()))\n",
    "            best_key = list(avg_cov.keys())[best_index]\n",
    "            dict_cov_best[dataset_name][size][pipeline] = dict_cov_patterns[dataset_name][size][pipeline][best_key]\n",
    "            dict_len_best[dataset_name][size][pipeline] = dict_len_patterns[dataset_name][size][pipeline][best_key]  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colors_blindness = sns.color_palette(\"colorblind\")\n",
    "colors_blindness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax1, ax2, ax3, ax4) = plt.subplots(1, 4, figsize=(15,4.3))\n",
    "axes = {'meps_19': ax1, 'bio': ax2, 'concrete': ax3, 'bike': ax4}\n",
    "\n",
    "dict_markers = {'iid': 'd',\n",
    "                'worst': \"v\", \n",
    "                'best': \"^\"}\n",
    "\n",
    "dict_datasets = {'meps_19': r'\\texttt{meps_19} ($d=139$, $l=5$)','meps_20': r'\\texttt{meps} ($d=139$, $l=5$)',\n",
    "                 'meps_21': r'\\texttt{meps} ($d=139$, $l=5$)','bio': r'\\texttt{bio} ($d=9$, $l=9$)',\n",
    "                 'concrete': r'\\texttt{concrete} ($d=8$, $l=8$)','bike': r'\\texttt{bike} ($d=18$, $l=4$)'}\n",
    "alphas_meps = {'meps_19': 1, 'meps_20':0.7, 'meps_21': 0.4}\n",
    "dict_colors = {'QR_TrainCal_NNet_Mask': colors_blindness[2],\n",
    "               'CQR_NNet_Mask': colors_blindness[1], \n",
    "               'CQR_MDA_Exact_NNet_Mask': colors_blindness[4],\n",
    "               'CQR_MDA_Nested_NNet_Mask': colors_blindness[9]}\n",
    "\n",
    "dict_coverages = {'iid': 'Marginal', 'worst': 'Lowest', 'best': 'Highest'}\n",
    "\n",
    "marker_size = 60\n",
    "\n",
    "small = False\n",
    "medium = True\n",
    "\n",
    "name_pipelines_to_plot = name_pipelines\n",
    "    \n",
    "for dataset_name in datasets_names:\n",
    "    \n",
    "    ax = axes[dataset_name]\n",
    "    ax.set_title(dict_datasets[dataset_name])\n",
    "\n",
    "    alpha_data = 1\n",
    "\n",
    "    for pipeline in name_pipelines_to_plot:\n",
    "\n",
    "        if medium:\n",
    "            ax.scatter(np.mean(dict_cov[dataset_name]['normal'][pipeline]),\n",
    "                       np.mean(dict_len[dataset_name]['normal'][pipeline]), \n",
    "                       marker=dict_markers['iid'], color=dict_colors[pipeline],s=marker_size,alpha=alpha_data)\n",
    "            ax.errorbar(np.mean(dict_cov[dataset_name]['normal'][pipeline]), \n",
    "                        np.mean(dict_len[dataset_name]['normal'][pipeline]),\n",
    "                        xerr=np.std(dict_cov[dataset_name]['normal'][pipeline])/np.sqrt(n_rep),\n",
    "                        yerr=np.std(dict_len[dataset_name]['normal'][pipeline])/np.sqrt(n_rep), \n",
    "                        color=dict_colors[pipeline], alpha=0.3)\n",
    "            ax.scatter(np.mean(dict_cov_worst[dataset_name]['normal'][pipeline]),\n",
    "                       np.mean(dict_len_worst[dataset_name]['normal'][pipeline]), \n",
    "                       marker=dict_markers['worst'], color=dict_colors[pipeline],s=marker_size,alpha=alpha_data)\n",
    "            ax.errorbar(np.mean(dict_cov_worst[dataset_name]['normal'][pipeline]), \n",
    "                        np.mean(dict_len_worst[dataset_name]['normal'][pipeline]),\n",
    "                        xerr=np.std(dict_cov_worst[dataset_name]['normal'][pipeline])/np.sqrt(n_rep),\n",
    "                        yerr=np.std(dict_len_worst[dataset_name]['normal'][pipeline])/np.sqrt(n_rep), \n",
    "                        color=dict_colors[pipeline], alpha=0.3)\n",
    "            ax.scatter(np.mean(dict_cov_best[dataset_name]['normal'][pipeline]),\n",
    "                       np.mean(dict_len_best[dataset_name]['normal'][pipeline]), \n",
    "                       marker=dict_markers['best'], color=dict_colors[pipeline],s=marker_size,alpha=alpha_data)\n",
    "            ax.errorbar(np.mean(dict_cov_best[dataset_name]['normal'][pipeline]), \n",
    "                        np.mean(dict_len_best[dataset_name]['normal'][pipeline]),\n",
    "                        xerr=np.std(dict_cov_best[dataset_name]['normal'][pipeline])/np.sqrt(n_rep),\n",
    "                        yerr=np.std(dict_len_best[dataset_name]['normal'][pipeline])/np.sqrt(n_rep), \n",
    "                        color=dict_colors[pipeline], alpha=0.3)\n",
    "\n",
    "\n",
    "        if small:\n",
    "            ax.scatter(np.mean(dict_cov[dataset_name]['small'][pipeline]),\n",
    "                       np.mean(dict_len[dataset_name]['small'][pipeline]), \n",
    "                       marker=dict_markers['iid'], color=dict_colors[pipeline],s=marker_size, facecolors='none',alpha=alpha_data)\n",
    "            ax.errorbar(np.mean(dict_cov[dataset_name]['small'][pipeline]), \n",
    "                        np.mean(dict_len[dataset_name]['small'][pipeline]),\n",
    "                        xerr=np.std(dict_cov[dataset_name]['small'][pipeline])/np.sqrt(n_rep),\n",
    "                        yerr=np.std(dict_len[dataset_name]['small'][pipeline])/np.sqrt(n_rep), \n",
    "                        color=dict_colors[pipeline], alpha=0.3)\n",
    "            ax.scatter(np.mean(dict_cov_worst[dataset_name]['small'][pipeline]),\n",
    "                       np.mean(dict_len_worst[dataset_name]['small'][pipeline]), \n",
    "                       marker=dict_markers['worst'], color=dict_colors[pipeline],s=marker_size, facecolors='none',alpha=alpha_data)\n",
    "            ax.errorbar(np.mean(dict_cov_worst[dataset_name]['small'][pipeline]), \n",
    "                        np.mean(dict_len_worst[dataset_name]['small'][pipeline]),\n",
    "                        xerr=np.std(dict_cov_worst[dataset_name]['small'][pipeline])/np.sqrt(n_rep),\n",
    "                        yerr=np.std(dict_len_worst[dataset_name]['small'][pipeline])/np.sqrt(n_rep), \n",
    "                        color=dict_colors[pipeline], alpha=0.3)\n",
    "            ax.scatter(np.mean(dict_cov_best[dataset_name]['small'][pipeline]),\n",
    "                       np.mean(dict_len_best[dataset_name]['small'][pipeline]), \n",
    "                       marker=dict_markers['best'], color=dict_colors[pipeline],s=marker_size, facecolors='none',alpha=alpha_data)\n",
    "            ax.errorbar(np.mean(dict_cov_best[dataset_name]['small'][pipeline]), \n",
    "                        np.mean(dict_len_best[dataset_name]['small'][pipeline]),\n",
    "                        xerr=np.std(dict_cov_best[dataset_name]['small'][pipeline])/np.sqrt(n_rep),\n",
    "                        yerr=np.std(dict_len_best[dataset_name]['small'][pipeline])/np.sqrt(n_rep), \n",
    "                        color=dict_colors[pipeline], alpha=0.3)\n",
    "\n",
    "\n",
    "for ax in [ax1,ax2,ax3,ax4]:\n",
    "    ax.axvline(x=1-alpha, color='black', ls=':')\n",
    "    ax.set_xlabel(\"Average coverage\")\n",
    "\n",
    "    \n",
    "ax1.set_ylabel(\"Average length\")\n",
    "\n",
    "\n",
    "# Methods legend\n",
    "\n",
    "handles = []\n",
    "names = list( map(dict_methods.get, name_pipelines_to_plot) )\n",
    "for idc,color in enumerate(list( map(dict_colors.get, name_pipelines_to_plot) )):\n",
    "    handles.append(mlines.Line2D([], [], color=color, marker='o', linestyle='None', markersize=8))\n",
    "\n",
    "if mask == 'Yes':\n",
    "    fig.legend(handles, names, ncol=4, bbox_to_anchor=(0.63,0.13),handletextpad=0.1, \n",
    "               labelspacing=0.2, borderpad=0.3, handlelength=1.2, borderaxespad=1.1)\n",
    "else:\n",
    "    fig.legend(handles, names, ncol=4, bbox_to_anchor=(0.63,0.1),handletextpad=0.1, \n",
    "               labelspacing=0.2, borderpad=0.3, handlelength=1.2, borderaxespad=1.1)\n",
    "    \n",
    "# Coverage legend\n",
    "\n",
    "handles = []\n",
    "labels = []\n",
    "for cov in list(dict_coverages.keys()):\n",
    "    handles.append(mlines.Line2D([], [], color='black', marker=dict_markers[cov], linestyle='None', markersize=8))\n",
    "    labels.append(dict_coverages[cov])\n",
    "\n",
    "if mask == 'Yes':\n",
    "    fig.legend(handles, labels, bbox_to_anchor=(0.95, 0.13),ncol=3,handletextpad=0.1, \n",
    "               labelspacing=0.2, borderpad=0.3, handlelength=1.2, borderaxespad=1.1)\n",
    "else:\n",
    "    fig.legend(handles, labels, bbox_to_anchor=(0.95, 0.1),ncol=3,handletextpad=0.1, \n",
    "               labelspacing=0.2, borderpad=0.3, handlelength=1.2, borderaxespad=1.1)\n",
    "\n",
    "\n",
    "fig.tight_layout()\n",
    "\n",
    "name_plot = 'plots/semi_synth'\n",
    "\n",
    "if mask == 'Yes':\n",
    "    plt.savefig(name_plot+'.pdf',bbox_inches='tight', dpi=300)\n",
    "else:\n",
    "    plt.savefig(name_plot+'_nomask.pdf',bbox_inches='tight', dpi=300)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}