notebooks/optimal/noisy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Algorithm performance on credit card data under misspecifications (Figure 4)\n",
    "\n",
    "In this notebook we produce plots to compare the performance of the algorithms presented in the paper in terms of utility for instances generated based on real data, when the probabilities $P(y=1|x)$ and the costs $c(x_i, x_j)$ are misspecified. The resulting plots are presented in Figure 4.\n",
    "\n",
    "**Required script execution:** noisy.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir(\"../\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import container\n",
    "plt.style.use('default')\n",
    "import seaborn as sns\n",
    "import glob\n",
    "import json\n",
    "from lib import utils\n",
    "from matplotlib.colors import LogNorm\n",
    "import matplotlib.ticker as mtick\n",
    "import copy\n",
    "from mpl_toolkits.axes_grid1.inset_locator import zoomed_inset_axes\n",
    "from mpl_toolkits.axes_grid1.inset_locator import mark_inset\n",
    "sns.set_context(\"paper\", font_scale=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "name=\"noisy\"\n",
    "datas=\"credit\"\n",
    "gamma=0.8563985631883158\n",
    "cost_method='max_percentile_shift'\n",
    "final_df = []\n",
    "cols = sns.color_palette(\"Set2\",9)\n",
    "col_dict = {\"Non-Strategic\": cols[0],\n",
    "            \"Bruteforce\": cols[1],\n",
    "            \"Threshold\": cols[2],\n",
    "            \"Iterative\": cols[3],\n",
    "            \"Iterative (Optimized)\": cols[4],\n",
    "            \"DP\": cols[5]\n",
    "           }\n",
    "linestyles_dict = {\"Non-Strategic\": \"-.\",\n",
    "            \"Bruteforce\": \"-\",\n",
    "            \"Threshold\": \"-\",\n",
    "            \"Iterative\": \"--\",\n",
    "            \"Iterative (Optimized)\": \"-.\",\n",
    "            \"DP\": \":\"\n",
    "           }\n",
    "markers_dict = {\"Non-Strategic\": \"v\",\n",
    "            \"Bruteforce\": \"s\",\n",
    "            \"Threshold\": \"^\",\n",
    "            \"Iterative\": \"o\",\n",
    "            \"Iterative (Optimized)\": \"P\",\n",
    "            \"DP\": \"D\"\n",
    "           }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Noisy P(y|x) utility plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = glob.glob(\"./outputs/optimal/{name}_*_data_{datas}_cost_{cost_method}_noisy_pyx_*_config.json\".format(name=name, datas=datas, cost_method=cost_method))\n",
    "data = []\n",
    "\n",
    "for ind,fi in enumerate(files):\n",
    "    with open(fi,\"r\") as file:\n",
    "        parts = fi.split('_')\n",
    "        algo = parts[1]\n",
    "        type_of_noise = parts[9]\n",
    "        beta = parts[11]\n",
    "        seed = parts[13]\n",
    "\n",
    "        obj = json.load(file)\n",
    "        if algo == 'thres':\n",
    "            util = obj['strategic_threshold']\n",
    "        elif algo =='iterative':\n",
    "            util = obj['strategic']\n",
    "            non_strategic_util = obj['non_strategic']\n",
    "            \n",
    "        data.append({'Algorithm': algo, 'beta': beta, 'seed': seed, 'utility': util})\n",
    "        if algo == 'iterative':\n",
    "            data.append({'Algorithm': 'non_strategic', 'beta': beta, 'seed': seed, 'utility': non_strategic_util})\n",
    "        \n",
    "df_standard = pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_standard['Algorithm'] = df_standard['Algorithm'].str.replace(\"non_strategic\", \"Non-Strategic\")\n",
    "df_standard['Algorithm'] = df_standard['Algorithm'].str.replace(\"thres\", \"Threshold\")\n",
    "df_standard['Algorithm'] = df_standard['Algorithm'].str.replace(\"iterative\", \"Iterative\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "utils.latexify(8,5, font_scale=3.0)\n",
    "ax = plt.gca()\n",
    "df = df_standard\n",
    "df['beta'] = pd.to_numeric(df['beta'])\n",
    "df['utility'] = pd.to_numeric(df['utility'])\n",
    "df.sort_values([\"Algorithm\"], inplace=True)\n",
    "algorithms=[\"Non-Strategic\", \"Threshold\", \"Iterative\"]\n",
    "colors = [col_dict[alg] for alg in algorithms]\n",
    "markers = [markers_dict[alg] for alg in algorithms]\n",
    "linestyles = [linestyles_dict[alg] for alg in algorithms]\n",
    "\n",
    "lines={}\n",
    "betas=sorted(df['beta'].unique())\n",
    "for alg in algorithms:\n",
    "    lines[alg]=[]\n",
    "    for x_val in betas:\n",
    "        line_l=pd.DataFrame(df[(df['Algorithm']==alg) & (df['beta']==x_val)], columns=['utility']).values.tolist()\n",
    "        lines[alg].append((x_val,np.mean(line_l),np.std(line_l)))\n",
    "\n",
    "ax=plt.subplot(111)\n",
    "new_handles = []\n",
    "for alg in algorithms:\n",
    "    line_sorted=sorted(lines[alg], key=lambda x:x[0])\n",
    "    x_vals=[x[0] for x in line_sorted]\n",
    "    y_vals=[x[1] for x in line_sorted]\n",
    "    std_vals=[x[2] for x in line_sorted]\n",
    "    # h, = ax.plot(x_vals, y_vals, linestyle=linestyles_dict[alg], marker=markers_dict[alg], label=alg, markersize=9, color=col_dict[alg], linewidth=2)\n",
    "    h,_,_ = ax.errorbar(x_vals, y_vals, yerr=std_vals, linestyle=linestyles_dict[alg], marker=markers_dict[alg],\n",
    "    label=alg, markersize=9, color=col_dict[alg], linewidth=2) #linestyle=linestyles_dict[alg], marker=markers_dict[alg], label=alg, markersize=9, color=col_dict[alg], linewidth=2)\n",
    "    new_handles.append(copy.copy(h))\n",
    "\n",
    "for ind, alg in enumerate(algorithms):\n",
    "    new_handles[ind].set_color('w')\n",
    "    new_handles[ind].set_label(alg)\n",
    "    new_handles[ind].set_markerfacecolor(col_dict[alg])\n",
    "    new_handles[ind].set_markeredgecolor(col_dict[alg])\n",
    "    new_handles[ind].set_markersize(9)\n",
    "    new_handles[ind].set_markeredgewidth(1)\n",
    "\n",
    "plt.legend(handles=new_handles, shadow=None, loc='upper right', labelspacing=0.1, handletextpad=0.3)\n",
    "plt.ylabel(\"Utility, $u(\\\\pi,\\\\gamma)$\")\n",
    "plt.xlabel(\"$\\\\phi$\")\n",
    "ax.spines['top'].set_visible(False)\n",
    "ax.spines['right'].set_visible(False)\n",
    "ax.set_xscale('log')\n",
    "ax.set_ylim([0, 0.05])\n",
    "plt.xticks(ticks=sorted(df['beta'].unique()), labels=sorted(df['beta'].unique()), rotation = 30)\n",
    "plt.savefig(\"./figures/optimal/noisy_pyx.pdf\", bbox_inches='tight', )\n",
    "plt.clf()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Noisy cost utility plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = glob.glob(\"./outputs/optimal/{name}_*_data_{datas}_cost_{cost_method}_noisy_cost_*_config.json\".format(name=name, datas=datas, cost_method=cost_method))\n",
    "data = []\n",
    "\n",
    "for ind,fi in enumerate(files):\n",
    "    with open(fi,\"r\") as file:\n",
    "        parts = fi.split('_')\n",
    "        algo = parts[1]\n",
    "        type_of_noise = parts[9]\n",
    "        beta = parts[11]\n",
    "        seed = parts[13]\n",
    "        \n",
    "        obj = json.load(file)\n",
    "        if algo == 'thres':\n",
    "            util = obj['strategic_threshold']\n",
    "        elif algo =='iterative':\n",
    "            util = obj['strategic']\n",
    "            non_strategic_util = obj['non_strategic']\n",
    "        \n",
    "        data.append({'Algorithm': algo, 'beta': beta, 'seed': seed, 'utility': util})\n",
    "        if algo == 'iterative':\n",
    "            data.append({'Algorithm': 'non_strategic', 'beta': beta, 'seed': seed, 'utility': non_strategic_util})\n",
    "        \n",
    "\n",
    "df_standard = pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_standard['Algorithm'] = df_standard['Algorithm'].str.replace(\"non_strategic\", \"Non-Strategic\")\n",
    "df_standard['Algorithm'] = df_standard['Algorithm'].str.replace(\"thres\", \"Threshold\")\n",
    "df_standard['Algorithm'] = df_standard['Algorithm'].str.replace(\"iterative\", \"Iterative\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "utils.latexify(8,5, font_scale=3.0)\n",
    "ax = plt.gca()\n",
    "df = df_standard\n",
    "df['beta'] = pd.to_numeric(df['beta'])\n",
    "df['utility'] = pd.to_numeric(df['utility'])\n",
    "df.sort_values([\"Algorithm\"], inplace=True)\n",
    "algorithms=[\"Non-Strategic\", \"Threshold\", \"Iterative\"]\n",
    "colors = [col_dict[alg] for alg in algorithms]\n",
    "markers = [markers_dict[alg] for alg in algorithms]\n",
    "linestyles = [linestyles_dict[alg] for alg in algorithms]\n",
    "\n",
    "lines={}\n",
    "betas=sorted(df['beta'].unique())\n",
    "for alg in algorithms:\n",
    "    lines[alg]=[]\n",
    "    for x_val in betas:\n",
    "        line_l=pd.DataFrame(df[(df['Algorithm']==alg) & (df['beta']==x_val)], columns=['utility']).values.tolist()\n",
    "        lines[alg].append((x_val,np.mean(line_l),np.std(line_l)))\n",
    "\n",
    "ax=plt.subplot(111)\n",
    "new_handles = []\n",
    "for alg in algorithms:\n",
    "    line_sorted=sorted(lines[alg], key=lambda x:x[0])\n",
    "    x_vals=[x[0] for x in line_sorted]\n",
    "    y_vals=[x[1] for x in line_sorted]\n",
    "    std_vals=[x[2] for x in line_sorted]\n",
    "    # h, = ax.plot(x_vals, y_vals, linestyle=linestyles_dict[alg], marker=markers_dict[alg], label=alg, markersize=9, color=col_dict[alg], linewidth=2)\n",
    "    h,_,_ = ax.errorbar(x_vals, y_vals, yerr=std_vals, linestyle=linestyles_dict[alg], marker=markers_dict[alg],\n",
    "    label=alg, markersize=9, color=col_dict[alg], linewidth=2) #linestyle=linestyles_dict[alg], marker=markers_dict[alg], label=alg, markersize=9, color=col_dict[alg], linewidth=2)\n",
    "    new_handles.append(copy.copy(h))\n",
    "\n",
    "for ind, alg in enumerate(algorithms):\n",
    "    new_handles[ind].set_color('w')\n",
    "    new_handles[ind].set_label(alg)\n",
    "    new_handles[ind].set_markerfacecolor(col_dict[alg])\n",
    "    new_handles[ind].set_markeredgecolor(col_dict[alg])\n",
    "    new_handles[ind].set_markersize(9)\n",
    "    new_handles[ind].set_markeredgewidth(1)\n",
    "\n",
    "plt.legend(handles=new_handles, shadow=None, loc='upper right', labelspacing=0.1, handletextpad=0.3)\n",
    "plt.ylabel(\"Utility, $u(\\\\pi,\\\\gamma)$\")\n",
    "plt.xlabel(\"$\\\\phi$\")\n",
    "ax.spines['top'].set_visible(False)\n",
    "ax.spines['right'].set_visible(False)\n",
    "ax.set_xscale('log')\n",
    "ax.set_ylim([0.015, 0.042])\n",
    "plt.xticks(ticks=sorted(df['beta'].unique()), labels=sorted(df['beta'].unique()), rotation = 30)\n",
    "plt.savefig(\"./figures/optimal/noisy_cost.pdf\", bbox_inches='tight', )\n",
    "plt.clf()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.1 ('env': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  },
  "vscode": {
   "interpreter": {
    "hash": "0ddb0c2da25db7768b0241d542d21cb35172a6678fcb32e19d61594f5477b3c6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}