top_results_analysis.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyse obtained results"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports and necessary definitions valid for all analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import seaborn as sns\n",
    "\n",
    "from matplotlib.backends.backend_pdf import PdfPages\n",
    "from matplotlib.colors import ListedColormap\n",
    "from matplotlib.patches import Rectangle\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"_experiments/top_methods/all_results.csv\")\n",
    "df = df.drop(\"Unnamed: 0\", axis=1)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Configure which data to use and where to save outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ss_methods = df[\"selection_metric\"].unique()\n",
    "protocols = sorted(df[\"protocol\"].unique(), reverse=True)  # this is only to keep the order: OR and AND\n",
    "\n",
    "# in some experiments mi_value was saved tih too big precision (differences \n",
    "# in 15th floating place) hence we need to round them \n",
    "df[\"mi_value\"] = df[\"mi_value\"].round(1)\n",
    "mi_vals = df[\"mi_value\"].unique()\n",
    "\n",
    "# permute networks so that they're ordered as following:\n",
    "nets = [\"arxiv\", \"timik1q2009\"]\n",
    "assert set(df[\"network\"].unique()) == set(nets)\n",
    "\n",
    "# set up a path where results are saved in\n",
    "workdir = Path(\"_results\")\n",
    "workdir.mkdir(exist_ok=True)\n",
    "\n",
    "ss_methods_abbrv_map ={\n",
    "    \"cbim\": \"cbim\",\n",
    "    \"cim\": \"cim\",\n",
    "    'degree_centrality': \"deg-c\",\n",
    "    'degree_centrality_discount': \"deg-c-d\",\n",
    "    'greedy': \"greedy\",\n",
    "    'k_shell': \"k-sh\",\n",
    "    'k_shell_mln': \"k-sh-m\",\n",
    "    'kpp_shell': \"kpp-sh\",\n",
    "    'neighbourhood_size': \"nghb-1s\",\n",
    "    'neighbourhood_2_hop_size': \"nghb-2s\",\n",
    "    'neighbourhood_size_discount': \"nghb-sd\",\n",
    "    'page_rank': \"p-rnk\",\n",
    "    'page_rank_mln': \"p-rnk-m\",\n",
    "    'random': \"random\",\n",
    "    'vote_rank': \"v-rnk\",\n",
    "    'vote_rank_mln': \"v-rnk-m\",\n",
    "}\n",
    "\n",
    "nets_abbrv_map = {\n",
    "    \"arxiv\": \"arxiv\",\n",
    "    \"timik1q2009\": \"timik\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for protocol OR we had budgets like [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30] \n",
    "# for protocol AND we had budgets like \n",
    "# [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40],\n",
    "# but we'd like to skip wery low budgets where all methods are unaccurate and keep only:\n",
    "# [15, 20, 25, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40] \n",
    "rows_to_drop = df.loc[\n",
    "    (df[\"protocol\"] == \"AND\") & \n",
    "    (\n",
    "        ~df[\"seeding_budget\"].isin(\n",
    "            {15, 20, 25, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40}\n",
    "        )\n",
    "    )\n",
    "]\n",
    "df = df.drop(rows_to_drop.index)\n",
    "s_budgets = df[\"seeding_budget\"].unique()\n",
    "\n",
    "# drop unselected nets\n",
    "rows_to_drop = df.loc[~df[\"network\"].isin(set(nets))]\n",
    "df = df.drop(rows_to_drop.index)\n",
    "\n",
    "\n",
    "ss_methods, protocols, mi_vals, s_budgets, nets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Optional, Union\n",
    "\n",
    "def prepare_ticklabels(series: pd.Index) -> Union[np.ndarray, str]:\n",
    "    try:\n",
    "        return series.to_numpy().round(2)\n",
    "    except:\n",
    "        return \"auto\"\n",
    "\n",
    "def plot_heatmap(\n",
    "    vis_df: pd.DataFrame,\n",
    "    heatmap_ax: plt.Axes,\n",
    "    bar_ax: plt.Axes,\n",
    "    vrange=(0, 100),\n",
    "    cmap=\"RdYlGn\",\n",
    "    mask: Optional[pd.DataFrame] = None,\n",
    "    fmt: Optional[str] = \".0f\",\n",
    "    **kwargs,\n",
    ") -> None:\n",
    "    sns.heatmap(\n",
    "        vis_df,\n",
    "        ax=heatmap_ax,\n",
    "        cbar_ax=bar_ax,\n",
    "        cmap=cmap,\n",
    "        vmin=vrange[0],\n",
    "        vmax=vrange[1],\n",
    "        annot=True,\n",
    "        annot_kws={\"size\": 7},\n",
    "        fmt=fmt,\n",
    "        yticklabels=prepare_ticklabels(vis_df.index),\n",
    "        xticklabels=prepare_ticklabels(vis_df.columns),\n",
    "        linewidth=.5,\n",
    "        mask=mask,\n",
    "        cbar= True if bar_ax is not None else False,\n",
    "        **kwargs,\n",
    "    )\n",
    "    heatmap_ax.invert_yaxis()\n",
    "    # heatmap_ax.tick_params(axis=\"x\", rotation=80)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Detailed heatmaps for all cases\n",
    "\n",
    "Charts of f(mi_value, seeding_budget) = gain for each network, protocol, ssm case"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### bulk plot (pdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define values to visualise as well as attributes of plots\n",
    "x = \"mi_value\"\n",
    "y = \"seeding_budget\" \n",
    "Z = {\n",
    "    \"gain\": {\"vrange\": (0, 100), \"cmap\": \"RdYlGn\"},\n",
    "    \"diffusion_len\": {\"vrange\": (0, df[\"diffusion_len\"].max()), \"cmap\": \"BuPu\"},\n",
    "}\n",
    "\n",
    "# create file descriptor where to save visualisations\n",
    "pdf = PdfPages(workdir.joinpath(f\"top_heatmaps_{'_'.join(Z)}_by_{x}_{y}.pdf\"))\n",
    "\n",
    "for net in sorted(nets):\n",
    "\n",
    "    # a dummy plot that contains just name of processed network\n",
    "    fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(12, 4))\n",
    "    ax.set_visible(False)\n",
    "    fig.suptitle(f\"{net} network - results\", x=0.5, y=.5, fontsize = 15)\n",
    "    fig.savefig(pdf, format=\"pdf\")\n",
    "    plt.close(fig)\n",
    "\n",
    "    for ssm in sorted(ss_methods):\n",
    "\n",
    "        for z, z_attrs in Z.items():\n",
    "            print(f\"processing: {net}, {ssm}, {z}\")\n",
    "\n",
    "            # prepare canvas - proto, proto, legend\n",
    "            fig, ax = plt.subplots(\n",
    "                nrows=1,\n",
    "                ncols=3,\n",
    "                figsize=(12, 4),\n",
    "                gridspec_kw={\"width_ratios\": [49, 49, 2]}\n",
    "            )\n",
    "            fig.tight_layout(pad=0.5, rect=(0.1, 0.1, 0.9, 0.9))\n",
    "            title = (\n",
    "                f\"{z} achieved by {ssm} s.s. method on {net} network; \"\n",
    "                \"protocols from left:\"\n",
    "            )\n",
    "\n",
    "            # prepare and plot heatmap for each proto\n",
    "            for idx, proto in enumerate(protocols):\n",
    "                df_plot = df.loc[\n",
    "                    (df[\"network\"] == net) &\n",
    "                    (df[\"protocol\"] == proto) &\n",
    "                    (df[\"selection_metric\"] == ssm)\n",
    "                ]\n",
    "                df_plot = pd.pivot_table(df_plot, index=x, columns=y, values=z)\n",
    "                if len(df_plot) == 0:  # greedy wasn\"t evaluated for all nets\n",
    "                    continue\n",
    "                plot_heatmap(\n",
    "                    df_plot, ax[idx], ax[2], z_attrs[\"vrange\"], z_attrs[\"cmap\"]\n",
    "                )\n",
    "                title += f\" {proto}\"\n",
    "\n",
    "            # add title and save plot to pdf\n",
    "            fig.suptitle(title)\n",
    "            fig.savefig(pdf, format=\"pdf\")\n",
    "            plt.close(fig)\n",
    "\n",
    "pdf.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ranking for each selection method f(mi_Value, seeding_budget) -> position in the ranking \n",
    "\n",
    "First create a tensor if shape (ss_methods x nets x mi_vals x s_budgets_proto)\n",
    "valued with gains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cube = {}\n",
    "\n",
    "for ssm in ss_methods:\n",
    "    for net in nets:\n",
    "        for proto in protocols:\n",
    "            ddf = df.loc[\n",
    "                (df[\"network\"] == net) &\n",
    "                (df[\"protocol\"] == proto) &\n",
    "                (df[\"selection_metric\"] == ssm)\n",
    "            ]\n",
    "            ddf = pd.pivot_table(ddf, index=\"mi_value\", columns=\"seeding_budget\", values=\"gain\").to_numpy()\n",
    "            if len(ddf) == 0:\n",
    "                raise ArithmeticError(f\"Incorrect form of the dataframe for: {ssm}, {net}, {proto}\")\n",
    "            if not cube.get(ssm):\n",
    "                cube[ssm] = {}\n",
    "            if not cube[ssm].get(net):\n",
    "                cube[ssm][net] = {}\n",
    "            cube[ssm][net][proto] = ddf\n",
    "\n",
    "len(cube)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ssm_axis = []\n",
    "nets_axis = []\n",
    "protocols_axis = []\n",
    "\n",
    "ssm_arrays = []\n",
    "\n",
    "for ssm, net_proto_dict in cube.items():\n",
    "    # print(ssm)\n",
    "    ssm_axis.append(ssm)\n",
    "    net_arrays = []\n",
    "\n",
    "    for idx, (net, proto_dict) in enumerate(net_proto_dict.items()):\n",
    "        # print(net)\n",
    "        if len(nets_axis) == len(nets):\n",
    "            assert nets_axis[idx] == net\n",
    "        else:\n",
    "            nets_axis.append(net)\n",
    "        proto_arrs = []\n",
    "\n",
    "        for idx, (proto, arr) in enumerate(proto_dict.items()):\n",
    "            # print(proto)\n",
    "            if len(protocols_axis) == len(protocols):\n",
    "                assert protocols_axis[idx] == proto\n",
    "            else:\n",
    "                protocols_axis.append(proto)\n",
    "\n",
    "            proto_arrs.append(arr)\n",
    "        \n",
    "        net_arrays.append(np.stack(proto_arrs))\n",
    "\n",
    "    ssm_arrays.append(np.stack(net_arrays))\n",
    "\n",
    "cube_gain = np.stack(ssm_arrays)\n",
    "\n",
    "# array with all gains concatenated for all test runs - check shape (L==R)\n",
    "cube_gain.shape, (len(ss_methods), len(nets), len(protocols), len(mi_vals), 14)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When tensor is obtained now compute rankings of ss methods, i.e. place of ss \n",
    "method for certain evaluated case: net, mi_val, s_budgets_proto by obtained gain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Dict, List\n",
    "from scipy.stats import rankdata\n",
    "\n",
    "def rank_result(arr: np.ndarray) -> np.ndarray:\n",
    "    \"\"\"\n",
    "    Create ranking e.g for this array:\n",
    "    [52.63, 3.50, 100., -4, 3.50, 100., 100., np.nan, 100., 22.54, 100.]\n",
    "    return [2, 4, 1, 5, 4, 1, 1, 6, 1, 3, 1]\n",
    "    \"\"\"\n",
    "    raw_rank = rankdata(arr, method=\"min\", nan_policy=\"omit\")\n",
    "    if np.isnan(raw_rank).any():\n",
    "        raw_rank[np.isnan(raw_rank)] = 0\n",
    "    inverted_rank = np.abs(raw_rank - max(raw_rank) - 1)\n",
    "    _ = {idx: position for position, idx in enumerate(sorted(np.unique(inverted_rank)), 1)}\n",
    "    return np.vectorize(lambda x: _[x])(inverted_rank)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranks = np.apply_along_axis(rank_result, 0, cube_gain)\n",
    "print(ranks.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ssm_axis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stat_to_dict(stat: np.ndarray, names_level_1: List, names_level_2) -> Dict:\n",
    "    return {\n",
    "        n1: {n2: stat[idx1][idx2] for idx2, n2 in enumerate(names_level_2)} \n",
    "        for idx1, n1 in enumerate(names_level_1)\n",
    "    }"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then reduce dimensionality of tensor by mean to obtain flat vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranks_reduded_by_mean_in_nets = np.mean(ranks, 1)\n",
    "print(ranks_reduded_by_mean_in_nets.shape)\n",
    "\n",
    "ranks_reduded_by_mean_in_nets_mi = np.mean(ranks_reduded_by_mean_in_nets, 2)\n",
    "print(ranks_reduded_by_mean_in_nets_mi.shape)\n",
    "\n",
    "ranks_reduded_by_mean_in_nets_mi_sb = np.mean(ranks_reduded_by_mean_in_nets_mi, 2)\n",
    "print(ranks_reduded_by_mean_in_nets_mi_sb.shape)\n",
    "\n",
    "overall_gain = stat_to_dict(ranks_reduded_by_mean_in_nets_mi_sb, ssm_axis, protocols_axis)\n",
    "\n",
    "print(\"Average performance of ssm in total\")\n",
    "overall_gain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranked_ssms = pd.DataFrame(overall_gain).T\n",
    "ranked_ssms[\"MEAN\"] = ranked_ssms.mean(axis=1)\n",
    "ranked_ssms = ranked_ssms.round(2)\n",
    "ranked_ssms.to_csv(workdir / f\"top_ssm_ranking.csv\")\n",
    "ranked_ssms\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# debugging\n",
    "_  = cube_gain[:, :1, ...]\n",
    "np.apply_along_axis(rank_result, 0, _)[:, 0, 1, 8, 1], _[:, 0, 1, 8, 1]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## General, averaged heatmap\n",
    "\n",
    "Charts of f(network, ssm) = average(??) for each protocol, where ?? is gain or\n",
    "diffusion length. There is also a ranking visualised from the average gain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = \"network\"\n",
    "y = \"selection_metric\"\n",
    "z = \"gain\"\n",
    "\n",
    "fig, ax = plt.subplots(\n",
    "    nrows=1, ncols=3, figsize=(10, 2), gridspec_kw={\"width_ratios\": [49, 49, 2]}\n",
    ")\n",
    "fig.tight_layout(pad=.5, rect=(0.05, 0.1, 0.95, 0.9))\n",
    "title = f\"Average {z} achieved by s.s. methods on nets; protocols from left:\"\n",
    "\n",
    "for idx, proto in enumerate(protocols):\n",
    "    df_plotted = df.loc[df[\"protocol\"] == proto]\n",
    "    df_plotted = pd.pivot_table(df_plotted, index=x, columns=y, values=z, aggfunc=np.mean)\n",
    "    df_plotted = df_plotted.reindex(index=nets)\n",
    "    df_plotted = df_plotted.rename(index=nets_abbrv_map, columns=ss_methods_abbrv_map)\n",
    "    \n",
    "    plot_heatmap(df_plotted, ax[idx], ax[2], mask=df_plotted.isnull(), square=True)\n",
    "    title += f\" {proto}\"\n",
    "    df_plotted.to_numpy().argmax(axis=1)\n",
    "\n",
    "fname = workdir / f\"top_means_by_{z}.pdf\"\n",
    "fig.subplots_adjust(wspace=.4)\n",
    "fig.suptitle(title)\n",
    "fig.savefig(fname, dpi=300)\n",
    "plt.close(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = \"network\"\n",
    "y = \"selection_metric\"\n",
    "z = \"gain\"\n",
    "\n",
    "fig, ax = plt.subplots(\n",
    "    nrows=1, ncols=3, figsize=(10, 2), gridspec_kw={\"width_ratios\": [49, 49, 2]}\n",
    ")\n",
    "fig.tight_layout(pad=.5, rect=(0.05, 0.1, 0.95, 0.9))\n",
    "title = f\"Rank w.r.t. avg. {z} achieved by s.s. methods on nets; protocols from left:\"\n",
    "\n",
    "for idx, proto in enumerate(df[\"protocol\"].unique()):\n",
    "    df_plotted = df.loc[df[\"protocol\"] == proto]\n",
    "    df_plotted = pd.pivot_table(df_plotted, index=x, columns=y, values=z, aggfunc=np.mean)\n",
    "    df_plotted = df_plotted.reindex(index=nets)\n",
    "    df_plotted = df_plotted.rename(index=nets_abbrv_map, columns=ss_methods_abbrv_map)\n",
    "\n",
    "    df_plotted2 = df_plotted.copy()\n",
    "    df_plotted2.loc[:] = np.apply_along_axis(rank_result, 1, np.nan_to_num(df_plotted2.to_numpy(), 0))\n",
    "\n",
    "    # https://matplotlib.org/stable/users/explain/colors/colormaps.html\n",
    "    plot_heatmap(df_plotted2, ax[idx], ax[2], mask=df_plotted.isnull(), cmap=\"bone\", vrange=(1, len(ss_methods)))\n",
    "    title += f\" {proto}\"\n",
    "    df_plotted.to_numpy().argmax(axis=1)\n",
    "\n",
    "    for net_idx, ssm_idx in enumerate(df_plotted2.to_numpy().argmin(axis=1)):\n",
    "        ax[idx].add_patch(Rectangle((ssm_idx, net_idx), 1, 1, fill=False, edgecolor=\"black\", lw=2))\n",
    "\n",
    "fname = workdir / f\"top_means_by_{z}_ranked.pdf\"\n",
    "fig.subplots_adjust(wspace=.4)\n",
    "fig.suptitle(title)\n",
    "fig.savefig(fname, dpi=300)\n",
    "plt.close(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = \"network\"\n",
    "y = \"selection_metric\"\n",
    "z = \"diffusion_len\"\n",
    "\n",
    "fig, ax = plt.subplots(\n",
    "    nrows=1, ncols=3, figsize=(10, 2), gridspec_kw={\"width_ratios\": [49, 49, 2]}\n",
    ")\n",
    "fig.tight_layout(pad=.5, rect=(0.05, 0.1, 0.95, 0.9))\n",
    "title = f\"Average {z} achieved by s.s. methods on nets; protocols from left:\"\n",
    "\n",
    "for idx, proto in enumerate(df[\"protocol\"].unique()):\n",
    "    df_plotted = df.loc[df[\"protocol\"] == proto]\n",
    "    df_plotted = pd.pivot_table(df_plotted, index=x, columns=y, values=z, aggfunc=np.mean)\n",
    "    df_plotted = df_plotted.reindex(index=nets)\n",
    "    df_plotted = df_plotted.rename(index=nets_abbrv_map, columns=ss_methods_abbrv_map)\n",
    "    \n",
    "    # plot_heatmap(df_plotted, ax[idx], ax[2], mask=df_plotted.isnull())\n",
    "    plot_heatmap(\n",
    "        df_plotted, ax[idx], ax[2],\n",
    "        mask=df_plotted.isnull(),\n",
    "        cmap=\"BuPu\",\n",
    "        vrange=(1, np.ceil(df_plotted.max().max()).astype(int)),\n",
    "        fmt=\".1f\",\n",
    "    )\n",
    "    title += f\" {proto}\"\n",
    "    df_plotted.to_numpy().argmax(axis=1)\n",
    "\n",
    "fname = workdir / f\"top_means_by_{z}.pdf\"\n",
    "fig.subplots_adjust(wspace=.4)\n",
    "fig.suptitle(title)\n",
    "fig.savefig(fname, dpi=300)\n",
    "plt.close(fig)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Wilcoxon tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from misc.wilcoxon_test import test_samples\n",
    "from itertools import combinations\n",
    "\n",
    "def get_sorted_gain_series(network, selection_method, protocol):\n",
    "    x = df.loc[\n",
    "        (df[\"network\"] == network) & \n",
    "        (df[\"selection_metric\"] == selection_method) & \n",
    "        (df[\"protocol\"] == protocol)\n",
    "    ].sort_values(by=[\"seeding_budget\", \"mi_value\"]).reindex()\n",
    "    if len(x) == 0:\n",
    "        raise ValueError\n",
    "    return x[\"gain\"].to_numpy()\n",
    "\n",
    "def get_selection_metrics_for_net(net):\n",
    "    return set(df.loc[df[\"network\"] == net][\"selection_metric\"].unique())\n",
    "\n",
    "ss_methods, nets, protocols"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### bulk plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create file descriptor where to save visualisations\n",
    "pdf = PdfPages(workdir / f\"top_heatmaps_wilcoxon_test.pdf\")\n",
    "\n",
    "for net in sorted(nets):\n",
    "\n",
    "    # prepare canvas\n",
    "    fig, ax = plt.subplots(\n",
    "        nrows=1, ncols=2, figsize=(10, 4), gridspec_kw={\"width_ratios\": [50, 50]}\n",
    "    )\n",
    "    fig.tight_layout(pad=0.5, rect=(0.05, 0.05, 0.95, 0.95))\n",
    "    title = f\"P-values of Wilcoxon test for network: {net}; protocols from left:\"\n",
    "\n",
    "    # prepare and plot heatmap for each proto\n",
    "    for idx, proto in enumerate(protocols):\n",
    "        title += f\" {proto}\"\n",
    "\n",
    "        p_dict = {}\n",
    "        for ss1, ss2 in combinations(sorted(ss_methods), 2):\n",
    "            if not {ss1, ss2}.issubset(get_selection_metrics_for_net(net)):\n",
    "                continue  # greedy have not been computed for all nets\n",
    "            gain_ss1 = get_sorted_gain_series(net, ss1, proto)\n",
    "            gain_ss2 = get_sorted_gain_series(net, ss2, proto)\n",
    "            p_value = test_samples(gain_ss1, gain_ss2)\n",
    "            if p_dict.get(ss1) is None:\n",
    "                p_dict[ss1] = {ss2: p_value}\n",
    "            else:\n",
    "                p_dict[ss1][ss2] = p_value\n",
    "        p_values = pd.DataFrame.from_dict(p_dict, orient='index')\n",
    "        p_values.loc[p_values.columns[-1]] = np.NaN\n",
    "        p_values.insert(0, p_values.iloc[0].name, np.NaN)\n",
    "\n",
    "        # shorten names of records\n",
    "        p_values = p_values.rename(\n",
    "            index=ss_methods_abbrv_map, columns=ss_methods_abbrv_map\n",
    "        )\n",
    "\n",
    "        # plot heatmap\n",
    "        plot_heatmap(\n",
    "            p_values, ax[idx], bar_ax=None, \n",
    "            cmap=ListedColormap(['whitesmoke']), fmt=\".2f\"\n",
    "        )\n",
    "\n",
    "        # mark the best value\n",
    "        rows, cols = np.where(p_values.to_numpy() >= 0.05)\n",
    "        for r_idx, c_idx in zip(rows, cols):\n",
    "            ax[idx].add_patch(\n",
    "                Rectangle((c_idx, r_idx), 1, 1, fill=True, color=\"lightcoral\", lw=0)\n",
    "            )\n",
    "\n",
    "    # add title and save plot to pdf\n",
    "    fig.subplots_adjust(wspace=.4)\n",
    "    fig.suptitle(title)\n",
    "    fig.savefig(pdf, format=\"pdf\")\n",
    "    plt.close(fig)\n",
    "\n",
    "pdf.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### merged stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"error\")\n",
    "\n",
    "tests = {}\n",
    "\n",
    "for net in sorted(nets):\n",
    "    tests[net] = {}\n",
    "\n",
    "    for idx, proto in enumerate(protocols):\n",
    "\n",
    "        p_dict = {}\n",
    "        for ss1, ss2 in combinations(sorted(ss_methods), 2):\n",
    "            if not {ss1, ss2}.issubset(get_selection_metrics_for_net(net)):\n",
    "                p_value = None  # greedy have not been computed for all nets\n",
    "            else:\n",
    "                gain_ss1 = get_sorted_gain_series(net, ss1, proto)\n",
    "                gain_ss2 = get_sorted_gain_series(net, ss2, proto)\n",
    "                try:\n",
    "                    p_value = test_samples(gain_ss1, gain_ss2)\n",
    "                except Warning as e:\n",
    "                    print(ss1, ss2, net, proto, p_value)\n",
    "            if p_dict.get(ss1) is None:\n",
    "                p_dict[ss1] = {ss2: p_value}\n",
    "            else:\n",
    "                p_dict[ss1][ss2] = p_value\n",
    "\n",
    "        p_values = pd.DataFrame.from_dict(p_dict, orient='index')\n",
    "        p_values.loc[p_values.columns[-1]] = np.NaN\n",
    "        p_values.insert(0, p_values.iloc[0].name, np.NaN)\n",
    "\n",
    "        # replace zeros of the diagonal with ones (self comparison should return 1)\n",
    "        np.fill_diagonal(p_values.values, 1)\n",
    "\n",
    "        tests[net][proto] = p_values\n",
    "\n",
    "warnings.resetwarnings()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# concatenate obtained statistics by counting occurances of p_value > 0.05\n",
    "reference_df = tests[nets[0]][\"AND\"] \n",
    "reference_cols = list(reference_df.columns)\n",
    "reference_idx = list(reference_df.index)\n",
    "\n",
    "# containers for counts of p_values > 0.05\n",
    "p_counts_or = np.zeros_like(reference_df.to_numpy())\n",
    "p_counts_and = p_counts_or.copy()\n",
    "\n",
    "# for each statistic computed for given net and proto determine p_values > 0.05\n",
    "# and count them in container\n",
    "for net, net_tests in tests.items():\n",
    "    for proto, test in net_tests.items():\n",
    "        assert list(test.columns) == reference_cols\n",
    "        assert list(test.index) == reference_idx\n",
    "        p_counts = (test > 0.05).to_numpy().astype(int)\n",
    "        if proto == \"OR\":\n",
    "            p_counts_or += p_counts\n",
    "        if proto == \"AND\":\n",
    "            p_counts_and += p_counts\n",
    "\n",
    "# obtained matrices ought to be triangular, so replace zeros above diagonal \n",
    "# with nans\n",
    "p_counts_or[np.tril_indices(p_counts_or.shape[0], -1)] = np.nan\n",
    "p_counts_and[np.tril_indices(p_counts_and.shape[0], -1)] = np.nan\n",
    "\n",
    "# convert matrices to dateframes\n",
    "p_counts_or = pd.DataFrame(p_counts_or, columns=reference_cols, index=reference_idx)\n",
    "p_counts_and = pd.DataFrame(p_counts_and, columns=reference_cols, index=reference_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# visualise obtained results\n",
    "fig, ax = plt.subplots(\n",
    "    nrows=1, ncols=3, figsize=(10, 4), gridspec_kw={\"width_ratios\": [49, 49, 2]}\n",
    ")\n",
    "fig.tight_layout(pad=0.5, rect=(0.05, 0.1, 0.95, 0.85))\n",
    "title = (\n",
    "    \"Occurences 'p-value > 0.05' for Wilcoxon tests performed on series \" \n",
    "    + \"'f(mi_value, seeding_budget)->gain' \\n aggregated over all evaluated \" \n",
    "    + \"networks; protocols from left:\"\n",
    ")\n",
    "\n",
    "vrange = (0, max(p_counts_or.max().max(), p_counts_and.max().max()))\n",
    "vrange = (0, len(ss_methods))\n",
    "\n",
    "for idx, (proto, proto_df) in enumerate(zip([\"OR\", \"AND\"], [p_counts_or, p_counts_and])):\n",
    "    proto_df = proto_df.rename(index=ss_methods_abbrv_map, columns=ss_methods_abbrv_map)\n",
    "    _proto_df = proto_df.to_numpy()\n",
    "    np.fill_diagonal(_proto_df, len(ss_methods))\n",
    "    proto_df.loc[:] = _proto_df\n",
    "    plot_heatmap(proto_df, ax[idx], ax[2], cmap=\"Reds\", fmt=\".0f\", vrange=vrange)\n",
    "    title += f\" {proto}\"\n",
    "\n",
    "fig.subplots_adjust(wspace=.4)\n",
    "fig.suptitle(title)\n",
    "fig.savefig(workdir / f\"top_p_value_occurences.pdf\", dpi=300)\n",
    "plt.close(fig)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ltm-seeding-mln",
   "language": "python",
   "name": "ltm-seeding-mln"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "370e4dcd90c6b278f2eaed32c70bf26cf961be4169d33535105c1f563db5caee"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}