all_results_analysis.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyse obtained results"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports and necessary definitions valid for all analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import seaborn as sns\n",
    "\n",
    "from matplotlib.backends.backend_pdf import PdfPages\n",
    "from matplotlib.colors import ListedColormap\n",
    "from matplotlib.patches import Rectangle\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>network</th>\n",
       "      <th>protocol</th>\n",
       "      <th>seeding_budget</th>\n",
       "      <th>mi_value</th>\n",
       "      <th>diffusion_len</th>\n",
       "      <th>active_actors_prct</th>\n",
       "      <th>seed_actors_prct</th>\n",
       "      <th>gain</th>\n",
       "      <th>selection_metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aucs</td>\n",
       "      <td>OR</td>\n",
       "      <td>1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>1.639344</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>kpp_shell</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ckm_physicians</td>\n",
       "      <td>OR</td>\n",
       "      <td>1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>85.477178</td>\n",
       "      <td>1.244813</td>\n",
       "      <td>85.294118</td>\n",
       "      <td>kpp_shell</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>er1</td>\n",
       "      <td>OR</td>\n",
       "      <td>1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>kpp_shell</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>er2</td>\n",
       "      <td>OR</td>\n",
       "      <td>1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>kpp_shell</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>er3</td>\n",
       "      <td>OR</td>\n",
       "      <td>1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>kpp_shell</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          network protocol  seeding_budget  mi_value  diffusion_len  \\\n",
       "0            aucs       OR               1       0.1            3.0   \n",
       "1  ckm_physicians       OR               1       0.1            5.0   \n",
       "2             er1       OR               1       0.1            5.0   \n",
       "3             er2       OR               1       0.1            4.0   \n",
       "4             er3       OR               1       0.1            3.0   \n",
       "\n",
       "   active_actors_prct  seed_actors_prct        gain selection_metric  \n",
       "0          100.000000          1.639344  100.000000        kpp_shell  \n",
       "1           85.477178          1.244813   85.294118        kpp_shell  \n",
       "2          100.000000          1.000000  100.000000        kpp_shell  \n",
       "3          100.000000          1.000000  100.000000        kpp_shell  \n",
       "4          100.000000          1.000000  100.000000        kpp_shell  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"_experiments/all_methods/all_results.csv\")\n",
    "df = df.drop(\"Unnamed: 0\", axis=1)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Configure which data to use and where to save outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Choice: multilayer\n"
     ]
    }
   ],
   "source": [
    "ss_methods = df[\"selection_metric\"].unique()\n",
    "protocols = df[\"protocol\"].unique()\n",
    "\n",
    "# in some experiments mi_value was saved tih too big precision (differences \n",
    "# in 15th floating place) hence we need to round them \n",
    "df[\"mi_value\"] = df[\"mi_value\"].round(1)\n",
    "mi_vals = df[\"mi_value\"].unique()\n",
    "\n",
    "# permute networks so that they're ordered as following:\n",
    "# - real networks, ER SF\n",
    "nets_ml = [\n",
    "    'aucs', 'ckm_physicians', 'eu_transportation', 'lazega', \n",
    "    'er2', 'er3', 'er5', 'sf2', 'sf3', 'sf5',\n",
    "]\n",
    "nets_sl = [\"eu_trans_1\", \"er1\", \"sf1\"]\n",
    "assert set(df[\"network\"].unique()) == set(nets_ml + nets_sl)\n",
    "\n",
    "# set up a path where results are saved in\n",
    "workdir = Path(\"_results\")\n",
    "workdir.mkdir(exist_ok=True)\n",
    "\n",
    "# <----------- SELECT networks to analyse (multilayer of singlelayer)\n",
    "_net_choice = \"ml\"\n",
    "if _net_choice == \"ml\":\n",
    "    nets = nets_ml\n",
    "    net_case = \"multilayer\"\n",
    "elif _net_choice == \"sl\":\n",
    "    nets = nets_sl\n",
    "    net_case = \"singlelayer\"\n",
    "elif _net_choice == \"all\":\n",
    "    nets = nets_ml + nets_sl\n",
    "    net_case = \"all\" \n",
    "else:\n",
    "    raise ValueError\n",
    "print(f\"Choice: {net_case}\")\n",
    "\n",
    "ss_methods_abbrv_map ={\n",
    "    \"cbim\": \"cbim\",\n",
    "    \"cim\": \"cim\",\n",
    "    'degree_centrality': \"deg-c\",\n",
    "    'degree_centrality_discount': \"deg-c-d\",\n",
    "    'greedy': \"greedy\",\n",
    "    'k_shell': \"k-sh\",\n",
    "    'k_shell_mln': \"k-sh-m\",\n",
    "    'kpp_shell': \"kpp-sh\",\n",
    "    'neighbourhood_size': \"nghb-1s\",\n",
    "    'neighbourhood_2_hop_size': \"nghb-2s\",\n",
    "    'neighbourhood_size_discount': \"nghb-sd\",\n",
    "    'page_rank': \"p-rnk\",\n",
    "    'page_rank_mln': \"p-rnk-m\",\n",
    "    'random': \"random\",\n",
    "    'vote_rank': \"v-rnk\",\n",
    "    'vote_rank_mln': \"v-rnk-m\",\n",
    "}\n",
    "\n",
    "nets_abbrv_map = {\n",
    "    \"aucs\": \"aucs\",\n",
    "    \"ckm_physicians\": \"ckmp\",\n",
    "    \"lazega\": \"lazega\",\n",
    "    \"eu_transportation\": \"eutr-A\",\n",
    "    \"eu_trans_1\": \"eutr-1\",\n",
    "    \"er1\": \"er-1\",\n",
    "    \"er2\": \"er-2\",\n",
    "    \"er3\": \"er-3\",\n",
    "    \"er5\": \"er-5\",\n",
    "    \"sf1\": \"sf-1\",\n",
    "    \"sf2\": \"sf-2\",\n",
    "    \"sf3\": \"sf-3\",\n",
    "    \"sf5\": \"sf-5\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array(['kpp_shell', 'neighbourhood_size_discount', 'k_shell_mln',\n",
       "        'vote_rank', 'degree_centrality', 'cbim', 'page_rank',\n",
       "        'degree_centrality_discount', 'page_rank_mln',\n",
       "        'neighbourhood_2_hop_size', 'cim', 'neighbourhood_size',\n",
       "        'vote_rank_mln', 'k_shell', 'random', 'greedy'], dtype=object),\n",
       " array(['OR', 'AND'], dtype=object),\n",
       " array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]),\n",
       " array([ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 15, 20, 25, 30, 31, 32, 33,\n",
       "        34, 35, 36, 37, 38, 39, 40]),\n",
       " ['aucs',\n",
       "  'ckm_physicians',\n",
       "  'eu_transportation',\n",
       "  'lazega',\n",
       "  'er2',\n",
       "  'er3',\n",
       "  'er5',\n",
       "  'sf2',\n",
       "  'sf3',\n",
       "  'sf5'])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# for protocol OR we had budgets like [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30] \n",
    "# for protocol AND we had budgets like \n",
    "# [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40],\n",
    "# but we'd like to skip wery low budgets where all methods are unaccurate and keep only:\n",
    "# [15, 20, 25, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40] \n",
    "rows_to_drop = df.loc[\n",
    "    (df[\"protocol\"] == \"AND\") & \n",
    "    (\n",
    "        ~df[\"seeding_budget\"].isin(\n",
    "            {15, 20, 25, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40}\n",
    "        )\n",
    "    )\n",
    "]\n",
    "df = df.drop(rows_to_drop.index)\n",
    "s_budgets = df[\"seeding_budget\"].unique()\n",
    "\n",
    "# drop unselected nets\n",
    "rows_to_drop = df.loc[~df[\"network\"].isin(set(nets))]\n",
    "df = df.drop(rows_to_drop.index)\n",
    "\n",
    "\n",
    "ss_methods, protocols, mi_vals, s_budgets, nets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Optional, Union\n",
    "\n",
    "def prepare_ticklabels(series: pd.Index) -> Union[np.ndarray, str]:\n",
    "    try:\n",
    "        return series.to_numpy().round(2)\n",
    "    except:\n",
    "        return \"auto\"\n",
    "\n",
    "def plot_heatmap(\n",
    "    vis_df: pd.DataFrame,\n",
    "    heatmap_ax: plt.Axes,\n",
    "    bar_ax: plt.Axes,\n",
    "    vrange=(0, 100),\n",
    "    cmap=\"RdYlGn\",\n",
    "    mask: Optional[pd.DataFrame] = None,\n",
    "    fmt: Optional[str] = \".0f\",\n",
    ") -> None:\n",
    "    sns.heatmap(\n",
    "        vis_df,\n",
    "        ax=heatmap_ax,\n",
    "        cbar_ax=bar_ax,\n",
    "        cmap=cmap,\n",
    "        vmin=vrange[0],\n",
    "        vmax=vrange[1],\n",
    "        annot=True,\n",
    "        annot_kws={\"size\": 7},\n",
    "        fmt=fmt,\n",
    "        yticklabels=prepare_ticklabels(vis_df.index),\n",
    "        xticklabels=prepare_ticklabels(vis_df.columns),\n",
    "        linewidth=.5,\n",
    "        mask=mask,\n",
    "        cbar= True if bar_ax is not None else False,\n",
    "    )\n",
    "    heatmap_ax.invert_yaxis()\n",
    "    # heatmap_ax.tick_params(axis=\"x\", rotation=80)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Detailed heatmaps for all cases\n",
    "\n",
    "Charts of f(mi_value, seeding_budget) = gain for each network, protocol, ssm case"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### single plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = \"mi_value\"\n",
    "y = \"seeding_budget\"\n",
    "z = \"gain\"\n",
    "\n",
    "net = \"aucs\"\n",
    "ssm = \"cbim\"\n",
    "\n",
    "fig, ax = plt.subplots(\n",
    "    nrows=1, ncols=3, figsize=(10, 4), gridspec_kw={\"width_ratios\": [49, 49, 2]}\n",
    ")\n",
    "fig.tight_layout(pad=0.5, rect=(0.05, 0.05, 0.95, 0.95))\n",
    "title = f\"{z} achieved by {ssm} s.s. method on {net} net; protocols from left:\"\n",
    "\n",
    "for idx, proto in enumerate(df[\"protocol\"].unique()):\n",
    "    df_plotted = df.loc[\n",
    "        (df[\"network\"] == net) &\n",
    "        (df[\"protocol\"] == proto) &\n",
    "        (df[\"selection_metric\"] == ssm)\n",
    "    ]\n",
    "    df_plotted = pd.pivot_table(df_plotted, index=x, columns=y, values=z)\n",
    "    plot_heatmap(df_plotted, ax[idx], ax[2])\n",
    "    title += f\" {proto}\"\n",
    "\n",
    "fig.suptitle(title)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### bulk plot (pdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define values to visualise as well as attributes of plots\n",
    "x = \"mi_value\"\n",
    "y = \"seeding_budget\" \n",
    "Z = {\n",
    "    \"gain\": {\"vrange\": (0, 100), \"cmap\": \"RdYlGn\"},\n",
    "    \"diffusion_len\": {\"vrange\": (0, df[\"diffusion_len\"].max()), \"cmap\": \"BuPu\"},\n",
    "}\n",
    "\n",
    "# create file descriptor where to save visualisations\n",
    "pdf = PdfPages(workdir.joinpath(f\"all_heatmaps_{'_'.join(Z)}_by_{x}_{y}.pdf\"))\n",
    "\n",
    "for net in sorted(nets):\n",
    "\n",
    "    # a dummy plot that contains just name of processed network\n",
    "    fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(12, 4))\n",
    "    ax.set_visible(False)\n",
    "    fig.suptitle(f\"{net} network - results\", x=0.5, y=.5, fontsize = 15)\n",
    "    fig.savefig(pdf, format=\"pdf\")\n",
    "    plt.close(fig)\n",
    "\n",
    "    for ssm in sorted(ss_methods):\n",
    "\n",
    "        for z, z_attrs in Z.items():\n",
    "            print(f\"processing: {net}, {ssm}, {z}\")\n",
    "\n",
    "            # prepare canvas - proto, proto, legend\n",
    "            fig, ax = plt.subplots(\n",
    "                nrows=1,\n",
    "                ncols=3,\n",
    "                figsize=(12, 4),\n",
    "                gridspec_kw={\"width_ratios\": [49, 49, 2]}\n",
    "            )\n",
    "            fig.tight_layout(pad=0.5, rect=(0.1, 0.1, 0.9, 0.9))\n",
    "            title = (\n",
    "                f\"{z} achieved by {ssm} s.s. method on {net} network; \"\n",
    "                \"protocols from left:\"\n",
    "            )\n",
    "\n",
    "            # prepare and plot heatmap for each proto\n",
    "            for idx, proto in enumerate(protocols):\n",
    "                df_plot = df.loc[\n",
    "                    (df[\"network\"] == net) &\n",
    "                    (df[\"protocol\"] == proto) &\n",
    "                    (df[\"selection_metric\"] == ssm)\n",
    "                ]\n",
    "                df_plot = pd.pivot_table(df_plot, index=x, columns=y, values=z)\n",
    "                if len(df_plot) == 0:  # greedy wasn\"t evaluated for all nets\n",
    "                    continue\n",
    "                plot_heatmap(\n",
    "                    df_plot, ax[idx], ax[2], z_attrs[\"vrange\"], z_attrs[\"cmap\"]\n",
    "                )\n",
    "                title += f\" {proto}\"\n",
    "\n",
    "            # add title and save plot to pdf\n",
    "            fig.suptitle(title)\n",
    "            fig.savefig(pdf, format=\"pdf\")\n",
    "            plt.close(fig)\n",
    "\n",
    "pdf.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ranking for each selection method f(mi_Value, seeding_budget) -> position in the ranking \n",
    "\n",
    "First create a tensor if shape (ss_methods x nets x mi_vals x s_budgets_proto)\n",
    "valued with gains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "proto = \"AND\"  # <---------- change this to plot stats for AND!!!\n",
    "s_budgets_proto = df.loc[df[\"protocol\"] == proto][\"seeding_budget\"].unique()\n",
    "cube = {}\n",
    "consist_shape = []\n",
    "\n",
    "for ssm in ss_methods:\n",
    "    for net in nets:\n",
    "        ddf = df.loc[\n",
    "            (df[\"network\"] == net) &\n",
    "            (df[\"protocol\"] == proto) &\n",
    "            (df[\"selection_metric\"] == ssm)\n",
    "        ]\n",
    "        ddf = pd.pivot_table(ddf, index=\"mi_value\", columns=\"seeding_budget\", values=\"gain\").to_numpy()\n",
    "        if len(consist_shape) == 0:\n",
    "            consist_shape = ddf.shape\n",
    "        if len(ddf) == 0:\n",
    "            print(ssm)\n",
    "            ddf = np.empty(shape=consist_shape)\n",
    "            ddf[:] = np.nan\n",
    "\n",
    "        if not cube.get(ssm):\n",
    "            cube[ssm] = {}\n",
    "        cube[ssm][net] = ddf\n",
    "\n",
    "len(cube)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_arrays = []\n",
    "ssm_axis = []\n",
    "nets_axis = []\n",
    "\n",
    "for ssm, net_dict in cube.items(): \n",
    "    ssm_axis.append(ssm)\n",
    "    _arrs = []\n",
    "    for idx, (net, arr) in enumerate(net_dict.items()):\n",
    "        if len(nets_axis) == len(nets):\n",
    "            assert nets_axis[idx] == net\n",
    "        else:\n",
    "            nets_axis.append(net)\n",
    "        _arrs.append(arr)\n",
    "\n",
    "    list_arrays.append(np.stack(_arrs))\n",
    "\n",
    "cube_gain = np.stack(list_arrays)\n",
    "\n",
    "# array with all gains concatenated for all test runs - check shape (L==R)\n",
    "cube_gain.shape, (len(ss_methods), len(nets), len(mi_vals), len(s_budgets_proto))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When tensor is obtained now compute rankings of ss methods, i.e. place of ss \n",
    "method for certain evaluated case: net, mi_val, s_budgets_proto by obtained gain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Dict, List\n",
    "from scipy.stats import rankdata\n",
    "\n",
    "def rank_result(arr: np.ndarray) -> np.ndarray:\n",
    "    \"\"\"\n",
    "    Create ranking e.g for this array:\n",
    "    [52.63, 3.50, 100., -4, 3.50, 100., 100., np.nan, 100., 22.54, 100.]\n",
    "    return [2, 4, 1, 5, 4, 1, 1, 6, 1, 3, 1]\n",
    "    \"\"\"\n",
    "    raw_rank = rankdata(arr, method=\"min\", nan_policy=\"omit\")\n",
    "    if np.isnan(raw_rank).any():\n",
    "        raw_rank[np.isnan(raw_rank)] = 0\n",
    "    inverted_rank = np.abs(raw_rank - max(raw_rank) - 1)\n",
    "    _ = {idx: position for position, idx in enumerate(sorted(np.unique(inverted_rank)), 1)}\n",
    "    return np.vectorize(lambda x: _[x])(inverted_rank)\n",
    "\n",
    "def stat_to_dict(stat: np.ndarray, names: List) -> Dict:\n",
    "    return {n: stat[idx] for idx, n in enumerate(names)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranks = np.apply_along_axis(rank_result, 0, cube_gain)\n",
    "print(ranks.shape)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then reduce dimensionality of tensor by mean to obtain flat vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranks_reduded_by_mean_in_nets = np.mean(ranks, 1)\n",
    "print(ranks_reduded_by_mean_in_nets.shape)\n",
    "\n",
    "ranks_reduded_by_mean_in_nets_sb = np.mean(ranks_reduded_by_mean_in_nets, 1)\n",
    "print(ranks_reduded_by_mean_in_nets_sb.shape)\n",
    "\n",
    "ranks_reduded_by_mean_in_nets_sb_mi = np.mean(ranks_reduded_by_mean_in_nets_sb, 1)\n",
    "print(ranks_reduded_by_mean_in_nets_sb_mi.shape)\n",
    "\n",
    "overall_gain = stat_to_dict(ranks_reduded_by_mean_in_nets_sb_mi, ssm_axis)\n",
    "\n",
    "print(\"Average performance of ssm in total\")\n",
    "overall_gain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we need to manually obtain indices of real, sf and er nets\n",
    "print(nets_axis)\n",
    "rn_names = (\"real\", nets_axis[:4])\n",
    "er_names = (\"er\",   nets_axis[4:7])\n",
    "sf_names = (\"sf\",   nets_axis[7:])\n",
    "\n",
    "print(rn_names)\n",
    "print(er_names)\n",
    "print(sf_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute the same ranks but splitted through types of nets\n",
    "ranks_real_nets = (ranks[:, :4, ...], *rn_names)\n",
    "ranks_er_nets = (ranks[:, 4:7, ...], *er_names)\n",
    "ranks_sf_nets = (ranks[:, 7:, ...], *sf_names)\n",
    "\n",
    "net_type_gain = {}\n",
    "\n",
    "for rank_ in (ranks_real_nets, ranks_er_nets, ranks_sf_nets):\n",
    "    arr_, net_type_, names_ = rank_[0], rank_[1], rank_[2]\n",
    "\n",
    "    rank_reduded_by_mean_in_nets = np.mean(arr_, 1)\n",
    "    print(ranks_reduded_by_mean_in_nets.shape)\n",
    "\n",
    "    rank_reduded_by_mean_in_nets_sb = np.mean(rank_reduded_by_mean_in_nets, 1)\n",
    "    print(rank_reduded_by_mean_in_nets_sb.shape)\n",
    "\n",
    "    rank_reduded_by_mean_in_nets_sb_mi = np.mean(rank_reduded_by_mean_in_nets_sb, 1)\n",
    "    print(rank_reduded_by_mean_in_nets_sb_mi.shape)\n",
    "\n",
    "    net_type_gain[net_type_] = stat_to_dict(rank_reduded_by_mean_in_nets_sb_mi, ss_methods)\n",
    "\n",
    "print(\"Average performance of ssm by network type\")\n",
    "net_type_gain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net_type_gain[\"all\"] = overall_gain\n",
    "ranked_ssms = pd.DataFrame(net_type_gain).round(2)\n",
    "\n",
    "# replace values for greedy since it was computed only on real networks\n",
    "greedy_row = ranked_ssms.loc[\"greedy\"].to_dict()\n",
    "greedy_row[\"er\"] = greedy_row[\"sf\"] = np.nan\n",
    "greedy_row[\"all\"] = greedy_row[\"real\"]\n",
    "greedy_row\n",
    "ranked_ssms.loc[\"greedy\"] = greedy_row\n",
    "\n",
    "ranked_ssms.to_csv(workdir / f\"all_ssm_ranking_{proto}_{net_case}.csv\")\n",
    "ranked_ssms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# debugging\n",
    "_  = cube_gain[:, :1, ...]\n",
    "np.apply_along_axis(rank_result, 0, _)[:, 0, 4, 8], _[:, 0, 4, 8]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## General, averaged heatmap\n",
    "\n",
    "Charts of f(network, ssm) = average(??) for each protocol, where ?? is gain or\n",
    "diffusion length. There is also a ranking visualised from the average gain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = \"network\"\n",
    "y = \"selection_metric\"\n",
    "z = \"gain\"\n",
    "\n",
    "fig, ax = plt.subplots(\n",
    "    nrows=1, ncols=3, figsize=(10, 4), gridspec_kw={\"width_ratios\": [49, 49, 2]}\n",
    ")\n",
    "fig.tight_layout(pad=.5, rect=(0.05, 0.15, 0.95, 0.95))\n",
    "title = f\"Average {z} achieved by s.s. methods on nets; protocols from left:\"\n",
    "\n",
    "for idx, proto in enumerate(df[\"protocol\"].unique()):\n",
    "    df_plotted = df.loc[df[\"protocol\"] == proto]\n",
    "    df_plotted = pd.pivot_table(df_plotted, index=x, columns=y, values=z, aggfunc=np.mean)\n",
    "    df_plotted = df_plotted.reindex(index=nets)\n",
    "    df_plotted = df_plotted.rename(index=nets_abbrv_map, columns=ss_methods_abbrv_map)\n",
    "    \n",
    "    plot_heatmap(df_plotted, ax[idx], ax[2], mask=df_plotted.isnull())\n",
    "    title += f\" {proto}\"\n",
    "    df_plotted.to_numpy().argmax(axis=1)\n",
    "\n",
    "fname = workdir / f\"all_means_{net_case}_by_{z}.pdf\"\n",
    "fig.subplots_adjust(wspace=.4)\n",
    "fig.suptitle(title)\n",
    "fig.savefig(fname, dpi=300)\n",
    "plt.close(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = \"network\"\n",
    "y = \"selection_metric\"\n",
    "z = \"gain\"\n",
    "\n",
    "fig, ax = plt.subplots(\n",
    "    nrows=1, ncols=3, figsize=(10, 4), gridspec_kw={\"width_ratios\": [49, 49, 2]}\n",
    ")\n",
    "fig.tight_layout(pad=.5, rect=(0.05, 0.15, 0.95, 0.95))\n",
    "title = f\"Rank w.r.t. avg. {z} achieved by s.s. methods on nets; protocols from left:\"\n",
    "\n",
    "for idx, proto in enumerate(df[\"protocol\"].unique()):\n",
    "    df_plotted = df.loc[df[\"protocol\"] == proto]\n",
    "    df_plotted = pd.pivot_table(df_plotted, index=x, columns=y, values=z, aggfunc=np.mean)\n",
    "    df_plotted = df_plotted.reindex(index=nets)\n",
    "    df_plotted = df_plotted.rename(index=nets_abbrv_map, columns=ss_methods_abbrv_map)\n",
    "\n",
    "    df_plotted2 = df_plotted.copy()\n",
    "    df_plotted2.loc[:] = np.apply_along_axis(rank_result, 1, np.nan_to_num(df_plotted2.to_numpy(), 0))\n",
    "\n",
    "    # https://matplotlib.org/stable/users/explain/colors/colormaps.html\n",
    "    plot_heatmap(df_plotted2, ax[idx], ax[2], mask=df_plotted.isnull(), cmap=\"bone\", vrange=(1, len(ss_methods)))\n",
    "    title += f\" {proto}\"\n",
    "    df_plotted.to_numpy().argmax(axis=1)\n",
    "\n",
    "    for net_idx, ssm_idx in enumerate(df_plotted2.to_numpy().argmin(axis=1)):\n",
    "        ax[idx].add_patch(Rectangle((ssm_idx, net_idx), 1, 1, fill=False, edgecolor=\"black\", lw=2))\n",
    "\n",
    "fname = workdir / f\"all_means_{net_case}_by_{z}_ranked.pdf\"\n",
    "fig.subplots_adjust(wspace=.4)\n",
    "fig.suptitle(title)\n",
    "fig.savefig(fname, dpi=300)\n",
    "plt.close(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = \"network\"\n",
    "y = \"selection_metric\"\n",
    "z = \"diffusion_len\"\n",
    "\n",
    "fig, ax = plt.subplots(\n",
    "    nrows=1, ncols=3, figsize=(10, 4), gridspec_kw={\"width_ratios\": [49, 49, 2]}\n",
    ")\n",
    "fig.tight_layout(pad=.5, rect=(0.05, 0.15, 0.95, 0.95))\n",
    "title = f\"Average {z} achieved by s.s. methods on nets; protocols from left:\"\n",
    "\n",
    "for idx, proto in enumerate(df[\"protocol\"].unique()):\n",
    "    df_plotted = df.loc[df[\"protocol\"] == proto]\n",
    "    df_plotted = pd.pivot_table(df_plotted, index=x, columns=y, values=z, aggfunc=np.mean)\n",
    "    df_plotted = df_plotted.reindex(index=nets)\n",
    "    df_plotted = df_plotted.rename(index=nets_abbrv_map, columns=ss_methods_abbrv_map)\n",
    "    \n",
    "    # plot_heatmap(df_plotted, ax[idx], ax[2], mask=df_plotted.isnull())\n",
    "    plot_heatmap(\n",
    "        df_plotted, ax[idx], ax[2],\n",
    "        mask=df_plotted.isnull(),\n",
    "        cmap=\"BuPu\",\n",
    "        vrange=(1, np.ceil(df_plotted.max().max()).astype(int)),\n",
    "        fmt=\".1f\",\n",
    "    )\n",
    "    title += f\" {proto}\"\n",
    "    df_plotted.to_numpy().argmax(axis=1)\n",
    "\n",
    "fname = workdir / f\"all_means_{net_case}_by_{z}.pdf\"\n",
    "fig.subplots_adjust(wspace=.4)\n",
    "fig.suptitle(title)\n",
    "fig.savefig(fname, dpi=300)\n",
    "plt.close(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = \"network\"\n",
    "y = \"selection_metric\"\n",
    "z1 = \"gain\"\n",
    "z2 = \"diffusion_len\"\n",
    "\n",
    "fig, ax = plt.subplots(\n",
    "    nrows=1, ncols=3, figsize=(10, 4), gridspec_kw={\"width_ratios\": [49, 49, 2]}\n",
    ")\n",
    "fig.tight_layout(pad=.5, rect=(0.05, 0.15, 0.95, 0.95))\n",
    "title = f\"Average G/DL achieved by s.s. methods on nets; protocols from left:\"\n",
    "\n",
    "for idx, proto in enumerate(df[\"protocol\"].unique()):\n",
    "    df_plotted = df.loc[df[\"protocol\"] == proto]\n",
    "    df_plotted1 = pd.pivot_table(df_plotted, index=x, columns=y, values=z1, aggfunc=np.mean)\n",
    "    df_plotted2 = pd.pivot_table(df_plotted, index=x, columns=y, values=z2, aggfunc=np.mean)\n",
    "\n",
    "    df_plotted1.loc[:] = (df_plotted1.to_numpy() / df_plotted2.to_numpy()).round(0)\n",
    "    df_plotted1 = df_plotted1.reindex(index=nets)\n",
    "    df_plotted1 = df_plotted1.rename(index=nets_abbrv_map, columns=ss_methods_abbrv_map)\n",
    "\n",
    "    plot_heatmap(\n",
    "        df_plotted1, ax[idx], ax[2],\n",
    "        mask=df_plotted1.isnull(),\n",
    "        cmap=\"BuPu\",\n",
    "        vrange=(1, np.ceil(df_plotted1.max().max()).astype(int)),\n",
    "        fmt=\".0f\",\n",
    "    )\n",
    "    title += f\" {proto}\"\n",
    "\n",
    "fname = workdir / f\"all_means_{net_case}_bydk_{z}.pdf\"\n",
    "fig.subplots_adjust(wspace=.4)\n",
    "fig.suptitle(title)\n",
    "fig.savefig(fname, dpi=300)\n",
    "plt.close(fig)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Wilcoxon tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array(['kpp_shell', 'neighbourhood_size_discount', 'k_shell_mln',\n",
       "        'vote_rank', 'degree_centrality', 'cbim', 'page_rank',\n",
       "        'degree_centrality_discount', 'page_rank_mln',\n",
       "        'neighbourhood_2_hop_size', 'cim', 'neighbourhood_size',\n",
       "        'vote_rank_mln', 'k_shell', 'random', 'greedy'], dtype=object),\n",
       " ['aucs',\n",
       "  'ckm_physicians',\n",
       "  'eu_transportation',\n",
       "  'lazega',\n",
       "  'er2',\n",
       "  'er3',\n",
       "  'er5',\n",
       "  'sf2',\n",
       "  'sf3',\n",
       "  'sf5'],\n",
       " array(['OR', 'AND'], dtype=object))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from misc.wilcoxon_test import test_samples\n",
    "from itertools import combinations\n",
    "\n",
    "def get_sorted_gain_series(network, selection_method, protocol):\n",
    "    x = df.loc[\n",
    "        (df[\"network\"] == network) & \n",
    "        (df[\"selection_metric\"] == selection_method) & \n",
    "        (df[\"protocol\"] == protocol)\n",
    "    ].sort_values(by=[\"seeding_budget\", \"mi_value\"]).reindex()\n",
    "    if len(x) == 0:\n",
    "        raise ValueError\n",
    "    return x[\"gain\"].to_numpy()\n",
    "\n",
    "def get_selection_metrics_for_net(net):\n",
    "    return set(df.loc[df[\"network\"] == net][\"selection_metric\"].unique())\n",
    "\n",
    "ss_methods, nets, protocols"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### single plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = nets[0]\n",
    "\n",
    "fig, ax = plt.subplots(\n",
    "    nrows=1, ncols=2, figsize=(10, 4), gridspec_kw={\"width_ratios\": [50, 50]}\n",
    ")\n",
    "fig.tight_layout(pad=0.5, rect=(0.05, 0.05, 0.95, 0.95))\n",
    "title = f\"P-values of Wilcoxon test for network: {net}; protocols from left:\"\n",
    "\n",
    "\n",
    "for idx, proto in enumerate(protocols):\n",
    "    title += f\" {proto}\"\n",
    "\n",
    "    p_dict = {}\n",
    "    for ss1, ss2 in combinations(sorted(ss_methods), 2):\n",
    "        gain_ss1 = get_sorted_gain_series(net, ss1, proto)\n",
    "        gain_ss2 = get_sorted_gain_series(net, ss2, proto)\n",
    "        p_value = test_samples(gain_ss1, gain_ss2)\n",
    "        if p_dict.get(ss1) is None:\n",
    "            p_dict[ss1] = {ss2: p_value}\n",
    "        else:\n",
    "            p_dict[ss1][ss2] = p_value\n",
    "    p_values = pd.DataFrame.from_dict(p_dict, orient='index')\n",
    "    p_values.loc[p_values.columns[-1]] = np.NaN\n",
    "    p_values.insert(0, p_values.iloc[0].name, np.NaN)\n",
    "\n",
    "    # shorten names of records\n",
    "    p_values = p_values.rename(\n",
    "        index=ss_methods_abbrv_map, columns=ss_methods_abbrv_map\n",
    "    )\n",
    "\n",
    "    # plot heatmap\n",
    "    plot_heatmap(\n",
    "        p_values, \n",
    "        ax[idx],\n",
    "        bar_ax=None,\n",
    "        cmap=ListedColormap(['whitesmoke']),\n",
    "        fmt=\".2f\"\n",
    "    )\n",
    "\n",
    "    # mark the best value\n",
    "    rows, cols = np.where(p_values.to_numpy() >= 0.05)\n",
    "    for r_idx, c_idx in zip(rows, cols):\n",
    "        ax[idx].add_patch(\n",
    "            Rectangle((c_idx, r_idx), 1, 1, fill=True, color=\"lightcoral\", lw=0)\n",
    "        )\n",
    "\n",
    "fig.subplots_adjust(wspace=.4)\n",
    "fig.suptitle(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# p_values.insert(0, p_values.iloc[0].name, None)\n",
    "# p_values.loc[p_values.columns[-1]] = None\n",
    "p_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gain_ss1.shape, gain_ss2.shape, ss1, ss2, net, proto"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### bulk plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create file descriptor where to save visualisations\n",
    "pdf = PdfPages(workdir / f\"all_heatmaps_wilcoxon_test_{net_case}.pdf\")\n",
    "\n",
    "for net in sorted(nets):\n",
    "\n",
    "    # prepare canvas\n",
    "    fig, ax = plt.subplots(\n",
    "        nrows=1, ncols=2, figsize=(10, 4), gridspec_kw={\"width_ratios\": [50, 50]}\n",
    "    )\n",
    "    fig.tight_layout(pad=0.5, rect=(0.05, 0.05, 0.95, 0.95))\n",
    "    title = f\"P-values of Wilcoxon test for network: {net}; protocols from left:\"\n",
    "\n",
    "    # prepare and plot heatmap for each proto\n",
    "    for idx, proto in enumerate(protocols):\n",
    "        title += f\" {proto}\"\n",
    "\n",
    "        p_dict = {}\n",
    "        for ss1, ss2 in combinations(sorted(ss_methods), 2):\n",
    "            if not {ss1, ss2}.issubset(get_selection_metrics_for_net(net)):\n",
    "                continue  # greedy have not been computed for all nets\n",
    "            gain_ss1 = get_sorted_gain_series(net, ss1, proto)\n",
    "            gain_ss2 = get_sorted_gain_series(net, ss2, proto)\n",
    "            p_value = test_samples(gain_ss1, gain_ss2)\n",
    "            if p_dict.get(ss1) is None:\n",
    "                p_dict[ss1] = {ss2: p_value}\n",
    "            else:\n",
    "                p_dict[ss1][ss2] = p_value\n",
    "        p_values = pd.DataFrame.from_dict(p_dict, orient='index')\n",
    "        p_values.loc[p_values.columns[-1]] = np.NaN\n",
    "        p_values.insert(0, p_values.iloc[0].name, np.NaN)\n",
    "\n",
    "        # shorten names of records\n",
    "        p_values = p_values.rename(\n",
    "            index=ss_methods_abbrv_map, columns=ss_methods_abbrv_map\n",
    "        )\n",
    "\n",
    "        # plot heatmap\n",
    "        plot_heatmap(\n",
    "            p_values, ax[idx], bar_ax=None, \n",
    "            cmap=ListedColormap(['whitesmoke']), fmt=\".2f\"\n",
    "        )\n",
    "\n",
    "        # mark the best value\n",
    "        rows, cols = np.where(p_values.to_numpy() >= 0.05)\n",
    "        for r_idx, c_idx in zip(rows, cols):\n",
    "            ax[idx].add_patch(\n",
    "                Rectangle((c_idx, r_idx), 1, 1, fill=True, color=\"lightcoral\", lw=0)\n",
    "            )\n",
    "\n",
    "    # add title and save plot to pdf\n",
    "    fig.subplots_adjust(wspace=.4)\n",
    "    fig.suptitle(title)\n",
    "    fig.savefig(pdf, format=\"pdf\")\n",
    "    plt.close(fig)\n",
    "\n",
    "pdf.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### merged stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"error\")\n",
    "\n",
    "tests = {}\n",
    "\n",
    "for net in sorted(nets):\n",
    "    tests[net] = {}\n",
    "\n",
    "    for idx, proto in enumerate(protocols):\n",
    "\n",
    "        p_dict = {}\n",
    "        for ss1, ss2 in combinations(sorted(ss_methods), 2):\n",
    "            if not {ss1, ss2}.issubset(get_selection_metrics_for_net(net)):\n",
    "                p_value = None  # greedy have not been computed for all nets\n",
    "            else:\n",
    "                gain_ss1 = get_sorted_gain_series(net, ss1, proto)\n",
    "                gain_ss2 = get_sorted_gain_series(net, ss2, proto)\n",
    "                try:\n",
    "                    p_value = test_samples(gain_ss1, gain_ss2)\n",
    "                except Warning as e:\n",
    "                    print(ss1, ss2, net, proto, p_value)\n",
    "            if p_dict.get(ss1) is None:\n",
    "                p_dict[ss1] = {ss2: p_value}\n",
    "            else:\n",
    "                p_dict[ss1][ss2] = p_value\n",
    "\n",
    "        p_values = pd.DataFrame.from_dict(p_dict, orient='index')\n",
    "        p_values.loc[p_values.columns[-1]] = np.NaN\n",
    "        p_values.insert(0, p_values.iloc[0].name, np.NaN)\n",
    "\n",
    "        # replace zeros of the diagonal with ones (self comparison should return 1)\n",
    "        np.fill_diagonal(p_values.values, 1)\n",
    "\n",
    "        tests[net][proto] = p_values\n",
    "\n",
    "warnings.resetwarnings()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# concatenate obtained statistics by counting occurances of p_value > 0.05\n",
    "reference_df = tests[nets[0]][\"AND\"] \n",
    "reference_cols = list(reference_df.columns)\n",
    "reference_idx = list(reference_df.index)\n",
    "\n",
    "# containers for counts of p_values > 0.05\n",
    "p_counts_or = np.zeros_like(reference_df.to_numpy())\n",
    "p_counts_and = p_counts_or.copy()\n",
    "\n",
    "# for each statistic computed for given net and proto determine p_values > 0.05\n",
    "# and count them in container\n",
    "for net, net_tests in tests.items():\n",
    "    for proto, test in net_tests.items():\n",
    "        assert list(test.columns) == reference_cols\n",
    "        assert list(test.index) == reference_idx\n",
    "        p_counts = (test > 0.05).to_numpy().astype(int)\n",
    "        if proto == \"OR\":\n",
    "            p_counts_or += p_counts\n",
    "        if proto == \"AND\":\n",
    "            p_counts_and += p_counts\n",
    "\n",
    "# obtained matrices ought to be triangular, so replace zeros above diagonal \n",
    "# with nans\n",
    "p_counts_or[np.tril_indices(p_counts_or.shape[0], -1)] = np.nan\n",
    "p_counts_and[np.tril_indices(p_counts_and.shape[0], -1)] = np.nan\n",
    "\n",
    "# convert matrices to dateframes\n",
    "p_counts_or = pd.DataFrame(p_counts_or, columns=reference_cols, index=reference_idx)\n",
    "p_counts_and = pd.DataFrame(p_counts_and, columns=reference_cols, index=reference_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# visualise obtained results\n",
    "fig, ax = plt.subplots(\n",
    "    nrows=1, ncols=3, figsize=(10, 4), gridspec_kw={\"width_ratios\": [49, 49, 2]}\n",
    ")\n",
    "fig.tight_layout(pad=0.5, rect=(0.05, 0.1, 0.95, 0.85))\n",
    "title = (\n",
    "    \"Occurences 'p-value > 0.05' for Wilcoxon tests performed on series \" \n",
    "    + \"'f(mi_value, seeding_budget)->gain' \\n aggregated over all evaluated \" \n",
    "    + \"networks; protocols from left:\"\n",
    ")\n",
    "\n",
    "vrange = (0, max(p_counts_or.max().max(), p_counts_and.max().max()))\n",
    "vrange = (0, len(nets))\n",
    "diagonal_base = [len(nets)] * len(ss_methods)\n",
    "\n",
    "\n",
    "for idx, (proto, proto_df) in enumerate(zip([\"OR\", \"AND\"], [p_counts_or, p_counts_and])):\n",
    "    \n",
    "    proto_df = proto_df.rename(index=ss_methods_abbrv_map, columns=ss_methods_abbrv_map)\n",
    "    _proto_df = proto_df.to_numpy()\n",
    "\n",
    "    diagonal = diagonal_base.copy()\n",
    "    diagonal[proto_df.columns.get_loc(\"greedy\")] = 4\n",
    "    np.fill_diagonal(_proto_df, diagonal)\n",
    "\n",
    "    proto_df.loc[:] = _proto_df\n",
    "    plot_heatmap(proto_df, ax[idx], ax[2], cmap=\"Reds\", fmt=\".0f\", vrange=vrange)\n",
    "    title += f\" {proto}\"\n",
    "\n",
    "fig.subplots_adjust(wspace=.4)\n",
    "fig.suptitle(title)\n",
    "fig.savefig(workdir / f\"all_p_value_occurences_{net_case}.pdf\", dpi=300)\n",
    "plt.close(fig)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ltm-seeding-mln",
   "language": "python",
   "name": "ltm-seeding-mln"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "370e4dcd90c6b278f2eaed32c70bf26cf961be4169d33535105c1f563db5caee"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}