notebooks/thesis-runtimes-plots.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Produce plots for the first two experimental sections: architecture selection and runtime library comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from typing import Dict\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "import seaborn as sns\n",
    "import glob\n",
    "import wandb\n",
    "\n",
    "sns.set()\n",
    "\n",
    "# matplotlib.use(\"pgf\")\n",
    "matplotlib.rcParams.update({\n",
    "    \"pgf.texsystem\": \"pdflatex\",\n",
    "    'font.family': 'serif',\n",
    "    'font.serif': 'Palatino',\n",
    "    'text.usetex': True,\n",
    "    'pgf.rcfonts': False,\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_dir = \"/Users/vaclav/prog/thesis/data/benchmark_data/\"\n",
    "\n",
    "\n",
    "def read_results(subdir):\n",
    "    df = pd.read_csv(os.path.join(base_dir, subdir, \"results.csv\"), index_col=0)\n",
    "\n",
    "    df[\"runtime\"] = df[\"name\"].str.split(\"_\", n=1, expand=True)[0]\n",
    "\n",
    "    sparse_rows_mask = df[\"name\"].str.contains(\"sparse\")\n",
    "    assert df.loc[sparse_rows_mask, \"name\"].str.contains(\"0.90_sparse\").all(), \"Unimplemented sparsity level\"\n",
    "    assert (df.loc[\n",
    "                sparse_rows_mask, \"runtime\"] == \"DeepSparse\").all(), \"All sparse models are expected to be DeepSparse\"\n",
    "    df[\"sparsity\"] = 0.0\n",
    "    df.loc[sparse_rows_mask, \"sparsity\"] = 0.90\n",
    "\n",
    "    df.loc[df[\"runtime\"] == \"DeepSparse\", \"runtime\"] = \"DeepSparse (0\\%)\"\n",
    "    df.loc[sparse_rows_mask, \"runtime\"] = \"DeepSparse (90\\%)\"\n",
    "\n",
    "    df[\"quantization\"] = \"off\"\n",
    "    df.loc[df[\"name\"].str.contains(\"quant_static\"), \"quantization\"] = \"static\"\n",
    "    df.loc[df[\"name\"].str.contains(\"quant_dynamic\"), \"quantization\"] = \"dynamic\"\n",
    "    # df.loc[\"sparsity\"] = df[\"name\"].str.split(\"_\", n=1, expand=True)[0]\n",
    "\n",
    "    # Only the largest model\n",
    "    df = df.loc[df[\"model_i\"] == df[\"model_i\"].max()]\n",
    "\n",
    "    df[\"inference_time_ms\"] = 1000 * df[\"inference_time_s\"]\n",
    "    df = df.sort_values(\"name\", kind=\"stable\")\n",
    "\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_dir = \"/Users/vaclav/prog/thesis/figures/\"\n",
    "\n",
    "\n",
    "# def make_barplot(df, filename):\n",
    "#     fig = plt.figure()\n",
    "#     sns.barplot(data=df, y=\"name\", x='inference_time_s', ax=fig.gca(), ci=95)\n",
    "#     plt.xlabel(\"Inference time (s)\")\n",
    "#     plt.ylabel(\"Runtime library\")\n",
    "#     fig.savefig(os.path.join(plot_dir, filename), bbox_inches=\"tight\")\n",
    "\n",
    "def make_table(df):\n",
    "    df_latex_table = (df\n",
    "                      .groupby([\"runtime\", \"quantization\"])\n",
    "                      .agg({\"inference_time_ms\": \"mean\"})  #lambda x: x.quantile(0.99)\n",
    "                      .reset_index()\n",
    "                      .pivot(index=\"runtime\",\n",
    "                             columns=\"quantization\",\n",
    "                             values=\"inference_time_ms\")\n",
    "                      )[[\"off\", \"static\", \"dynamic\"]]\n",
    "\n",
    "    # df_latex_table.columns.name = \"Quantization\"\n",
    "    # df_latex_table.index.name = \"Runtime\"\n",
    "    df_latex_table.columns.name = None\n",
    "    df_latex_table.index.name = None\n",
    "\n",
    "    styler = (\n",
    "        df_latex_table.style\n",
    "            .format_index(escape=\"latex\")\n",
    "            .format(precision=2, na_rep=\"---\")\n",
    "    )\n",
    "\n",
    "    return styler.to_latex(hrules=True)\n",
    "\n",
    "\n",
    "def make_plots(df, name, display_name, pdf_metadata: Dict[str, str] = None):\n",
    "    if pdf_metadata is None:\n",
    "        pdf_metadata = {}\n",
    "\n",
    "    ci = 95\n",
    "    g = sns.catplot(\n",
    "        data=df, kind=\"bar\",\n",
    "        y=\"runtime\", x=\"inference_time_ms\", hue=\"quantization\",\n",
    "        ci=ci, palette=\"dark\", alpha=.6, height=4, aspect=4 / 3,\n",
    "    )\n",
    "    plt.xlabel(\"Inference time (ms)\")\n",
    "    plt.ylabel(\"Runtime name\")\n",
    "\n",
    "    plt.title(display_name)\n",
    "\n",
    "    pdf_metadata[\"ci\"] = ci\n",
    "    pdf_metadata[\"iterations\"] = df[\"iteration\"].max()\n",
    "\n",
    "    g.fig.savefig(\n",
    "        os.path.join(plot_dir, f\"{name}.pdf\"),\n",
    "        bbox_inches=\"tight\",\n",
    "        # Metadata doc: https://matplotlib.org/stable/api/backend_pdf_api.html#matplotlib.backends.backend_pdf.PdfFile\n",
    "        metadata={\n",
    "            \"Title\": \" \".join([f\"{k}={v}\" for k, v in pdf_metadata.items()])\n",
    "        },\n",
    "        backend=\"pgf\",\n",
    "    )\n",
    "\n",
    "    # print(make_table(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dirname = \"0711-dense-2\"\n",
    "df_dense = read_results(dirname)\n",
    "# df_dense = df_dense.loc[df_dense[\"quantization\"] == \"off\"]\n",
    "make_plots(df_dense, \"dense\", \"Dense model\", {\"dirname\": dirname})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dirname = \"0630-inverted_bottleneck\"\n",
    "df_ib = read_results(dirname)\n",
    "# df_ib = df_ib.loc[df_ib[\"quantization\"] == \"off\"]\n",
    "make_plots(df_ib, \"ib\", \"Inverted bottleneck model\", {\"dirname\": dirname})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dirname = \"0704-dilated_cnn\"\n",
    "# dirname = \"0715-dilated_cnn-2\" # input size 32\n",
    "dirname = \"0715-dilated_cnn-3\" # input size 128\n",
    "df_dilated_cnn = read_results(dirname)\n",
    "# df_dilated_cnn = df_dilated_cnn.loc[df_dilated_cnn[\"quantization\"] == \"off\"]\n",
    "make_plots(df_dilated_cnn, \"dilated_cnn\", \"Dilated CNN model\", {\"dirname\": dirname})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Architectural benchmarks\n",
    "\n",
    "- 0725-ddspae-2\n",
    "- 0715-ddspae-tiny\n",
    "- 0721-ddspae-cnn-8\n",
    "- TODO: fullrave\n",
    "- TODO: fullrave noiseless"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_columns(df, model_name):\n",
    "    df_out = df[[]].copy()\n",
    "\n",
    "    is_ddsp_like = \"ddsp\" in model_name or \"newt\" in model_name\n",
    "\n",
    "    if is_ddsp_like:\n",
    "        # There should be no encoder, because the encoding is done in the preprocessor\n",
    "        assert df[\"Autoencoder.encoder\"].mean() < 0.01 * df[\"Autoencoder\"].mean()\n",
    "\n",
    "        # The encoder is a pitch detector\n",
    "        df_out[\"Encoder (pitch detector)\"] = df[\"Autoencoder.preprocessor\"]\n",
    "\n",
    "        df_out[\"Decoder \" + (\"(CNN)\" if \"cnn\" in model_name else \"(RNN)\")] = df[\"Autoencoder.decoder\"]\n",
    "        df_out[\"Synthesizer\"] = df[\"Autoencoder.processor_group\"]\n",
    "    else:\n",
    "        # df_out[\"Preprocessor\"] = df[\"Autoencoder.preprocessor\"]\n",
    "        df_out[\"Encoder (CNN)\"] = df[\"Autoencoder.encoder\"]\n",
    "        df_out[\"Decoder (CNN)\"] = df[\"Autoencoder.decoder\"]\n",
    "        df_out[\"Multi-band decomposition\"] = df[\"Autoencoder.preprocessor\"] + df[\"Autoencoder.processor_group\"]\n",
    "\n",
    "    return df_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = [\n",
    "    {\"dataset\": \"Violin\", \"name\": \"0725-ddspae-2\", \"display_name\": \"DDSP-full\"},\n",
    "    {\"dataset\": \"Violin\", \"name\": \"0715-ddspae-tiny\", \"display_name\": \"DDSP-tiny\"},\n",
    "    {\"dataset\": \"Violin\", \"name\": \"0725-ddspae-cnn-1\", \"display_name\": \"DDSP-CNN\"},\n",
    "    {\"dataset\": \"Violin\", \"name\": \"0726-fullrave-noiseless\", \"display_name\": \"RAVE-like\"},\n",
    "    {\"dataset\": \"Violin\", \"name\": \"0726-ddspae-cnn\", \"display_name\": \"DDSP-CNN-IB\"},  # IB, ch=32\n",
    "    {\"dataset\": \"Violin\", \"name\": \"0725-newt\", \"display_name\": \"NEWT-like\"},\n",
    "    # TRUMPET\n",
    "    {\"dataset\": \"Trumpet\", \"name\": \"0805-ddspae\", \"display_name\": \"DDSP-full\"},\n",
    "    {\"dataset\": \"Trumpet\", \"name\": \"0805-ddspae-tiny\", \"display_name\": \"DDSP-tiny\"},\n",
    "    {\"dataset\": \"Trumpet\", \"name\": \"0804-ddspae-cnn-3\", \"display_name\": \"DDSP-CNN\"},\n",
    "    {\"dataset\": \"Trumpet\", \"name\": \"0809-fullrave-noiseless-6\", \"display_name\": \"RAVE-like\"}, # still training\n",
    "    {\"dataset\": \"Trumpet\", \"name\": \"0809-ddspae-cnn\", \"display_name\": \"DDSP-CNN-IB\"},  # IB, ch=32\n",
    "    {\"dataset\": \"Trumpet\", \"name\": \"0805-newt\", \"display_name\": \"NEWT-like\"},\n",
    "]\n",
    "\n",
    "eval_dir = '/Users/vaclav/prog/thesis/data/eval_data/'\n",
    "\n",
    "api = wandb.Api()\n",
    "\n",
    "for model in models:\n",
    "    model_name = model[\"name\"]\n",
    "    # !wandb artifact get 'neural-audio-synthesis-thesis/nas-evaluation/eval-'$model_name':latest' --root $eval_dir\n",
    "\n",
    "    artifact = api.artifact(f\"neural-audio-synthesis-thesis/nas-evaluation/eval-{model_name}:latest\")\n",
    "    csv_dir = artifact.checkout()\n",
    "\n",
    "    run = artifact.logged_by()\n",
    "    model[\"loss\"] = run.summary[\"losses/total_loss\"]\n",
    "\n",
    "    # glob.glob(os.path.join(eval_dir, f\"eval-{model_name}.csv\"))\n",
    "    model[\"csv_path\"] = os.path.join(csv_dir, f\"eval-{model_name}.csv\")\n",
    "    \n",
    "    print(artifact.created_at, model[\"name\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(run.config[\"operative_config\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for model in models:\n",
    "    dataset_artifact = {\n",
    "        \"Violin\": \"violin4\",\n",
    "        \"Trumpet\": \"urmp_tpt2\",\n",
    "    }[model[\"dataset\"]]\n",
    "    print(f\"$script {dataset_artifact}:latest {model['name']}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_benchmark_results(must_contain, aliases=[]):\n",
    "    df_all = []\n",
    "\n",
    "    for model in models:\n",
    "        model_name = model[\"name\"]\n",
    "        if must_contain not in model_name:\n",
    "            continue\n",
    "\n",
    "        if model[\"dataset\"] != \"Violin\" and False:\n",
    "            continue  # This would be redundant\n",
    "\n",
    "        df = pd.read_csv(model[\"csv_path\"], index_col=0)\n",
    "\n",
    "        for col, display_name in aliases:\n",
    "            df2 = df.copy()\n",
    "            df2[\"name\"] = display_name\n",
    "            df_all.append(df2[[\"name\", col]].rename(columns={col: \"time\"}))\n",
    "            print(f\"{col}: {df2[col].mean():.2f}\")\n",
    "\n",
    "        if must_contain == \"ddsp\":\n",
    "            # Disambiguate DDSP variants\n",
    "            df[\"name\"] = f\"Decoder ({model['display_name'][5:]})\"\n",
    "            df_all.append(df[[\"name\", \"Autoencoder.decoder\"]].rename(columns={\"Autoencoder.decoder\": \"time\"}))\n",
    "\n",
    "    df_all = pd.concat(df_all)\n",
    "    df_all = df_all.rename(columns={\"time\": \"Inference time (s)\", \"name\": \"Component\"})\n",
    "\n",
    "    return df_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_benchmark_results(df_all, filename):\n",
    "    g = sns.catplot(data=df_all, y=\"Component\", x=\"Inference time (s)\", kind=\"bar\", orient=\"h\", ci=95, aspect=2, height=2)\n",
    "    g.ax.bar_label(g.ax.containers[0], fmt=\"%.3f\", padding=10)\n",
    "\n",
    "    pdf_metadata = {\"models\": [x[\"name\"] for x in models]}\n",
    "\n",
    "    g.fig.savefig(\n",
    "        os.path.join(plot_dir, filename),\n",
    "        bbox_inches=\"tight\",\n",
    "        # Metadata doc: https://matplotlib.org/stable/api/backend_pdf_api.html#matplotlib.backends.backend_pdf.PdfFile\n",
    "        metadata={\n",
    "            \"Title\": \" \".join([f\"{k}={v}\" for k, v in pdf_metadata.items()])\n",
    "        },\n",
    "        backend=\"pgf\",\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all = prepare_benchmark_results(\n",
    "    must_contain=\"ddsp\",\n",
    "    aliases=[\n",
    "        (\"Autoencoder.preprocessor\", \"Pitch detector\"),\n",
    "        (\"Autoencoder.processor_group\", \"Synthesizer\")\n",
    "    ]\n",
    ")\n",
    "plot_benchmark_results(df_all, filename=\"ddsp-like-initial-benchmark.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all = prepare_benchmark_results(\n",
    "    must_contain=\"newt\",\n",
    "    aliases=[\n",
    "        (\"Autoencoder.preprocessor\", \"Pitch detector\"),\n",
    "        (\"Autoencoder.decoder\", \"Decoder\"),\n",
    "        (\"Autoencoder.processor_group\", \"NEWT Synthesizer\"),\n",
    "    ]\n",
    ")\n",
    "plot_benchmark_results(df_all, filename=\"newt-initial-benchmark.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all = prepare_benchmark_results(\n",
    "    must_contain=\"rave\",\n",
    "    aliases=[\n",
    "        (\"Autoencoder.preprocessor\", \"Multi-band analysis\"),\n",
    "        (\"Autoencoder.encoder\", \"Encoder\"),\n",
    "        (\"Autoencoder.decoder\", \"Decoder\"),\n",
    "        (\"Autoencoder.processor_group\", \"Multi-band synthesis\"),\n",
    "    ]\n",
    ")\n",
    "plot_benchmark_results(df_all, filename=\"rave-initial-benchmark.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(\n",
    "    [[m[\"display_name\"], m[\"dataset\"], m[\"loss\"]] for m in models],\n",
    "    columns=[\"Model\", \"Dataset\", \"Loss\"]\n",
    ")\n",
    "df = df.pivot(index=\"Model\", columns=\"Dataset\")\n",
    "df.columns = df.columns.get_level_values(1)\n",
    "df.columns.name = None\n",
    "df = df.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "styler = (\n",
    "    df.style\n",
    "        # .format_index(escape=\"latex\")\n",
    "        .hide(axis=\"index\")\n",
    "        .format(precision=2, na_rep=\"---\")\n",
    ")\n",
    "\n",
    "print(styler.to_latex(hrules=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for model in models:\n",
    "    df = pd.read_csv(model[\"csv_path\"], index_col=0)\n",
    "    df = prepare_columns(df, model[\"name\"])\n",
    "    g = sns.catplot(data=df, kind=\"bar\", orient=\"h\", ci=95, aspect=2, height=2)\n",
    "    # g.set(xlim=(0, 3))\n",
    "    g.ax.bar_label(g.ax.containers[0], fmt=\"%.2f\", padding=10)\n",
    "    plt.title(model.get(\"display_name\", model[\"name\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g.ax.containers[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame([[m[\"display_name\"], m[\"loss\"]] for m in models if \"ddsp\" in m[\"name\"]], columns=[\"Model\", \"Loss\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "styler = (\n",
    "    df.style\n",
    "        # .format_index(escape=\"latex\")\n",
    "        .hide(axis=\"index\")\n",
    "        .format(precision=2, na_rep=\"---\")\n",
    ")\n",
    "\n",
    "print(styler.to_latex(hrules=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.catplot(data=df, x=0, y=1, kind=\"bar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for model in models:\n",
    "    df = pd.read_csv(model[\"csv_path\"], index_col=0)\n",
    "    df = prepare_columns(df, model[\"name\"])\n",
    "    g = sns.catplot(data=df, kind=\"bar\", orient=\"h\", ci=95, aspect=2, height=2)\n",
    "    # g.set(xlim=(0, 3))\n",
    "    g.ax.bar_label(g.ax.containers[0], fmt=\"%.2f\", padding=10)\n",
    "    plt.title(model.get(\"display_name\", model[\"name\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g.ax.containers[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}