anaconda_package_data_dashboard.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Anaconda package download dashboard\n",
    "\n",
    "In order to create a download dashboard, we'll first get the last month of download data using ``intake`` as described in the previous notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import intake\n",
    "\n",
    "cat_url = 'https://raw.githubusercontent.com/ContinuumIO/anaconda-package-data/master/catalog/anaconda_package_data.yaml'\n",
    "cat = intake.Catalog(cat_url)\n",
    "\n",
    "today = datetime.date.today()\n",
    "first = today.replace(day=1)\n",
    "last_month = first - datetime.timedelta(days=1)\n",
    "try:\n",
    "    monthly = cat.anaconda_package_data_by_month(year=last_month.year, month=last_month.month).to_dask()\n",
    "    month = last_month\n",
    "except:\n",
    "    # if the last month isn't available, get the month before\n",
    "    month_before = last_month.replace(day=1) - datetime.timedelta(days=1)\n",
    "    monthly = cat.anaconda_package_data_by_month(year=month_before.year, month=month_before.month).to_dask()\n",
    "    month = month_before\n",
    "\n",
    "package_names = list(monthly.pkg_name.unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next we'll set up a parameterized class to create various plots and widgets to control those plots. We'll use a parameterized class to make it easier to keep track of the computed data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import colorcet as cc\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import panel as pn\n",
    "import param\n",
    "pd.options.plotting.backend = 'holoviews'\n",
    "\n",
    "\n",
    "class PackageExplorer(param.Parameterized):\n",
    "    title = f\"## Monthly conda downloads for {month.strftime('%B %Y')}\"\n",
    "    help_text = 'Select a package and optionally a version and groupby column to explore anaconda download data'\n",
    "    \n",
    "    package = param.String(default='pandas')\n",
    "    version = param.Selector(objects=['*.*'], default='*.*')\n",
    "    groupby = param.Selector(objects=['pkg_python', 'pkg_platform', 'data_source'])\n",
    "    backend = param.Selector(objects=['matplotlib', 'holoviews'], default='holoviews')\n",
    "    \n",
    "    data = param.Parameter(default=None, precedence=-1)\n",
    "    percent_df = param.Parameter(default=None, precedence=-1)\n",
    "    conda_badge = param.Parameter(default=None, precedence=-1)\n",
    "    saved = param.Boolean(default=False, precedence=-1)\n",
    "    \n",
    "    def __init__(self, **params):\n",
    "        super().__init__(**params)\n",
    "        self.update_saved()\n",
    "        self.update_data()\n",
    "\n",
    "    @property\n",
    "    def filename(self):\n",
    "        return f'data/{self.package}.parq'\n",
    "    \n",
    "    @param.depends('package', watch=True)\n",
    "    def update_saved(self):\n",
    "        self.saved = os.path.exists(self.filename)\n",
    "    \n",
    "    @param.depends('package', watch=True)\n",
    "    def update_data(self):\n",
    "        self.data = monthly[monthly['pkg_name'] == self.package].compute()\n",
    "    \n",
    "    @param.depends('data', 'version', 'groupby', watch=True)\n",
    "    def update_percent_df(self):\n",
    "        data = self.data\n",
    "        if self.version != '*.*':\n",
    "            data = data[data['pkg_version'] == self.version]\n",
    "        tot = data.groupby(self.groupby).counts.sum()\n",
    "        pct = tot / tot.sum()\n",
    "        self.percent_df = pd.DataFrame({'percents': pct, 'downloads': tot})\n",
    "    \n",
    "    @param.depends('data', watch=True)\n",
    "    def update_version(self):\n",
    "        versions = sorted(self.data.pkg_version.unique(), reverse=True)       \n",
    "        self.param.version.objects = ['*.*'] + versions\n",
    "        self.version = '*.*'\n",
    "        \n",
    "    @param.depends('percent_df', watch=True)\n",
    "    def update_conda_badge(self, top_of_colormap=1e6):\n",
    "        colors = cc.palette_n.rainbow[-20:80:-1]\n",
    "        step = len(colors) / np.log10(top_of_colormap)\n",
    "\n",
    "        downloads = self.percent_df.downloads.sum()\n",
    "        \n",
    "        # get color\n",
    "        if downloads > top_of_colormap:\n",
    "            color_index = -1\n",
    "        elif downloads > 0:\n",
    "            color_index = int(np.log10(downloads) * step)\n",
    "        else:\n",
    "            color_index = 0\n",
    "        color = colors[color_index][1:]\n",
    "        \n",
    "        # format downloads string\n",
    "        if downloads > 1e6:\n",
    "            downloads = '{}M'.format(int(downloads/1e6))\n",
    "        elif downloads > 1e3:\n",
    "            downloads = '{}k'.format(int(downloads/1e3))\n",
    "        else:\n",
    "            downloads = int(downloads)\n",
    "\n",
    "        self.conda_badge = f\"https://img.shields.io/badge/conda-{downloads}/month-{color}.svg\"\n",
    "\n",
    "    @param.depends('saved')\n",
    "    def save_button(self):\n",
    "        def on_click(*args):\n",
    "            data = self.data.drop(['pkg_name'], axis=1)\n",
    "            cols = ['pkg_version', 'pkg_python', 'pkg_platform']\n",
    "            data.to_parquet(self.filename, partition_cols=cols)\n",
    "            self.saved = True\n",
    "        button = pn.widgets.Button(name='Save Data', width=150, disabled=self.saved)\n",
    "        button.on_click(on_click)\n",
    "        return button\n",
    "    \n",
    "    @param.depends('saved')\n",
    "    def download_data(self):\n",
    "        display = 'none' if self.saved is False else 'block'\n",
    "        url = f'../notebooks/{self.filename}'\n",
    "        name = 'Download Saved File'\n",
    "        tip = 'This download will only work in the notebook or on binder, not in a served panel app.'\n",
    "        return f'<div style=\"display:{display}\"><a href={url} download title=\"{tip}\">{name}</a></div>'\n",
    "    \n",
    "    @param.depends('percent_df')\n",
    "    def table(self):\n",
    "        df = self.percent_df.copy()\n",
    "        df['percents'] = df.percents.apply(lambda x: f'{x:.2%}')\n",
    "        return df\n",
    "\n",
    "    @param.depends('percent_df')\n",
    "    def groupby_plot(self):\n",
    "        df = self.percent_df\n",
    "        title = f'{self.package}: {self.version} grouped by {self.groupby}'\n",
    "        return df.plot.barh(y='percents', title=title, ylim=(0, 1), xaxis=False, \n",
    "                            xlabel='', flip_yaxis=True, responsive=True, min_width=300)\n",
    "\n",
    "    @param.depends('conda_badge')\n",
    "    def conda_badge_str(self):\n",
    "        return pn.pane.Str(self.conda_badge, width_policy='max')\n",
    "        \n",
    "    @param.depends('conda_badge')\n",
    "    def conda_badge_md(self):\n",
    "        return self.conda_badge\n",
    "    \n",
    "    @param.depends('data', 'version')\n",
    "    def time_plot(self):\n",
    "        data = self.data\n",
    "        if self.version != '*.*':\n",
    "            data = data[data['pkg_version'] == self.version]\n",
    "        df = data.set_index('time').resample('1H').counts.sum()\n",
    "        title = f'{self.package}: {self.version} hourly downloads'\n",
    "        return df.plot.line(title=title, responsive=True, min_height=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we'll instantiate the class and create a layout."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explorer = PackageExplorer(name='')\n",
    "package_widget = pn.widgets.AutocompleteInput(options=package_names, value='pandas', name='Package name')\n",
    "\n",
    "dashboard = pn.Column(\n",
    "    explorer.title,\n",
    "    pn.Row(\n",
    "        pn.Column(\n",
    "            explorer.help_text,\n",
    "            pn.panel(explorer.param.package, widgets={'package': package_widget}),\n",
    "            pn.Row(explorer.save_button, \n",
    "                   pn.panel(explorer.download_data, align='end')),\n",
    "            explorer.param.version,\n",
    "            pn.panel('Groupby', margin=(10, 0, 0, 10)),\n",
    "            pn.panel(explorer.param.groupby, widgets={'groupby': pn.widgets.RadioButtonGroup}),\n",
    "        margin=0), \n",
    "        pn.Row(\n",
    "            explorer.table,\n",
    "            pn.panel(explorer.groupby_plot, margin=(25, 0, 0, 10),\n",
    "                     align='end', height_policy='max', width_policy='max'),\n",
    "            align='end'),\n",
    "        height_policy='max'),\n",
    "    pn.Row(\n",
    "        'Use this link to embed the badge:',\n",
    "        explorer.conda_badge_str,\n",
    "        pn.panel(explorer.conda_badge_md),\n",
    "        margin=(10, 0)),\n",
    "    explorer.time_plot,\n",
    "    width_policy='max',\n",
    ")\n",
    "dashboard.servable('Package Download Dashboard')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}