daily_update.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import itertools\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from ruamel.yaml import YAML\n",
    "from bokeh.plotting import figure, show, ColumnDataSource\n",
    "from bokeh.models import NumeralTickFormatter\n",
    "from bokeh.models.tools import HoverTool\n",
    "from bokeh.palettes import Category10, Category20\n",
    "from bokeh.resources import INLINE\n",
    "import panel as pn\n",
    "\n",
    "pn.extension()\n",
    "pn.config.sizing_mode = 'stretch_width'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "now = datetime.datetime.now()\n",
    "dt_string = now.strftime(\"%d-%m-%Y %H:%M\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "header = pn.pane.Markdown(f\"\"\"\n",
    "# COVID-19 -- World data\n",
    "\n",
    "Next page: [COVID-19 -- Argentina](https://epassaro.github.io/covid-19/ar)\n",
    "\n",
    "\"\"\", width=800, margin=(0, 0, 10, 0)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "footer = pn.pane.Markdown(f\"\"\"\n",
    "Data provided by the [John Hopkins University repository](https://github.com/CSSEGISandData/COVID-19). Source code available on [GitHub](https://github.com/epassaro/covid-19) under the [GNU GPLv3 License](https://raw.githubusercontent.com/epassaro/covid-19/master/LICENSE).\n",
    "<br> **Last updated on: ** _{dt_string}_ UTC. Powered by [Panel](https://panel.holoviz.org/index.html) and [GitHub Pages](https://pages.github.com/).\n",
    "\n",
    "\"\"\", width=800, margin=(0, 0, 10, 0)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fname = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yaml = YAML()\n",
    "\n",
    "with open('plots.yml') as f:\n",
    "    countries = yaml.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_top_ten(fname, how='total'):\n",
    "    df = pd.read_csv(fname)\n",
    "    df = df.drop(columns=['Province/State', 'Lat', 'Long'])\n",
    "    df = df.groupby(by=['Country/Region']).sum()\n",
    "\n",
    "    if how == 'new':\n",
    "        df['diff'] = df.iloc[:,-1] - df.iloc[:,-2]\n",
    "        df = df.sort_values(by='diff', ascending=False)\n",
    "\n",
    "    else:\n",
    "        df = df.iloc[:,-1]\n",
    "        df = df.sort_values(ascending=False)\n",
    "\n",
    "    top_ten = df.head(10).index.tolist()\n",
    "\n",
    "    return top_ten"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_table(fname, countries, key):\n",
    "\n",
    "    map_names = {'US': 'United States',\n",
    "                 'Korea, South': 'South Korea'}\n",
    "\n",
    "    df = pd.read_csv(fname)\n",
    "    df = df.drop(columns=['Province/State', 'Lat', 'Long'])\n",
    "    df = df.groupby(by=['Country/Region']).sum()    \n",
    "    df = df.transpose()\n",
    "\n",
    "    df.columns.name = None\n",
    "    df.index.name = 'date'\n",
    "    df.index = pd.to_datetime(df.index)\n",
    "\n",
    "    df_list = []\n",
    "    for country in countries[key]:\n",
    "\n",
    "        d = df[country]\n",
    "        n = d.diff()\n",
    "\n",
    "        n[n < 0] = 0 # remove negative diffs (dirty)\n",
    "        n[n >= 5*n.std() ] = 0 # remove outliers (dirty)\n",
    "\n",
    "        r = n.rolling(7).mean()\n",
    "        \n",
    "        n.name = f'diff{map_names.get(country, country)}'       \n",
    "        r.name = f'roll{map_names.get(country, country)}'   \n",
    "\n",
    "        df_list.append(d)\n",
    "        df_list.append(n)\n",
    "        df_list.append(r)\n",
    "\n",
    "\n",
    "    df = pd.concat(df_list, axis=1)\n",
    "    df = df.rename(columns=map_names)\n",
    "\n",
    "    df = df.reset_index()\n",
    "    df['datestr'] = df['date'].map(lambda x: x.strftime('%d-%m-%Y'))\n",
    "    df = df.set_index('date')\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_plots(df, name):\n",
    "\n",
    "    n = len(df.columns) // 3\n",
    "\n",
    "    if n == 2:  # dumb fix for 2 countries plot (use 3 colors palette)\n",
    "        n = 3\n",
    "\n",
    "    if n <= 10:\n",
    "        palette = itertools.cycle(Category10[n])\n",
    "\n",
    "    else:\n",
    "        palette = itertools.cycle(Category20[n])\n",
    "\n",
    "    source = ColumnDataSource(df)\n",
    "    p = figure(plot_width=680, plot_height=520, name='Total Cases', x_axis_type='datetime')\n",
    "    q = figure(plot_width=680, plot_height=520, name='New Cases', x_axis_type='datetime')\n",
    "\n",
    "    hovers = []\n",
    "    for country in df[ [c for c in df.columns if c[:4] not in ['date', 'diff', 'roll']] ]:\n",
    "\n",
    "        color = next(palette)\n",
    "        p.line(x='date', y=country, source=source,\n",
    "               line_width=1.5, \n",
    "               legend_label=country, \n",
    "               color=color)\n",
    "\n",
    "        p.circle(x='date', y=country, source=source,\n",
    "                 legend_label=country,\n",
    "                 color=color,\n",
    "                 name=country) # Tooltip are shown only on points\n",
    "\n",
    "        q.vbar(x='date', top=f'diff{country}', source=source,\n",
    "                 legend_label=country,\n",
    "                 width=datetime.timedelta(days=1),\n",
    "                 line_width=0,\n",
    "                 color=color,\n",
    "                 alpha=0.5,\n",
    "                 name=country)\n",
    "\n",
    "        q.line(x='date', y=f'roll{country}', source=source,\n",
    "                 legend_label=country,\n",
    "                 line_width=1.5,\n",
    "                 color=color,\n",
    "                 alpha=0.75,\n",
    "                 name=country)\n",
    "\n",
    "        TOOLTIPS = [('Country', country),\n",
    "                    ('Date', '@{datestr}'),\n",
    "                    ('New cases', f'@{{diff{country}}}{{0,0}}'),\n",
    "                    ('7-d mov. average', f'@{{roll{country}}}{{0,0.0}}'),\n",
    "                    ('Total cases', f'@{{{country}}}{{0,0}}')] # Double curly braces for names containing whitespaces\n",
    "\n",
    "        hover = HoverTool(tooltips=TOOLTIPS, names=[country], description=country)\n",
    "        hovers.append(hover)\n",
    "\n",
    "    p.add_tools(*hovers)\n",
    "    p.legend.click_policy = 'hide'\n",
    "    p.legend.location = 'top_left'\n",
    "    p.legend.label_text_font_size = '8pt'\n",
    "    p.yaxis.formatter = NumeralTickFormatter(format='0,0')\n",
    "    p.xaxis.axis_label = \"Date\"\n",
    "    p.yaxis.axis_label = \"Total cases\"\n",
    "    p.title.text = f\"Cumulative confirmed COVID-19 cases ({name})\"\n",
    "\n",
    "    q.add_tools(*hovers)\n",
    "    q.legend.click_policy = 'hide'\n",
    "    q.legend.location = 'top_left'\n",
    "    q.legend.label_text_font_size = '8pt'\n",
    "    q.yaxis.formatter = NumeralTickFormatter(format='0,0')\n",
    "    q.xaxis.axis_label = \"Date\"\n",
    "    q.yaxis.axis_label = \"New cases\"\n",
    "    q.title.text = f\"Daily new confirmed COVID-19 cases ({name})\"\n",
    "\n",
    "    return pn.Tabs(q, p, name=name, margin=(10, 20, 0, 20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "countries['Top New'] = get_top_ten(fname, how='new')\n",
    "countries['Top Total'] = get_top_ten(fname)\n",
    "\n",
    "countries.move_to_end('Top Total', last=False)\n",
    "countries.move_to_end('Top New', last=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "_ = [ make_plots(create_table(fname, countries, k), k) for k in countries ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabs = pn.Tabs(*_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pn.Column(header, \n",
    "          tabs,\n",
    "          footer,\n",
    "          margin=(20, 40, 0, 40), # top, right, bottom, left\n",
    "         ).save('index.html', \n",
    "                title='COVID-19 - World data', \n",
    "                resources=INLINE)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}