local_corridor_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualization/Analysis of Local GTFS-RT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "\n",
    "from datetime import datetime, timezone, timedelta\n",
    "from dateutil import tz\n",
    "\n",
    "\n",
    "import branca.colormap as cm\n",
    "import folium\n",
    "from folium.plugins import HeatMap\n",
    "from folium.plugins import HeatMapWithTime\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pytz\n",
    "import seaborn as sns\n",
    "\n",
    "TZ = pytz.timezone('America/Los_Angeles')\n",
    "sns.set_style('whitegrid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "corridor_segments = {\n",
    "    0: {\n",
    "        2749,\n",
    "        2751,\n",
    "        2752,\n",
    "        2753,\n",
    "        2754,\n",
    "        2755,\n",
    "        2756,\n",
    "        2757\n",
    "    },\n",
    "    1: {\n",
    "        12200,\n",
    "        12201,\n",
    "        12202,\n",
    "        12203,\n",
    "        12204\n",
    "    },\n",
    "    2: {\n",
    "        13532,\n",
    "        13533,\n",
    "        13534,\n",
    "        13535,\n",
    "        13536\n",
    "    },\n",
    "    3: {\n",
    "        14585,\n",
    "        14586,\n",
    "        14587,\n",
    "        14589,\n",
    "        14590,\n",
    "        14591,\n",
    "        14243\n",
    "    },\n",
    "    4: {\n",
    "        8510,\n",
    "        8511,\n",
    "        8512,\n",
    "        8513,\n",
    "        8514\n",
    "    },\n",
    "    5: {\n",
    "        13221,\n",
    "        13220,\n",
    "        13219,\n",
    "        13218,\n",
    "        13217\n",
    "    },\n",
    "    6: {\n",
    "        18045,\n",
    "        18046,\n",
    "        18047,\n",
    "        18048,\n",
    "        18049\n",
    "    },\n",
    "    7: {\n",
    "        19235,\n",
    "        19234,\n",
    "        11333,\n",
    "        11331,\n",
    "        11330,\n",
    "        11329,\n",
    "        19212\n",
    "    },\n",
    "    8: {\n",
    "        12369,\n",
    "        12366,\n",
    "        12365,\n",
    "        12364,\n",
    "        12363\n",
    "    }\n",
    "}\n",
    "corridor_names = ['2nd Ave; Pike to James',\n",
    "                 'Pike; 3rd to 9th',\n",
    "                 'Westlake; Denny to Mercer',\n",
    "                 'E Olive/E John; Denny to 10th',\n",
    "                 '9th Ave; Alder to Columbia',\n",
    "                 'University Way; Campus Pkwy to 45th',\n",
    "                  'Pacific; 15th to Montlake',\n",
    "                  'NW Market/Leary; 24th to 15th',\n",
    "                  'Rainier; S Bayview to MLK',\n",
    "                  '108th Ave; 4th to 12th']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process the Local Summarize_RDS Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "Unable to allocate 964. MiB for an array with shape (15, 8421801) and data type float64",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-09824a60c791>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;31m# If already processed, load the data here and comment out above\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0mall_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{path}/../all_data.csv\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'trip_short_name'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'route_short_name'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;31m# Add date to the data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    684\u001b[0m     )\n\u001b[1;32m    685\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 686\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    687\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    688\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    456\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    457\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 458\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    459\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    460\u001b[0m         \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m   1209\u001b[0m             \u001b[0mnew_rows\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1210\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1211\u001b[0;31m         \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcol_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1213\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_currow\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mnew_rows\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data, index, columns, dtype, copy)\u001b[0m\n\u001b[1;32m    466\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    467\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 468\u001b[0;31m             \u001b[0mmgr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minit_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    469\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMaskedArray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    470\u001b[0m             \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmrecords\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mmrecords\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/pandas/core/internals/construction.py\u001b[0m in \u001b[0;36minit_dict\u001b[0;34m(data, index, columns, dtype)\u001b[0m\n\u001b[1;32m    281\u001b[0m             \u001b[0marr\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_datetime64tz_dtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0marr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    282\u001b[0m         ]\n\u001b[0;32m--> 283\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0marrays_to_mgr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_names\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    284\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    285\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/pandas/core/internals/construction.py\u001b[0m in \u001b[0;36marrays_to_mgr\u001b[0;34m(arrays, arr_names, index, columns, dtype, verify_integrity)\u001b[0m\n\u001b[1;32m     91\u001b[0m     \u001b[0maxes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mcreate_block_manager_from_arrays\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marr_names\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/pandas/core/internals/managers.py\u001b[0m in \u001b[0;36mcreate_block_manager_from_arrays\u001b[0;34m(arrays, names, axes)\u001b[0m\n\u001b[1;32m   1681\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1682\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1683\u001b[0;31m         \u001b[0mblocks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mform_blocks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnames\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1684\u001b[0m         \u001b[0mmgr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBlockManager\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblocks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1685\u001b[0m         \u001b[0mmgr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_consolidate_inplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/pandas/core/internals/managers.py\u001b[0m in \u001b[0;36mform_blocks\u001b[0;34m(arrays, names, axes)\u001b[0m\n\u001b[1;32m   1739\u001b[0m     \u001b[0mblocks\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mBlock\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1740\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitems_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"FloatBlock\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1741\u001b[0;31m         \u001b[0mfloat_blocks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_multi_blockify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitems_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"FloatBlock\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1742\u001b[0m         \u001b[0mblocks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfloat_blocks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1743\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/pandas/core/internals/managers.py\u001b[0m in \u001b[0;36m_multi_blockify\u001b[0;34m(tuples, dtype)\u001b[0m\n\u001b[1;32m   1833\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtup_block\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgrouper\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1834\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1835\u001b[0;31m         \u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplacement\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_stack_arrays\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtup_block\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1836\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1837\u001b[0m         \u001b[0mblock\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmake_block\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplacement\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mplacement\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/pandas/core/internals/managers.py\u001b[0m in \u001b[0;36m_stack_arrays\u001b[0;34m(tuples, dtype)\u001b[0m\n\u001b[1;32m   1861\u001b[0m     \u001b[0mshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0m_shape_compat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfirst\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1862\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1863\u001b[0;31m     \u001b[0mstacked\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1864\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1865\u001b[0m         \u001b[0mstacked\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_asarray_compat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mMemoryError\u001b[0m: Unable to allocate 964. MiB for an array with shape (15, 8421801) and data type float64"
     ]
    }
   ],
   "source": [
    "# Location for the local data\n",
    "path = './transit_vis/data/to_upload'\n",
    "\n",
    "# Join all the data which is saved in different csv for each day\n",
    "# files = os.listdir(path)\n",
    "# frames = [pd.read_csv(f\"{path}/{filename}\", dtype={'trip_short_name': str, 'route_short_name': str}) for filename in files[0:30]]\n",
    "# all_data = pd.concat(frames)\n",
    "# all_data.to_csv(f\"{path}/../all_data.csv\")\n",
    "# all_data.head()\n",
    "\n",
    "# If already processed, load the data here and comment out above\n",
    "all_data = pd.read_csv(f\"{path}/../all_data.csv\", dtype={'trip_short_name': str, 'route_short_name': str})\n",
    "\n",
    "# Add date to the data\n",
    "all_data['day'] = pd.to_datetime(all_data['locationtime'], unit='s')\n",
    "all_data['day'] = all_data['day'].dt.strftime('%Y-%m-%d')\n",
    "\n",
    "# Add corridor id to the data\n",
    "def get_dict_key(value, corridor_dict):\n",
    "    for corridor in corridor_dict:\n",
    "        if value in corridor_dict[corridor]:\n",
    "            return corridor\n",
    "    return -1\n",
    "all_data['corridor_id'] = all_data['seg_compkey'].apply(get_dict_key, args=(corridor_segments,))\n",
    "\n",
    "# REMOVE ALL DATA NOT IN CORRIDORS\n",
    "# all_data = all_data[all_data['corridor_id']>=0]\n",
    "\n",
    "# Add travel time to the data\n",
    "# Use difference in the first/last time each unique trip is recorded on a corridor's segments for each day\n",
    "tt_data = all_data[all_data['corridor_id']>=0]\n",
    "tt_data = tt_data.groupby(['corridor_id','day','tripid']).agg(['max','min'])[['locationtime']]\n",
    "tt_data.reset_index(inplace=True)\n",
    "tt_data['tt'] = tt_data[('locationtime','max')] - tt_data[('locationtime','min')]\n",
    "tt_data.columns = tt_data.columns.droplevel(1)\n",
    "\n",
    "# Some interferences are matched to GTFS data that was assigned to segments outside of the corridor; these are left empty\n",
    "# This only affects traversal times. The tt cannot be calculated for tracked locations outside of a corridor\n",
    "all_data = pd.merge(all_data, tt_data[['corridor_id','day','tripid','tt']], on=['corridor_id','day','tripid'], how='left')\n",
    "\n",
    "all_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Distributions of Metrics for Buses with/without Interference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "path = './transit_vis/data/to_upload'\n",
    "\n",
    "interference_data = pd.read_csv(f\"{path}/../interference_instances_new.csv\")\n",
    "interference_data['start_time'] = pd.to_datetime(interference_data['int_start_time']).dt.tz_localize(tz.tzlocal()).astype(int).astype(str).str.slice(0,10).astype(int)\n",
    "interference_data['stop_time'] = pd.to_datetime(interference_data['int_stop_time']).dt.tz_localize(tz.tzlocal()).astype(int).astype(str).str.slice(0,10).astype(int)\n",
    "interference_data['corridor_id'] = interference_data['corridor_id'] - 1\n",
    "interference_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Track down the datapoints in GTFS-RT that correspond to each interference instance\n",
    "idx_interference = []  # The index of the interference in the interference data\n",
    "data_interference = []  # Data for +/- xmins from the interference in GTFS-RT\n",
    "datarow_interference = []  # What row of the +/- xmins GTFS-RT data the interference corresponds to\n",
    "\n",
    "for idx, row in interference_data.iterrows():\n",
    "\n",
    "    # Get the tracked datapoints corresponding to +/- xmins of the data; FROM THE START of the interference\n",
    "    found = False\n",
    "    margin_mins = 2\n",
    "    margin = margin_mins*60  # xmins in seconds\n",
    "    data = all_data[all_data['vehicleid'] == row.vehicle_id]\n",
    "    data = data[(data['locationtime']>=(row.start_time - margin)) & (data['locationtime']<=(row.start_time + margin))]\n",
    "    \n",
    "    # Get the metric across the time period where the interference occurred\n",
    "    if len(data) > 0:\n",
    "        data = data.sort_values(by='locationtime', ascending=True)\n",
    "        for j, time in enumerate(data['locationtime']):\n",
    "            if row.start_time <= time:\n",
    "                idx_interference.append(idx)\n",
    "                datarow_interference.append(j-1)\n",
    "                # Get time variable for each interference that is +/- seconds from 0; where 0 is the time of the interference\n",
    "                data['time_from_interference'] = data['locationtime'] - data['locationtime'].iloc[j-1]\n",
    "                # Get schedule deviation variable for each location with interference at 0\n",
    "                data['deviation_from_interference'] = data['scheduledeviation'] - data['scheduledeviation'].iloc[j-1]\n",
    "                data['interference_id'] = str(idx)\n",
    "                data_interference.append(data)\n",
    "                found = True\n",
    "                break\n",
    "        if not found:\n",
    "            print(f\"Interference {idx} vehicle id {row.vehicle_id} Fell outside data\")\n",
    "    else:\n",
    "        print(f\"Interference {idx} vehicle id {row.vehicle_id} No data found\")\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(6,4,figsize=(10,10))\n",
    "fig.tight_layout(h_pad=3)\n",
    "\n",
    "for i, idx in enumerate(idx_interference):\n",
    "    sns.ecdfplot(data=data_interference[i], x='speed_m_s', ax=axes.flat[i]).set(xlim=(-120,60), title=f\"Interference #{idx}: {interference_data['difference'][idx]}: {interference_data['corridor_name'][idx]}\")\n",
    "    axes.flat[i].axvline(data_interference[i]['speed_m_s'].iloc[datarow_interference[i]], color='red')\n",
    "    axes.flat[i].axvline(np.median(data_interference[i]['speed_m_s']), color='blue')\n",
    "\n",
    "fig.savefig(fname=\"interference_speeds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(9,4,figsize=(10,10))\n",
    "fig.tight_layout(h_pad=3)\n",
    "\n",
    "for i, idx in enumerate(idx_interference):\n",
    "    sns.ecdfplot(data=data_interference[i], x='deviation_change_s', ax=axes.flat[i]).set(xlim=(-120,60), title=f\"Interference #{idx}: {interference_data['difference'][idx]}: {interference_data['corridor_name'][idx]}\")\n",
    "    axes.flat[i].axvline(data_interference[i]['deviation_change_s'].iloc[datarow_interference[i]], color='red')\n",
    "    axes.flat[i].axvline(np.median(data_interference[i]['deviation_change_s']), color='blue')\n",
    "\n",
    "fig.savefig(fname=\"interference_deviation_changes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(3,3,figsize=(20,20))\n",
    "fig.tight_layout(h_pad=3)\n",
    "\n",
    "for i in range(9):  # No Bellevue corridor because using the Seattle Streets dataset to map match coords\n",
    "    full_data = all_data[all_data['corridor_id']==i]\n",
    "    sns.ecdfplot(data=full_data, x='tt', ax=axes.flat[i]).set(xlim=(-1,600), title=f\"{corridor_names[i]}\")\n",
    "    for j in range(len(idx_interference)):\n",
    "        if data_interference[j]['corridor_id'].iloc[datarow_interference[j]]==i:\n",
    "            axes.flat[i].axvline(data_interference[j]['tt'].iloc[datarow_interference[j]], color='red')\n",
    "    axes.flat[i].axvline(np.median(full_data['tt']), color='blue')\n",
    "    \n",
    "fig.savefig(fname=\"corridor_traversal_times\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create column for de-meaned values in the timeseries data\n",
    "for i in range(len(data_interference)):\n",
    "    norm = np.mean(data_interference[i]['scheduledeviation'])\n",
    "    data_interference[i]['scheduledeviation_norm'] = data_interference[i]['scheduledeviation'] - norm\n",
    "\n",
    "fig, axes = plt.subplots(1,1,figsize=(20,10))\n",
    "\n",
    "# Event study timeseries plot per interference\n",
    "sns.lineplot(data=pd.concat(data_interference),\n",
    "             x=\"time_from_interference\",\n",
    "             y=\"scheduledeviation_norm\",\n",
    "             hue=\"interference_id\",\n",
    "             palette='Set2',\n",
    "             marker='o',\n",
    "             ax=axes).set(xlim=(-120,120), xlabel='Time from Interference (s)', ylabel='Schedule Deviation (s)', title=f\"De-Meaned Schedule Deviation +/-{margin_mins}min from Moment of Interference\")\n",
    "\n",
    "# LOWESS regression for data before interference\n",
    "sns.regplot(data=pd.concat(data_interference)[pd.concat(data_interference)['time_from_interference']<=0],\n",
    "             x=\"time_from_interference\",\n",
    "             y=\"scheduledeviation_norm\",\n",
    "             lowess=True,\n",
    "             color='black').set(xlim=(-110,200), xlabel='Time from Interference (s)', ylabel='Schedule Deviation from Interference (s)')\n",
    "\n",
    "# LOWESS regression for data after interference\n",
    "sns.regplot(data=pd.concat(data_interference)[pd.concat(data_interference)['time_from_interference']>0],\n",
    "             x=\"time_from_interference\",\n",
    "             y=\"scheduledeviation_norm\",\n",
    "             lowess=True,\n",
    "             color='black').set(xlim=(-110,200), xlabel='Time from Interference (s)', ylabel='Schedule Deviation from Interference (s)')\n",
    "plt.legend(loc='upper right')\n",
    "\n",
    "fig.savefig(fname=\"event_study_plot\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find Top 10/Top Random Coaches that Spend Most Time in Corridors, and Avg Unique Corridor Traversals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter to period of interest\n",
    "# start_time = 1607072400 # 12/4/20 1AM\n",
    "# end_time = 1608282000 # 12/18/20 1AM\n",
    "start_time = 1607518800 # 12/9/20 5AM\n",
    "end_time = 1607583600 # 12/9/20 11PM\n",
    "all_data = all_data.loc[(all_data['locationtime'] > start_time) & (all_data['locationtime'] < end_time)]\n",
    "\n",
    "# Get list of all segments in our corridors\n",
    "segments_dict = list(corridor_segments.values())\n",
    "all_corridor_segments = []\n",
    "for item in segments_dict:\n",
    "    for seg in item:\n",
    "        all_corridor_segments.append(seg)\n",
    "\n",
    "# Filter to only tracked locations in our corridors\n",
    "data = all_data.loc[all_data['seg_compkey'].isin(all_corridor_segments)]\n",
    "del all_data\n",
    "\n",
    "#data.to_csv(\"for_borna.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the times that buses are in our corridors\n",
    "z = []\n",
    "dates = data['locationtime'].values\n",
    "dates\n",
    "for time in dates:\n",
    "    x = int(datetime.fromtimestamp(time).strftime(\"%-M\"))/60\n",
    "    y = int(datetime.fromtimestamp(time).strftime(\"%H\"))\n",
    "    z.append(x+y)\n",
    "plt.hist(z, bins=62)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the average number of traversals per corridor\n",
    "# Want count of unique trip ids for each segment id\n",
    "traversal_data = data.groupby(['seg_compkey']).nunique('tripid').reset_index()\n",
    "for corridor in corridor_segments:\n",
    "    corridor_data = traversal_data.loc[traversal_data['seg_compkey'].isin(list(corridor_segments[corridor]))]\n",
    "    print(corridor_data['lat'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the top ten vehicles with most tracked locations\n",
    "vehicle_counts = data.groupby('vehicleid').count().reset_index()[['vehicleid', 'lat']]\n",
    "vehicle_list = vehicle_counts.sort_values('lat', ascending=False)\n",
    "vehicle_list = vehicle_list.iloc[:10,:]\n",
    "vehicle_list = vehicle_list['vehicleid'].values\n",
    "corridor_data_10 = data[data['vehicleid'].isin(vehicle_list)]\n",
    "print(f\"Number of tracks for best 10 routes: {len(corridor_data_10)}\")\n",
    "\n",
    "# Get the data corresponding to ten random vehicles\n",
    "vehicle_list = vehicle_counts.sample(10)\n",
    "vehicle_list = vehicle_list.sort_values('lat', ascending=False)\n",
    "vehicle_list = vehicle_list.iloc[0:1,:]\n",
    "vehicle_list = vehicle_list['vehicleid'].values\n",
    "corridor_data_rand = data[data['vehicleid'].isin(vehicle_list)]\n",
    "print(f\"Number of tracks for best random route: {len(corridor_data_rand)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate Densities by Random Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1000 iterations for random 10 vehicle ids from past two weeks\n",
    "best_route_name = []\n",
    "best_route_count = []\n",
    "for i in range(0, 1000):\n",
    "    sample = vehicle_counts.sample(10)\n",
    "    sample = sample.sort_values('lat', ascending=False)\n",
    "    sample = sample.iloc[0:1,:]\n",
    "    best_route_name.append(sample['vehicleid'].values[0])\n",
    "    best_route_count.append(sample['lat'].values[0])\n",
    "\n",
    "# The top 20 vehicles\n",
    "top_route_name = []\n",
    "top_route_count = []\n",
    "sample = vehicle_counts.sort_values('lat', ascending=False)\n",
    "sample = sample.iloc[:10,:]\n",
    "top_route_name.append(sample['vehicleid'].values)\n",
    "top_route_count.append(sample['lat'].values)\n",
    "top_route_count = list(top_route_count[0])\n",
    "\n",
    "# All vehicles\n",
    "all_route_count = list(vehicle_counts['lat'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Distribution of number of tracks across network\n",
    "fig = plt.figure(figsize=(10,6))\n",
    "sns.kdeplot(best_route_count, bw_adjust=1)\n",
    "sns.kdeplot(top_route_count, bw_adjust=1)\n",
    "sns.kdeplot(all_route_count, bw_adjust=1)\n",
    "fig.legend(labels=['Top Sampled Count','Top Un-Sampled Count','All Counts'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate Spatial Heatmaps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.unique(corridor_data_rand['route_short_name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the map\n",
    "f_map = folium.Map(\n",
    "    location=[47.606209, -122.332069],\n",
    "    zoom_start=11,\n",
    "    prefer_canvas=True)\n",
    "\n",
    "# Create and add layer for all the sampled coordinate data\n",
    "# lats = sampled['lat'].tolist()\n",
    "# lons = sampled['lon'].tolist()\n",
    "# points = zip(lats, lons)\n",
    "# points = list(points)\n",
    "# HeatMap(\n",
    "#     data=points,\n",
    "#     name='Points',\n",
    "#     radius=8,\n",
    "#     min_opacity=0.2\n",
    "# ).add_to(f_map)\n",
    "\n",
    "# Create and add layer for all the sampled coordinate data in our corridors\n",
    "lats = corridor_data_rand['lat'].tolist()\n",
    "lons = corridor_data_rand['lon'].tolist()\n",
    "points = zip(lats, lons)\n",
    "points = list(points)\n",
    "HeatMap(\n",
    "    data=points,\n",
    "    name='Points',\n",
    "    radius=8,\n",
    "    min_opacity=0.2\n",
    ").add_to(f_map)\n",
    "\n",
    "folium.LayerControl().add_to(f_map)\n",
    "\n",
    "# Save map and plot in notebook\n",
    "f_map.save(f\"./local_sampled_map.html\")\n",
    "f_map"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}