Rick_TCNS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn import manifold\n",
    "\n",
    "from scipy.optimize import minimize\n",
    "from scipy.spatial.distance import pdist, squareform\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "# matlab io\n",
    "import scipy.io as sio\n",
    "\n",
    "import multiprocessing as mp\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "orderdf = pd.read_csv('ignored_assets/orders.csv', parse_dates = [6,7])\n",
    "coredf = orderdf[orderdf['dest_district_hash'].isin(orderdf['start_district_hash'].unique())].copy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "T=24"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Pick one day: according to the ICRA paper, January 21, 2016 has the highest activity\n",
    "coredfs=coredf[(coredf['timestamp']>='2016-01-21') & (coredf['timestamp']<'2016-01-22')].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "districts = coredfs['start_district_hash'].unique()\n",
    "N = len(districts)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# pos_i is already in inferred_locations.csv.pos_i, a Nx2 vector of node locations\n",
    "pos_i=np.zeros([N,2])\n",
    "posdf=pd.read_csv('inferred_locations.csv')\n",
    "posdf=posdf.set_index('start_district_hash')\n",
    "for i in range(N):\n",
    "    pos_i[i,0]=posdf['x'][districts[i]]\n",
    "    pos_i[i,1]=posdf['y'][districts[i]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Append travel time to every trip\n",
    "def district_loc(district_hash):\n",
    "    return np.array([posdf.loc[district_hash].x, posdf.loc[district_hash].y])\n",
    "def travel_distance(_record):\n",
    "    return np.linalg.norm(district_loc(_record['start_district_hash'])- district_loc(_record['dest_district_hash']))\n",
    "def travel_distance_pd(df):\n",
    "    return df.apply(travel_distance,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#my_travel_distances = coredfs.apply(travel_distance,axis=1); #distances are in km\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parallelize_dataframe(df, func,num_partitions=mp.cpu_count()):\n",
    "    df_split = np.array_split(df, num_partitions)\n",
    "    assert mp.cpu_count()>2, \"With 2 or fewer cores, you probably don't want to run this!\"\n",
    "    pool = mp.Pool(mp.cpu_count()-2)\n",
    "    df_out = pd.concat(pool.map(func, df_split))\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "    return df_out\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_travel_distances = parallelize_dataframe(coredfs,travel_distance_pd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "coredfs['travel_distance']=my_travel_distances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2016-01-21 0:00\n",
      "2016-01-21 1:00\n",
      "2016-01-21 2:00\n",
      "2016-01-21 3:00\n",
      "2016-01-21 4:00\n",
      "2016-01-21 5:00\n",
      "2016-01-21 6:00\n",
      "2016-01-21 7:00\n",
      "2016-01-21 8:00\n",
      "2016-01-21 9:00\n",
      "2016-01-21 10:00\n",
      "2016-01-21 11:00\n",
      "2016-01-21 12:00\n",
      "2016-01-21 13:00\n",
      "2016-01-21 14:00\n",
      "2016-01-21 15:00\n",
      "2016-01-21 16:00\n",
      "2016-01-21 17:00\n",
      "2016-01-21 18:00\n",
      "2016-01-21 19:00\n",
      "2016-01-21 20:00\n",
      "2016-01-21 21:00\n",
      "2016-01-21 22:00\n",
      "2016-01-21 23:00\n"
     ]
    }
   ],
   "source": [
    "# Compute departure rates hour-by-hour. The output is lambda_tv, a NxNxT matrix of departure rates denoting the departure rate per minute\n",
    "lambda_tv = np.zeros([N,N,T]) \n",
    "#For each departure location, compute probability of destinations. Output is pij_tv, a NxNXT matrix of o-d probabilities\n",
    "pij_tv = np.zeros([N,N,T])\n",
    "# speed: compute for each trip, then avg. # speed is in km/minute\n",
    "speed = np.zeros([T,1])\n",
    "#compute avg expected travel time. Output is Tij_tv, a NxNxT vector of travel times (in minutes).\n",
    "Tij_tv = np.zeros([N,N,T])\n",
    "\n",
    "\n",
    "for t in range(T):\n",
    "    print ('2016-01-21 '+str(t)+':00')\n",
    "    coredfs_h = coredfs[(coredfs['timestamp']>=('2016-01-21 '+str(t)+':00:00')) & (coredfs['timestamp']<('2016-01-21 '+str(t)+':59:59'))]\n",
    "    tempcdfs=coredfs_h.groupby(['start_district_hash','dest_district_hash']).count()['order_id']\n",
    "    tempttime = coredfs_h.groupby(['start_district_hash','dest_district_hash']).mean()['expected_travel_time']\n",
    "    \n",
    "    #temptottime = coredfs_h.groupby(['start_district_hash']).sum()['expected_travel_time']\n",
    "    #temptotdist = coredfs_h.groupby(['start_district_hash']).sum()['travel_distance']\n",
    "    for od in range(N):        \n",
    "        for dd in range(N):\n",
    "            try:\n",
    "                lambda_tv[od,dd,t] = tempcdfs[districts[od],districts[dd]]/60.\n",
    "            except:\n",
    "                lambda_tv[od,dd,t] = 0\n",
    "            try:\n",
    "                Tij_tv[od,dd,t] = tempttime[districts[od],districts[dd]]\n",
    "            except:\n",
    "                Tij_tv[od,dd,t] = np.nan\n",
    "        if sum(lambda_tv[od,:,t]):\n",
    "            for dd in range(N):\n",
    "                pij_tv[od,dd,t]=lambda_tv[od,dd,t]/sum(lambda_tv[od,:,t])\n",
    "                \n",
    "    speed[t]=sum(coredfs_h['travel_distance'])/sum(coredfs_h['expected_travel_time'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "for od in range(N):        \n",
    "    for dd in range(N): \n",
    "        meanTT=np.average(Tij_tv[od,dd,np.logical_not(np.isnan(Tij_tv[od,dd,:]))])\n",
    "        if np.isnan(meanTT): #Fuck it\n",
    "            for t in range(T):\n",
    "                meanTTt = np.linalg.norm(pos_i[od]-pos_i[dd],ord=2)/speed[t]\n",
    "                Tij_tv[od,dd,t]=meanTTt\n",
    "        else:\n",
    "            Tij_tv[od,dd,np.isnan(Tij_tv[od,dd,:])]=meanTT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "sio.savemat('timeVaryingSimDataHangzhou'+str(N)+'Stations.mat',{\n",
    "        'lambda_tv':lambda_tv,\n",
    "        'pij_tv': pij_tv,\n",
    "        'pos_i': pos_i,\n",
    "        'speed_tv': speed,\n",
    "        'Tij_tv': Tij_tv\n",
    "        })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[   4.57644518,    3.88815841,    3.79484261, ...,    3.39195164,\n",
       "            3.42162461,    4.07302934],\n",
       "        [  13.27858641,   13.73075371,    5.50563231, ...,    9.00058355,\n",
       "            8.26458767,   10.30590695],\n",
       "        [  55.52290214,   64.38790332,   55.52290214, ...,   55.52290214,\n",
       "           55.52290214,   55.52290214],\n",
       "        ..., \n",
       "        [  54.7372643 ,   52.31704862,   46.70633325, ...,   40.23012453,\n",
       "           42.18425234,   55.93046075],\n",
       "        [  48.49688915,   46.35259252,   41.38153222, ...,   35.64364999,\n",
       "           37.37499555,   49.55405407],\n",
       "        [  67.28465124,   64.30965111,   57.41279516, ...,   49.4520494 ,\n",
       "           51.85412063,   68.75136331]],\n",
       "\n",
       "       [[  13.07793518,   14.75863078,    8.47618534, ...,   10.14796313,\n",
       "            9.8556484 ,   14.20865135],\n",
       "        [   5.05229614,    4.56349202,    4.08051993, ...,    4.17524005,\n",
       "            4.13467359,    5.23705858],\n",
       "        [  29.53715913,   29.53715913,   29.53715913, ...,   28.21469925,\n",
       "           29.68220133,   40.79233627],\n",
       "        ..., \n",
       "        [  48.89622399,   46.73427071,   41.72227753, ...,   35.93714822,\n",
       "           37.68275009,   49.96209386],\n",
       "        [   7.93184316,   77.9186946 ,   39.50368948, ...,   39.50368948,\n",
       "           39.50368948,   39.50368948],\n",
       "        [  45.67808504,   45.67808504,   45.67808504, ...,   45.67808504,\n",
       "           45.67808504,   45.67808504]],\n",
       "\n",
       "       [[  24.37875325,   24.37875325,   24.37875325, ...,   24.37875325,\n",
       "           24.37875325,   24.37875325],\n",
       "        [  31.2597136 ,   10.26473821,   31.2597136 , ...,   25.42855602,\n",
       "           49.92395403,   41.05895284],\n",
       "        [   3.44244269,    2.57632757,    2.31680612, ...,    2.80827744,\n",
       "            2.79511351,    3.11941395],\n",
       "        ..., \n",
       "        [  90.02030279,   86.0400427 ,   76.81272192, ...,   66.1620203 ,\n",
       "           69.37575739,   91.98262054],\n",
       "        [  40.86026182,   39.05361971,   34.86533405, ...,   30.0309751 ,\n",
       "           31.48969202,   41.75095885],\n",
       "        [ 129.04885254,  123.34293974,  110.11508866, ...,   94.84674608,\n",
       "           99.45380772,  131.86193854]],\n",
       "\n",
       "       ..., \n",
       "       [[  54.7372643 ,   52.31704862,   46.70633325, ...,   40.23012453,\n",
       "           42.18425234,   55.93046075],\n",
       "        [  34.52684671,   34.52684671,   34.52684671, ...,   34.52684671,\n",
       "           34.52684671,   34.52684671],\n",
       "        [  90.02030279,   86.0400427 ,   76.81272192, ...,   66.1620203 ,\n",
       "           69.37575739,   91.98262054],\n",
       "        ..., \n",
       "        [   6.80038906,    6.80038906,    6.80038906, ...,   22.30247666,\n",
       "            6.80038906,    3.96592158],\n",
       "        [  96.15099981,   91.8996701 ,   82.04393655, ...,   70.66788495,\n",
       "           74.10048876,   98.24695825],\n",
       "        [  61.30632882,   58.59566105,   52.31159907, ...,   45.05817517,\n",
       "           47.24681947,   62.64272176]],\n",
       "\n",
       "       [[  48.49688915,   46.35259252,   41.38153222, ...,   35.64364999,\n",
       "           37.37499555,   49.55405407],\n",
       "        [  47.41565391,   45.31916424,   40.45893343, ...,   34.84897694,\n",
       "           36.54172226,   48.4492494 ],\n",
       "        [  40.86026182,   39.05361971,   34.86533405, ...,   30.0309751 ,\n",
       "           31.48969202,   41.75095885],\n",
       "        ..., \n",
       "        [  96.15099981,   91.8996701 ,   82.04393655, ...,   70.66788495,\n",
       "           74.10048876,   98.24695825],\n",
       "        [   1.16644752,    1.16644752,    1.16644752, ...,    1.16644752,\n",
       "            1.16644752,    1.16644752],\n",
       "        [ 114.61741278,  109.54958807,   97.80099801, ...,   84.24010312,\n",
       "           88.3319604 ,  117.11591341]],\n",
       "\n",
       "       [[  67.28465124,   64.30965111,   57.41279516, ...,   49.4520494 ,\n",
       "           51.85412063,   68.75136331],\n",
       "        [  76.65572031,   73.26637708,   65.40896159, ...,   56.33948305,\n",
       "           59.07610272,   78.3267087 ],\n",
       "        [ 129.04885254,  123.34293974,  110.11508866, ...,   94.84674608,\n",
       "           99.45380772,  131.86193854],\n",
       "        ..., \n",
       "        [  61.30632882,   58.59566105,   52.31159907, ...,   45.05817517,\n",
       "           47.24681947,   62.64272176],\n",
       "        [ 114.61741278,  109.54958807,   97.80099801, ...,   84.24010312,\n",
       "           88.3319604 ,  117.11591341],\n",
       "        [   2.76614699,    2.76614699,    2.76614699, ...,    2.76614699,\n",
       "            4.19921109,    2.76614699]]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Tij_tv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}