mc_es.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sutton's book Chapter 5: Monte Carlo with Exploring Start\n",
    "# Blackjack as in https://www.youtube.com/watch?v=SWdPf21v5Ak\n",
    "import random\n",
    "import numpy as np\n",
    "from scipy.stats import beta\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "DECK_OF_CARDS = [1,2,3,4,5,6,7,8,9,10,10,10,10] * 4 #J, Q, K count as 10.\n",
    "DEALDER_STICK_THRESHOLD = 17 # same as Sutton's and Vega's rule\n",
    "\n",
    "def sum_hand(hand):\n",
    "    min_s = sum(hand)\n",
    "    max_s = sum(hand)\n",
    "    if 1 in hand:\n",
    "        max_s = min_s  + 10\n",
    "    if max_s <= 21 and max_s > min_s:\n",
    "        ## Has a usable ACE.\n",
    "        return (max_s, 1)\n",
    "    else:\n",
    "        return (min_s, 0)\n",
    "    \n",
    "def busted(hand):\n",
    "    (s, _) = sum_hand(hand)\n",
    "    return s > 21    \n",
    "    \n",
    "def convert_cards_to_state(my_cards, dealer_card):\n",
    "    (s, usable_ace) = sum_hand(my_cards)\n",
    "    # state is 0-based, for easy indexing with ndarray.\n",
    "    return (s-1, dealer_card-1, usable_ace)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BasePlayer(object):\n",
    "    # Return 1 if hit, 0 if stick.\n",
    "    def action(self, cards, rival_card):\n",
    "        raise NotImplementedError(\"action() is not implemented.\")\n",
    "\n",
    "class Dealer(BasePlayer):\n",
    "    def __init__(self):\n",
    "        self.stick_threshold = DEALDER_STICK_THRESHOLD\n",
    "    def action(self, cards):\n",
    "        (s, _) = sum_hand(cards) #s>=1 \n",
    "        return int(self.stick_threshold > s) #stick for 17 or greater, same as Sutton's "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Policy(object):\n",
    "    def __init__(self):\n",
    "        # stats is a 5D tensor. \n",
    "        # The first 3 dimensions describe state. The 4th dimension describes\n",
    "        # action (0 for stick, 1 for hit). The 5th dimension describes the\n",
    "        # reward: {-1, 0, 1}.\n",
    "        # self.states[i][j][k][a][r] saves counts of how many times we have\n",
    "        # reward `r` when we take action `a` for state s = (i, j, k) (s is 0-based).\n",
    "        self.stats = np.zeros((21, 10, 2, 2, 3), dtype='int64') \n",
    "        # q is a 4D tensor that represents q(s, a).\n",
    "        self.q = np.zeros((21, 10, 2, 2), dtype='float64')\n",
    "        # pi is a 4D tensor that represents pi(a|s), last dimension is the action\n",
    "        self.pi = np.zeros((21, 10, 2, 2), dtype='float64')\n",
    "        #pi[:,:,:,1] is the probability of hit while pi[:,:,:,0] is the probability of stick;use deterministic action here\n",
    "        self.pi[0:11,:,:,:] = [0., 1.] #Always hit if the cards sum to [1,11]; note that this part of pi should never get updated\n",
    "        self.pi[11:19,:,:,:]=[0.,1.] #same as Sutton's initial policy, hit if [12,19]; stick if 20,21\n",
    "        self.pi[19:,:,:,:] = [1., 0.] \n",
    "        \n",
    "    def update(self, state_action_seq, final_reward):\n",
    "        #update q(s,a) snd pi(a|s) using the episode(ie., state_action_seq) and the reward \n",
    "        #For blackjack, the states in the state_action_seq are all different; first visit MC and every visit MC are the same\n",
    "        for state_action in state_action_seq:  \n",
    "            (i,j,k,a) = state_action #i=sum(play_cards), 0<=i<=20, 0<=j<=9, k=0 or 1,  \n",
    "            # reward is in {-1, 0, 1}    \n",
    "            self.stats[i,j,k,a,final_reward + 1] += 1\n",
    "            # q stores the return estimate E[R|(s,a)] so far.\n",
    "            self.q[i,j,k,a] = (self.stats[i,j,k,a,2] - self.stats[i,j,k,a,0]) * 1.0 / np.sum(self.stats[i,j,k,a,:])\n",
    "                        \n",
    "            #greedy update pi(a|s) based on estimated q(s,a)\n",
    "            if i <= 10:\n",
    "                # No update: we always hit.\n",
    "                continue\n",
    "            if i == 20:\n",
    "                # No update: we always stick.\n",
    "                continue                     \n",
    "            # If q[i,i,k,0] > q[i,j,k,1], argsort returns [1,0].            \n",
    "            self.pi[i,j,k,:] = np.argsort(self.q[i,j,k,:]) \n",
    "  \n",
    "    def action(self, state):#greedy \n",
    "        return np.argmax(self.pi[state])\n",
    "    \n",
    "    # Visualize matrix as a grayscale image, assuming entries are in [0,1].\n",
    "    def imshow(self, ax, matrix, title):\n",
    "        ax.set_title(title)\n",
    "        ax.imshow(matrix, cmap='gray',vmin=0., vmax=1., interpolation=\"none\")        \n",
    "\n",
    "        ax.set_xlabel('Dealer Card')\n",
    "        ax.set_xticks(np.arange(10))\n",
    "        ax.set_xticklabels(np.arange(1,11,1).astype('S2'))\n",
    "        \n",
    "        ax.set_ylabel('Player Card')\n",
    "        ax.set_yticks(np.arange(11))\n",
    "        ax.set_yticklabels(np.arange(21,10,-1).astype('S2'))\n",
    "        \n",
    "        ax.spines['right'].set_color('none')\n",
    "        ax.spines['top'].set_color('none')      \n",
    "    \n",
    "    # Visualize the hard decision matrix \n",
    "    def visualize(self):\n",
    "        fig = plt.figure(figsize=(12, 9), dpi=80)\n",
    "        \n",
    "        usable_ace = 0\n",
    "        ax = plt.subplot(221)\n",
    "        decision = (self.pi[10:,:,usable_ace,1] > self.pi[10:,:,usable_ace,0]).astype(int)\n",
    "        self.imshow(ax, np.flipud(decision), 'Decision (No usable Ace)')\n",
    "        \n",
    "        usable_ace = 1\n",
    "        ax = plt.subplot(223)\n",
    "        decision = (self.pi[10:,:,usable_ace,1] > self.pi[10:,:,usable_ace,0]).astype(int)\n",
    "        self.imshow(ax, np.flipud(decision), 'Decision (with usable Ace)')\n",
    "        \n",
    "        plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PolicyPlayer(BasePlayer):\n",
    "    def __init__(self, policy):\n",
    "        self.policy = policy\n",
    "        self.state_action_pairs = []\n",
    "        \n",
    "    def action(self, cards, rival_card,a=None):\n",
    "        s = convert_cards_to_state(cards, rival_card) #s is 0-based \n",
    "        if a is None: \n",
    "            a = self.policy.action(s)\n",
    "        (i,j,k) = s  # \n",
    "        self.state_action_pairs.append((i,j,k,a))\n",
    "        return a\n",
    "    \n",
    "    def reset_states(self):\n",
    "        self.state_action_pairs = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A simple debugger (to avoid clutter in main code.)\n",
    "class Dbg(object):\n",
    "    # level can be 0 or 1. (0 means silent.)\n",
    "    def __init__(self, level):\n",
    "        self.level = level\n",
    "    def print_hands(self, game):\n",
    "        if self.level == 0:\n",
    "            return\n",
    "        print 'dealer cards: ', game.dealer_cards \n",
    "        print 'player cards: ', game.player_cards\n",
    "        \n",
    "    def print_hand(self,cards):\n",
    "        if self.level == 0:\n",
    "            return\n",
    "        print 'cards: ', cards \n",
    "        \n",
    "    def on_dealer_action(self, action):\n",
    "        if self.level == 0:\n",
    "            return\n",
    "        print 'dealer action: ', action\n",
    "    def on_player_action(self, action):\n",
    "        if self.level == 0:\n",
    "            return\n",
    "        print 'player action: ', action\n",
    "    def print_bust_status(self, dealer, player):\n",
    "        if self.level == 0:\n",
    "            return\n",
    "        if player and dealer:\n",
    "            print 'both busted'\n",
    "        elif player:\n",
    "            print 'player busted'\n",
    "        elif dealer:\n",
    "            print 'dealer busted'\n",
    "        else:\n",
    "            pass\n",
    "    def print_sum_of_hands(self, dealer_sum, player_sum):\n",
    "        if self.level == 0:\n",
    "            return\n",
    "        print 'dealer sum: {}, player sum: {}'.format(dealer_sum, player_sum)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Game(object):\n",
    "    \"\"\"\n",
    "    Blackjack shoe game as in https://www.youtube.com/watch?v=SWdPf21v5Ak\n",
    "    \"\"\"\n",
    "    def __init__(self, dealer, player, debug_level=0):\n",
    "        self.dealer = dealer\n",
    "        self.player = player\n",
    "        self.cards = DECK_OF_CARDS[:]\n",
    "        #random.shuffle(self.cards) #Sutton assumes infinite cards. Use sampling with replacement instead. \n",
    "        self.dbg = Dbg(debug_level)\n",
    "        \n",
    "    def on_player_action(self, action):\n",
    "        if action == 1:\n",
    "           #Sutton assumes infinite cards.\n",
    "            self.player_cards.extend(np.random.choice(self.cards,size=1))\n",
    "        self.dbg.on_player_action(action)\n",
    "    def on_dealer_action(self, action):\n",
    "        if action == 1:\n",
    "            self.dealer_cards.extend(np.random.choice(self.cards,size=1))\n",
    "        self.dbg.on_dealer_action(action)\n",
    "            \n",
    "    def play(self,dealer_init_cards, player_init_cards, a):\n",
    "        \"\"\"\n",
    "        each play generates an episode starting from state the initial states and action \n",
    "        dealer_init_cards, player_init_cards: List[Int] of size 2; must be non-busted state; \n",
    "        a: player first action,  1 for hit ; 0 for stick, \n",
    "        return: 1 if the player wins, -1 if the player loses, 0 for draw\n",
    "        \"\"\" \n",
    "        self.player.reset_states() #reset the player's memory of state_action pair\n",
    "        self.dealer_cards = list(dealer_init_cards)\n",
    "        self.player_cards = list(player_init_cards)\n",
    "        \n",
    "        assert not busted(dealer_init_cards) and not busted(player_init_cards), \"Error: Invalid initial state!\"\n",
    "        \n",
    "        #player's turn\n",
    "        self.player.action(self.player_cards,self.dealer_cards[0],a)\n",
    "        while a:\n",
    "            self.on_player_action(a)  # update the states \n",
    "            if busted(self.player_cards): \n",
    "                return -1 \n",
    "            a = self.player.action(self.player_cards, self.dealer_cards[0])\n",
    "    \n",
    "        self.on_player_action(a)\n",
    "        \n",
    "        #dealer's turn\n",
    "        a = self.dealer.action(self.dealer_cards)   \n",
    "        while a:\n",
    "            self.on_dealer_action(a)  # update the states \n",
    "            if busted(self.dealer_cards): \n",
    "                return 1\n",
    "            a = self.dealer.action(self.dealer_cards) #dealer sticks to the same rule regardless of the player's cards\n",
    "            \n",
    "        #both dealer and player stick \n",
    "        d_sum, _ = sum_hand(self.dealer_cards)\n",
    "        p_sum, _ = sum_hand(self.player_cards)\n",
    "        return np.sign(p_sum - d_sum)\n",
    "        \n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Learner:\n",
    "    def __init__(self):\n",
    "        self.player_policy = Policy()\n",
    "        self.player = PolicyPlayer(self.player_policy)\n",
    "        self.dealer = Dealer()\n",
    "        self.game = Game(self.dealer,self.player)\n",
    "    def train(self):\n",
    "        \"\"\"\n",
    "        learn the optimal policy using MC with exploring starts \n",
    "        return: optimal policy pi(a|s)\n",
    "        \"\"\"\n",
    "        np.random.seed(4)\n",
    "        max_iter = int(1e6) #reduce max_iter to 1e6 generates correct result except for the case player=12, dealer=4, use_able=0 \n",
    "        tol = 1e-2\n",
    "        max_epoch =100\n",
    "        for i_epoch in range(max_epoch):\n",
    "            q_old = np.array(self.player_policy.q) \n",
    "            for i in range(max_iter): \n",
    "                dealer_init_cards = np.random.choice(self.game.cards,size=2,replace=True)\n",
    "                player_init_cards = np.random.choice(self.game.cards,size=2, replace=True)\n",
    "                for a in [0,1]:\n",
    "                    rt = self.game.play(dealer_init_cards, player_init_cards,a)\n",
    "                    self.player_policy.update(self.player.state_action_pairs,rt) \n",
    "            q_delta = np.max(np.abs(q_old-self.player_policy.q))\n",
    "            print \"epoch=%d, q_delta=%.6f\" % (i_epoch,q_delta)\n",
    "            if q_delta<tol: \n",
    "                break \n",
    "        else: \n",
    "            print \"Warning: not converge for max_epoch(%d), tol=%.4f\" %(max_epoch,tol)\n",
    "        return self.player_policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch=0, q_delta=0.939809\n",
      "epoch=1, q_delta=0.071547\n",
      "epoch=2, q_delta=0.042219\n",
      "epoch=3, q_delta=0.040160\n",
      "epoch=4, q_delta=0.028726\n",
      "epoch=5, q_delta=0.014514\n",
      "epoch=6, q_delta=0.020872\n",
      "epoch=7, q_delta=0.013394\n",
      "epoch=8, q_delta=0.013407\n",
      "epoch=9, q_delta=0.013193\n",
      "epoch=10, q_delta=0.012797\n",
      "epoch=11, q_delta=0.008625\n"
     ]
    }
   ],
   "source": [
    "learner = Learner()\n",
    "optimal_policy = learner.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21, 10, 2, 2)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimal_policy.pi.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  1.,  1.,  0.,  0.,  0.,  1.,  1.,  1.,  1.],\n",
       "       [ 1.,  0.,  0.,  0.,  0.,  0.,  1.,  1.,  1.,  1.],\n",
       "       [ 1.,  0.,  0.,  0.,  0.,  0.,  1.,  1.,  1.,  1.],\n",
       "       [ 1.,  0.,  0.,  0.,  0.,  0.,  1.,  1.,  1.,  1.],\n",
       "       [ 1.,  0.,  0.,  0.,  0.,  0.,  1.,  1.,  1.,  1.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimal_policy.pi[11:,:,0,1] #usable=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.],\n",
       "       [ 1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.],\n",
       "       [ 1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.],\n",
       "       [ 1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.],\n",
       "       [ 1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.],\n",
       "       [ 1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.],\n",
       "       [ 1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  1.,  1.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimal_policy.pi[11:,:,1,1] #usable=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export the result generated maxiter=1e8 as the true solution \n",
    "#np.save(\"optimal_policy_pi.npy\",optimal_policy.pi)\n",
    "#np.save(\"optimal_policy_q.npy\",optimal_policy.q)\n",
    "#np.save(\"optimal_policy_stats.npy\",optimal_policy.stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.0, 0.0, array([-0.20757332, -0.21537975]))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#player_cards=12, dealer up cards =3, useable=False is a difficult case due to the probability are very closed\n",
    "i,j,k = 11,3,0\n",
    "optimal_policy.pi[i,j,k,0], optimal_policy.pi[i,j,k,1], optimal_policy.q[i,j,k]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21, 10, 2)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimal_policy.v = optimal_policy.q.max(axis=3)\n",
    "optimal_policy.v.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [-0.45466273, -0.09183119, -0.07024945, -0.06062271, -0.02084466,\n",
       "        -0.00145693, -0.08468038, -0.14755877, -0.23431427, -0.34311606],\n",
       "       [-0.45590667, -0.1245706 , -0.10328596, -0.07857794, -0.0093961 ,\n",
       "        -0.00552486, -0.11686267, -0.1807654 , -0.26012891, -0.35881289],\n",
       "       [-0.48523331, -0.12935592, -0.10338055, -0.08293783, -0.03286696,\n",
       "        -0.03611227, -0.1543676 , -0.2181916 , -0.29925172, -0.38036582],\n",
       "       [-0.49287467, -0.1116444 , -0.07760715, -0.04097901, -0.00889503,\n",
       "         0.02871672, -0.07247203, -0.21461247, -0.28775855, -0.3700755 ],\n",
       "       [-0.42225488, -0.02220213,  0.00382782,  0.03614377,  0.06881362,\n",
       "         0.11160669,  0.08609117, -0.06144295, -0.20921382, -0.30195848],\n",
       "       [-0.33295747,  0.08075287,  0.09792778,  0.12830107,  0.15256158,\n",
       "         0.19523732,  0.16689178,  0.10199844, -0.05297119, -0.21412216],\n",
       "       [-0.21644195,  0.18202688,  0.20287808,  0.22800471,  0.25891704,\n",
       "         0.29116982,  0.25092226,  0.20039079,  0.11091023, -0.04489209],\n",
       "       [-0.10382822,  0.23281433,  0.25301795,  0.28448891,  0.30864591,\n",
       "         0.33288886,  0.29728965,  0.22995859,  0.15799882,  0.05942858],\n",
       "       [-0.52348186, -0.25477161, -0.23786669, -0.20757332, -0.16593613,\n",
       "        -0.15866066, -0.2122719 , -0.27109152, -0.33880845, -0.41832285],\n",
       "       [-0.55414699, -0.29314235, -0.25452091, -0.20590873, -0.16629431,\n",
       "        -0.1580905 , -0.26964009, -0.32290553, -0.3816289 , -0.46271444],\n",
       "       [-0.5857245 , -0.28993104, -0.25621163, -0.21627105, -0.16758482,\n",
       "        -0.15153735, -0.3199636 , -0.36950201, -0.43116481, -0.50016578],\n",
       "       [-0.61383851, -0.28862128, -0.25153824, -0.21318293, -0.16683198,\n",
       "        -0.14995769, -0.37148539, -0.41819609, -0.47060244, -0.53720807],\n",
       "       [-0.64262983, -0.28601469, -0.25725113, -0.21157775, -0.16887982,\n",
       "        -0.15439085, -0.41622823, -0.45532316, -0.51150173, -0.57071003],\n",
       "       [-0.63831075, -0.15022316, -0.11580351, -0.08424755, -0.04388153,\n",
       "         0.00964556, -0.10569445, -0.38259379, -0.4212274 , -0.46458746],\n",
       "       [-0.37744611,  0.12500113,  0.15321071,  0.18056667,  0.1976831 ,\n",
       "         0.28344392,  0.39846567,  0.10400545, -0.18192439, -0.24043145],\n",
       "       [-0.11356664,  0.38823192,  0.40193476,  0.42081936,  0.43616285,\n",
       "         0.49544924,  0.61107315,  0.59630296,  0.29228805, -0.01993811],\n",
       "       [ 0.14319242,  0.64077778,  0.64829383,  0.65992943,  0.67037506,\n",
       "         0.70384338,  0.7714156 ,  0.79485153,  0.75707887,  0.43501185],\n",
       "       [ 0.63767099,  0.88056586,  0.88099495,  0.88868969,  0.8922456 ,\n",
       "         0.902095  ,  0.92645358,  0.93111045,  0.93909833,  0.8892794 ]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimal_policy.v[:,:,0] #usable=False, the first three rows are 0 because these states won't occur. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [-0.27968289,  0.08711434,  0.09434307,  0.11778671,  0.16926253,\n",
       "         0.19480752,  0.16159592,  0.10704433,  0.00257022, -0.13049691],\n",
       "       [-0.29716238,  0.03619588,  0.06712304,  0.11780969,  0.11065094,\n",
       "         0.1567118 ,  0.12585812,  0.04388909, -0.01986404, -0.15965482],\n",
       "       [-0.33409574,  0.02262517,  0.04166328,  0.09113525,  0.10164857,\n",
       "         0.14811222,  0.07719913,  0.01932926, -0.05779646, -0.19621816],\n",
       "       [-0.36249062, -0.00874614,  0.02498706,  0.05667249,  0.08609521,\n",
       "         0.11962395,  0.0207346 , -0.03204085, -0.10769915, -0.22966779],\n",
       "       [-0.37839273, -0.01146925,  0.00638171,  0.038582  ,  0.08432079,\n",
       "         0.11827247, -0.00450299, -0.06554878, -0.12668446, -0.25406319],\n",
       "       [-0.38845601,  0.01233672,  0.02685776,  0.05221604,  0.08497274,\n",
       "         0.12351558,  0.0584663 , -0.06915154, -0.14574729, -0.25468783],\n",
       "       [-0.33317052,  0.12864104,  0.14839789,  0.18559317,  0.19592417,\n",
       "         0.28386958,  0.40082042,  0.09819324, -0.09130173, -0.2031297 ],\n",
       "       [-0.11034789,  0.3938261 ,  0.40978344,  0.43689853,  0.44260063,\n",
       "         0.49640794,  0.61590828,  0.58888216,  0.28604434, -0.01803739],\n",
       "       [ 0.14579611,  0.63471929,  0.64567442,  0.65920948,  0.67284944,\n",
       "         0.70027598,  0.77410181,  0.79211148,  0.76234287,  0.43459446],\n",
       "       [ 0.63943516,  0.88154181,  0.88474038,  0.88908177,  0.89283576,\n",
       "         0.90098634,  0.92643228,  0.9296155 ,  0.93956997,  0.88805052]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimal_policy.v[:,:,1] #usable=True, the first 11 rows are 0 because these states won't occur. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAAMKCAYAAAC7kZ5ZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3XmcXFWZ//HPNwEEgiQsEhBlCQGBnxppEMQxLCKIGQwBGUwjsrmAKGBwRmRwBXdFFhlcUCDI0AzIliiENSDLOJCwKYLIHlnCEnbCkvTz++PcIpXq6u7q6qo61Z3v+/WqV1L3nrrnqeqqp06de+45igjMzKz1RuQOwMxsWeUEbGaWiROwmVkmTsBmZpk4AZuZZeIEbGaWiROwmVkmTsBmZpk4AZuZZeIEvIyTtL6kbkn7DfBx10qa3ay4aqh/b0nPSFo5Vww5FK/7nTWUq+vv2o4kfVTSi5LWyB1LozkBZyZp/+KDUrotlPSopFmSDpO0SgvCqOd69AC6Gx1ILSSNAL4NnBQRr5Rtf6h4DU+q8pjti317tjDUZsg+d4Ck0ZJelbRY0ruaXV9EXA7cBxzd7LpazQm4PQTwdWBf4BDg5GLbicBfJL2naRVHPAysBPxugA/dGfho4yOqyWRgE+C0iu1R3D4nae0qj8uevIaJfyN9+T4BfKpFdf4KOFjSqBbV1xJOwO1jVkScExHTI+JHEfExYCdgLeASSW9pVsUR8XoMcFamiFgUEYuaFVM/DgBujIjHq+y7CxgJfK3KPjUzqGXIvsAfgS5gnxbVeQGwIin5DxtOwG0sIq4FjgPWJ73p3yTpXZJ+X/SDLpR0i6SPVx6j+Ll4gqQHi5+N8yRNl7R6sb9HX6GksZLOKMq+KukxSRdLWq+szLWSrqmo622SfivpiSKm2yv7IMvqO1LS5yTdV9Rxs6St+ntNii+iXYGreinyEHAWvbeCK4+3haTLJD1f9DNeJWmbGh5X6tLYrpfnN9DXc7KkPxTdT68Wr8vXi+6WavV3SLpR0iuSHpB0cH8xF4+r6X3Tx+PfCUwkJd//AcZJ+kAvZbeRdKmkBZJeknSHpMPriScingLuBHavNdahwAm4/f2O1HLbpbRB0v8D/gy8C/gBcCTwEnCxpN3Lyo0CbgC+CMwCDgd+UTzuHX3UeSHpjf5b4AvAScAqwHplZZZqMUtaEbiO9JP0d8C/A88BZ0o6rEodnyrK/BI4BtgAuEDSyD7iAtgSWAG4tY8y3wOWp3oruDzmzYE/Ae8BfggcW8RxraT39xMH1N6lUcvreQDwInA86e80p4jnB1WOtzqpBToH+A9gHvALSQf0FUSt75t+7FM85o8RcQtwP1W6ISTtTHo/bErqSjsSuAb410HEMxf4YI1xDg0R4VvGG7A/sBjo6KPMs8CcsvtXAbcBy1WUuwG4p+z+d4pjT+7j2OuT+vP2K+6PLu4f2U/cs4Fryu4fUdQ1tWzbSOBG4HlgVEV9TwKrlpX9ePH4Sf3Ue1BRbvMq+x4EZhT//y3wMjC2uL99Ue+eZeUvAhYC65dtW7uId3Y/cWxfxLFdg17Pt1TZ9gtSUl6+4nVfDBxRtm150hfS48DIanEM5H3TT5x3AGeV3f8uMB8YUbZtBPAAKTm/tY9jDSge0hfqYmDNRnz22uHmFvDQ8BLwVgBJqwE7AucDoyWtUboBVwAbS1qneNyewB0RMWMAdS0EXgd2kDRmAI/7GPBERJxb2hARi0knFFchJaxy50bEC2X3rye19Mf1U09pKNKz/ZT7Ln20gouf9jsDF0U6EVmK+QngHOBDaswIlJpez4h4rSy2VYq/5w3AyqRWZLlFwK/LHvsG6STVWqRfCD0M8H1TlaT3kn4tnFO2uQtYk6VPyG5B+iVxYkS82MB4Sn/zNfuKcyhxAh4aViG1hADGkxLVccBTFbdvF2XWKv7dCPjrQCqKiNeBo0gJdb6k6yT9h6Sx/Tx0feAfVbbfXcS7fsX2eRX1Plf8d7UaQ+3zhFpEPEjqCvl8L7G/jZTc7q2y727SZ+OdNcbSVxw1vZ6SNpd0kaTngBdIf8/SyJTRFYd9LCIWVmy7l/SabNBLKAN53/RmX1Jj4CFJG0naCHgNeJiluyE2InXP3NXHseqJp/Q3HzajWZbLHYD1TdK6pA/gfcWm0pfmT4HLe3nYfb1sr0lEnCRpBjCF1LI5Fjha0o4Rccdgjl1mcS/b+xup8Ezx72rAY/2U/R7waVICvKT20GrSWxLo0Yfd3+spaTSpL/o50nDEB4BXSa3ZH9KYhlIj3jdTgVHA3yq2B/A2SStH2bjsJsRT+nJ+usY62p4TcPvbj/QGn1Xcf6D4942IuKb6Q950P/DueiotWpAnACcULZ07gK8U8VTzMOnnaaXNyvY3wj2kJL0hfbewiIgHJJ0NHAzcXLH7KeAV0gmgSpuR+k/nVdlX8mwRR2W3wga9xNLX67kjKbnsHhE3lh5TlKvm7ZJWqmgFv4v0Pnmol8cM5H3Tg6QdSCduv076G5RbjdQlMoXUPXE/6bV5N+nEW6Pi2RB4OiKe6bfkEOEuiDYm6cMsaRGdA28Ox7mWNCi9xzArSeX9YxcAEwZwhhtJK6nnmOMHSV0gfY1FvhRYW9Iny441EjiseOx1tcbQj7mkPtV+h6wVvksaNfFVylqtEdFN6mvcvWI42FigE7g+Il7q47gPU5yEq9h+aHk9Nb6ei0gJa0TZ41YojlXNcqQLdkpllyd9yTxFen16GOD7pppS98NPI+LCittvSa3VUjfErcVz/HLRum9UPFsC/9tPnEOKW8DtQcAkSZuR/iZjgQ+TThI9SBrF8HpZ+S+STlr9RdJppAQ9FtgWWJd0EgTgJ8BewPmSziB9ONcgjTg4OCL+UiWWTYCrJZ1H+qm5iHQyby3SCZfe/JqUBM5UGs/7EGnQ/LakM/Yv1/xq9CEiXpN0BfARlvQV9lW+1Aren57dBl8vjnOjpFNJCfXzLEnYfR33BUnnA4dLgtTq243Ut1yultfzJlKL+ixJJxfb9q0Sb8njwFclbUDq+50KvBf4XHHisze1vm+WUnwZ7AlcWfE+LDeD9FqsGRFPS/pCse324r33OOlk4uaRLjIaUDyS3lY8x5/38fyGntzDMJb1G0uGoZVuC4FHSV0OX6QYvlXlcRsAZxRlXwUeIfVzTqkoN4Y07vSR4tgPk4ZorVbsX7+otzRsanXSyIW7SCeDFpASxJ4Vx50NXF2xbU3gN6RhSQuB24FPV5Qp1TetynNaDHyjhtdsCimRrVux/QHgkirlNyK1mhdVeR4TSK3350mt0iuBrWv8260BnFc87mngv0jdF/W8nh8gDdl7idT18X3Sl8NSQ92K1/0OUnK6kTTU7gHgkF5e5/0qttf0vql4zB7Fsfbvo8x2RZkvlW3btngfl04s3gZ8oc738SHF61z18zBUbyqenNmQUQwhuws4PyK+mTseaz5Jt5LGnf977lgayQnYhiRJewOnAutF7WfebQiS9FHSeOFxETFsRkCAE7CZWTYeBWFmlokTsJlZJsMyAUtaWWm6vmVquRozG1qG6zjgTSkGpBdjNM3MWiYiako8w7IFbGY2FDgBm5ll4gRsZpaJE7CZWSZOwGZmmTgBm5ll4gRsZpaJE7CZWSbZE7CkoyXdLOkFSfOLhQk3qSizh6TLJT0tqbtYndXMbEjLnoCBiaRZ7rchTUC9PHCFpJXKyowizZy/1LIyZmZDWfZLkSNiUvl9SQcAT5LWf7qhKHN2sW99+l8118xsSGiHFnClMaRW7oLcgZiZNVNbJWClmXNOBG6IiL/ljsfMrJnaKgGTlpjZnLTKq5nZkCSps5Zy2fuASySdAkwCJkbE47njMTOrV0R01VKuLRJwkXx3B7aPiEf6Ke5REGY2LGRPwJJOBTqBycDLksYWu56PiFeLMqsB6wHrkkZBbFr0Fz8REfMzhG1mNmjZV0WW1E31Vu2BEXFWUWZ/4Iwq5b4TEcdWOWYHxYoYZmatVuuKGNkTcDM4AZtZTl6SyMyszTkBm5ll4gRsZpaJE7CZWSZOwGZmmTgBm5ll4gRsZpaJE7CZWSZOwGZmmTgBm5ll4gRsZpaJE7CZWSZOwGZmmTgBm5ll4gRsZpaJE7CZWSZOwGZmmTgBm5llkj0BSzpa0s2SXpA0X9JFkjapUu5YSY9JekXSlZLG54jXzKxRsidgYCLwc2Ab4CPA8sAVklYqFZB0FPAl4PPA1sDLwOWSVmh9uGZmjdF2i3JKWhN4EtguIm4otj0G/CQiTijurwrMB/aPiPOqHMOLcppZNkN5Uc4xpOXnFwBI2hBYG7i6VCAiXgD+D9g2R4BmZo3QVglYkoATgRsi4m/F5rVJCXl+RfH5xT4zsyGprRIwcCqwOTA1dyBmZvWS1FlLubZJwJJOASYBO0TE42W7ngAEjK14yNhin5lZW4mIrlrKtUUCLpLv7sCOEfFI+b6IeJCUaHcqK78qadTETa2M08yskZbLHYCkU4FOYDLwsqRSS/f5iHi1+P+JwNcl3Qc8BBwH/BO4pMXhmpk1TPYEDBxCOsl2bcX2A4GzACLix5JWBn5FGiVxPfCxiHi9hXGamTVU240DbgSPAzaznIbyOGAzs2WCE7CZWSZOwGZmmTgBm5ll4gRsZpaJE7CZWSZOwGZmmTgBm5ll4gRsZpaJE7CZWSZOwGZmmTgBm5ll4gRsZpaJE7CZWSZOwGZmmTgBm5ll4gRsZpaJE7CZWSZtkYAlTZQ0Q9KjkrolTa7Yv5akM4v9L0u6VNL4XPGamTVCWyRgYBRwO3AoaYHOSpcAGwAfB94HPAJcJWmlVgVoZtZo7bAqMhExC5gFIGmpxewkbQxsA2weEfcU274APEFazv701kZrZtYY7dIC7stbSK3i10obIi3l/BrwoVxBmZkN1lBIwPcA84AfSBojaQVJRwHvANbJG5qZWf3aPgFHxCJgD2ATYAHwErA9cCnQnTE0M7OqJHXWUq4t+oD7ExG3AR2S3gqsEBHPSPozcEvm0MzMeoiIrlrKtX0LuFxEvFgk342BrYCLc8dkZlavtmgBSxoFjAdKIyDGSZoALIiIeZL2Ap4iDT97L3AicGFEXJ0lYDOzBmiLBExqzc4mjXYI4Phi+3TgINLJtp8BawGPF9u/2/owzcwaR2lE1/AiqQOYmzsOM1s2RYT6LzXE+oDNzIYTJ2Azs0ycgM3MMnECNjPLxAnYzCwTJ2Azs0ycgM3MMnECNjPLxAnYzCwTJ2Azs0ycgM3MMnECNjPLxAnYzCwTJ2Azs0ycgM3MMnECNjPLxAnYzCwTJ2Azs0zaIgFLmihphqRHJXVLmlyxf5SkUyTNk/SKpLskHZwrXjOzRmiLBAyMAm4HDiUtylnpBGAXYB9g0+L+KZJ2a1mEZmYN1harIkfELGAWgKRqi9ltC0yPiOuL+7+RdAiwNfCH1kRpZtZY7dIC7s9NwGRJbweQtCOwMXB51qjMzAahLVrANTgM+DXwT0mLgMXA5yLixrxhmZnVb6i0gA8HtgF2AzqArwCnSvpw1qjMzKqQ1FlLubZvAUtaEfgeMCUiLis2/1XSFsC/A9dkC87MrIqI6Kql3FBoAS9f3BZXbF/M0IjfzKyqtmgBSxoFjAdKIyDGSZoALIiIeZKuA34q6TDgYWAHYD/gyzniNTNrBEVUG3bb4iCk7YHZ9BwDPD0iDpK0FvAD0ljg1UlJ+FcRcVIvx+sA5jYxZDOzXkVEteG0PbRFAm40J2Azy6nWBOw+VDOzTJyAzcwycQI2M8vECdjMLBMnYDOzTJyAzcwycQI2M8vECdjMLBMnYDOzTJyAzcwycQI2M8vECdjMLBMnYDOzTJyAzcwyqWlCdkmH13rAiDi5/nDMzJYdNc0HLOnBik1vA1YGnivujwFeAZ6MiHENjbAOng/YzHJq6HzAEbFh6QYcA9wObBYRq0fE6sBmwK3AN+oN2MxsWTPgFTEk3Q/sFRG3VWzfEvh9kaSzcgvYzHJq5ooY61C973gkMLaO4yFpoqQZkh6V1C1pcsX+bkmLi3/Lb1+ppz4zs3ZQTwK+GvhV0coE3mz9/gK4qs44RpG6NQ6l58KcAGuTEv/axe0goBv4fZ31mZllV8+y9AcB04E5kt4oO87lwGfrCSIiZgGzACT1aLpHxJPl9yVNAWZHxMP11Gdm1g4GnIAj4ilgkqSNSSffAO6JiHsbGlkviiXqJwGfbkV9ZmbNMqAELGl54B5gt4i4G/hHU6Lq2wHAC8BFGeo2M2uYAfUBR8QbwIpNiqVWBwJnR8TrmeMwM6tKUmct5eo5CfdfwFGS6uk/HhRJE4FNgN+0um4zs1pFRFct5epJou8HdgJ2kfQX4OWKives45i1+gwwNyL+2sQ6zMxaop4E/BxwQSODkDQKGA+URkCMkzQBWBAR84oyqwJ7AdMaWbeZWS4DvhKuKUFI2wOz6TkGeHpEHFSU+RxwArBORLzYz/F8JZyZZVPrlXBtkYAbzQnYzHKqNQHXdSJN0l7A3sB6wAoVFXdUfZCZmS1lwKMgirmBzwDmA1sANwPPAOOAyxoanZnZMFbPMLRDgc9HxGHA68CPI2Jn4GRgdCODMzMbzupJwOsBNxX/Xwi8tfj/74CaBh+bmVl9CfgJYPXi/48AHyj+vyFLhpGZmVk/6knA1wCl+XrPAE6QdCXwP3h+BjOzmtWzIsYIYERELCruTwU+SJqY51ftMEeDh6GZWU4eB+wEbGaZNHwccDH/77HAwRHxQsW+0aQVMY6NiHsGEmgz3XLLLXR0tG5Y8siRI1tWlw0/3d3duUOwFhtIH/B/APMqky9ARDwPzAO+1qjAzMyGu4Ek4O2B8/vYfx6w7eDCMTNbdgwkAa8HPNnH/qeBdw4uHDOzZcdAEvDzwEZ97B9PWirIzMxqMJAE/CfgsD72Hw5cP7hwzMyWHQOZDe0HwP9K+j3wY+DvxfZNga8CHyWNBzYzsxrUnIAj4rZiGsrTgT0qdj8D7B0RtzYyODOz4WxA8wFHxB8krQ/sypIlhO4FroiIV5oQn5nZsDXgCdkjYiENnvOhWO34P4AtgXWAKRExo6LMZsAPScPhlgPuAj4REf9sZCxmZq1Sz2Q8zTAKuJ0013CPa6MlbUQ6wfc3YDvgPcBxwKstjNHMrKHqWpKo0SJiFjALQFK1a6i/C/wxIo4u2/ZgK2IzM2uWdmkB96pIyP8K/EPSLEnzJf1Z0u65YzMzG4wBJWBJy0naT9LYZgVUxVrAKsBRwKXAzqQ+6AuLvmMzsyFpoKMgFkn6JbBZk+KppvQlcXFEnFz8/05JHwQOwRd/mNkQVU8f8M3A+4CHGxxLb54GFgF3V2y/G/iXvh545JFHMnr00uuETp06lc5OL11nZoPX1dXFueee22P7zJkzOyOiq7/H17Mixt6kq+JOIE16/nL5/oi4c0AH7Hn8biqGoUm6EbgvIvYv23Yh8EpE7FvlGB3AXM8HbEOJ5wMePnoZTNBDPS3gUro/uWxbkC7KCGDAWUjSKJZc2AEwTtIEYEFEzAN+Apwr6XpgNvAxYDfSmGAzsyGpngS8YcOjgK1IiTWK2/HF9unAQRFxsaRDgP8ETiLNQ7FnRPxvE2IxM2uJeq6Ea3jfb0RcRz8jMiLiTODMRtdtZpZLXeOAJX1a0o2SHivmhkDSlz0218ysdgNOwJK+APyMNCZ3DEv6fJ8Dvty40MzMhrd6WsCHAZ+LiO8Bi8u2zyHN0WBmZjWoJwFvCNxWZftrpEl1zMysBvUk4AdJF2JU2pWeF0uYmVkv6hmG9jPgvyStSBq3u7WkTuBo4LONDM7MbDirZxjabyQtJE0RuTJwDvAYcERE9Lwmz8zMqqprPuCI+G/gvyWtDKwSEU82Niwzs+GvnmFo3ymN/Y2IV5x8zczqU08LeHfgGEnXAb8FLoiI1xobVmNIosY5Mcyy83t12TPgFnBEvA94P2lRzJOAJyT9QtL7Gx2cmdlwVtelyBFxW0QcDrwd+AzwDuBGSXdKOkLS6L6PYGZmg10TTsDywArF/58FvgTMk/TJQR7bzGxYq3cyni0lnQI8TpqY/TZgs4jYPiI2Bo5h6fmCzcysQj2jIP4C/Jl0SfJngHdGxNci4r6yYl3A2xoTopnZ8FTPKIjzgNMj4tHeCkTE0wyBJe/NzHKq50q445oRiJnZsqauK+EkvQOYDKxHOgH3pog4sgFxmZkNe/X0Ae9EWpPtC8BXgB2BA4GDqD5LWi3HnChphqRHJXVLmlyx/4xie/nt0nrqMjNrF/X00/4A+GlEvAd4FfgE8E7gOuD8OuMYBdwOHEpalLOay4CxwNrFrbPOuszM2kI9XRCbsST5LQJWioiXJH0TuAT4xUAPGBGzgFkA6v16zNci4qk64jUza0v1tIBfZkm/7+PARmX71hx0RL3bQdJ8SfdIOlXS6k2sy8ys6eppAf8Z+BBp9YtLgeMlvQfYs9jXDJcBF5BW49iI1A1yqaRtI6K3Lgszs7amgeYvSeNIcwDfKWkUcDzwQeAfwJER8fCgApK6gSkRMaOPMhsC9wM7RcTsKvs7gLkTJ05kzJgxS+2bOnUqnZ3N6T4eMcJDn61+bksMPV1dXXR1dfXYPnPmzH0ioueOCgNOwM1WSwIuyj0JHBMRp1XZ1wHMnTNnDh0dHU2KtCcnYBuMdvss2qDUNLfokMwYxTjkNUh90GZmQ1JNfcCSnqX34WFLiYgBnxwrujLGs+RbY5ykCcCC4vYtUh/wE0W5HwH3ApcPtC4zs3ZR60m4Lzc1CtgKmE1K8kHqVwaYThob/F5gP2AMaQHQy4FvRsQbTY7LzKxpakrAETG9mUFExHX03R2yazPrNzPLoeY+YEkjJH1V0o2SbpH0Q0krNTM4M7PhbCAn4Y4Bvg+8CDwKHAH8VzOCMjNbFgwkAe8HHBoRu0bEFODjwKckDcmRFGZmuQ0kea5HuiINgIi4inTC7O2NDsrMbFkwkAS8HGn2s3JvkBblNDOzARrIXBACzpT0Wtm2FYFfSnq5tCEi9mxUcGZmw9lAEnC1oWhnNyoQM7NlTc0JOCIObGYgZmbLGo9gMDPLxAnYzCwTJ2Azs0ycgM3MMnECNjPLxAnYzCwTJ2Azs0ycgM3MMnECNjPLxAnYzCyTtkjAkiZKmiHpUUndkib3UfaXRZnDWxmjmVmjtUUCBkYBt5MW4Ox19WVJewDbkFbkMDMb0gYyG1rTRMQsYBaAJFUrI2ld4CTgo8ClrYvOzKw52qUF3KciKZ8F/Dgi7s4dj5lZIwyJBAx8DXg9Ik7JHYiZWaO0RRdEXyRtCRwObDHQx06bNo0xY8YstW3q1Kl0dnY2KDozW5Z1dXXR1dXVY/vMmTM7I6LnjgqK6PWcVxaSuoEpETGjuH8EcDxLn5wbCXQDj0TEuCrH6ADmzpkzh46OjhZEnYwYMVR+UFg7arfPog1K1XNZldq+BUzq+72yYtsVxfYzWh+OmVljtEUCljQKGM+Sb41xkiYACyJiHvBsRfk3gCci4h+tjdTMrHHaIgEDWwGzSd0MQepygLQQ6EFVyvu3mpkNeW2RgCPiOgYwIqNav6+Z2VDjs0ZmZpk4AZuZZeIEbGaWiROwmVkmTsBmZpk4AZuZZeIEbGaWiROwmVkmTsBmZpk4AZuZZeIEbGaWiROwmVkmTsBmZpk4AZuZZeIEbGaWiROwmVkmTsBmZpk4AZuZZdIWCVjSREkzJD0qqVvS5Ir935J0t6SXJC2QdKWkrXPFa2bWCG2RgIFRwO3AoVRfcPPvwBeBdwP/AjwEXCFpjVYFaGbWaO2yKOcsYBaAJFXZf275fUlHAp8B3ktaTdnMbMhplxZwzSQtDxwMPAfckTkcM7O6tUULuBaS/hU4F1gZeAzYOSIW5I3KzKx+QyYBA9cAE4A1gc8B50vaOiKe7u0B06ZNY8yYMUttmzp1Kp2dnU0N1MyWDV1dXXR1dfXYPnPmzM6I6LmjgiKqnfPKR1I3MCUiZvRT7l7gtxHxoyr7OoC5c+bMoaOjo0mR9jRixJDr0bE20m6fRRuUHueyqhnKGWME8JbcQZiZ1astuiAkjQLGs+RbY5ykCcAC4BngGGAG8DipC+JLwNuB81sfrZlZY7RFAga2Ig0ni+J2fLF9OvAFYFNgP1LyfQa4BfhQRNzd+lDNzBqjLRJwRFxH390hn2hVLGZmrTKU+4DNzIY0J2Azs0ycgM3MMnECNjPLxAnYzCwTJ2Azs0ycgM3MMnECNjPLpC0uxGgWSVSZ371puru7W1ZXSY4JXEaOHNnyOs2GI7eAzcwycQI2M8vECdjMLBMnYDOzTJyAzcwycQI2M8vECdjMLBMnYDOzTJyAzcwyaYsELGmipBmSHpXULWly2b7lJP1I0p2SXirKTJe0Ts6YzcwGqy0SMDAKuB04lLQoZ7mVgfcB3wG2APYA3gVc0soAzcwarS3mgoiIWcAsAFVM3hARLwAfLd8m6UvA/0l6R0T8s2WBmpk1ULu0gAdqDKml/FzuQMzM6jXkErCktwA/BM6JiJdyx2NmVq+26IKolaTlgPNJrd9D+ys/bdo0Ro8evdS2zs5OOjs7mxOgmS1Turq66Orq6rF95syZnRHRc0cF5ZhPti+SuoEpETGjYnsp+W4AfDginu3jGB3A3Llz59LR0dHMcJeS47X0fMDDR7t9Fm1QapqIfEi0gMuS7zhgx76Sr5nZUNEWCVjSKGA8S741xkmaACwAHgcuIA1F2w1YXtLYotyCiHij1fGamTVCWyRgYCtgNqlvN4Dji+3TSeN/P15sv73YruL+jsCfWhqpmVmDtEUCjojr6HtExpAbrWFm1h8nNjOzTJyAzcwycQI2M8vECdjMLBMnYDOzTJyAzcwycQI2M8vECdjMLJO2uBBjuKiYS37Y1tnd3d3yOs2GI7eAzcwycQI2M8vECdjMLBMnYDOzTJyAzcwycQI2M8vECdjMLBMnYDOzTJyAzcwyaYsELGmipBmSHpXULWlyxf49JF0u6eli/3tzxWpm1ihtkYCBUaQFNw8lLbZZbf/1wFd72W9mNuS0xVwQETELmAWgKpMbRMTZxb71WbJ0vZnZkNYuLWAzs2WOE7CZWSZt0QXRLNOmTWOBuo1ZAAAgAElEQVT06NFLbevs7KSzszNTRGY2nHR1ddHV1dVj+8yZMzsjoueOCopor3NakrqBKRExo8q+9YEHgfdFxJ19HKMDmDt37lw6OjqaF+wyqt3eM8NFjrmdrWlq+mMOxS4If/rNbFhoiy4ISaOA8Sz51hgnaQKwICLmSVoNWA9YtyizaTFa4omImJ8laDOzQWqXFvBWwG3AXFIL93jgVuA7xf7Jxf6Zxf6uYv/BLY/UzKxB2qIFHBHX0ceXQURMB6a3LiIzs+ZrlxawmdkyxwnYzCwTJ+Aqqo3rG071uU7XOdTqG651OgFX4Tdzc5x77rmus0n8nh2adToBm5ll4gRsZpaJE7CZWSZtMQ64CVYEuPvuu+t68PPPP8+tt97a0IDaqb7B1lnvXBDPPfdcy5/nUKpzMHNB+D3bXnVuueWWHcA9EfFKX+XabjKeRpC0D/DfueMws2XalhHRZ/Yergl4DeCjwEPAq3mjMbNl1LLZAjYzGwp8Es7MLBMnYDOzTJyAzcwycQI2M8vECdjMLBMn4IKkiZJmSHpUUrekyU2u72hJN0t6QdJ8SRdJ2qTJdR4i6Q5Jzxe3myTt2sw6K+r/WvHa/qzJ9XyrqKf89rcm1/l2Sb+T9LSkV4rXuWkrwkp6sMpz7Jb08ybWOULScZIeKJ7jfZK+3qz6yupdRdKJkh4q6r1B0lYNPH6/n31Jx0p6rKj/SknjG1G3E/ASo4DbgUNpzcKfE4GfA9sAHwGWB66QtFIT65wHHAV0AFsC1wCXSNqsiXUCIOn9wOeBO5pdV+GvwFhg7eL2oWZVJGkMcCPwGmn8+WbAV4Bnm1UnaRmvtctuO5Pet+c1sc6vkZYBOxTYFPgq8FVJX2pinQC/BXYCPgW8G7gSuErSOg06fp+ffUlHAV8ivX+3Bl4GLpe0wqBrjgjfKm5ANzC5xXWuWdT7oRbX+wxwYJPrWAX4O/BhYDbwsybX9y3g1ha+hj8Ermvl361KDCcC9za5jpnAaRXbfg+c1cQ6VwTeAHat2D4HOLYJ9fX47AOPAdPK7q8KLAT2Hmx9bgG3jzGkb98Frais+Dk5FVgZ+N8mV/dfwMyIuKbJ9ZTbuPhJeb+ksyW9s4l1fRyYI+m8ojvpVkmfbWJ9S5G0PKl1+NsmV3UTsJOkjYt6JwD/AlzaxDqXA0aSfl2UW0gTf9WUSNqQ9Avj6tK2iHgB+D9g28Eef7hOxjOkKM3CciJwQ0Q0u6/y3aSEuyLwIrBHRNzTxPqmAu8j/WRulT8DB5Ba3esA3wb+JOndEfFyE+obB3yBtJr390g/U0+W9FpE/K4J9VXaAxhN8xeu/SGp9XePpMWkLsxjIqJpM9BHxEuS/hf4hqR7gPnAPqTk949m1VtmbVLDaH7F9vnFvkFxAm4PpwKbk1oTzXYPMIH0gd0LOEvSds1IwpLeQfpi+UhEvNHo4/cmIi4vu/tXSTcDDwN7A2c0ocoRwM0R8Y3i/h3FF90hQCsS8EHAZRHxRJPr+SQp+U0F/kb6Yj1J0mNN/qLZFzgdeBRYBNwKnEM6jzGkuQsiM0mnAJOAHSLi8WbXFxGLIuKBiLgtIo4hnRQ7oknVbQm8DbhV0huS3gC2B46Q9LoGM//iAETE88C9QEPOXFfxOFA59+ndwHpNqu9NktYjncQ9rdl1AT8GfhgR50fEXRHx38AJwNHNrDQiHoyIHUkny94ZER8AVgAeaGa9hScAkU7olhtb7BsUJ+CMiuS7O7BjRDySKYwRwFuadOyrgPeQWkoTitsc4GxgQhRnNJpN0iqk5NusL7gbgXdVbHsXqdXdbAeRfg43sx+2ZGVgccW2blqURyJiYUTMl7QaabTJxS2o80FSot2ptE3SqqTRSzcN9vjugihIGkX6kJZaZeOKkwwLImJeE+o7FegEJgMvSyp9wz4fEU2ZQlPS94HLgEeAt5JO3GwP7NKM+or+1qX6tCW9DDwTEfXNll8DST8hnbF/GFgX+A7pTHqzVlg8AbhR0tGkYWDbAJ8FPtek+oA3zx0cAJwZEd3NrKswE/i6pH8Cd5GGM04DftPMSiXtQvpc/h3YmNQS/xtwZoOO399n/0TS876PNMXtccA/gUsGXXmzho8MtRspEXWTvuHLb6c3qb5qdS0G9mvic/wN6WfbQtK3+hXAh1v8Ol9D84ehdRUfkIWkL5tzgA2bXOck4E7gFVJyOqgFr+XOxXtmfIv+dqOAnwEPksbC/oP05bZck+v9N+C+4u/5KHAS8NYGHr/fzz7pRO5jxd/38ka95p4P2MwsE/cBm5ll4gRsZpaJE7CZWSZOwGZmmTgBm5ll4gRsZpaJE7CZWSZOwGZmmTgB2zKhWMLn8NxxNIKk7Yulc1bNHYsNjhOwZSHpjCKJLC5mRntC0hWSDmzVLGmNJOnzkv4s6UVJzyqt93dEE5eY8iWsw4ATsOV0GWlS6/WBXUnzRJwEzJTUVu/NYtWJ3vadTZoj4SJgB9Ksb8eRJlraeRB1erKsYa6t3uS2zHktIp6KiMcj4vaI+CFpes5JpFm+AJA0WtJvJD2ptJrzVZLeW7Z/nKSLi1b0i0Xrc6ee1S1RwzG/Jek2SZ+RVJrAqNpx9qaYpDwifhQRcyPikYiYGRE7kdbAQ9JWRQv/KUnPSbpW0hYVx+pWWrn6EkkvAf9ZbJ8k6e/FirxXAxsM4DW2NuYEbG0lImaTJonfs2zz74E1SHPAdpBWRLiqWI0Y0qKffwR2JM09fBkwo1iRozf9HRPSFIV7kpb8eV8vx9kHuCci/tDL83mx+O9bSdMnfpA0XeW9wKXFVIjlvgVcSFr99/TiOVxAmvpwAmlGux/28bxsKGnFNHa++VZ5Iy0NdGEv+7qAvxb//xBpefflK8r8A/hsH8f/C3Bo2f0HgcNrPSYpEb4KrN7P87gLuKiO5z8CeB6YVLatG/hpRbnvAX+p2PYD0nSJq+b+O/o2uJv7mKwdiSUnmd5Laj0uqDg3tyKwEbw5ofZ3SF0X65AWGliR3pcE6veYhYcjor9Vqms6YShpLVIy3R5Yi7TS70pVYpxbcX8z0gq85Zq9irW1iBOwtaPNSC1WSN0Lj5ESV2Wye67493jSkjFfAe4n9ddeQFo3rJpajglp0vH+3AtsWkO5s4DVgMNIk8S/Rlq9uTLGZqzabG3KCdjaiqQPk9aRO77YdCtppMTi6H3dvA+SluWZURxjFfo+UVXLMWt1DtAl6eMRMbNyp6RVI+KFIsYvRLFis6R3AmvWcPy7gY9XbNt2kDFbm/BJOMvpLZLGSnq7pC0k/SdpocUZFMu5R8RVpJ/cF0vaWdL6kj4o6buSOorj/APYU9KEYi2v/6aProEaj1mTiDiPtA5cl6SjJW0paT1Ju0m6ijQsrRTjpyVtKmkb0sKkr9RQxS+BjSX9WNImkvYB9h9IjNa+nIAtp11JXQEPkkYubA98KSKmRHG2qTAJ+BNwOmlhxnNIfafzi/1Hkk6q3UgaLTCL1MotV3nhQn/HrFlEdBYx7A5cSxrF8U3gStK6e5BWL16N1Mc7nTTe+cl+YiTSopCfKI59O/B5mrwMvLWO14QzM8vELWAzs0ycgM3MMnECNjPLxAnYzCwTJ+BlQDHMqlvSfgN83LWSZjcrrhrq31vSM5JWHuRxSvPnbjeAsnv2V7ZdlP19j6yh7LcldbcirmaT1CXpf3LHMRhOwC0gaf/iA1K6LZT0qKRZkg4rLhxotnqGuwRpfoKWK6aj/DZwUkTUMl62P0s9f0mdko6opewwEzTp+Uk6tHh/t+pS6R8Bn5D0nhbV13C+Eq51AvgG8BCwPOlKrB2AE4EjJU2OiL80peKIh4uJwd8Y4EPrnsu2ASYDmwCnDfZAEXGdpJUi4vWyzfsA/480HrfSkJsQvk3sQxrTvbWkcRHxQDMri4jbJc0hXYJ+QDPraha3gFtrVkScExHTI80d+zHSHAZrAZdIekuzKo6I12OAg74jYlFELGpWTP04ALgxIh5vxMEqkq81mKQNSZdbHwk8DXyqRVWfR7oKclDdVLk4AWcWEdeSVk9YH9i3fJ+kd0n6fdEPulDSLZIq5wUoTS5+gtK6Z69KmidpuqTVi/09+oCLS4DPKMq+KukxpUnN1ysrc62kayrqepuk3ypNfr5Q0u2VfcvlfZKSPifpvqKOmyVt1d9rUnwR7QpcVbH9AklzK7bNLOrarWzb1sW2jxb3l+oDLvq1/xUoxdmtNOl6SQAjJB1TvD4LlSZsL58prbfYz5T0YJXtPfpei8ugr1dawuhFSfdI+l7Z/uUlHStpjtIk7i9J+pOkHfqo/8uSHlKavP1aSf+vv5iLx+1b1PNK8X7rUt/zKVf6FLCANC/z7+klASs5QtKdxev6pKTLVHEJ+ADiuZI0uVLOX2t1cxdEe/gd8H1gF+C3AMUH5wbgn6T5X18G9ibNX7BnRFxSlBtVlHtX8djbSJO8TAbeQfpQVHMhadaxk4GHSa3wnUmX45YmqKnsN10RuA4YB/yc1J3yb8CZkkZHxM8r6vgU6cPxy+JYRwEXFD9PF/fxemxJmiWs8nLi64HJklaJiJeKbR8kzY07EShNir5dse3GsseWP5fvAqOBdYEvk7ocXirbL9LlvouBnxRljyLN39DfRDi99bEutV3S5sBM0uXF3yDNjja+eD4lq5IuYe4Cfk2aQvMzwCxJW0fEnRV17E96vU8hTa15BHC1pPdExFO9BSzpGOBY4FxSl8/bgMOB6yRtUUwm1J99gAsiYpGkLuAQSVtGROX0mqcXcf6xqGs50t/uAxR/7wHG8zfS7Hf/QroMfWjJPSHxsnAjveEWAx19lHkWmFN2/ypSMl2uotwNpBUYSve/Uxx7ch/HXp90Mm2/4v7o4v6R/cQ9G7im7P4RRV1Ty7aNJCW654FRFfU9Sdmk4aRZvRZTNgl5L/UeVJTbvGL7lsVxP1rcf3dx/1zgprJyF1e8ltsXx9uubNtM4IEqdW9fHPOvwMiy7YdVi6nK48/o5bjfIs2+VvlartbHsVTl778q8DhwWpW/70vA2mXb30/FJO9V4liPdG7gqIp6NgdeB75Ww/u79HfZsWzbI8DPKsrtWJT7WR/HGnA8wD3AHwb6uWyHm7sg2sdLpBYOklYjvVnPB0ZLWqN0I03usrGkdYrH7QncEcVUjDVaSHoz76Cll+Dpz8eAJyLi3NKGSC3Zk0ktr+0ryp8bS7dWricllXH91LNG8e+zFdtvI71OpeFkE4F5pLl2tyxa6JBWvLi+32fTt9Nj6VZ6rbHXqjTv8B5S9VWgI1kEb/50X430y2AOaRmlShdFxBNlj7+FNJn7pD7i+ATpeZ1f8T57kjSD2441PJdPAU+QJiIq+R9gasVz+wQpAR/b4HiepbapPduOE3D7WAUorR82nvQmPA54quL27aLMWsW/G5FaazWLdELqKFJCnS/pOkn/IWlsPw9dn/QhqHR3Ee/6FdvnVdRbSjqr1RjqUokpIrpJ00hOLDZNJCXGG0kt8Q8UP+1XZ/AJeF7F/dKXQa2x9+d/SHGfRvobdEn6t8pkrDSE8Q7S8kjPkBLRv5J+xVS6r8q2e+l7buTxpDxwH0u/z54kTTS/Vu8PfXO44CdJv5bGSdqo6Cu/mTTSp3xx1HHAY2Xvg0bFU76CypDiPuA2IGld0geq9AEqfTH+FLi8l4dV+7DVLCJOkjQDmEJamPJY4GhJO0bEHYM5dpne+nn7G+b1TPHvaqTpKsvdAPxncaJuInBcRDwv6a/F/SdJH8bBJuB6Y+8tEYxcqlDEq8B2knYkJdRdSYnsakm7RERI2pdi7Tzgx6Tntpi0WnKjWuIjSK3SXak+5vulKtvKfZi0DNRUoLNiX5Bax1dVPqjB8axG+qIZcpyA28N+pDfrrOJ+6Yz8GxFxTfWHvOl+Ul/ogEXEg8AJwAlFq+UO0pjK3q6Ye5i0WkWlzcr2N8I9pES3IWnRy3LXk36GdwJvZ0mi/ROpa2I+cG/0cdKp0KwW07NAtW6dDaoGkVaBng38u6SjSScIdwSuIf0cvz8i9ip/jKTefsJvXGXbJqSTpb25n/RaPxQR9Xyp70t6zQ+l55fTJ0hdLIdExGtFXbtIGtNHK3hA8UgaCbyToXgCDndBZKe0BM/XSUn3HIAieVwLHCxp7SqPKe/vugCYIGn3AdS5knqOOX6Q1AXS11jkS4G1JX2y7FgjSSeoXiSNkGiEuaQ+6mpD1v4PWETqQlkQEXcX268nnUnfjtpavy9T/Wf8YN1P6rd/80ux6K+fUl6o6M+tdAcp+ZT+Bj1a4UqrafQ2EmOKpLeXld0a2Ib0d+vNhaSW5req7VQxlLGXfSsCewAzI+KiiLiw/EYajbEqaUQOpPfqiN7qqjOezUkjPm6sUrztuQXcOgImSdqM9LqPJf1825mU/CbH0hcLfJGUSP4i6TRSgh5L+vCtC2xRlPsJsBfppMUZpOS1BmnEwcFR/eq6TUg/dc8jDeNZRDqZtxZpyFNvfg0cTBp2thVLhqFtCxwREQ1ZUDIiXpN0BfARlvR5l/YtVBoL/AHS0kUlfwJGAStTPQFXts7mAntLOh64BXgpIv7Q82EDdi7pEtmLJZ1cxHQIadWN8hNn31Qal/xH0i+HscAXSKMHbijK/IF0kcHFRblxpNf/LtI5g0r3ATdI+gVLhqE9RXqPVBURD0j6OvB9pYspLiZ9mY4jfWn8CvhZLw/fnXTiuLcTwH8u6v8UcH5EXCvpd8DhkjYh/eIbQeo6uiYiTq0jnl1IX6YD6eZoH7mHYSwLN5YMQyvdFgKPkt6AX6QYvlXlcRuQ+gAfJZ2EeYT0U2tKRbkxpEtqHymO/TBpTPBqxf71i3pLw9BWJ41cuAt4gTRW+CZgz4rjzgaurti2JvAb0s/OhaRxrJ+uKFOqb1qV57QY+EYNr9kU0hfDulX2/ag4zlcqtt9bPGaDiu3VhqGtTBp//Uyx74GKspWvxVKvYT+x70RqzS4kfcF10nP41w6k1t68oty8Ip6NKo51FOnL9xXS6IePFe+J+6u93qRxzQ8V5WcD76443reARb283tcV74cXivfGScD4Pp7nJaQ+2RX7KHN68d4tvRdFulruruJ5P0H6onlfPfGQTsqemfszXu/NSxJZWyrOrt9Fajl9M3c81n4kvY/0pbRFNGkelWZzAra2JWlv4FRgvWjMjGg2jBRX3BFpUdQhyQnYzCwTj4IwM8vECdjMLJNhmYAlrSypQ0N0jlAzWzYMyz7gYm7Rufvuuy9jx/Y3vUFPd999N5tttln/BRuk1fW5Ttc51OobanX+9Kc/rWlVlWGdgHPHYWbLpoioKQEPyy4IM7OhwAnYzCwTJ2Azs0ycgM3MMnECNjPLxAnYzCwTJ2Azs0ycgM3MMsmegCUdLelmSS9Imi/pomK2/PIye0i6XNLTkrolvTdXvGZmjZI9AZOWI/k5ae2qjwDLA1dIWqmszCjSMjNfZYguP21mVin7mnARMan8vqQDSMtvb0mxNlZEnF3sW5/+lwU3MxsS2qEFXGkMqZW7IHcgZmbN1FYJWJKAE4EbIuJvueMxM2umtkrApPW/Ngem5g7EzKxekmpapy57H3CJpFOAScDEiHg8dzxmZvWKiK5ayrVFAi6S7+7A9hHxSD/FPQrCzIaF7AlY0qlAJzAZeFlSaQmL5yPi1aLMasB6wLqkURCbFv3FT0TE/Axhm5kNWvYVMSR1U71Ve2BEnFWU2R84o0q570TEsVWO6RUxzCybWlfEyJ6Am8EJ2Mxy8pJEZmZtzgnYzCwTJ2Azs0ycgM3MMnECNjPLxAnYzCwTJ2Azs0ycgM3MMnECNjPLxAnYzCwTJ2Azs0ycgM3MMnECNjPLxAnYzCwTJ2Azs0ycgM3MMnECNjPLxAnYzCyT7AlY0tGSbpb0gqT5ki6StEmVcsdKekzSK5KulDQ+R7xmZo2SPQEDE4GfA9sAHwGWB66QtFKpgKSjgC8Bnwe2Bl4GLpe0QuvDNTNrjLZblFPSmsCTwHYRcUOx7THgJxFxQnF/VWA+sH9EnFflGF6U08yyGcqLco4hLT+/AEDShsDawNWlAhHxAvB/wLY5AjQza4S2SsCSBJwI3BARfys2r01KyPMris8v9pmZDUltlYCBU4HNgam5AzEzq5ekzlrKtU0ClnQKMAnYISIeL9v1BCBgbMVDxhb7zMzaSkR01VKuLRJwkXx3B3aMiEfK90XEg6REu1NZ+VVJoyZuamWcZmaNtFzuACSdCnQCk4GXJZVaus9HxKvF/08Evi7pPuAh4Djgn8AlLQ7XzKxhsidg4BDSSbZrK7YfCJwFEBE/lrQy8CvSKInrgY9FxOstjNPMrKHabhxwI3gcsJnlNJTHAZuZLROcgM3MMnECNjPLxAnYzCwTJ2Azs0ycgM3MMnECNjPLxAnYzCwTJ2Azs0ycgM3MMnECNjPLxAnYzCwTJ2Azs0ycgM3MMnECNjPLxAnYzCwTJ2Azs0zaIgFLmihphqRHJXVLmlyxfy1JZxb7X5Z0qaTxueI1M2uEtkjAwCjgduBQ0vpwlS4BNgA+DrwPeAS4StJKrQrQzKzR2mFRTiJiFjALQNJSaylJ2pi0BP3mEXFPse0LpKXqO4HTWxutmVljtEsLuC9vIbWKXyttiLSS6GvAh3IFZWY2WEMhAd8DzAN+IGmMpBUkHQW8A1gnb2hmZvVr+wQcEYuAPYBNgAXAS8D2wKVAd8bQzMwGpe0TMEBE3BYRHcBoYJ2ImASsCTyQNzIzs54kddZSbkgk4JKIeDEinilOzG0FXJw7JjOzShHRVUu5thgFIWkUMB4ojYAYJ2kCsCAi5knaC3iKNPzsvcCJwIURcXWWgM3MGqAtEjCpNTubNNohgOOL7dOBg0gn234GrAU8Xmz/buvDNDNrHKURXcOLpA5gbu44zGzZFBHqv9QQ6wM2MxtOnIDNzDJxAjYzy8QJ2MwsEydgM7NMnIDNzDJxAjYzy8QJ2MwsEydgM7NM2uVS5Ka45ZZb6OjoaFl9I0eObFldZkNVd7dnkS1xC9jMLBMnYDOzTJyAzcwycQI2M8vECdjMLBMnYDOzTJyAzcwycQI2M8ukLRKwpImSZkh6VFK3pMkV+0dJOkXSPEmvSLpL0sG54jUza4S2SMDAKOB24FDSopyVTgB2AfYBNi3unyJpt5ZFaGbWYG1xKXJEzAJmAUiqtpjdtsD0iLi+uP8bSYcAWwN/aE2UZmaN1S4t4P7cBEyW9HYASTsCGwOXZ43KzGwQ2qIFXIPDgF8D/5S0CFgMfC4ibswblplZ/YZKAj4c2AbYDXgE2A44VdJjEXFNbw868sgjGT169FLbpk6dSmdnZzNjNbNlRFdXF+eee26P7TNnzuyMiK7+Hq+Iaue88pHUDUyJiBnF/RWB54ttl5WVOw1YNyImVTlGBzDX01GatZ9lYTrKXs5l9TAU+oCXL26LK7YvZmjEb2ZWVVt0QUgaBYwHSt8a4yRNABZExDxJ1wE/lXQY8DCwA7Af8OUc8ZqZNUJbJGBgK2A2aQxwAMcX26cDBwGfBH4AnA2sTkrCR0fEr1sfqplZY7RFAo6I6+ijOyEingQ+07qIzMyaz32oZmaZOAGbmWXiBGxmlokTsJlZJk7AZmaZOAGbmWXiBGxmlokTsJlZJk7AZmaZtMWVcM0yYsQIRoxo3XfMsjDLk9lg1ThR2DLBLWAzs0ycgM3MMnECNjPLxAnYzCyTmk7CSTq81gNGxMn1h2NmtuyodRTEtIr7bwNWBp4r7o8BXgGeBJyAzcxqUFMXRERsWLoBxwC3A5tFxOoRsTqwGXAr8I3mhWpmNrzU0wd8HHBYRPy9tKH4/zTgu40KzMxsuKsnAa9D9a6LkcDYeoKQNFHSDEmPSuqWNLlif7ekxcW/5bev1FOfmVk7qCcBXw38SlJHaYOkLYFfAFfVGccoUrfGoaRFOSutTUr8axe3g4Bu4Pd11mdmll09lyIfRFqteI6kN8qOcznw2XqCiIhZwCwAVblOsViU802SpgCzI+LheuozM2sHA07AEfEUMEnSxqSTbwD3RMS9DY2sF5LWAiYBn25FfWZmzTKgBCxpeeAeYLeIuBv4R1Oi6tsBwAvARRnqNjNrmAEl4Ih4Q9KKzQqmRgcCZ0fE6/0VnDZtGqNHj15qW2dnJ52dnc2KzcyWIV1dXXR1dfXYPnPmzM6I6LmjgiKqnfPq4wHSfwKbAJ+NiEUDenBtx+8GpkTEjCr7JgLXAhMi4q99HKMDmDt37lw6Ojp6K9ZwA30tzZZFy8h0lDU9yXpOwr0f2AnYRdJfgJfLd0bEnnUcs1afAeb2lXzNzIaKehLwc8AFjQxC0ihgPEu+NcZJmgAsiIh5RZlVgb3oeVm0mdmQVM8oiAObEMdWwGzSGOAAji+2TycNewP4ZPHvuU2o38ys5dpiSaKIuI5+LgqJiNOA01oTkZlZ89WVgCXtBewNrAesUL4vIlp31svs/7d352FyVWUex78/ZJMADYKyyCIBBOZBAh0EQQICIhmUsDiDFo6MAiJkWGycYRFHRFwADYiKwzwiELZiQBS6FcIuKItIh4BA2CNggCAEwhYw0O/8cW5Ip7q6U11dt2519+/zPPWk6p7b93270/3WrXPPPcdsGBv0rcjZ3MDnAXOArYC7gBeBscA1Dc3OzGwEq2cuiMnAIRFxBPAP4LSI2I00D3DbgF9pZmbvqqcArwfcnj2fD6yUPb8Q8B0OZmY1qqcAPwe8L3v+FPCx7PkG1Dj42MzM6ivANwEL5+s9DzhD0vXA/+H5GczMalbPKIhDyAp3RJwl6UVge6AT+N8G5mZmNqLVcyNGD2ky9IWvL8U3R5iZDVrNXRCSNpZUzm4Jrmxrk3SJpE0bm56Z2cg1mDPg/wKejohXKhsiYp6kp4HjSPP1jkqjZJYnM2uQwVyE2wm4fID2y4DthpaOmdnoMZgCvB7w/ADtLwDrDtWtwLAAABZkSURBVC0dM7PRYzAFeB6w4QDtG5GWCjIzsxoMpgDfChwxQPuRwB+Glo6Z2egxmItwPwDukPQr4DTg4Wz7psAxwO6k8cBmZlaDmgtwRNyTTUN5LrBPRfOLwH4RMb2RyZmZjWSDXRX5t5LWByayaAmhR4DrIuKNHPIzMxux6rkTbj6e88HMbMjqmYyn4SRNkNQpabakHkmTquyzmaSrJL0s6TVJf5K0ThH5mpk1QksUYGAMMIM02XtUNkrakDTC4kFgR+AjwMnAm03M0cysoVplUc5pwDQAVb+f97vA7yLi+F7bZjUjNzOzvAzqDFjS0pIOkLRGXglViSng08CjkqZJmiPpTkl7NSsHM7M8DKoAR8TbwNnA8vmkU9UHgBWBY4Grgd1IFwF/LWlCE/MwM2uoerog7gK2BJ5scC79WfgmcWVE/CR7fp+k7YFDGeDuu46ODtraFl8ntFQqUSp56TozG7pyuUy5XO6zvaurqxQRfRsqKKLPNa+Bv0Daj3RX3BlAN/B67/aIuG9QB+x7/B5g74jozF4vk8X4dkR8v9d+pwAfj4g+Z8GS2oHu7u5u2tvbh5KOmVk9apqbtp4z4IWrX/yk17bIAgbwnjqO2a+IWCDpz8AmFU0fpnln4WZmDVdPAd6g0UlIGsOiO+sAxkoaB8yNiKeBHwKXSvoDcDPwz8BnSHMUm5kNS/XcCZfHWefWpMIa2WNKtn0qcGBEXCnpUOAbwJmkiYD2jYg7csjFzKwp6hoHLOmLpAtgGwDbRcSTkr4GzIqIqwZ7vIi4hSWMyIiI84HzB5+tmVlrGvSdcJIOA04nDQlbhUV9vi8DX2tcamZmI1s9tyIfAXwlIr4HvNNr+92kW4TNzKwG9RTgDYB7qmx/izSng5mZ1aCeAjyLdCNGpYnAzKGlY2Y2etRzEe504CxJy5OGjW0jqQQcDxzcyOTMzEayeoahnSNpPmmGshWAS4BngKMi4tIBv9jMzN5V1zC0iLgYuFjSCsCKEfF8Y9MyMxv56hmGdlK2LhwR8YaLr5lZfeq5CLcX8LikGyXtL2m5RidlZjYaDLoAR8SWwEeBB0i3BT8n6X8kfbTRyZmZjWR1rQkXEfdExJHA2sBBwDrAbZLuk3SUpLaBj2BmZkNdlFPAMsCy2fOXgMOBpyV9bojHNjMb0eoqwJLGS/oZ8CxpYvZ7gM0iYqeI2Bg4gcXnCzYzswr1jIL4C3An6Zbkg4B1I+K4iHis125l4P2NSdHMbGSqZxzwZcC5ETG7vx0i4gWG3r1hZjai1XMn3Ml5JGJmNtrUOyH7OsAkYD3SBbh3RcTRDcjLzGzEG3QBlrQr0Ak8AWwK3A98iDQKYnojkzMzG8nq6af9AfCjiPgI8CbwWWBd4Bbg8nqSkDRBUqek2ZJ6JE2qaD8v2977cXU9sczMWkU9BXgz4ILs+dvAeyPiNeBbwLF15jEGmAFMJi3KWc01wBrAmtmjVGcsM7OWUE8f8Oss6vd9FtiQdFsywOr1JBER04BpAJLUz25vRcTf6zm+mVkrqucM+E5gh+z51cAUSScA52ZtefmEpDmSHpL0c0nvyzGWmVnu6jkDPhpYMXt+Yvb8c8CjWVsergGuIC2HtCGpH/pqSdtFRH9dFmZmLU2tVr8k9QB7R0TnAPtsADwO7BoRN1dpbwe6d9xxR9raFp8XqFQqUSq5+9jMhq5cLlMul/ts7+rq2j8i+jZUGJYFONvveeCEiPhFlbZ2oLu7u5v29vacMjUz61d/17IWU1MXhKSX6H90wmIiIve+2exGkNVIFwHNzIalWvuAv5ZnEpLGABux6F1jrKRxwNzscSKpD/i5bL9TgUeAa/PMy8wsTzUV4IiYmnMeWwM3k86yA5iSbZ9KGhu8BXAAsAppBeZrgW9FxIKc8zIzy03NoyAkLQX8J2lNuGWBG4GTImL+UJOIiFsYeEjcxKHGMDNrNYMZB3wC8H3gVWA2cBRwVh5JmZmNBoMpwAcAkyNiYkTsDewJfCE7MzYzs0EaTPFcj3RDBAARcQOpv3btRidlZjYaDKYAL02a/ay3BaRFOc3MbJAGcyuygPMlvdVr2/LA2ZJeX7ghIvZtVHJmZiPZYApwtaFoFzUqETOz0abmAhwRX84zETOz0cYjGMzMCuICbGZWEBdgM7OCuACbmRXEBdjMrCAuwGZmBXEBNjMriAuwmVlBXIDNzAriAmxmVhAXYDOzgrREAZY0QVKnpNmSeiRNGmDfs7N9jmxmjmZmjdYSBRgYA8wgLcAZ/e0kaR9gW9KSSGZmw9pgpqPMTURMA6YBSFK1fSR9EDgT2B24unnZmZnlo1XOgAeUFeULgNMiYmbR+ZiZNcKwKMDAccA/IuJnRSdiZtYoLdEFMRBJ44Ejga0G+7UdHR20tbUttq1UKlEqlRqUnZmNZuVymXK53Gd7V1dXKSL6NlRQRL/XvAohqQfYOyI6s9dHAVNY/OLce4Ae4KmIGFvlGO1Ad3d3N+3t7U3I2sxsMVWvZVVq+TNgUt/v9RXbrsu2n9f8dMzMGqMlCrCkMcBGLHrXGCtpHDA3Ip4GXqrYfwHwXEQ82txMzcwapyUKMLA1cDOpmyFIXQ6QVmI+sMr+rdVvYmZWh5YowBFxC4MYkVGt39fMbLgZLsPQzMxGHBdgM7OCuACbmRXEBdjMrCAuwGZmBXEBNjMriAuwmVlBXIDNzAriAmxmVhAXYDOzgrgAm5kVxAXYzKwgLsBmZgVxATYzK4gLsJlZQVyAzcwK4gJsZlaQlijAkiZI6pQ0W1KPpEkV7SdKminpNUlzJV0vaZui8jUza4SWKMDAGGAGMJnq6709DPwHsDnwceCvwHWSVmtWgmZmjdYqa8JNA6YBSFKV9kt7v5Z0NHAQsAVpMU8zs2GnVc6AayZpGeCrwMvAvQWnY2ZWt5Y4A66FpE8DlwIrAM8Au0XE3GKzMjOr37ApwMBNwDhgdeArwOWStomIF/r7go6ODtra2hbbViqVKJVKuSZqZqNDuVymXC732d7V1VWKiL4NFRRR7ZpXcST1AHtHROcS9nsE+GVEnFqlrR3o7u7upr29PadMzcz61edaVjXDrg+4l6WA5YpOwsysXi3RBSFpDLARi941xkoaB8wFXgROADqBZ0ldEIcDawOXNz9bM7PGaIkCDGxNGk4W2WNKtn0qcBiwKXAAqfi+CPwZ2CEiZjY/VTOzxmiJAhwRtzBwd8hnm5WLmVmzDOc+YDOzYc0F2MysIC7AZmYFcQE2MyuIC7CZWUFcgM3MCuICbGZWEBdgM7OCuACbmRXEBdjMrCAuwGZmBXEBNjMriAuwmVlBXIDNzAriAmxmVhAXYDOzgrgAm5kVxAXYzKwgLVGAJU2Q1ClptqQeSZN6tS0t6VRJ90l6LdtnqqS1iszZzGyoWqIAA2OAGcBk0qKcva0AbAmcBGwF7ANsAlzVzATNzBqtVRblnAZMA5CkirZXgN17b5N0OPAnSetExN+alqiZWQO1yhnwYK1COlN+uehEzMzqNewKsKTlgFOASyLitaLzMTOrV0t0QdRK0tLA5aSz38lL2r+jo4O2trbFtpVKJUqlUj4JmtmoUi6XKZfLfbZ3dXWVIqJvQwVFVF7zKpakHmDviOis2L6w+H4I2CUiXhrgGO1Ad3d3N+3t7Xmma2ZWjZa8yzA5A+5VfMcCOw9UfM3MhouWKMCSxgAbsehdY6ykccBc4FngCtJQtM8Ay0haI9tvbkQsaHa+ZmaN0BIFGNgauJnUtxvAlGz7VNL43z2z7TOy7cpe7wzc2tRMzcwapCUKcETcwsAjMobdaA0zsyVxYTMzK4gLsJlZQVyAzcwK4gJsZlYQF2Azs4K4AJuZFcQF2MysIC7AZmYFcQE2MyuIC7CZWUFcgM3MCuICbGZWEBdgM7OCuACbmRXEBdjMrCAuwGZmBXEBNjMriAuwmVlBWqIAS5ogqVPSbEk9kiZVtO8j6VpJL2TtWxSVq5lZo7REAQbGkBbcnExabLNa+x+AY/ppNzMbdlplUc5pwDQASarSflHWtj6Llq43MxvWWuUM2Mxs1HEBNjMrSEt0QeSlo6ODtra2xbaVSiVKpVJBGZnZSFIulymXy322d3V1lSKib0MFRbTWNS1JPcDeEdFZpW19YBawZUTcN8Ax2oHu7u5u2tvb80vWzKy6mq5VDccuiNZ6xzAzq1NLdEFIGgNsxKJ3jbGSxgFzI+JpSasC6wEfzPbZNBst8VxEzCkkaTOzIWqVM+CtgXuAbtIZ7hRgOnBS1j4pa+/K2stZ+1ebnqmZWYO0xBlwRNzCAG8GETEVmNq8jMzM8tcqZ8BmZqOOC7CZWUFcgM3MCuICXEW1gdUjKZ5jOuZwizdSY7oAV+FfZsd0zNaKN1JjugCbmRXEBdjMrCAuwGZmBWmJGzFysDzAzJkz6/riefPmMX369IYm1ErxHNMxh1u84RZz/Pjx7cBDEfHGQPu13GxojSBpf+DiovMws1FtfEQMWL1HagFeDdgd+CvwZrHZmNkoNTrPgM3MhgNfhDMzK4gLsJlZQVyAzcwK4gJsZlYQF+CMpAmSOiXNltQjaVLO8Y6XdJekVyTNkfQbSR/OOeahku6VNC973C5pYp4xK+Ifl/1sT885zolZnN6PB3OOubakCyW9IOmN7Oec24qwkmZV+R57JP00x5hLSTpZ0hPZ9/iYpG/mFa9X3BUl/VjSX7O4f5S0dQOPv8S/fUnfkfRMFv96SRs1IrYL8CJjgBnAZJqz8OcE4KfAtsAngWWA6yS9N8eYTwPHAu3AeOAm4CpJm+UYEwBJHwUOAe7NO1bmfmANYM3ssUNegSStAtwGvEUa/rgZ8HXgpbxikpbxWrPXYzfS7+1lOcY8jrQM2GRgU+AY4BhJh+cYE+CXwK7AF4DNgeuBGySt1aDjD/i3L+lY4HDS7+82wOvAtZKWHXLkiPCj4gH0AJOaHHP1LO4OTY77IvDlnGOsCDwM7ALcDJyec7wTgelN/BmeAtzSzP+3Kjn8GHgk5xhdwC8qtv0KuCDHmMsDC4CJFdvvBr6TQ7w+f/vAM0BHr9crA/OB/YYaz2fArWMV0rvv3GYEyz5Ofh5YAbgj53BnAV0RcVPOcXrbOPtI+bikiyStm2OsPYG7JV2WdSdNl3RwjvEWI2kZ0tnhL3MOdTuwq6SNs7jjgI8DV+cYc2ngPaRPF73NJ8dPNQtJ2oD0CePGhdsi4hXgT8B2Qz3+SJ0LYliRJNIZzB8jIu++ys1JBXd54FVgn4h4KMd4nwe2JH1kbpY7gS+RzrrXAr4N3Cpp84h4PYd4Y4HDSKt5f4/0MfUnkt6KiAtziFdpH6CN/BeuPYV09veQpHdIXZgnRMSleQWMiNck3QH8t6SHgDnA/qTi92hecXtZk3RiNKdi+5ysbUhcgFvDz4F/Ip1N5O0hYBzpD/ZfgAsk7ZhHEZa0DumN5ZMRsaDRx+9PRFzb6+X9ku4CngT2A87LIeRSwF0R8d/Z63uzN7pDgWYU4AOBayLiuZzjfI5U/D4PPEh6Yz1T0jM5v9H8G3AuMBt4G5gOXEK6jjGsuQuiYJJ+BuwBfCIins07XkS8HRFPRMQ9EXEC6aLYUTmFGw+8H5guaYGkBcBOwFGS/pGd+ecuIuYBjwANuXJdxbNA5dR7M4H1cor3LknrkS7i/iLvWMBpwCkRcXlEPBARFwNnAMfnGTQiZkXEzqSLZetGxMeAZYEn8oybeQ4Q6YJub2tkbUPiAlygrPjuBewcEU8VlMZSwHI5HfsG4COkM6Vx2eNu4CJgXGRXNPImaUVS8c3rDe42YJOKbZuQzrrzdiDp43Ce/bALrQC8U7GthybVkYiYHxFzJK1KGm1yZRNiziIV2l0XbpO0Mmn00u1DPb67IDKSxpD+SBeelY3NLjLMjYinc4j3c6AETAJel7TwHXZeROQyg5uk7wPXAE8BK5Eu3OwEfCqPeFl/62J92pJeB16MiPoma66BpB+Srtg/CXwQOIl0JT2vBb7OAG6TdDxpGNi2wMHAV3KKB7x77eBLwPkR0ZNnrEwX8E1JfwMeIA1n7ADOyTOopE+R/i4fBjYmnYk/CJzfoOMv6W//x6Tv+zHSDIsnA38Drhpy8LyGjwy3B6kQ9ZDe4Xs/zs0pXrVY7wAH5Pg9nkP62Daf9K5+HbBLk3/ON5H/MLRy9gcyn/RmcwmwQc4x9wDuA94gFacDm/Cz3C37ndmoSf93Y4DTgVmksbCPkt7cls457r8Cj2X/n7OBM4GVGnj8Jf7tky7kPpP9/17bqJ+5p6M0MyuI+4DNzAriAmxmVhAXYDOzgrgAm5kVxAXYzKwgLsBmZgVxATYzK4gLsJlZQVyAzcwK4gJso0K2htqRRefRCJJ2ytYuW7noXGxoXICtEJLOy4rIO9nUlM9Juk7Sl5s1TWUjSTpE0p2SXpX0ktKCq0fluMaf5xAYAVyArUjXkFYVWB+YSJqo50ygS1JL/W5my/7013YRaZKa3wCfIE27eTJpprvdhhDTsxWOcC31S26jzlsR8feIeDYiZkTEKaT5kfcgTbMIgKQ2SedIel7SPEk3SNqiV/tYSVdmZ9GvZmefu/YNt0gNxzxR0j2SDpK0cAa5asfZj2yViIg4NSK6I+KpiOiKiF1Ji5AiaevsDP/vkl6W9HtJW1Ucq0fSoZKukvQa8I1s+x6SHs6WRL8R+NAgfsbWwlyAraVExM2kVTr27bX5V8BqpEm420lL0tyQLQcPadXl3wE7kyZ/vwbozJZE6s+Sjglpjth9SWuubdnPcfYHHoqI3/bz/byaPV2JNH/t9qT5gh8Brs7mou3tRODXpOXXz82+hytIc8+OI00pesoA35cNJ82YR9QPPyofpLXZft1PWxm4P3u+A/ASsEzFPo8CBw9w/L8Ak3u9ngUcWesxSYXwTeB9S/g+HgB+U8f3vxQwD9ij17Ye4EcV+30P+EvFth+Q5qtduej/Rz+G9nAfk7Uisegi0xaks8e5Fdfmlgc2hHdXNDiJ1HWxFmmll+Xpf022JR4z82REzK0h1yWS9AFSMd0J+ABpqfX3Vsmxu+L1ZqQl0Hu7o5aY1vpcgK0VbUY6Y4XUvfAMqXBVFruXs3+nkNbs+jrwOKm/9grSwo3V1HJMSKs+LMkjwKY17HcBsCpwBGmVjreAO6vkWEtMGyFcgK2lSNqFtJDnlGzTdNJIiXei/4VLtyeti9aZHWNFBr5QVcsxa3UJUJa0Z0R0VTZKWjkiXslyPCwirs22rwusXsPxZwJ7Vmzbbog5W4vwRTgr0nKS1pC0tqStJH2DtNJtJ3AhQETcQPrIfaWk3SStL2l7Sd+V1J4d51FgX0njssUUL2aAroEaj1mTiLiMtBBnWdLxksZLWk/SZyTdQBqWtjDHL0raVNK2pJWh36ghxNnAxpJOk/RhSfsD/z6YHK11uQBbkSaSugJmkUYu7AQcHhF7R3a1KbMHcCtwLmll3EtIfadzsvajSRfVbiONFphGOsvtrfLGhSUds2YRUcpy2Av4PWkUx7eA60kLn0JaPn5VUh/vVNJ45+eXkCORVuX9bHbsGcAhwPGDzdFakxflNDMriM+AzcwK4gJsZlYQF2Azs4K4AJuZFcQF2MysIC7AZmYFcQE2MyuIC7CZWUFcgM3MCuICbGZWEBdgM7OC/D/rhucPV+KMCwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f67a467f1d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "optimal_policy.visualize()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Q: What's the house edge for this optimal policy ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Use MC to compute prob(i,j,k)\n",
    "prob = np.zeros((21,10,2))\n",
    "np.random.seed(4)\n",
    "max_iter = int(1e5) #reduce max_iter to 1e6 generates correct result except for the case player=12, dealer=4, use_able=0 \n",
    "for _ in range(max_iter):\n",
    "    dealer_init_cards = np.random.choice(learner.game.cards,size=2,replace=True)\n",
    "    player_init_cards = np.random.choice(learner.game.cards,size=2, replace=True)\n",
    "    i,j,k = convert_cards_to_state(player_init_cards, dealer_init_cards[0])\n",
    "    prob[i,j,k] = prob[i,j,k] + 1.0 \n",
    "prob = prob/max_iter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.99999999999999989"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prob.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "House edge:  -0.0465476640519\n"
     ]
    }
   ],
   "source": [
    "#E[Reture]= sum_{i,j,k} Prob[i,j,k] * E[Reture|i,j,k], E[Return|i,j,k]= optimal_policy.v\n",
    "print \"House edge: \", np.sum(prob*optimal_policy.v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}