Sutton-Barto-book-exercise-answers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sutton & Barto Reinforcement Learning Book"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3.7.3'"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from platform import python_version\n",
    "python_version()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chapter 4: Dynamic Programming\n",
    "\n",
    "## Example 4.1 Demonstration of iterative policy evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<IMG SRC=\"images/gridworld.png\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### First set up the GridWorld environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GridWorld:\n",
    "    \n",
    "    name = 'GridWorld'\n",
    "    roles = [1]\n",
    "    possible_n_players = [1]\n",
    "    help_text = {\n",
    "        'Move format': \"(row, col)\",\n",
    "        'Move not available': \"That action is not available.\",\n",
    "        'Number of players': \"This game is for 1 player.\"\n",
    "    }\n",
    "    actions = {\n",
    "        'l': (0, -1),\n",
    "        'u': (-1, 0),\n",
    "        'r': (0, 1),\n",
    "        'd': (1, 0)\n",
    "    }\n",
    "\n",
    "    def __init__(self, size=(4, 4), moves=None):\n",
    "\n",
    "        self.size = size\n",
    "\n",
    "        # Create all states\n",
    "        self.terminal_states = [(0, 0), (3, 3)]\n",
    "        self.states = [(row, col) for row in range(size[0])\n",
    "                       for col in range(size[1])\n",
    "                       if (row, col) not in self.terminal_states]\n",
    "\n",
    "        # Start in random state\n",
    "        self.start_state = random.choice(self.states)\n",
    "\n",
    "        # Store rewards for all possible possible rewards\n",
    "        self.rewards = {**{s: -1.0 for s in self.states}, \n",
    "                        **{s: 0.0 for s in self.terminal_states}}\n",
    "\n",
    "        # Store next state and reward for all possible\n",
    "        # state transitions\n",
    "        self.turn = self.roles[0]\n",
    "        \n",
    "        # Game state transitions are represented as \n",
    "        # (s, a): [p_1: (s2_1, r2_1), p_2: (s2_2, r2_2), ...]\n",
    "        # where (s, a) is the current state and action, s2_i is \n",
    "        # the possible next_state, p_i is the probability o\n",
    "        # that next state, and r2_i is the reward.  For a \n",
    "        # deterministic environment, the list should contain only\n",
    "        # one item: (1.0, (s2, r)).\n",
    "        self.transitions = {(s, a): [(1.0, (self.next_state(s, (self.turn, a)),\n",
    "                                            self.rewards[s]))]\n",
    "                            for s in self.states \n",
    "                            for a in self.actions}\n",
    "        self.n_players = 1\n",
    "        self.winner = None\n",
    "        self.game_over = False\n",
    "        self.reset()\n",
    "        if moves is not None:\n",
    "            for move in moves:\n",
    "                self.make_move(move)\n",
    "\n",
    "    def reset(self):\n",
    "        \n",
    "        self.moves = []\n",
    "        self.state = self.start_state\n",
    "        self.winner = None\n",
    "        self.game_over = False\n",
    "\n",
    "    def show_state(self):\n",
    "\n",
    "        x = np.array(['_']*np.prod(game.size)).reshape(game.size)\n",
    "        i_terminal_states = np.array(game.terminal_states).T\n",
    "        x[i_terminal_states[0], i_terminal_states[1]] = 'X'\n",
    "        x[game.state] = 'O'\n",
    "        for row in x:\n",
    "            print(\" \".join(row))\n",
    "\n",
    "    def available_moves(self, state=None):\n",
    "\n",
    "        return list(self.actions.keys())\n",
    "\n",
    "    def update_state(self, move):\n",
    "        \n",
    "        self.state = self.next_state(self.state, move)\n",
    "\n",
    "    def next_state(self, state, move):\n",
    "\n",
    "        role, action = move\n",
    "\n",
    "        next_state = tuple(np.array(state) +\n",
    "                           np.array(self.actions[action]))\n",
    "\n",
    "        # Check if out of bounds\n",
    "        if (next_state[0] < 0) or (next_state[0] >= self.size[0]) or \\\n",
    "            (next_state[1] < 0) or (next_state[1] >= self.size[1]):\n",
    "            next_state = state\n",
    "\n",
    "        if state in self.terminal_states:\n",
    "            next_state = terminal_states[0]\n",
    "\n",
    "        return next_state\n",
    "\n",
    "    def make_move(self, move, show=False):\n",
    "        \n",
    "        assert self.game_over is False, \"Game is over\"\n",
    "\n",
    "        self.update_state(move)\n",
    "        self.moves.append(move)\n",
    "\n",
    "        if show:\n",
    "            role, action = move\n",
    "            print(\"Player %s made move %s\" % (str(role), str(action)))\n",
    "\n",
    "        self.check_if_game_over()\n",
    "\n",
    "    def get_rewards(self):\n",
    "        \"\"\"Returns reward resulting from the last action.\"\"\"\n",
    "\n",
    "        return {1: -1.0}\n",
    "\n",
    "    def get_terminal_rewards(self):\n",
    "        \"\"\"Returns the reward after the terminal state.\"\"\"\n",
    "\n",
    "        assert self.game_over\n",
    "\n",
    "        return {1: 0.0}   \n",
    "\n",
    "    def check_if_game_over(self):\n",
    "\n",
    "        if self.state in self.terminal_states:\n",
    "            self.game_over, self.winner = True, self.turn\n",
    "\n",
    "        return self.game_over\n",
    "\n",
    "    def generate_state_key(self, state, role):\n",
    "\n",
    "        return self.state\n",
    "\n",
    "    def __repr__(self):\n",
    "\n",
    "        return \"GridWorld(%s)\" % self.size.__repr__()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridWorld((4, 4))"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game = GridWorld()\n",
    "game"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([1], [1])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.roles, game.possible_n_players"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['l', 'u', 'r', 'd']"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.available_moves()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[((0, 1), -1.0),\n",
       " ((0, 2), -1.0),\n",
       " ((0, 3), -1.0),\n",
       " ((1, 0), -1.0),\n",
       " ((1, 1), -1.0),\n",
       " ((1, 2), -1.0),\n",
       " ((1, 3), -1.0),\n",
       " ((2, 0), -1.0),\n",
       " ((2, 1), -1.0),\n",
       " ((2, 2), -1.0),\n",
       " ((2, 3), -1.0),\n",
       " ((3, 0), -1.0),\n",
       " ((3, 1), -1.0),\n",
       " ((3, 2), -1.0),\n",
       " ((0, 0), 0.0),\n",
       " ((3, 3), 0.0)]"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(game.rewards.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(((0, 1), 'l'), [(1.0, ((0, 0), -1.0))]),\n",
       " (((0, 1), 'u'), [(1.0, ((0, 1), -1.0))]),\n",
       " (((0, 1), 'r'), [(1.0, ((0, 2), -1.0))]),\n",
       " (((0, 1), 'd'), [(1.0, ((1, 1), -1.0))]),\n",
       " (((0, 2), 'l'), [(1.0, ((0, 1), -1.0))]),\n",
       " (((0, 2), 'u'), [(1.0, ((0, 2), -1.0))]),\n",
       " (((0, 2), 'r'), [(1.0, ((0, 3), -1.0))]),\n",
       " (((0, 2), 'd'), [(1.0, ((1, 2), -1.0))])]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(game.transitions.items())[0:8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X _ _ _\n",
      "_ O _ _\n",
      "_ _ _ _\n",
      "_ _ _ X\n"
     ]
    }
   ],
   "source": [
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X _ _ _\n",
      "_ _ O _\n",
      "_ _ _ _\n",
      "_ _ _ X\n"
     ]
    }
   ],
   "source": [
    "game.make_move((1, 'r'))\n",
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: -1.0}"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.get_rewards()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "while not game.game_over:\n",
    "    game.make_move((1, random.choice(game.available_moves())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: 0.0}"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.get_terminal_rewards()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create value functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length: 16\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{(0, 1): 0.0,\n",
       " (0, 2): 0.0,\n",
       " (0, 3): 0.0,\n",
       " (1, 0): 0.0,\n",
       " (1, 1): 0.0,\n",
       " (1, 2): 0.0,\n",
       " (1, 3): 0.0,\n",
       " (2, 0): 0.0,\n",
       " (2, 1): 0.0,\n",
       " (2, 2): 0.0,\n",
       " (2, 3): 0.0,\n",
       " (3, 0): 0.0,\n",
       " (3, 1): 0.0,\n",
       " (3, 2): 0.0,\n",
       " (0, 0): 0.0,\n",
       " (3, 3): 0.0}"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# State value-function\n",
    "values = {s: 0.0 for s in game.states + game.terminal_states}\n",
    "print(\"Length:\", len(values))\n",
    "values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "def show_values(values, decimals=1):\n",
    "\n",
    "    rows = max([k[0] for k, v in values.items()]) + 1\n",
    "    cols = max([k[1] for k, v in values.items()]) + 1\n",
    "    x = np.full((rows, cols), np.nan)\n",
    "    for k, v in values.items():\n",
    "        x[k] = round(v, decimals)\n",
    "    print(x)\n",
    "\n",
    "show_values(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length: 56\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(((0, 1), 'l'), 0.0),\n",
       " (((0, 1), 'u'), 0.0),\n",
       " (((0, 1), 'r'), 0.0),\n",
       " (((0, 1), 'd'), 0.0),\n",
       " (((0, 2), 'l'), 0.0),\n",
       " (((0, 2), 'u'), 0.0),\n",
       " (((0, 2), 'r'), 0.0),\n",
       " (((0, 2), 'd'), 0.0)]"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Q (state, action) value function\n",
    "q_values = {}\n",
    "for s in game.states:\n",
    "    for a in game.actions:\n",
    "        q_values[(s, a)] = 0.0\n",
    "print(\"Length:\", len(q_values))\n",
    "list(q_values.items())[0:8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "def greedy_policy_with_state_values(game, state, action, values):\n",
    "    \"\"\"Returns the probability of taking the action in\n",
    "    given state for a greedy policy with given state value\n",
    "    function values.\n",
    "    \"\"\"\n",
    "\n",
    "    actions = game.available_moves(state)\n",
    "\n",
    "    # Requires the game's state-transition matrix\n",
    "    action_values = {a: values[game.transitions[(state, a)][0][1][0]] \n",
    "                     for a in actions}\n",
    "    v_max = max(action_values.items(), key=lambda x: x[1])\n",
    "\n",
    "    return 1.0 if action == v_max[0] else 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOEAAACICAYAAAARSIzoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAKtUlEQVR4nO3dfYwdVRnH8e+P8lLA8toKBFgWBCXlpVpawKBA/xBaSkFETStKqJIKwYiSGNAEUCGxBCWiKNAUQinhRQNFlILFBAHFBloQgWq11hJJkbao0BZjLT7+MbNwbe/unNvd2bN37++T3Nx758yZ+2yTp3PmzMwzigjMLJ/tcgdg1umchGaZOQnNMnMSmmXmJDTLzEloltn2uQMYSKNHj47u7u7cYZhtZenSpesiYkyzttqSUNKtwOnAmog4skm7gOuB04A3gfMi4pmybXLZNgKYGxGzU36zu7ubJUuWDNBfYDZwJL3UW1udw9HbgMl9tE8BDitfs4AbASSNAH5Qto8FZkgaW2OcZlnVloQR8Tjw9z5WORO4PQqLgT0k7QccC6yIiJURsQm4u1zXbFjKOTGzP/DXhu8vl8t6W242LOWcmFGTZdHH8uYbkWZRDGfp6uoamMisJd2XPTjg21w1e+qAb3OoyrknfBk4sOH7AcDqPpY3FRFzImJCREwYM6bp5JPZkJYzCR8AzlXheOD1iHgFeBo4TNLBknYEppfrmg1LdZ6iuAs4GRgt6WXgSmAHgIi4CVhIcXpiBcUpipll22ZJXwB+TnGK4taIeLGuOM1yqy0JI2JGRXsAF/XStpAiSc2GPV+2ZpaZk9AsMyehWWZOQrPMnIRmmTkJzTJzEppl5iQ0y8xJaJaZk9AsMyehWWZOQrPMnIRmmTkJzTJzEppl5iQ0y8xJaJZZrUkoabKk5ZJWSLqsSftXJP22fL0g6S1Je5VtqyQ9X7a5rLYNW3XWmOmppP0RigpqT0t6ICKW9awTEdcC15brTwO+HBGNBYMnRcS6umI0Gwrq3BO2Wkl7BnBXjfGYDUlJSSjpXklTJbWStMmVtCXtQvHcinsbFgewSNLSssCv2bCUmlQ3Ap8C/iRptqTDE/q0Ukl7GvDrLYaiJ0TEeIoHw1wk6cSmPyLNkrRE0pK1a9cmhGU2tCQlYUT8IiLOAcYDq4BHJD0paaakHXrp1kol7elsMRSNiNXl+xpgAcXwtllsrsBtbS15eClpb+A84HzgWYrnB44HHumlS1IlbUm7AycBP2lYtqukUT2fgVOAF1JjNWsnSbOjku4DDgfmA9PKcvUA9/R2+qC3StqSLijbbypXPQtYFBEbG7rvAywoniPK9sCdEfFwa3+aWXtIPUUxt6yK/TZJO0XEvyNiQm+dmlXSbki+nu+3UTxQtHHZSmBcYmxmbS11OHp1k2W/GchAzDpVn3tCSftSnFbYWdIHeGfGczdgl5pjM+sIVcPRUykmYw4ArmtYvh74Wk0xmXWUPpMwIuYB8ySdHRH39rWumW2bquHopyPiDqBb0iVbtkfEdU26mVkLqoaju5bv76o7ELNOVTUcvbl8/8bghGPWeaqGo9/rqz0ivjiw4Zh1nqrh6NJBicKsg6XMjppZjaqGo9+NiC9J+ilNbkOKiDNqi8ysQ1QNR+eX79+uOxCzTlU1HF1avj9W3o50OMUecXlZssLM+in1VqapwE3AnymuHz1Y0ucj4qE6gzPrBKm3Mn2HovLZCgBJ7wEeBJyEZv2UeivTmp4ELK0E1tQQj1nHqZod/Vj58UVJC4EfURwTfoKifIWZ9VPVnnBa+RoJvEpRC+ZkYC2wZ9XGEypwnyzp9YYq3Fek9jUbLqpmR2du64ZTKnCXnoiI07exr1nbS50dHQl8DjiCYq8IQER8to9ub1fgLrfRU4E7JZH609esraROzMwH9qW40/4xijvt11f0Sa3A/UFJz0l6SNIRLfY1a3upSXhoRFwObCyvJ50KHFXRJ6UC9zPAQRExDvg+cH8LfYsVXYHb2lxqEv6nfP+npCOB3YHuij6VFbgj4o2I2FB+XgjsIGl0St+GbbgCt7W11CScI2lP4HKKKtrLgGsq+lRW4Ja0r8oKv5KOLeN5LaWv2XCRNDETEXPLj48BhyT2SanA/XHgQkmbgX8B0yMigKZ9W/i7zNpG6uzo3sDXgRMojs2eAK6KiNf66ldVgTsibgBuSO1rNhylDkfvprhM7WyKvdc64J66gjLrJKkXcO8VEVc1fL9a0kfrCMis06TuCR+VNF3SduXrkxR3UZhZP1VdwL2e4hhQwCXAHWXTdsAG4MpaozPrAFXXjo4arEDMOlXqMSGSzgB6nhv/y4j4WT0hmXWWpGNCSbOBiylO0i8DLi6XmVk/pe4JTwPeHxH/BZA0j+K59b7Pz6yfUmdHAfZo+Lz7QAdi1qlS94TfAp6V9CjFTOmJwFdri8qsg1QmYXmB9a+A44GJFEl4aUT8rebYzDpCZRJGREi6PyKOwXcymA241GPCxZIm1hqJWYdKPSacBFwgaRWwkWJIGhFxdF2BmXWK1CScUmsUZh2s6trRkcAFwKHA88AtEbF5MAIz6xRVx4TzgAkUCTiF4pkUZjaAqoajYyPiKABJtwBPtbJxSZOB6ylKVMyNiNlbtJ8DXFp+3QBcGBHPlW2rKMoqvgVsjogJrfy2WbuoSsKeKms9NWOSN5xYRfsvwEkR8Q9JU4A5wHEN7ZMiYl3yj5q1oaokHCfpjfKzgJ3L7z2zo7v10beyinZEPNmw/mKK0oZmHaXqfsIR/dh2syrax/WyLhRl9hufdxjAIkkB3BwRc/oRi9mQlXw/4TZopYr2JIok/FDD4hMiYrWkdwOPSPpDRDzepO8sYBZAV1dX/6M2G2St3EXRqqQq2pKOBuYCZzaWUIyI1eX7GmABxfB2K67Abe2uziRMqcDdBdwHfCYi/tiwfFdJo3o+A6cAL9QYq1k2tQ1HEytwXwHsDfywnHntORWxD7CgXLY9cGdEPFxXrGY51XlMmFKB+3zg/Cb9VgLj6ozNbKioczhqZgmchGaZOQnNMnMSmmXmJDTLzElolpmT0CwzJ6FZZk5Cs8ychGaZOQnNMnMSmmXmJDTLzElolpmT0CwzJ6FZZk5Cs8xqTUJJkyUtl7RC0lbPt1fhe2X77ySNT+1rNlzUloQNFbinAGOBGZLGbrHaFOCw8jULuLGFvmbDQp17wrcrcEfEJqCnAnejM4Hbo7AY2EPSfol9zYaFOgs9pVTgbrbO/ol9gf8v/gtskLS8HzE3MxoY6s/DaIcYoYU4dU3NkfStjn/Pg3pryF2Bu7d1kqt3l+XxayuRL2nJUH8iVDvECI6zN3UmYUoF7t7W2TGhr9mwkLUCd/n93HKW9Hjg9Yh4JbGv2bCQuwL3QuA0YAXwJjCzr751xVqhHZ4G1Q4xguNsShFND7XMbJD4ihmzzJyEZpk5Ca025YTbgdVrdjYfE7YpSVc0Wx4R3xzsWPoiaWlEHJM7jt5IuqSv9oi4ru4Yan00mtVqY8PnkcDpwO8zxdKXxZImRsTTuQPpxajy/X3ARN45FTYN2Orx7HXwnrCBpPU0vzJHQETEboMcUjJJOwEPRMSpuWNpJGkZ8F7gJYr/OHr+LY/OGtgWJC0Czo6I9eX3UcCPI2Jy3b/tPWGDiBhVvdaQtQtwSO4gmpiSO4BEXcCmhu+bgO7B+GEnYZuS9Dzv7LVHAGOAIXU8CBARL+WOIdF84ClJCyj+Xc8C5g3GD3s42qYkNV6Vvxl4NSI254pnOChvKv9w+fXxiHh2UH7XSWiWl88TmmXmJDTLzElolpmT0CwzJ6FZZv8Dz/TXBC8l37AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 216x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Test above function\n",
    "state = (1, 1)\n",
    "actions = ['l', 'u', 'r', 'd']\n",
    "test_values = {\n",
    " (1, 0): 1.0,\n",
    " (0, 1): 7.0,\n",
    " (1, 2): 9.0,\n",
    " (2, 1): 8.0\n",
    "}\n",
    "pd.Series([greedy_policy_with_state_values(game, state, a, test_values)\n",
    "          for a in actions], index=actions).plot.bar(figsize=(3, 2))\n",
    "plt.ylabel(\"Probability\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "def greedy_policy_with_q_values(game, state, action, q_values):\n",
    "    \"\"\"Returns the probability of taking the action in\n",
    "    given state for a greedy policy with given state-action\n",
    "    value function q_values.\"\"\"\n",
    "    \n",
    "    actions = game.available_moves(state)\n",
    "    action_values = {a: q_values[(state, a)] for a in actions}\n",
    "    v_max = max(action_values.items(), key=lambda x: x[1])\n",
    "\n",
    "    return 1.0 if action == v_max[0] else 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOEAAACICAYAAAARSIzoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAKtUlEQVR4nO3dfYwdVRnH8e+P8lLA8toKBFgWBCXlpVpawKBA/xBaSkFETStKqJIKwYiSGNAEUCGxBCWiKNAUQinhRQNFlILFBAHFBloQgWq11hJJkbao0BZjLT7+MbNwbe/unNvd2bN37++T3Nx758yZ+2yTp3PmzMwzigjMLJ/tcgdg1umchGaZOQnNMnMSmmXmJDTLzEloltn2uQMYSKNHj47u7u7cYZhtZenSpesiYkyzttqSUNKtwOnAmog4skm7gOuB04A3gfMi4pmybXLZNgKYGxGzU36zu7ubJUuWDNBfYDZwJL3UW1udw9HbgMl9tE8BDitfs4AbASSNAH5Qto8FZkgaW2OcZlnVloQR8Tjw9z5WORO4PQqLgT0k7QccC6yIiJURsQm4u1zXbFjKOTGzP/DXhu8vl8t6W242LOWcmFGTZdHH8uYbkWZRDGfp6uoamMisJd2XPTjg21w1e+qAb3OoyrknfBk4sOH7AcDqPpY3FRFzImJCREwYM6bp5JPZkJYzCR8AzlXheOD1iHgFeBo4TNLBknYEppfrmg1LdZ6iuAs4GRgt6WXgSmAHgIi4CVhIcXpiBcUpipll22ZJXwB+TnGK4taIeLGuOM1yqy0JI2JGRXsAF/XStpAiSc2GPV+2ZpaZk9AsMyehWWZOQrPMnIRmmTkJzTJzEppl5iQ0y8xJaJaZk9AsMyehWWZOQrPMnIRmmTkJzTJzEppl5iQ0y8xJaJZZrUkoabKk5ZJWSLqsSftXJP22fL0g6S1Je5VtqyQ9X7a5rLYNW3XWmOmppP0RigpqT0t6ICKW9awTEdcC15brTwO+HBGNBYMnRcS6umI0Gwrq3BO2Wkl7BnBXjfGYDUlJSSjpXklTJbWStMmVtCXtQvHcinsbFgewSNLSssCv2bCUmlQ3Ap8C/iRptqTDE/q0Ukl7GvDrLYaiJ0TEeIoHw1wk6cSmPyLNkrRE0pK1a9cmhGU2tCQlYUT8IiLOAcYDq4BHJD0paaakHXrp1kol7elsMRSNiNXl+xpgAcXwtllsrsBtbS15eClpb+A84HzgWYrnB44HHumlS1IlbUm7AycBP2lYtqukUT2fgVOAF1JjNWsnSbOjku4DDgfmA9PKcvUA9/R2+qC3StqSLijbbypXPQtYFBEbG7rvAywoniPK9sCdEfFwa3+aWXtIPUUxt6yK/TZJO0XEvyNiQm+dmlXSbki+nu+3UTxQtHHZSmBcYmxmbS11OHp1k2W/GchAzDpVn3tCSftSnFbYWdIHeGfGczdgl5pjM+sIVcPRUykmYw4ArmtYvh74Wk0xmXWUPpMwIuYB8ySdHRH39rWumW2bquHopyPiDqBb0iVbtkfEdU26mVkLqoaju5bv76o7ELNOVTUcvbl8/8bghGPWeaqGo9/rqz0ivjiw4Zh1nqrh6NJBicKsg6XMjppZjaqGo9+NiC9J+ilNbkOKiDNqi8ysQ1QNR+eX79+uOxCzTlU1HF1avj9W3o50OMUecXlZssLM+in1VqapwE3AnymuHz1Y0ucj4qE6gzPrBKm3Mn2HovLZCgBJ7wEeBJyEZv2UeivTmp4ELK0E1tQQj1nHqZod/Vj58UVJC4EfURwTfoKifIWZ9VPVnnBa+RoJvEpRC+ZkYC2wZ9XGEypwnyzp9YYq3Fek9jUbLqpmR2du64ZTKnCXnoiI07exr1nbS50dHQl8DjiCYq8IQER8to9ub1fgLrfRU4E7JZH609esraROzMwH9qW40/4xijvt11f0Sa3A/UFJz0l6SNIRLfY1a3upSXhoRFwObCyvJ50KHFXRJ6UC9zPAQRExDvg+cH8LfYsVXYHb2lxqEv6nfP+npCOB3YHuij6VFbgj4o2I2FB+XgjsIGl0St+GbbgCt7W11CScI2lP4HKKKtrLgGsq+lRW4Ja0r8oKv5KOLeN5LaWv2XCRNDETEXPLj48BhyT2SanA/XHgQkmbgX8B0yMigKZ9W/i7zNpG6uzo3sDXgRMojs2eAK6KiNf66ldVgTsibgBuSO1rNhylDkfvprhM7WyKvdc64J66gjLrJKkXcO8VEVc1fL9a0kfrCMis06TuCR+VNF3SduXrkxR3UZhZP1VdwL2e4hhQwCXAHWXTdsAG4MpaozPrAFXXjo4arEDMOlXqMSGSzgB6nhv/y4j4WT0hmXWWpGNCSbOBiylO0i8DLi6XmVk/pe4JTwPeHxH/BZA0j+K59b7Pz6yfUmdHAfZo+Lz7QAdi1qlS94TfAp6V9CjFTOmJwFdri8qsg1QmYXmB9a+A44GJFEl4aUT8rebYzDpCZRJGREi6PyKOwXcymA241GPCxZIm1hqJWYdKPSacBFwgaRWwkWJIGhFxdF2BmXWK1CScUmsUZh2s6trRkcAFwKHA88AtEbF5MAIz6xRVx4TzgAkUCTiF4pkUZjaAqoajYyPiKABJtwBPtbJxSZOB6ylKVMyNiNlbtJ8DXFp+3QBcGBHPlW2rKMoqvgVsjogJrfy2WbuoSsKeKms9NWOSN5xYRfsvwEkR8Q9JU4A5wHEN7ZMiYl3yj5q1oaokHCfpjfKzgJ3L7z2zo7v10beyinZEPNmw/mKK0oZmHaXqfsIR/dh2syrax/WyLhRl9hufdxjAIkkB3BwRc/oRi9mQlXw/4TZopYr2JIok/FDD4hMiYrWkdwOPSPpDRDzepO8sYBZAV1dX/6M2G2St3EXRqqQq2pKOBuYCZzaWUIyI1eX7GmABxfB2K67Abe2uziRMqcDdBdwHfCYi/tiwfFdJo3o+A6cAL9QYq1k2tQ1HEytwXwHsDfywnHntORWxD7CgXLY9cGdEPFxXrGY51XlMmFKB+3zg/Cb9VgLj6ozNbKioczhqZgmchGaZOQnNMnMSmmXmJDTLzElolpmT0CwzJ6FZZk5Cs8ychGaZOQnNMnMSmmXmJDTLzElolpmT0CwzJ6FZZk5Cs8xqTUJJkyUtl7RC0lbPt1fhe2X77ySNT+1rNlzUloQNFbinAGOBGZLGbrHaFOCw8jULuLGFvmbDQp17wrcrcEfEJqCnAnejM4Hbo7AY2EPSfol9zYaFOgs9pVTgbrbO/ol9gf8v/gtskLS8HzE3MxoY6s/DaIcYoYU4dU3NkfStjn/Pg3pryF2Bu7d1kqt3l+XxayuRL2nJUH8iVDvECI6zN3UmYUoF7t7W2TGhr9mwkLUCd/n93HKW9Hjg9Yh4JbGv2bCQuwL3QuA0YAXwJjCzr751xVqhHZ4G1Q4xguNsShFND7XMbJD4ihmzzJyEZpk5Ca025YTbgdVrdjYfE7YpSVc0Wx4R3xzsWPoiaWlEHJM7jt5IuqSv9oi4ru4Yan00mtVqY8PnkcDpwO8zxdKXxZImRsTTuQPpxajy/X3ARN45FTYN2Orx7HXwnrCBpPU0vzJHQETEboMcUjJJOwEPRMSpuWNpJGkZ8F7gJYr/OHr+LY/OGtgWJC0Czo6I9eX3UcCPI2Jy3b/tPWGDiBhVvdaQtQtwSO4gmpiSO4BEXcCmhu+bgO7B+GEnYZuS9Dzv7LVHAGOAIXU8CBARL+WOIdF84ClJCyj+Xc8C5g3GD3s42qYkNV6Vvxl4NSI254pnOChvKv9w+fXxiHh2UH7XSWiWl88TmmXmJDTLzElolpmT0CwzJ6FZZv8Dz/TXBC8l37AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 216x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Test above function\n",
    "state = (1, 1)\n",
    "actions = ['l', 'u', 'r', 'd']\n",
    "test_q_values = {\n",
    " ((1, 1), 'l'): 1.0,\n",
    " ((1, 1), 'u'): 7.0,\n",
    " ((1, 1), 'r'): 9.0,\n",
    " ((1, 1), 'd'): 8.0\n",
    "}\n",
    "pd.Series([greedy_policy_with_q_values(game, state, a, test_q_values)\n",
    "          for a in actions], index=actions).plot.bar(figsize=(3, 2))\n",
    "plt.ylabel(\"Probability\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_policy(game, state, action, values=None):\n",
    "\n",
    "    actions = game.available_moves(state=state)\n",
    "\n",
    "    return 1.0/len(game.actions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANoAAACICAYAAACBW/UnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAJLklEQVR4nO3dXawcdR3G8e/TIhSkBaRNMH3xgFSbKkWhBS4UwoWhBaEqaoqoESW1FwQIMRFNwCgkYqIEMUhtoEktF0UDmioliAlCDDS0lQRSTLVWCA1v4mvBhNrweDFTujldzs45nPlvz57nk2x23nb215M8/f9n9j8zsk1EtGtKvwuImAwStIgCErSIAhK0iAIStIgCErSIAg7rdwGjNXPmTA8NDfW7jIiutm3b9ortWcOXT7igDQ0NsXXr1n6XEdGVpGe7LU/XMaKABC2igAQtooAJd4w2GkPX3jfu+3zmpgvGfZ/jXedEqBEmV51p0SIKSNAiCkjQIgpI0CIKSNAiCkjQIgpI0CIKSNAiCkjQIgpI0CIKSNAiCkjQIgpI0CIKSNAiCkjQIgpI0CIKaDVokpZK2iFpp6Rru6y/VNKT9etRSae2WU9Ev7QWNElTgduAZcBC4BJJC4dt9lfgHNuLgBuANW3VE9FPbbZoZwA7be+yvRfYACzv3MD2o7b/Wc9uBua0WE9E37QZtNnAcx3zu+tlb+UrwP0t1hPRN23enEddlnV96qGkc6mC9pG3WL8SWAkwb9688aovopg2W7TdwNyO+TnA88M3krQIuANYbvvv3XZke43txbYXz5p10N2WIw55bQZtCzBf0omSDgdWABs7N5A0D7gX+ILtP7VYS0RftdZ1tL1P0hXAA8BUYK3t7ZJW1etXA9cDxwM/lgSwz/bitmqK6JdGQZN0D7AWuN/2G013bnsTsGnYstUd05cDlzfdX8RE1bTreDvwOeDPkm6StKDFmiIGTqOg2f6t7UuB04BngAfrkRyXSXpHmwVGDILGJ0MkHQ98iaqr9wTwQ6rgPdhKZREDpOkx2r3AAmA9cKHtF+pVd0vKUwEjemh61vGO+sTGmyQdYfv1nCWM6K1p1/HGLsseG89CIgbZiC2apBOoxiceKenDHBhWNQM4quXaIgZGr67jeVQnQOYAN3cs3wN8s6WaIgbOiEGzvQ5YJ+li2/cUqili4PTqOn7e9l3AkKRrhq+3fXOXj0XEML26ju+s349uu5CIQdar6/iT+v3bZcqJGEy9uo63jrTe9pXjW07EYOrVddxWpIqIAdfkrGNEvE29uo632L5a0q/ocr8P2xe1VlnEAOnVdVxfv3+/7UIiBlmvruO2+v3h+r4fC6hath31vRojooGml8lcAKwG/kI13vFESV+1nfswRjTQ9DKZHwDn2t4JIOm9wH3khqcRjTS9TObl/SGr7QJebqGeiIHU66zjp+rJ7ZI2AT+jOkb7DNV9GyOigV5dxws7pl8Czqmn/wYc10pFEQOo11nHy0oVEjHImp51nEb1EIoPANP2L7f95ZbqihgoTU+GrAdOoLri+mGqK673tFVUxKBpGrSTbV8HvFaPf7wAOKW9siIGS9Og/a9+/5ekDwLHAEOtVBQxgJr+YL1G0nHAdVSPXjq6no6IBhoFzfYd9eTDwEntlRMxmBp1HSUdL+lHkv4gaZukW+p78UdEA02P0TZQDbm6GPg08Apwd1tFRQyapsdo77J9Q8f8jZI+0UZBEYOoaYv2kKQVkqbUr89Sjd4fkaSlknZI2inp2i7rF0h6TNLrkr422uIjJopeg4r3UA0iFnANcFe9agrwKvCtET47FbgN+BiwG9giaaPtpzs2+wdwJZDWMQbaiC2a7em2Z9TvU2wfVr+m2J7RY99nADtt76qvxt4ALB+2/5dtb+HA73QRA6npMRqSLgLOrmd/Z/vXPT4yG3iuY343cOboynvzu1cCKwHmzZs3ll1E9FXT0/s3AVcBT9evq+plI36sy7KD7qTVhO01thfbXjxr1qyx7CKir5q2aOcDH7L9BoCkdVTPsT7oBEeH3cDcjvk5wPNjKTJiomv8sHjg2I7pYxpsvwWYL+nE+g5aK6iGb0VMOk1btO8CT0h6iKpLeDbwjZE+YHufpCuAB4CpwFrb2yWtqtevrp8oupXqCaJvSLoaWGj7P2P750QcmnoGTZKA3wNnAUuogvZ12y/2+mz9gPlNw5at7ph+kapLGTHQegbNtiX90vbppOsXMSZNj9E2S1rSaiURA6zpMdq5wCpJzwCvUXUfbXtRW4VFDJKmQVvWahURA67XWMdpwCrgZOAp4E7b+0oUFjFIeh2jrQMWU4VsGdU9+CNilHp1HRfaPgVA0p3A4+2XFDF4erVob46qT5cxYux6tWinSto/SkPAkfX8/rOOvS6ViQh633t/aqlCIgbZaAYVR8QYJWgRBSRoEQUkaBEFJGgRBSRoEQUkaBEFJGgRBSRoEQUkaBEFJGgRBSRoEQUkaBEFJGgRBSRoEQUkaBEFJGgRBSRoEQUkaBEFJGgRBSRoEQUkaBEFtBo0SUsl7ZC0U9JBz7tW5dZ6/ZOSTmuznoh+aS1okqYCt1Hds38hcImkhcM2WwbMr18rgdvbqiein9ps0c4AdtreZXsvsAFYPmyb5cBPXdkMHCvp3S3WFNEXTZ+PNhazgec65ncDZzbYZjbwQudGklZStXgAr0raMb6lMhN4pcmG+t44f/PoNKqzzzXC5K7zPd0Wthk0dVnmMWyD7TXAmvEoqhtJW20vbmv/4yV1jq+SdbbZddwNzO2YnwM8P4ZtIia8NoO2BZgv6URJhwMrgI3DttkIfLE++3gW8G/bLwzfUcRE11rX0fY+SVcADwBTgbW2t0taVa9fDWwCzgd2Av8FLmurnh5a65aOs9Q5vorVKfugQ6KIGGcZGRJRQIIWUUCCFm9bfTJrbu8tJ68cox3iJF3fbbnt75SuZSSSttk+vd91dCPpmpHW27657Rra/ME6xsdrHdPTgI8Df+xTLSPZLGmJ7S39LqSL6fX7+4ElHPiZ6ULgkRIFTLoWTdIeuow+oRqlYtszCpc0KpKOADbaPq/ftXSS9DTwPuBZqv8c9v89F/W1sA6SfgNcbHtPPT8d+LntpW1/96Rr0WxP773VIe0o4KR+F9HFsn4X0MA8YG/H/F5gqMQXT7qgTTSSnuJACzwVmAUcUsdnALaf7XcNDawHHpf0C6q/6SeBdSW+eNJ1HScaSZ2jwfcBL9ne1696Jrr64uKP1rOP2H6iyPcmaBHty+9oEQUkaBEFJGgRBSRoEQUkaBEF/B/MFIrdykmawgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 216x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Test above function\n",
    "state = (1, 1)\n",
    "actions = ['l', 'u', 'r', 'd']\n",
    "pd.Series([random_policy(game, state, a)\n",
    "          for a in actions], index=actions).plot.bar(figsize=(3, 2))\n",
    "plt.ylabel(\"Probability\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "def policy_action(game, policy, state, values):\n",
    "\n",
    "    actions = game.available_moves(state)\n",
    "    probs = [policy(game, state, a, values) for a in actions]\n",
    "\n",
    "    return np.random.choice(actions, p=probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['l', 'l', 'l', 'd', 'l', 'd', 'd', 'd']"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Test above function\n",
    "[policy_action(game, random_policy, (1, 1), values) for i in range(8)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_values(game, policy, values, lr):\n",
    "    \"\"\"Sweeps through all game states and uses the\n",
    "    Bellman equation (Equation 4.5 in Sutton & Barto\n",
    "    book) to update the state value function assuming\n",
    "    the given policy.\n",
    "\n",
    "    Requires game object to have a transitions\n",
    "    dictionary of all (next_states, rewards) given\n",
    "    all (states, actions).\n",
    "    \"\"\"\n",
    "\n",
    "    v = values.copy()\n",
    "\n",
    "    for s in game.states:\n",
    "\n",
    "        actions = game.available_moves(s)\n",
    "        sum_values = 0.0\n",
    "\n",
    "        for a in actions:\n",
    "            p_action = policy(game, s, a, values)\n",
    "            for p_state, (s2, reward) in game.transitions[(s, a)]:\n",
    "                if s2 in game.states:\n",
    "                    sum_values += p_action*p_state*(reward + lr*values[s2])\n",
    "                elif s2 in game.terminal_states:\n",
    "                    sum_values += p_action*p_state*reward\n",
    "        v[s] = sum_values\n",
    "\n",
    "    return v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_policy(policy, game, values, lr=1.0, theta=0.01, \n",
    "                    max_iter=1000, show=True, show_values=None):\n",
    "    \"\"\"Carries out repeated value-updates with the\n",
    "    given policy until maximum iterations reached\n",
    "    or the maximum absolute change in values is less\n",
    "    than theta.  Returns updated values.\n",
    "    \"\"\"\n",
    "\n",
    "    iteration = 0\n",
    "    if show:\n",
    "        print(\"\\nk =\", iteration)\n",
    "        if show_values:\n",
    "            show_values(values)\n",
    "\n",
    "    while iteration < max_iter:\n",
    "\n",
    "        updated_values = update_values(game, policy, values, lr=lr)\n",
    "\n",
    "        delta = np.abs(\n",
    "            np.array(list(updated_values.values())) -\n",
    "            np.array(list(values.values()))\n",
    "        ).max()\n",
    "        \n",
    "        values = updated_values\n",
    "        iteration += 1\n",
    "        \n",
    "        if show:\n",
    "            print(\"\\nk =\", iteration)\n",
    "            if show_values:\n",
    "                show_values(values)\n",
    "\n",
    "        if delta < theta:\n",
    "            break\n",
    "    \n",
    "    if iteration == max_iter:\n",
    "        print(\"\\nMaximum iterations reached.\")\n",
    "    else:\n",
    "        print(\"\\nConverged to delta < %f after %d iterations\" % \n",
    "              (theta, iteration))\n",
    "    \n",
    "    return values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Estimate optimal values using equi-probable random policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "values = {s: 0.0 for s in game.states + game.terminal_states}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "k = 0\n",
      "[[0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]]\n",
      "\n",
      "k = 1\n",
      "[[ 0. -1. -1. -1.]\n",
      " [-1. -1. -1. -1.]\n",
      " [-1. -1. -1. -1.]\n",
      " [-1. -1. -1.  0.]]\n",
      "\n",
      "k = 2\n",
      "[[ 0.  -1.8 -2.  -2. ]\n",
      " [-1.8 -2.  -2.  -2. ]\n",
      " [-2.  -2.  -2.  -1.8]\n",
      " [-2.  -2.  -1.8  0. ]]\n",
      "\n",
      "k = 3\n",
      "[[ 0.  -2.4 -2.9 -3. ]\n",
      " [-2.4 -2.9 -3.  -2.9]\n",
      " [-2.9 -3.  -2.9 -2.4]\n",
      " [-3.  -2.9 -2.4  0. ]]\n",
      "\n",
      "k = 4\n",
      "[[ 0.  -3.1 -3.8 -4. ]\n",
      " [-3.1 -3.7 -3.9 -3.8]\n",
      " [-3.8 -3.9 -3.7 -3.1]\n",
      " [-4.  -3.8 -3.1  0. ]]\n",
      "\n",
      "k = 5\n",
      "[[ 0.  -3.7 -4.7 -4.9]\n",
      " [-3.7 -4.5 -4.8 -4.7]\n",
      " [-4.7 -4.8 -4.5 -3.7]\n",
      " [-4.9 -4.7 -3.7  0. ]]\n",
      "\n",
      "k = 6\n",
      "[[ 0.  -4.2 -5.5 -5.8]\n",
      " [-4.2 -5.2 -5.6 -5.5]\n",
      " [-5.5 -5.6 -5.2 -4.2]\n",
      " [-5.8 -5.5 -4.2  0. ]]\n",
      "\n",
      "k = 7\n",
      "[[ 0.  -4.7 -6.3 -6.7]\n",
      " [-4.7 -5.9 -6.4 -6.3]\n",
      " [-6.3 -6.4 -5.9 -4.7]\n",
      " [-6.7 -6.3 -4.7  0. ]]\n",
      "\n",
      "k = 8\n",
      "[[ 0.  -5.2 -7.  -7.5]\n",
      " [-5.2 -6.5 -7.1 -7. ]\n",
      " [-7.  -7.1 -6.5 -5.2]\n",
      " [-7.5 -7.  -5.2  0. ]]\n",
      "\n",
      "k = 9\n",
      "[[ 0.  -5.7 -7.7 -8.2]\n",
      " [-5.7 -7.2 -7.8 -7.7]\n",
      " [-7.7 -7.8 -7.2 -5.7]\n",
      " [-8.2 -7.7 -5.7  0. ]]\n",
      "\n",
      "k = 10\n",
      "[[ 0.  -6.1 -8.4 -9. ]\n",
      " [-6.1 -7.7 -8.4 -8.4]\n",
      " [-8.4 -8.4 -7.7 -6.1]\n",
      " [-9.  -8.4 -6.1  0. ]]\n",
      "\n",
      "Maximum iterations reached.\n"
     ]
    }
   ],
   "source": [
    "values = evaluate_policy(random_policy, game, values, max_iter=10, \n",
    "                         show=True, show_values=show_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Greedy policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(0, 1): -6.137969970703125,\n",
       " (0, 2): -8.35235595703125,\n",
       " (0, 3): -8.967315673828125,\n",
       " (1, 0): -6.137969970703125,\n",
       " (1, 1): -7.737396240234375,\n",
       " (1, 2): -8.427825927734375,\n",
       " (1, 3): -8.35235595703125,\n",
       " (2, 0): -8.35235595703125,\n",
       " (2, 1): -8.427825927734375,\n",
       " (2, 2): -7.737396240234375,\n",
       " (2, 3): -6.137969970703125,\n",
       " (3, 0): -8.967315673828125,\n",
       " (3, 1): -8.35235595703125,\n",
       " (3, 2): -6.137969970703125,\n",
       " (0, 0): 0.0,\n",
       " (3, 3): 0.0}"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array([['_', 'l', 'l', 'l'],\n",
      "       ['u', 'l', 'l', 'd'],\n",
      "       ['u', 'u', 'r', 'd'],\n",
      "       ['u', 'r', 'r', '_']], dtype='<U1')\n"
     ]
    }
   ],
   "source": [
    "def show_actions(game, policy, values):\n",
    "\n",
    "    x = np.array(['_']*np.prod(game.size), dtype='<U1').reshape(game.size)\n",
    "    for s in game.states:\n",
    "        x[s] = policy_action(game, policy, s, values)\n",
    "    \n",
    "    print(x.__repr__())\n",
    "\n",
    "# Optimal actions\n",
    "show_actions(game, greedy_policy_with_state_values, values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 1),\n",
       " (0, 2),\n",
       " (0, 3),\n",
       " (1, 0),\n",
       " (1, 1),\n",
       " (1, 2),\n",
       " (1, 3),\n",
       " (2, 0),\n",
       " (2, 1),\n",
       " (2, 2),\n",
       " (2, 3),\n",
       " (3, 0),\n",
       " (3, 1),\n",
       " (3, 2)]"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(0, 1): 'l',\n",
       " (0, 2): 'l',\n",
       " (0, 3): 'ld',\n",
       " (1, 0): 'u',\n",
       " (1, 1): 'lu',\n",
       " (1, 2): 'lurd',\n",
       " (1, 3): 'd',\n",
       " (2, 0): 'u',\n",
       " (2, 1): 'lurd',\n",
       " (2, 2): 'rd',\n",
       " (2, 3): 'd',\n",
       " (3, 0): 'ur',\n",
       " (3, 1): 'r',\n",
       " (3, 2): 'r'}"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define true optimal actions so we can confirm\n",
    "# if a policy is optimal\n",
    "array_of_actions = np.array([\n",
    "    ['', 'l', 'l', 'ld'],\n",
    "    ['u', 'lu', 'lurd', 'd'],\n",
    "    ['u', 'lurd', 'rd', 'd'],\n",
    "    ['ur', 'r', 'r', '']\n",
    "], dtype='<U4')\n",
    "optimal_actions = {index: x for index, x in \n",
    "                   np.ndenumerate(array_of_actions)\n",
    "                   if index not in game.terminal_states}\n",
    "optimal_actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_policy(game, policy, values, optimal_actions):\n",
    "    \"\"\"Returns an accuracy score for policy according to\n",
    "    how many optimal actions it makes in each state based\n",
    "    on optimal_actions.\n",
    "    \"\"\"\n",
    "\n",
    "    a_array = np.zeros(game.size, dtype=bool)\n",
    "    for s, opt in optimal_actions.items():\n",
    "        a = policy_action(game, policy, s, values)\n",
    "        if a in opt:\n",
    "            a_array[s] = True\n",
    "        else:\n",
    "            a_array[s] = False\n",
    "    \n",
    "    return a_array.sum()/len(game.states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_policy(game, greedy_policy_with_state_values, values, optimal_actions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Calculate Q-values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "def q_values_from_values(game, values, lr=1.0):\n",
    "\n",
    "    q_values = {}\n",
    "    for s in game.states:\n",
    "\n",
    "        sum_values = 0\n",
    "        actions = game.available_moves(s)\n",
    "        for a in actions:\n",
    "            sum_values = 0.0\n",
    "            for p_state, (s2, reward) in game.transitions[(s, a)]:\n",
    "                if s2 in game.states:\n",
    "                    sum_values += p_state*(reward + lr*values[s2])\n",
    "                elif s2 in game.terminal_states:\n",
    "                    sum_values += p_state*reward\n",
    "\n",
    "            # Q-value is value of next state + reward (?)\n",
    "            v_next = sum_values\n",
    "            q_values[(s, a)] = v_next\n",
    "\n",
    "    return q_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1.0, ((1, 0), -1.0))]"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.transitions[((1, 0), 'l')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{((0, 1), 'l'): -1.0,\n",
       " ((0, 1), 'u'): -7.137969970703125,\n",
       " ((0, 1), 'r'): -9.35235595703125,\n",
       " ((0, 1), 'd'): -8.737396240234375,\n",
       " ((0, 2), 'l'): -7.137969970703125,\n",
       " ((0, 2), 'u'): -9.35235595703125,\n",
       " ((0, 2), 'r'): -9.967315673828125,\n",
       " ((0, 2), 'd'): -9.427825927734375,\n",
       " ((0, 3), 'l'): -9.35235595703125,\n",
       " ((0, 3), 'u'): -9.967315673828125,\n",
       " ((0, 3), 'r'): -9.967315673828125,\n",
       " ((0, 3), 'd'): -9.35235595703125,\n",
       " ((1, 0), 'l'): -7.137969970703125,\n",
       " ((1, 0), 'u'): -1.0,\n",
       " ((1, 0), 'r'): -8.737396240234375,\n",
       " ((1, 0), 'd'): -9.35235595703125,\n",
       " ((1, 1), 'l'): -7.137969970703125,\n",
       " ((1, 1), 'u'): -7.137969970703125,\n",
       " ((1, 1), 'r'): -9.427825927734375,\n",
       " ((1, 1), 'd'): -9.427825927734375,\n",
       " ((1, 2), 'l'): -8.737396240234375,\n",
       " ((1, 2), 'u'): -9.35235595703125,\n",
       " ((1, 2), 'r'): -9.35235595703125,\n",
       " ((1, 2), 'd'): -8.737396240234375,\n",
       " ((1, 3), 'l'): -9.427825927734375,\n",
       " ((1, 3), 'u'): -9.967315673828125,\n",
       " ((1, 3), 'r'): -9.35235595703125,\n",
       " ((1, 3), 'd'): -7.137969970703125,\n",
       " ((2, 0), 'l'): -9.35235595703125,\n",
       " ((2, 0), 'u'): -7.137969970703125,\n",
       " ((2, 0), 'r'): -9.427825927734375,\n",
       " ((2, 0), 'd'): -9.967315673828125,\n",
       " ((2, 1), 'l'): -9.35235595703125,\n",
       " ((2, 1), 'u'): -8.737396240234375,\n",
       " ((2, 1), 'r'): -8.737396240234375,\n",
       " ((2, 1), 'd'): -9.35235595703125,\n",
       " ((2, 2), 'l'): -9.427825927734375,\n",
       " ((2, 2), 'u'): -9.427825927734375,\n",
       " ((2, 2), 'r'): -7.137969970703125,\n",
       " ((2, 2), 'd'): -7.137969970703125,\n",
       " ((2, 3), 'l'): -8.737396240234375,\n",
       " ((2, 3), 'u'): -9.35235595703125,\n",
       " ((2, 3), 'r'): -7.137969970703125,\n",
       " ((2, 3), 'd'): -1.0,\n",
       " ((3, 0), 'l'): -9.967315673828125,\n",
       " ((3, 0), 'u'): -9.35235595703125,\n",
       " ((3, 0), 'r'): -9.35235595703125,\n",
       " ((3, 0), 'd'): -9.967315673828125,\n",
       " ((3, 1), 'l'): -9.967315673828125,\n",
       " ((3, 1), 'u'): -9.427825927734375,\n",
       " ((3, 1), 'r'): -7.137969970703125,\n",
       " ((3, 1), 'd'): -9.35235595703125,\n",
       " ((3, 2), 'l'): -9.35235595703125,\n",
       " ((3, 2), 'u'): -8.737396240234375,\n",
       " ((3, 2), 'r'): -1.0,\n",
       " ((3, 2), 'd'): -7.137969970703125}"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_values = q_values_from_values(game, values)\n",
    "q_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "greedy_policy_with_q_values(game, (1, 0), 'r', q_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "greedy_policy_with_q_values(game, (1, 0), 'u', q_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_policy(game, greedy_policy_with_q_values, q_values, optimal_actions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exercise 4.1\n",
    "\n",
    "In Example 4.1, if $\\pi$ is the equiprobable random policy,\n",
    "\n",
    "- What is $q_\\pi(11,down)$?\n",
    "- What is $q_\\pi(7,down)$?\n",
    "\n",
    "$q_\\pi(11,down)$:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1.0"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_values[((2, 3), 'd')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$q_\\pi(7,down)$:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-7.137969970703125"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_values[((1, 3), 'd')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example 4.2 Jack’s Car Rental\n",
    "\n",
    "Jack manages two locations for a nationwide car rental company. Each day, some number of customers arrive at each location to rent cars. If Jack has a car available, he rents it out and is credited \\\\$10 by the national company. If he is out of cars at that location, then the business is lost. Cars become available for renting the day after they are returned. To help ensure that cars are available where they are needed, Jack can move them between the two locations overnight, at a cost of \\\\$2 per car moved. We assume that the number of cars requested and returned at each location are Poisson random variables, meaning that the probability that the number is\n",
    "\n",
    "$n$ is $\\frac{\\lambda^n}{n!}\\mathrm{e}^{-\\lambda}$, where $\\lambda$ is the expected number.\n",
    "\n",
    "Suppose $\\lambda$ is 3 and 4 for rental requests at the first and second locations and 3 and 2 for returns. To simplify the problem slightly, we assume that there can be no more than 20 cars at each location (any additional cars are returned to the nationwide company, and thus disappear from the problem) and a maximum of five cars can be moved from one location to the other in one night. We take the discount rate to be $\\gamma = 0.9$ and formulate this as a continuing finite MDP, where the time steps are days, the state is the number of cars at each location at the end of the day, and the actions are the net numbers of cars moved between the two locations overnight."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.special import factorial\n",
    "\n",
    "n_max = 20\n",
    "n = np.arange(0, n_max + 1)\n",
    "lam = 4\n",
    "\n",
    "def prob_request(n, lam):\n",
    "    \n",
    "    return lam**n/factorial(n)*np.exp(-lam)\n",
    "\n",
    "# This is equivalent to poisson.pmf(k=n, mu=lam)\n",
    "\n",
    "from scipy.stats import poisson\n",
    "\n",
    "np.isclose(poisson.pmf(k=n, mu=lam), \n",
    "           prob_request(n, lam)).all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.special import factorial\n",
    "\n",
    "class JacksCarRental:\n",
    "\n",
    "    name = \"Jack's Car Rental\"\n",
    "    roles = [1]\n",
    "    possible_n_players = [1]\n",
    "    help_text = {\n",
    "        'Move format': \"(row, col)\",\n",
    "        'Move not available': \"That action is not available.\",\n",
    "        'Number of players': \"This game is for 1 player.\"\n",
    "    }\n",
    "    \n",
    "    def __init__(self, n_sizes=[20, 20], lam_requests=[3, 4], \n",
    "                 lam_returns=[3, 2], move_max=5, credit=10,\n",
    "                 move_cost=2, start_state=[10, 10], \n",
    "                 moves=None):\n",
    "\n",
    "        self.n_locations = len(n_sizes)\n",
    "        self.n_sizes = n_sizes\n",
    "        self.lam_requests = lam_requests\n",
    "        self.lam_returns = lam_returns\n",
    "        self.credit = credit\n",
    "        self.move_cost = move_cost\n",
    "        self.start_state = start_state\n",
    "        self.n = np.arange(0, n_max + 1)\n",
    "\n",
    "        # Probabilities of number of requests/returns at\n",
    "        # each location\n",
    "        # TODO: Delete, don't need this\n",
    "        #self.p_requests = [poisson.pmf(k=n, mu=lam)\n",
    "        #                   for lam in lam_requests]\n",
    "        #self.p_returns = [poisson.pmf(k=n, mu=lam)\n",
    "        #                  for lam in lam_returns]\n",
    "\n",
    "        # Create all states\n",
    "        self.states = [(c1, c2) \n",
    "                       for c1 in range(n_sizes[0] + 1) \n",
    "                       for c2 in range(n_sizes[1] + 1)]\n",
    "\n",
    "        self.turn = self.roles[0]\n",
    "        \n",
    "        self.n_players = 1\n",
    "        self.winner = None\n",
    "        self.game_over = False\n",
    "        self.reset()\n",
    "        if moves is not None:\n",
    "            for move in moves:\n",
    "                self.make_move(move)\n",
    "\n",
    "    def reset(self):\n",
    "        \n",
    "        self.moves = []\n",
    "        self.state = self.start_state\n",
    "        self.winner = None\n",
    "        self.game_over = False\n",
    "\n",
    "    def show_state(self):\n",
    "\n",
    "        print(self.state)\n",
    "\n",
    "    def available_moves(self, state=None):\n",
    "        \"\"\"Determine how many cars are available for moving.\n",
    "        Return as a list of number of cars movable.\n",
    "        \"\"\"\n",
    "\n",
    "        if state is None:\n",
    "            state = self.state\n",
    "        s1, s2 = state\n",
    "\n",
    "        actions = list(range(-min(s2, n_max - s1), \n",
    "                             min(s1, n_max - s2) + 1))\n",
    "\n",
    "        return actions\n",
    "    \n",
    "    def update_state(self, move):\n",
    "        \n",
    "        self.state = self.next_state(self.state, move)\n",
    "\n",
    "    def next_state(self, state, move):\n",
    "\n",
    "        role, cars_moved = move\n",
    "        s1, s2 = state\n",
    "\n",
    "        cars_requested = [np.random.poisson(lam=lam) \n",
    "                          for lam in self.lam_requests]\n",
    "        cars_returned = [np.random.poisson(lam=lam) \n",
    "                         for lam in self.lam_returns]\n",
    "\n",
    "        cars_rented = [None, None]\n",
    "        for i in range(2):\n",
    "\n",
    "            # Rent cars if available\n",
    "            cars_rented[i] = min(state[i], cars_requested[i])\n",
    "\n",
    "            # Return cars if space available\n",
    "            cars_returned[i] = min(n_max - state[i], cars_returned[i])\n",
    "\n",
    "        s1 = s1 + cars_rented[0] - cars_returned[0] - cars_moved\n",
    "        s2 = s2 + cars_rented[1] - cars_returned[1] + cars_moved\n",
    "\n",
    "        self.cost = 2*cars_moved - 10*sum(cars_rented)\n",
    "\n",
    "        return (s1, s2)\n",
    "    \n",
    "    def make_move(self, move, show=False):\n",
    "        \n",
    "        assert self.game_over is False, \"Game is over\"\n",
    "\n",
    "        self.update_state(move)\n",
    "        self.moves.append(move)\n",
    "\n",
    "        if show:\n",
    "            role, action = move\n",
    "            print(\"Player %s made move %s\" % (str(role), str(action)))\n",
    "\n",
    "        self.check_if_game_over()\n",
    "\n",
    "    def get_rewards(self):\n",
    "        \"\"\"Returns reward resulting from the last action.\"\"\"\n",
    "\n",
    "        return {1: self.cost}\n",
    "\n",
    "    def get_terminal_rewards(self):\n",
    "        \"\"\"Returns the reward after the terminal state.\"\"\"\n",
    "\n",
    "        assert self.game_over\n",
    "\n",
    "        return {1: 0.0}   \n",
    "\n",
    "    def check_if_game_over(self):\n",
    "\n",
    "        if self.state in self.terminal_states:\n",
    "            self.game_over, self.winner = True, self.turn\n",
    "\n",
    "        return self.game_over\n",
    "\n",
    "    def generate_state_key(self, state, role):\n",
    "\n",
    "        return self.state\n",
    "\n",
    "    def __repr__(self):\n",
    "\n",
    "        return \"GridWorld(%s)\" % self.size.__repr__()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZhU1Z3/8fdH7NiyiBFcoqigwSVBbAFRomKTUZGEoDFGYlyi0aBJjCTzw1HMaIijDnHJZExQNMa4BzVq4kIUh9CC44K2okFRQYWkcScj6UbZv78/7u1O0XR1VXdXdVfD5/U89VTd5Zx77qnq+vY999Q5igjMzMxKzRYdXQAzM7OmOECZmVlJcoAyM7OS5ABlZmYlyQHKzMxKkgOUmZmVJAcosxIn6TBJr3V0OczamwOUWQ6SFks6oh2PF5I+W78cEXMiYu8iHesGSa9JWi/ptGIcw6y1HKDMNm8vAt8Dnu/ogpg15gBl1gaSviNpkaS/S3pA0s4Z2z4v6bF023uSLkzXD5X0lKSPJL0j6VeSPpVum50mf1FSnaSxkiol1WTku6+kqjT9y5LGZGy7WdIUSQ9LqpX0jKQ9s5U/IqZExExgZaHrxqytHKDMWknSF4H/BE4APgMsAaal23oA/wM8AuwMfBaYmSZdB/wI6A0MA/6F5CqGiBie7rN/RHSPiLsaHbMMeBCYAewA/AC4Q1JmE+CJwE+BTwOLgMsKdtJm7cgByqz1TgJuiojnI2IVMBEYJqkvMBp4NyKujoiVEVEbEc8ARER1RDwdEWsjYjFwPXB4nsc8GOgOTI6I1RHxZ+AhkqBU776ImBsRa4E7gIq2n6pZ+9uyowtg1ontTMa9m4iok7QM2AXYFXijqUSS9gJ+DgwBupL8HVa34Jh/i4j1GeuWpMes927G649JAppZp+MrKLPWexvYvX5BUjegF7AU+BuQ7d7PdcCrQP+I2Aa4EFALjrmrpMy/3d3SY5ptUhygzPJTJqk847ElcCdwuqQKSVsBlwPPpM12DwE7SfqhpK0k9ZB0UJpXD+AfQJ2kfYDvNjrWe8AeWcrxDLAC+DdJZZIqga+Q3vtqKUmfklROEiDrz9HfC1YS/EE0y8904JOMx6S099tFwL3AOyRXTN8AiIha4EiS4PEusBAYkeY1AfgmUAv8GtigIwQwCbgl7aV3QuaGiFgNjAFGAR8C1wKnRsSrrTyvGen5fAG4IX09vNkUZu1EnrDQzMxKka+gzMysJDlAmZlZSXKAMjOzkuQAZWZmJanT/VC3d+/e0bdv36Llv2LFCrp161a0/DcHrsPCcD22neuwMIpdj9XV1R9GxPaN13e6ANW3b1+ee+65ouVfVVVFZWVl0fLfHLgOC8P12Hauw8Iodj1KWtLUejfxmZlZSXKAMjOzktTpmvg2FVWqanrDrBFNrwcqK//5o+qs6ZvJIzO9mVmpc4AyM8thzZo11NTUsHLl5jmvY8+ePVmwYEGb8ykvL6dPnz6UlZXltb8DlJlZDjU1NfTo0YO+ffsi5Tvw/KajtraWHj16tCmPiGDZsmXU1NTQr1+/vNL4HpSZWQ4rV66kV69em2VwKhRJ9OrVq0VXoQ5QZmZ5cHBqu5bWYVEDlKSjJb0maZGkC5rYfpKkl9LHk5L2L2Z5zMys8yjaPShJXYApJHPi1ADPSnogIl7J2O0t4PCI+D9Jo0jmozlo49zMzEpHs71oW6EyKnPu06VLF/bbbz/Wrl1Lv379uO2229h2220LWo6Wuvzyy7nwwguLln8xr6CGAosi4s10krVpwDGZO0TEkxHxf+ni00CfIpbHzKzT2nrrrZk3bx7z589nu+22Y8qUKR1dJC6//PKi5l/MALUL8LeM5Zp0XTZnAH8qYnnMzDYJw4YNY+nSpQ3LV155JQceeCADBw7kJz/5ScP6yy67jL333psjjjiCE088kauuugqAysrKhiHjPvzwQ+rHN123bh3nnXdeQ17XX389AO+++y7Dhw+noqKCAQMGMGfOHC644AI++eQTKioqOOmkk1ixYgVf/vKX2X///RkwYAB33dV4ouiWK2Y386buhjX5S1FJI0gC1KFZto8DxgHsuOOOVFVVFaiIG6urqytq/g3Huaou24asaTLLlTV9M3m0x3lB+9Xhps712HaFqsOePXtSW1vb9gJlkW/etbW1rFu3jkcffZRTTjmF2tpaZs6cySuvvMLMmTOJCMaOHcsjjzxC165dufPOO5k9ezZr167lsMMOY8CAAQ15rFixgtraWurq6ogIamtr+e1vf0t5eTl//vOfWbVqFUcddRRf+MIX+OMf/0hlZSXnnXce69at4+OPP+bHP/4xv/rVr5gzZw4A999/P9tvvz3Tpk0DYPny5U2e18qVK/N+T4oZoGqAXTOW+wBvN95J0kDgRmBURCxrKqOIuIHk/hRDhgyJYg5a2F6DS1aNqGp6w6wJWdNsMJJEtvTN5NFeI0l4gM7CcD22XaHqcMGCBW3+HVBz8sn7k08+4bDDDmPx4sUMHjyYY445hi5duvDEE08wa9Yshg8fDiRBeenSpdTW1vK1r32NHXfcEYBjjz2Wrbbaih49etClSxe6detGjx49WLVqFZLo0aMHs2fP5qWXXuLBBx8EkiDzzjvvMGTIEM455xy22GILjj32WCoqKjYq+9ChQ7nooou49NJLGT16NIcddliT51FeXs4BBxyQV70Us4nvWaC/pH6SPgV8A3ggcwdJuwH3AadExOtFLIuZWadWfw9qyZIlrF69uuEeVEQwceJE5s2bx7x581i0aBFnnHEGkL1b95Zbbsn69esBNvhdUkTwy1/+siGvt956i6OOOopDDjmE2bNns8suu3DKKadw6623bpTnXnvtRXV1Nfvttx8TJ07kkksuafM5Fy1ARcRa4BzgUWABcHdEvCzpbElnp7tdDPQCrpU0T1Lx5tEwM9sE9OzZk2uuuYarrrqKNWvWMHLkSG666Sbq6pJm/6VLl/L+++8zfPhw7r//fj755BNqa2sbroogmbaouroagN///vcN60eOHMl1113HmjVrAHj99ddZsWIFf/3rX9lhhx34zne+wxlnnMHzzz8PQFlZWcO+b7/9Nl27duXkk09mwoQJDfu0RV5NfJLuBW4C/hQR6/PNPCKmA9MbrZua8fpM4Mx88zMzKwX5dAsvpgMOOID999+fadOmccopp7BgwQKGDRsGQPfu3bn99tsZNGgQY8eOpaKigt13332DJrcJEyZwwgkncNttt/HFL36xYf2ZZ57J4sWLGTRoEBHB9ttvzx/+8AfmzJnD2LFjKSsro3v37g1XUOPGjWPgwIEMGjSIU089lfPOO48tttiCsrIyrrvuujafpyJy35eQdARwOnAwcA9wc0S82uajt8KQIUNiU5iwcFMezdz3TgrD9dh2hbwHte+++7a9QB1o0qRJdO/enQkTst/nzqYQY/HVa6ouJVVHxJDG++bVxBcR/xMRJwGDgMXAY+nID6dLym9YWjMzsxbIuxefpF7AycApwAvAHSTdwr8FVBajcGZmVhiTJk3q6CK0WL73oO4D9gFuA74SEe+km+5yxwYzMyuGfK+gbkw7PDSQtFVErGqq3dDMzKyt8g1Ql9KoNx7wFMk9KeuEWtPJAjxtvJm1n2YDlKSdSMbP21rSAfxz+KJtgK5FLpuZmW3Gcl1BjQROIxmm6OcZ62uB4o2xbmZWwqqqCjt5YT4tEy2dbuOjjz7izjvv5Hvf+14hi9qumu1mHhG3RMQI4LSIGJHxGBMR97VTGc3MNnstnW7jo48+4tprr23xcdatW9faIhZcswFK0snpy76S/rXxox3KZ2ZmjeQz3cYFF1zAG2+8QUVFBeeddx5VVVWMHj26Ic0555zDzTffDCRDH11yySUceuih3HPPPVRWVnL++eczdOhQ9tprL5588kkAXn75ZYYOHUpFRQUDBw5k4cKFRT3PXE183dLn7kUthZmZ5WXdunXMnDmzYUDYGTNmsHDhQubOnUtEMGbMGGbPns3kyZOZP38+8+bNA3JPt1NeXs4TTzwBwNSpU1m7di1z585l+vTpTJ48mZEjRzJ16lTGjx/PSSedxOrVq4t+tdVsgIqI69Pnnxa1FGZm1qz6yQHrp9s48sgjgSRAzZgxo2EKi7q6OhYuXMhuu+3WovzHjh27wfJxxx0HwODBg1myZAmQXLlddtll1NTUcNxxx9G/f/+2nlazcvXiu6a57RFxbmGL0zm4i7aZtbf6e1DLly9n9OjRTJkyhXPPPbdhuo2zzjprg/0XL168wXLmFBuw4TQbAN26ddtgeauttgKSzhn1V0rf/OY3Oeigg3j44YcZOXIkN9544waDzRZarrH4qnM8zMysHeU73UaPHj02mNF2991355VXXmHVqlUsX76cmTNntvjYb775JnvssQfnnnsuY8aM4aWXXirYeTUlVxPfLUU9uplZJ9TRrSH5TLex5557csghhzBgwABGjRrFlVdeyQknnMDAgQPp379/3rPaZrrrrru4/fbbKSsrY6edduLiiy8u9KltIFcT3y8i4oeSHgQ2ekciYkzRSmZmZg3qr5DqZU5AOH78eMaPH79RmjvvvHOD5SuuuIIrrrhio/0aNwdmdqjo3bs38+fPB2DixIlMnDixpUVvtVy9+G5Ln68qdkHMzMwy5Wriq06fH5f0KZIRzQN4LSJWt0P5zMxsM5XvdBtfBqYCb5CMx9dP0lkR8adiFs7MrFREBFJhhzja3OQzg3umfEczvxoYERGLACTtCTwMOECZ2SavvLycZcuW0atXLwepVooIli1bRnl5ed5p8g1Q79cHp9SbwPstKZyZWWfVp08fampq+OCDDzq6KB1i5cqVLQos2ZSXl9OnT5+898/Vi++49OXLkqYDd5Pcg/o68GxrC2lm1pmUlZXRr1+/ji5Gh6mqqmpVt/S2ynUF9ZWM1+8Bh6evPwA+XZQSmZmZkbsX3+ntVRAzM7NM+fbiKwfOAD4PNDRERsS3i1QuMzPbzOUai6/ebcBOJDPsPk4yw25tsynMzMzaIN8A9dmIuAhYkY7P92Vgv1yJJB0t6TVJiyRd0MT2fSQ9JWmVpAktK7qZmW3K8g1Qa9LnjyQNAHoCfZtLIKkLMAUYBXwOOFHS5xrt9nfgXDyUkpmZNZJvgLpB0qeBi4AHgFeAn+VIMxRYFBFvpsMiTQOOydwhIt6PiGf5ZwA0MzMDQC0deiLvjKXjgaMj4sx0+RTgoIg4p4l9JwF1EdHklZSkccA4gB133HHwtGnTilJmSEYM7t69+Rnu66rrsm/c+7Wsm7p3H5w7j7ambyaPtqbPN491n11Bly41OdNb8/L5LFrzXIeFUex6HDFiRHVEDGm8Pt9efL2AScAhJD/UnQP8R0Qsay5ZE+taFQ0j4gbgBoAhQ4ZEZWVla7LJS1VVFbnyrxpRlX3jrOy30jLnkMmaR1vTN5NHW9Pnm0fdg6/RvXvuMljz8vksWvNch4XRUfWYbxPfNJKhjb4GHA98CNyVI00NsGvGch/g7ZYW0MzMNk/5BqjtIuI/IuKt9HEpsG2ONM8C/SX1S6fq+AbJ/SszM7Oc8h0sdpakb5CMxQfJVdTDzSWIiLWSzgEeBboAN0XEy5LOTrdPlbQT8BywDbBe0g+Bz0XEP1pxLmZmtgnJNVhsLcl9IwH/CtyebtoCqAN+0lz6iJgOTG+0bmrG63dJmv7MzMw2kGssvh7tVRAzM7NM+TbxIWkMMDxdrIqIh4pTJDMzszw7SUiaDIwn+YHuK8D4dJ2ZmVlR5HsF9SWgIiLWA0i6BXgB2Gh8PTMzs0LIt5s5bNitvGehC2JmZpYp3yuo/wRekDSLpEffcGBi0UplZmabvZwBSpKAJ4CDgQNJAtT5aRdxMzOzosgZoCIiJP0hIgbjkSDMzKyd5HsP6mlJBxa1JGZmZhnyvQc1Ajhb0mJgBUkzX0TEwGIVzMzMNm/5BqhRRS2FmZlZI7nG4isHzgY+C/wF+E1ErG2PghVTlaqybqt7sJqqqhEbrfc8RoXX3PvArI3fA/D7YLY5yXUP6hZgCElwGgVcXfQSmZmZkbuJ73MRsR+ApN8Ac4tfJDMzs9xXUGvqX2wKTXtmZtZ55LqC2l9S/eSBArZOl+t78W1T1NKZmdlmK9d8UF3aqyBmZmaZWjJYrJmZWbtxgDIzs5LkAGVmZiXJAcrMzEpSvkMdmZUcj0RhtmnzFZSZmZUkBygzMytJDlBmZlaSinoPStLRwH8DXYAbI2Jyo+1Kt38J+Bg4LSKeL2aZzDJlvY+V5R4W+D6WWXsp2hWUpC7AFJJR0D8HnCjpc412GwX0Tx/jgOuKVR4zM+tcinkFNRRYFBFvAkiaBhwDvJKxzzHArRERJNPKbyvpMxHxThHLZVYw7kloVjzFDFC7AH/LWK4BDspjn12ADQKUpHEkV1gAdZJeK2xRM3yF3sCHG29Qfumztwzll0db0zebRzudQ9Y6LEQZOjp9O5aB5urR8uQ6LIxi1+PuTa0sZoBq6q+w8b+O+exDRNwA3FCIQuUi6bmIGNIex9pUuQ4Lw/XYdq7DwuioeixmL74aYNeM5T7A263Yx8zMNkPFDFDPAv0l9ZP0KeAbwAON9nkAOFWJg4Hlvv9kZmZQxCa+iFgr6RzgUZJu5jdFxMuSzk63TwWmk3QxX0TSzfz0YpWnBdqlKXET5zosDNdj27kOC6ND6lFJBzozM7PS4pEkzMysJDlAmZlZSXKASkk6WtJrkhZJuqCjy9NZSVos6S+S5kl6rqPL01lIuknS+5LmZ6zbTtJjkhamz5/uyDKWuix1OEnS0vTzOE/SlzqyjKVO0q6SZklaIOllSePT9R3yWXSAIu9hmSx/IyKiwr8/aZGbgaMbrbsAmBkR/YGZ6bJldzMb1yHAf6Wfx4qImN7OZeps1gL/LyL2BQ4Gvp9+F3bIZ9EBKtEwLFNErAbqh2UyaxcRMRv4e6PVxwC3pK9vAY5t10J1Mlnq0FogIt6pH7A7ImqBBSSj+3TIZ9EBKpFtyCVruQBmSKpOh6iy1tux/neB6fMOHVyezuocSS+lTYBuJs2TpL7AAcAzdNBn0QEqkdeQS5aXQyJiEElz6fclDe/oAtlm7TpgT6CCZIzPqzu2OJ2DpO7AvcAPI+IfHVUOB6iEh1wqkIh4O31+H7ifpPnUWuc9SZ8BSJ/f7+DydDoR8V5ErIuI9cCv8ecxJ0llJMHpjoi4L13dIZ9FB6hEPsMyWQ6SuknqUf8aOAqY33wqa8YDwLfS198C/tiBZemU6r9UU1/Fn8dmpZPI/gZYEBE/z9jUIZ9FjySRSruf/oJ/Dst0WQcXqdORtAfJVRMkw2jd6XrMj6TfAZUk0xq8B/wE+ANwN7Ab8Ffg6xHhTgBZZKnDSpLmvQAWA2d5vM/sJB0KzAH+AqxPV19Ich+q3T+LDlBmZlaS3MRnZmYlyQHKzMxKkgOUmZmVJAcoMzMrSQ5QZmZWkhygrNUkhaSrM5YnSJpUoLxvlnR8IfLKcZyvpyM3zyr2sUqVpNMk7dzCNH0zRw1P11WnvyMs2nELQVKlpIfa+7jWcg5Q1hargOMk9e7ogmRKR6fP1xnA9yJiRAGPv2UL95ekjvxbPA1oU6BIx21bmg62nLm+ufeixcdtad1a5+YAZW2xFrgB+FHjDY2vgCTVpc+Vkh6XdLek1yVNlnSSpLnpPFJ7ZmRzhKQ56X6j0/RdJF0p6dl0ANCzMvKdJelOkh8ZNi7PiWn+8yX9LF13MXAoMFXSlU2k+bc0zYuSJqfrvpMe+0VJ90rqmnG+P0+vxH4m6fCMOYheqB9hIyPvvumV27XA88Cuko6S9JSk5yXdk46HVj9X2auSnpB0Tf1//0rmOpqQkef8NFAg6eS0TudJuj6tty5pOeen5/Wj9D0aAtyR7ru1pMHpe1Qt6dGMIW4Gp+f9FPD9RtU1Cnik/r2WdImkZ4BhTeWX5biL6//ZkTREUlXGed4gaQZwa3rldZ+kR5TMT3RFRh3krEPguMbvtZWoiPDDj1Y9gDpgG5Jf6PcEJgCT0m03A8dn7ps+VwIfAZ8BtgKWAj9Nt40HfpGR/hGSf6L6k4yXWA6MA/493Wcr4DmgX5rvCqBfE+XcmeTX79uTjHDxZ+DYdFsVMKSJNKOAJ4Gu6fJ26XOvjH0uBX6QUd6HgC7p8oMkA+cCdAe2bJR/X5Jf6h+cLvcGZgPd0uXzgYvTc/5bWgci+TX/Q+k+k4AJGXnOT/PdNz1+Wbr+WuBUYDDwWMb+2zauA6AsPe/t0+WxJCOrALwEHJ6+vhKYn5HXH4E90tcBnJBHfhvUPcnnqHf6eghQlXGe1cDW6fJpwJskn7lyYAnJWJotrkM/Svvhy2Vrk4j4h6RbgXOBT/JM9mykw81IegOYka7/C5DZ1HZ3JIN8LpT0JrAPyfh+AzOuznqSfPGsBuZGxFtNHO9Aki+7D9Jj3gEMJxlKKJsjgN9GxMfpedYP6zJA0qXAtiSB59GMNPdExLr09f8CP0+PdV9E1DRxjCUR8XT6+mCSyTL/VxLAp4Cn0nN+KyIWpmW/nSRIN+dfSILRs2leW5MM7vkgsIekXwIP8896z7Q3MAB4LE3bBXhHUk+SgPZ4ut9tJEEcJfed+kTEm+m2dSSDjWbNL0f5m/JARGR+vmZGxPL0+K8Au5O8J4WqQysBDlBWCL8gaab6bca6taRNyEq+LTJvnq/KeL0+Y3k9G34mG4/DFST/Af8gIjIDA5IqSa6gmtLUdCq5qInjQ3KldGxEvCjpNJIrt3oNx4+IyZIeBr4EPC3piIh4tVFemeUVydXNiRsUQqofR64pDXWcKs/I65aImLjRSUn7AyNJmuhOAL7deBfg5YgY1ijdts2U4zDgiYzllRmBusn8ssg8n/JG2xq/t5mfoXUkn5vW1KGVMN+DsjZLry7uJulwUG8xyX/xkMzGWdaKrL8uaQsl96X2AF4juWL5rpIpAZC0l5KR05vzDHC4pN5KbtqfCDyeI80M4NsZ95i2S9f3ILmiKANOypZY0p4R8ZeI+BlJM+Q+OY73NHCIpM+m6btK2gt4Feinf96by/zyXQwMSvcfRNLUCcmU3MdL2qG+7JJ2T+/vbBER9wIX1acFatPzgqSOt5c0LE1bJunzEfERsFzJYKI0OvejgT9lOa8m82viuPXnU/+Z+VqW/JrTmjq0EuYAZYVyNck9gHq/JgkKc4GDyH5105zXSALJn4CzI2IlcCPwCvC8km7O15OjJSBtTpwIzAJeBJ6PiGanC4iIR0imGHhO0jyS+2uQfLE/AzxG8sWXzQ/TzggvkjR9ZvsCrz/eByT3Vn4n6SWSL9t90nMeBzyc3uBfkpHsXmC7tHzfBV5P83oF+HeSmY1fSsv6GZJZoqvS/W9O64T09dR0fRfgeJKOHi8C84AvpPudDkxJO0lkNrdVkiXgR9KrL1t+DceVtDXwU+C/Jc0huSpqkVbWoZUwj2Zu1omkTZkTImJ0R5cFQFIf4NcRMaqjy2KbHt+DMrNWSzt/ODhZUfgKyszMSpLvQZmZWUlygDIzs5LkAGVmZiXJAcrMzEqSA5SZmZUkBygzMytJDlBmZlaSHKDMzKwkOUCZmVlJcoAyM7OS5ABlVuIkHSbptY4uh1l7c4Ayy0HSYklHtOPxon5OI4CImBMRexfhOHtJ+qOkDyT9XdKjkgp+HLPWcoAy23xtSzLn1d7AjsBcoNl5sszakwOUWRtI+o6kRekVyAOSds7Y9nlJj6Xb3pN0Ybp+qKSnJH0k6R1Jv5L0qXTb7DT5i5LqJI2VVCmpJiPffSVVpelfljQmY9vNkqZIelhSraRnMmaS3UBEzI2I30TE3yNiDfBfwN6SehWhqsxazAHKrJUkfRH4T+AEkhlrlwDT0m09gP8BHgF2Bj5LMhU7JLPF/ohkBuJhwL8A3wOIiOHpPvtHRPeIuKvRMcuAB0mmpN8B+AFwR6OmuRNJZqf9NLAIuCzPUxoOvBsRy/Lc36yoHKDMWu8k4KaIeD4iVpFMoT5MUl9gNMmX/dURsTIiaiPiGYCIqI6IpyNibUQsJpm2/vA8j3kw0B2YHBGrI+LPwEMkQanefenV0VrgDqAiV6bpzLhTgH/NsxxmRecZdc1ab2fg+fqFiKiTtAzYBdgVeKOpRJL2An4ODAG6kvwdVrfgmH+LiPUZ65akx6z3bsbrj0kCWlaStie5Irs2In6XZznMis5XUGat9zawe/2CpG5AL2Ap8DegyXs/wHXAq0D/iNgGuBBQC465q6TMv93d0mO2mKRPkwSnByIi36ZAs3bhAGWWnzJJ5RmPLYE7gdMlVUjaCrgceCZttnsI2EnSDyVtJamHpIPSvHoA/wDqJO0DfLfRsd4D9shSjmeAFcC/SSqTVAl8hfTeV0tI2gZ4FPjfiLigpenNis0Byiw/04FPMh6TImImcBFwL/AOyRXTNwAiohY4kiR4vAssBEakeU0AvgnUAr8GNugIAUwCbkl76Z2QuSEiVgNjgFHAh8C1wKkR8WorzumrwIEkQbYu47FbK/IyKzhFREeXwczMbCO+gjIzs5LkAGVmZiXJAcrMzEqSA5SZmZWkTvdD3d69e0ffvn2Llv+KFSvo1q1b0fLfHLgOC8P12Hauw8Iodj1WV1d/GBHbN17f6QJU3759ee6554qWf1VVFZWVlUXLf3PgOiwM12PbuQ4Lo9j1KGlJU+vdxGdmZiXJAcrMzEpSp2viK3VVVdmHVKuszP2j6LamNzPbVDhAmZnlsGbNGmpqali5cmVHF6VD9OzZkwULFrQ5n/Lycvr06UNZWVle+ztAmZnlUFNTQ48ePejbty9SvgPPbzpqa2vp0aNHm/KICJYtW0ZNTQ39+vXLK43vQZmZ5bBy5Up69eq1WQanQpFEr169WnQV6gBlZpYHB6e2a2kdOkCZmVlJ8j0oM7MWqlJVQfOrjMqc+3Tp0oX99tuPtWvX0q9fP2677Ta23XbbgpajpS6//HIuvPDCouXvKygzs05g6623Zt68ecyfP5/tttuOKVOmdHSRuPzyy4uavwOUmVknM2zYMJYuXdqwfOWVV3LggQcycOBAfvKTnzSsv+yyy9h77/AYKQUAABa+SURBVL054ogjOPHEE7nqqqsAqKysbBgy7sMPP6R+fNN169Zx3nnnNeR1/fXXA/Duu+8yfPhwKioqGDBgAHPmzOGCCy7gk08+oaKigpNOOokVK1bw5S9/mf33358BAwZw112NJ4puOTfxmZl1IuvWrWPmzJmcccYZAMyYMYOFCxcyd+5cIoIxY8Ywe/ZsunXrxrRp03jhhRdYu3YtgwYNYvDgwc3m/Zvf/IaePXvy7LPPsmrVKg455BCOOuoo7rnnHkaOHMmPf/xj1q1bx8cff8xhhx3Gr371K+bNmwfAvffey84778zDDz8MwPLly9t8rg5QZmadQP3VyuLFixk8eDBHHnkkkASoGTNmcMABBwBQV1fHwoULqa2t5atf/Spdu3YFYMyYMTmPMWPGDF566SV+//vfA0mQWbhwIYMGDeKcc85hzZo1HHvssVRUVGyUdr/99mPChAmcf/75jB49msMOO6zN5+wmPjOzTqD+HtSSJUtYvXp1wz2oiGDixInMmzePefPmsWjRooarq2zdurfcckvWr18PsMHvkiKCX/7ylw15vfXWWxx11FEccsghzJ49m1122YVTTjmFW2+9daM899prL6qrq9lvv/2YOHEil1xySZvP2QHKzKwT6dmzJ9dccw1XXXUVa9asYeTIkdx0003U1dUBsHTpUt5//32GDx/O/fffzyeffEJtbS0PPvhgQx59+/aluroaoOFqCWDkyJFcd911rFmzBoDXX3+dFStW8Ne//pUddtiB73znO5xxxhk8//zzAJSVlTXs+/bbb9O1a1dOPvlkJkyY0LBPW+TVxCfpXuAm4E8Rsb7NRzUz68Ty6RZeTAcccAD7778/06ZN45RTTmHBggUMGzYMgO7du3P77bczaNAgxo4dS0VFBbvvvvsGTW4TJkzghBNO4LbbbuOLX/xiw/ozzzyTxYsXM2jQICKC7bffnj/84Q/MmTOHsWPHUlZWRvfu3RuuoMaNG8fAgQMZNGgQp556Kueddx5bbLEFZWVlXHfddW0+T0XkHiFb0hHA6cDBwD3AzRHxapuP3gpDhgyJUp6w0KOZe5K4QnE9tl2h6nDBggXsu+++bS9QB5o0aRLdu3dnwoQJLU5biLH46jVVl5KqI2JI433zauKLiP+JiJOAQcBi4DFJT0o6XVJ+w9KamZm1QN69+CT1Ak4GTgFeAO4ADgW+BVQWo3BmZlYYkyZN6ugitFheV1CS7gPmAF2Br0TEmIi4KyJ+AHRvJt3Rkl6TtEjSBU1sP0nSS+njSUn7t/ZEzMxs05LvFdSNETE9c4WkrSJiVVPthun2LsAU4EigBnhW0gMR8UrGbm8Bh0fE/0kaBdwAHNTiszAzs01Ovt3ML21i3VM50gwFFkXEmxGxGpgGHJO5Q0Q8GRH/ly4+DfTJszxmZraJa/YKStJOwC7A1pIOAOq7mG1D0tzXnF2Av2Us19D81dEZwJ+ylGMcMA5gxx13pKqqKsehW6+urq5N+dfVXZV1Wz75tjV9KWhrHVrC9dh2harDnj17Ultb2/YCdVLr1q0r2PmvXLky7/ckVxPfSOA0kiubn2esrwVyjbHeVH/pJvtJSxpBEqAObWp7RNxA0vzHkCFDophdb9vezXxE1m35dTPPL31zw/3n8xuNtqZvNm93jy4I12PbFbKbeWY36+Z+DtIa+Xw3tHS6jY8++og777yT733ve20uXyG7mZeXlzcMy5RLs018EXFLRIwATouIERmPMRFxX468a4BdM5b7AG833knSQOBG4JiIWJZXqc3MNjMtnW7jo48+4tprr23xcdatW9faIhZcswFK0snpy76S/rXxI0fezwL9JfWT9CngG8ADjfLfDbgPOCUiXm/lOZiZbVbymW7jggsu4I033qCiooLzzjuPqqoqRo8e3ZDmnHPO4eabbwaSoY8uueQSDj30UO655x4qKys5//zzGTp0KHvttRdPPvkkAC+//DJDhw6loqKCgQMHsnDhwqKeZ64mvm7pc9au5NlExFpJ5wCPAl2AmyLiZUlnp9unAhcDvYBr00EN12brFWhmZvlPtzF58mTmz5/fMB1Grvs+5eXlPPHEEwBMnTqVtWvXMnfuXKZPn87kyZMZOXIkU6dOZfz48Zx00kmsXr266FdbzQaoiLg+ff5pazJPu6ZPb7RuasbrM4EzW5O3ZZetfbyzDJVkZhtr6XQbu+22W4vyHzt27AbLxx13HACDBw9myZIlQHLldtlll1FTU8Nxxx1H//7923pazcrVi++a5rZHxLmFLY6ZmTWl/h7U8uXLGT16NFOmTOHcc89tmG7jrLPO2mD/xYsXb7CcOcUGbDjNBkC3bt02WN5qq62ApHNG/ZXSN7/5TQ466CAefvhhRo4cyY033rjBYLOFlut3UNU5HmZm1o7ynW6jR48eG3QN33333XnllVdYtWoVy5cvZ+bMmS0+9ptvvskee+zBueeey5gxY3jppZcKdl5NydXEd0tRj25m1gl1dHN5PtNt7LnnnhxyyCEMGDCAUaNGceWVV3LCCScwcOBA+vfvn3dX70x33XUXt99+O2VlZey0005cfPHFhT61DeRq4vtFRPxQ0oM08RumiMg9h7CZmbVZ/RVSvcwJCMePH8/48eM3SnPnnXdusHzFFVdwxRVXbLRf4+bAzA4VvXv3Zv78+QBMnDiRiRMntrTorZarF99t6XP24Q3MzMyKIFcTX3X6/Hj6W6Z9SK6kXkvH1zMzMyuKfKd8/zIwFXiDZAijfpLOiogmx86zzUPW4ZJmtWsxzNpFRJD+XtNaKZ8Z3DPlO93G1cCIiFgEIGlP4GGyDO5qZrYpKS8vZ9myZfTq1ctBqpUigmXLllFeXp53mnwD1Pv1wSn1JvB+SwpnZtZZ9enTh5qaGj744IOOLkqHWLlyZYsCSzbl5eX06ZP/rEq5evEdl758WdJ04G6Se1BfJxlrz1rJzWNmnUdZWRn9+vXr6GJ0mKqqqlZ1S2+rXFdQX8l4/R5wePr6A+DTRSmRmZkZuXvxnd5eBTEzM8uUby++cpIJBT8PNDRERsS3i1SuDlNXV93kpIEd/ctxM7PNTa6x+OrdBuxEMsPu4ySTD26+8x+bmVnR5RugPhsRFwEr0vH5vgzsV7ximZnZ5i7fALUmff5I0gCgJ9C3KCUyMzMj/99B3SDp08BFJNO2d09fm5mZFUVeASoibkxfPg7sUbzimJmZJfJq4pPUS9IvJT0vqVrSLyT1KnbhzMxs85XvPahpJEMbfQ04HvgQuCtXIklHS3pN0iJJFzSxfR9JT0laJWlCSwpuZmabtnzvQW0XEf+RsXyppGObSyCpCzAFOBKoAZ6V9EBEvJKx29+Bc4Fm8zIzs81PvgFqlqRvkIzFB8lV1MM50gwFFkXEmwCSpgHHAA0BKiLeB95Pp/OwzUzW8QiByqhst3KYWWlSc/NzSKolGRxWQDdgfbppC6AuIrZpJu3xwNERcWa6fApwUESc08S+k9L8mpy5V9I4YBzAjjvuOHjatGm5z6yVli9/jy5dajZa37374LzS19VVZ92WmUdddV3TO+39WtvSN5NHvum7D+6ePe8M2c8BunfPnUchyrApq6ury6seLTvXYWEUux5HjBhRHRFDGq/PNRZfjzYcs6lJU1o1XlBE3ADcADBkyJCorKxsQ7Ga99BDV9O9+8a3w/Id6qipYZKayqNqRFXTO83Kfisur/TN5JFv+nyvXrKfA+TzHhWiDJuyqqqqvOrRsnMdFkZH1WO+TXxIGgMMTxerIuKhHElqgF0zlvsAb7eseGZmtrnKd7DYycCBwB3pqvGSDo2IjXrmZXgW6C+pH7AU+AbwzbYU1tpHVVX2GUM9aK6ZtZd8r6C+BFRExHoASbcALwBZA1RErJV0DvAo0AW4KSJelnR2un2qpJ2A54BtgPWSfgh8LiL+0eozMjOzTULeTXzAtiTdwiEZiy+niJgOTG+0bmrG63dJmv7MzMw2kG+A+k/gBUmzSDo/DAcmFq1UZma22csZoCQJeAI4mOQ+lIDz06sfMzOzosgZoCIiJP0hIgaTjGS+2WvuB6bMardimJlt0vIdi+9pSQcWtSRmZmYZ8r0HNQI4W9JiYAVJM19ExMBiFczMzDZv+QaoUUUthZmZWSPNBihJ5cDZwGeBvwC/iYi17VEws1w82KzZpi3XPahbgCEkwWkUcHXRS2RmZkbuJr7PRcR+AJJ+A8wtfpHMzMxyX0GtqX/hpj0zM2tPua6g9pdUPy6egK3T5fpefFnngzIzM2uLXPNBdWmvgpiZmWXK94e6ZmZm7aolo5mb5aWurjrrzMKeT8rM8uUAZZu1bL+l8u+ozDqem/jMzKwkOUCZmVlJcoAyM7OStFneg2p2PqcH260YZmbWjM0yQFnpq6pSk+tLrRegB6w1K56iNvFJOlrSa5IWSbqgie2SdE26/SVJg4pZHjMz6zyKdgUlqQswBTgSqAGelfRARLySsdsooH/6OAi4Ln022yw029w8q92KYVaSitnENxRYFBFvAkiaBhwDZAaoY4BbIyJIppXfVtJnIuKdIpbLNgOdpYmwEPxbLttUFTNA7QL8LWO5ho2vjpraZxdggwAlaRwwLl2sk/RaYYua4Sv0Bj7ceEPTX3gbaXoAhfzzaGv6ZvNop3PIWoeFKEM7pc9zt6LmMaK5emyH428a2laHVq/Y9bh7UyuLGaCa+vNo/O9rPvsQETcANxSiULlIei4ihrTHsTZVrsPCcD22neuwMDqqHovZSaIG2DVjuQ/wdiv2MTOzzVAxA9SzQH9J/SR9CvgG8ECjfR4ATk178x0MLPf9JzMzgyI28UXEWknnAI8CXYCbIuJlSWen26cC04EvAYuAj4HTi1WeFmiXpsRNnOuwMFyPbec6LIwOqUclHejMzMxKi8fiMzOzkuQAZWZmJckBKpVrWCbLj6TFkv4iaZ6k5zq6PJ2FpJskvS9pfsa67SQ9Jmlh+vzpjixjqctSh5MkLU0/j/Mkfakjy1jqJO0qaZakBZJeljQ+Xd8hn0UHKDYYlmkU8DngREmf69hSdWojIqLCvz9pkZuBoxutuwCYGRH9gZnpsmV3MxvXIcB/pZ/HioiY3s5l6mzWAv8vIvYFDga+n34Xdshn0QEq0TAsU0SsBuqHZTJrFxExG/h7o9XHALekr28Bjm3XQnUyWerQWiAi3omI59PXtcACktF9OuSz6ACVyDbkkrVcADMkVadDVFnr7Vj/u8D0eYcOLk9ndU46W8JNbibNn6S+wAHAM3TQZ9EBKpHXkEuWl0MiYhBJc+n3JQ3v6ALZZu06YE+ggmSMz6s7tjidg6TuwL3ADyPiHx1VDgeohIdcKpCIeDt9fh+4n6T51FrnPUmfAUif3+/g8nQ6EfFeRKyLiPXAr/HnMSdJZSTB6Y6IuC9d3SGfRQeoRD7DMlkOkrpJ6lH/GjgKmN98KmvGA8C30tffAv7YgWXplOq/VFNfxZ/HZkkS8BtgQUT8PGNTh3wWPZJEKu1++gv+OSzTZR1cpE5H0h4kV02QDKN1p+sxP5J+B1SSTGvwHvAT4A/A3cBuwF+Br0eEOwFkkaUOK0ma9wJYDJzl8T6zk3QoMAf4C7A+XX0hyX2odv8sOkCZmVlJchOfmZmVJAcoMzMrSQ5QZmZWkhygzMysJDlAmZlZSXKAslaTFJKuzlieIGlSgfK+WdLxhcgrx3G+no7cPKvYxypVkk6TtHML0/TNHDU8XVed/o6waMctBEmVkh5q7+NayzlAWVusAo6T1LujC5IpHZ0+X2cA34uIEQU8/pYt3F+SOvJv8TSgTYEiHbdtaTrYcub65t6LFh+3pXVrnZsDlLXFWuAG4EeNNzS+ApJUlz5XSnpc0t2SXpc0WdJJkuam80jtmZHNEZLmpPuNTtN3kXSlpGfTAUDPysh3lqQ7SX5k2Lg8J6b5z5f0s3TdxcChwFRJVzaR5t/SNC9Kmpyu+0567Bcl3Supa8b5/jy9EvuZpMMz5iB6oX6EjYy8+6ZXbtcCzwO7SjpK0lOSnpd0TzoeWv1cZa9KekLSNfX//SuZ62hCRp7z00CBpJPTOp0n6fq03rqk5ZyfnteP0vdoCHBHuu/Wkgan71G1pEczhrgZnJ73U8D3G1XXKOCR+vda0iWSngGGNZVfluMurv9nR9IQSVUZ53mDpBnAremV132SHlEyP9EVGXWQsw6B4xq/11aiIsIPP1r1AOqAbUh+od8TmABMSrfdDByfuW/6XAl8BHwG2ApYCvw03TYe+EVG+kdI/onqTzJeYjkwDvj3dJ+tgOeAfmm+K4B+TZRzZ5Jfv29PMsLFn4Fj021VwJAm0owCngS6psvbpc+9Mva5FPhBRnkfArqkyw+SDJwL0B3YslH+fUl+qX9wutwbmA10S5fPBy5Oz/lvaR2I5Nf8D6X7TAImZOQ5P8133/T4Zen6a4FTgcHAYxn7b9u4DoCy9Ly3T5fHkoysAvAScHj6+kpgfkZefwT2SF8HcEIe+W1Q9ySfo97p6yFAVcZ5VgNbp8unAW+SfObKgSUkY2m2uA79KO2HL5etTSLiH5JuBc4FPskz2bORDjcj6Q1gRrr+L0BmU9vdkQzyuVDSm8A+JOP7Dcy4OutJ8sWzGpgbEW81cbwDSb7sPkiPeQcwnGQooWyOAH4bER+n51k/rMsASZcC25IEnkcz0twTEevS1/8L/Dw91n0RUdPEMZZExNPp64NJJsv8X0kAnwKeSs/5rYhYmJb9dpIg3Zx/IQlGz6Z5bU0yuOeDwB6Sfgk8zD/rPdPewADgsTRtF+AdST1JAtrj6X63kQRxlNx36hMRb6bb1pEMNpo1vxzlb8oDEZH5+ZoZEcvT478C7E7ynhSqDq0EOEBZIfyCpJnqtxnr1pI2ISv5tsi8eb4q4/X6jOX1bPiZbDwOV5D8B/yDiMgMDEiqJLmCakpT06nkoiaOD8mV0rER8aKk00iu3Oo1HD8iJkt6GPgS8LSkIyLi1UZ5ZZZXJFc3J25QCKl+HLmmNNRxqjwjr1siYuJGJyXtD4wkaaI7Afh2412AlyNiWKN02zZTjsOAJzKWV2YE6ibzyyLzfMobbWv83mZ+htaRfG5aU4dWwnwPytosvbq4m6TDQb3FJP/FQzIbZ1krsv66pC2U3JfaA3iN5Irlu0qmBEDSXkpGTm/OM8DhknoruWl/IvB4jjQzgG9n3GPaLl3fg+SKogw4KVtiSXtGxF8i4mckzZD75Dje08Ahkj6bpu8qaS/gVaCf/nlvLvPLdzEwKN1/EElTJyRTch8vaYf6skvaPb2/s0VE3AtcVJ8WqE3PC5I63l7SsDRtmaTPR8RHwHIlg4nS6NyPBv6U5byazK+J49afT/1n5mtZ8mtOa+rQSpgDlBXK1ST3AOr9miQozAUOIvvVTXNeIwkkfwLOjoiVwI3AK8DzSro5X0+OloC0OXEiMAt4EXg+IpqdLiAiHiGZYuA5SfNI7q9B8sX+DPAYyRdfNj9MOyO8SNL0me0LvP54H5DcW/mdpJdIvmz3Sc95HPBweoN/SUaye4Ht0vJ9F3g9zesV4N9JZjZ+KS3rZ0hmia5K9785rRPS11PT9V2A40k6erwIzAO+kO53OjAl7SSR2dxWSZaAH0mvvmz5NRxX0tbAT4H/ljSH5KqoRVpZh1bCPJq5WSeSNmVOiIjRHV0WAEl9gF9HxKiOLottenwPysxaLe384eBkReErKDMzK0m+B2VmZiXJAcrMzEqSA5SZmZUkBygzMytJDlBmZlaS/j+Zfh94qTyeewAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "game = JacksCarRental()\n",
    "\n",
    "p_requests = [poisson.pmf(k=n, mu=lam)\n",
    "              for lam in game.lam_requests]\n",
    "p_returns = [poisson.pmf(k=n, mu=lam)\n",
    "            for lam in game.lam_returns]\n",
    "             \n",
    "for location in range(2):\n",
    "\n",
    "    # Plot pdfs for location 1\n",
    "    ax = plt.subplot(211 + location)\n",
    "    ax.bar(n-0.2, p_requests[location], width=0.4, color='m', \n",
    "           align='center', label=\"Requests\")\n",
    "    ax.bar(n+0.2, p_returns[location], width=0.4, color='y',\n",
    "           align='center', label=\"Returns\")\n",
    "    ax.set_title(\"Location %d\" % (location + 1))\n",
    "    plt.xlabel(\"Number of cars requested/returned\")\n",
    "    plt.ylabel(\"Probability\")\n",
    "    plt.legend()\n",
    "    plt.grid()\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "441"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(game.states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 0),\n",
       " (0, 1),\n",
       " (0, 2),\n",
       " (0, 3),\n",
       " (0, 4),\n",
       " (0, 5),\n",
       " (0, 6),\n",
       " (0, 7),\n",
       " (0, 8),\n",
       " (0, 9)]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.states[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[((0, 0), 0),\n",
       " ((0, 1), 0),\n",
       " ((0, 2), 0),\n",
       " ((0, 3), 0),\n",
       " ((0, 4), 0),\n",
       " ((0, 5), 0),\n",
       " ((0, 6), 0),\n",
       " ((0, 7), 0),\n",
       " ((0, 8), 0),\n",
       " ((0, 9), 0)]"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create state value-function\n",
    "values = {s: 0 for s in game.states}\n",
    "list(values.items())[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11, 13)"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.next_state(game.state, (1, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-5, 19)"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Test\n",
    "game.next_state(game.state, (1, 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "def show_values(values, r=1):\n",
    "    \"\"\"Displays the values rounded to r decimal places.\n",
    "    \"\"\"\n",
    "    \n",
    "    v_array  = np.zeros((n_max+1, n_max+1))\n",
    "    for s, v in values.items():\n",
    "        v_array[s] = v\n",
    "    print(v_array.round(r))\n",
    "\n",
    "show_values(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: Needs testing\n",
    "#evaluate_policy(random_policy, states, actions, transitions=None,  \n",
    "#                values=values, lr=1.0, theta=0.01, max_iter=10,\n",
    "#                show=True, show_values=show_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chapter 6: TD Learning\n",
    "\n",
    "### Example 6.2 Random walk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gamelearner import GameController, RandomPlayer, TDLearner\n",
    "from randomwalk import RandomWalkGame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1]"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game = RandomWalkGame()\n",
    "game.possible_n_players"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['T1', 'A', 'B', 'C', 'D', 'E', 'T2']"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T1 _ _ C _ _ T2\n"
     ]
    }
   ],
   "source": [
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Player 1 made move 'r'\n"
     ]
    }
   ],
   "source": [
    "game.make_move((1, 'r'), show=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T1 _ _ _ D _ T2\n"
     ]
    }
   ],
   "source": [
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: 0.0}"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.get_rewards()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T1 _ _ _ _ E T2\n"
     ]
    }
   ],
   "source": [
    "game.make_move((1, 'r'))\n",
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: 0.0}"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.get_rewards()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T1 _ _ _ D _ T2\n"
     ]
    }
   ],
   "source": [
    "game.make_move((1, 'l'))\n",
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T1 _ _ _ _ E T2\n"
     ]
    }
   ],
   "source": [
    "game.make_move((1, 'r'))\n",
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T1 _ _ _ _ _ T2\n"
     ]
    }
   ],
   "source": [
    "game.make_move((1, 'r'))\n",
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.game_over"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: 0.0}"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.get_rewards()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: 1.0}"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.get_terminal_rewards()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T1 _ _ _ _ _ T2\n",
      "Game over!\n",
      "RANDOM won in 41 moves\n"
     ]
    }
   ],
   "source": [
    "game = RandomWalkGame()\n",
    "player = RandomPlayer()\n",
    "ctrl = GameController(game, [player])\n",
    "ctrl.play(show=False)\n",
    "ctrl.announce_result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Game of Random Walk with 1 players ['TD']\n",
      "T1 _ _ C _ _ T2\n",
      "TD's turn (l/r): r\n",
      "TD got 0.0 reward.\n",
      "T1 _ _ _ D _ T2\n",
      "TD's turn (l/r): l\n",
      "TD got 0.0 reward.\n",
      "T1 _ _ C _ _ T2\n",
      "TD's turn (l/r): r\n",
      "TD got 0.0 reward.\n",
      "T1 _ _ _ D _ T2\n",
      "TD's turn (l/r): r\n",
      "TD got 0.0 reward.\n",
      "T1 _ _ _ _ E T2\n",
      "TD's turn (l/r): r\n",
      "TD got 0.0 reward.\n",
      "TD got 1.0 reward.\n",
      "T1 _ _ _ _ _ T2\n",
      "Game over!\n",
      "TD won in 5 moves\n"
     ]
    }
   ],
   "source": [
    "game.reset()\n",
    "player = TDLearner(initial_value=0.5)\n",
    "ctrl = GameController(game, [player])\n",
    "ctrl.play(show=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'D': 0.5, 'T2': 0.505}"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "player.value_function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TD(0) Updates on Random Walk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Equation (6.2)\n",
    "\n",
    "$V(S_t) \\leftarrow V(S_t) + \\alpha [R_{t+1} + \\gamma V(S_{t+1}) - V(S_t)]$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "size = 5\n",
    "game = RandomWalkGame(size=size)\n",
    "gamma = 1.0\n",
    "learning_rate = 0.1\n",
    "\n",
    "initial_value = 0.5\n",
    "value_function = {s: initial_value for s in game.states}\n",
    "value_function['T1'] = 0.0\n",
    "value_function['T2'] = 0.0\n",
    "\n",
    "episodes_to_record = [0, 1, 10, 100]\n",
    "value_sets = {}\n",
    "\n",
    "np.random.seed(5)\n",
    "\n",
    "n_iters = 1001\n",
    "for iteration in range(n_iters):\n",
    "    \n",
    "    if iteration in episodes_to_record:\n",
    "        value_sets[iteration] = value_function.copy()\n",
    "\n",
    "    past_states = [game.state]\n",
    "\n",
    "    while True:\n",
    "\n",
    "        # Behaviour policy\n",
    "        move = np.random.choice(game.available_moves())\n",
    "        game.make_move([1, move])\n",
    "        past_states.append(game.state)\n",
    "\n",
    "        reward = game.get_rewards()[1]\n",
    "\n",
    "        # TD(0) update\n",
    "        if not game.game_over:\n",
    "            value_function[past_states[-2]] += \\\n",
    "                    learning_rate*(\n",
    "                            reward + gamma*value_function[past_states[-1]] -\n",
    "                            value_function[past_states[-2]]\n",
    "                    )\n",
    "        else:\n",
    "            reward = game.get_terminal_rewards()[1]\n",
    "            value_function[past_states[-2]] += \\\n",
    "                learning_rate*(reward - value_function[past_states[-2]])\n",
    "            break\n",
    "\n",
    "    game.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOydd3hUxdrAf5NGeoP00EJC6AiEJkgRKYqAFbBfuV7EcgULKFbE7yqICCgoVgRb8IqoiBfEgoqoNKUECAkhkN573935/jibsAkJSUg2m03m9zx5smdmzsx7Zs/OO/POzDtCSolCoVAo2i82lhZAoVAoFJZFKQKFQqFo5yhFoFAoFO0cpQgUCoWinaMUgUKhULRzlCJQKBSKdo5SBG0EIUQHIcRxIYR/A9OeFEL4mlGe3UKIe8yVf3MihJBCiFBLy2GNCCH+IYTYY2k5Govp+2mtz9CcKEVgYYQQhSZ/BiFEicn1bUKIJUKICiFEgfHvlBBirRAioEZWc4FfpJSpxnyFEGK5ECLL+PeyEEIASCnLgPeBx+uQ6RYhRHxlepNwOyFEuhDi2uaviYYhhPhACPF/NcK6GRtzOzOX3SLlNBWjjEXGdyhJCPGqEMLW0nI1BSHETiHEIpPrIONz1hZWb2dIUR2lCCyMlNK18g84B0wzCfvYmGyzlNIN8AauB/yBgzWUwb3AhybXc4HrgIHAAOBaY5pKPgHuEkJ0qEWsrYAnMLZG+BRAAjsu4VEVLctA4zs1FpgFzLGwPE3lF6q/j2OAk7WExVR2hhQNRykCK0JKWSGljEL7YWcAjwIIIboAPYA/TZLfBayUUiZKKZOAlcA/TPJKBHKAEbWUUwp8BtxZI+pO4GMppU4I4SWE+EYIkSGEyDF+Dq5NbuOo5iOT62o9ayGEhxDiPSFEirEH+39N6cEaRzOLjaayHCHEBiGEo0n8QmNZyUKIOTXunSqE+EsIkS+ESBBCLDGJ/sX4P9fY2x5pvGeOEOKEsaydQoiuxnAhhFhlHEXlCSGOCCH61SFzoBDiayFEthAiVgjxrxr195kQYpNxVBglhIhoSF1IKWOB34DLTPK72yhvgRAiTghxr0ncOCFEohDiUaPcKUKIu03iOxrlzBdC7EN770yf43IhxH7j8+4XQlxuErfb+N3uNdbfNmN+Hxvz2y+E6FbHo/wCjBJCVLZZVwCrgYgaYb8Yy2rw+1kTIcQKIcQeIYRHQ9K3BZQisEKklHrgK7QXH6A/ECel1Jkk6wscNrk+bAwz5QTaiKE2NgI3CSGcQGusgWnAJmO8DbAB6Ap0AUqAtZfyPMaydEAoMAiYBDR1fuE2YDJaQ9UTeBpACDEFeAyYCIQBV9W4rwhN4XkCU4H7hBDXGePGGP97GkdsvxvjngRuAHyAX4FPjekmGe/pacxvFpBVh7yfAolAIHAT8KIQYoJJ/HQg0pjP1zSwroUQvdDek1iT4HS0EaI7cDewSggx2CTeH/AAgoB/AuuEEF7GuHVAKRCANsqoUqRCCG9gO/Aa0BF4FdguhOhokvds4A5j3j2A39HeI2+09/G5Oh5lH9CB8+/rGGCX8blMwyqVdaPfTyGEjRDiHbQR9CQpZd7F0rcllCKwXpLRfjygNQ4FNeJdAdMXOQ9wFaKa3b/AeO8FSCl/A9LQTFEAM4FTUsq/jfFZUsotUspiKWUB8B8uNCXVixDCD7gaWCClLJJSpgOr0BqMprBWSpkgpcw2ynaLyXNskFIek1IWAUtMb5JS7pZSHpVSGqSUR9Aa6Is9173AS1LKE0ZF/CJwmXFUUAG4Ab0AYUyTUjMDIURnYDTwuJSy1FjH76I1mJXskVJ+a+wEfEjdCrySQ0KIIrTGdTfwhskzbpdSnpYaPwPfcb5TgVHupcYR6LdAIRBuHKXdCDxr/K6OoSnxSqaimWY+lFLqpJSfoplvppmk2WAsOw/4H3BaSvm9se7+i9YRuADjvNafwBijwvGUUsahKd7KsD7Az8b0jX0/7dG+a28082zxRdK2OZQisF6CgGzj5xy0BseUQrQeXyXuQKGs7mXQDci9SBmbOG8eugOTH70QwlkI8ZYQ4qwQIh+tJ+Z5CSadrmg/whQhRK4QIhd4C6hrRZPOmN4Ue8Bg/KskweTzWbSeNsb/NeOqEEIMF0L8ZDQp5AHzgE71yL/GRPZsQABBUsof0Xqh64A0IcTbQgj3WvIIBLKNDZapXEEm16Z272LAUVx80nowWmdgFjAccDF5xquFEH8YzVC5wDU1njGrxuiy2JiXD2BH3fUXWOO6tudIM/lcUsu160We6Re0Xv8VQOUqnz0mYQlSyrPGZ2zs+xkKzACel1KWX0SGNolSBFaI0SY6Da03BHAECKnRMERRvdc40BhmSm+qm49qsgmYYLSFj0CbYK7kUSAcGC6ldOe82URwIUWAs8m16aqOBKAM6CSl9DT+uUspa5qxKjkHdKsR1h2tETBVBJ1NPndBG0EBpNQSZ8onaKaXzlJKD2C9yTPV5qo3AbjXRHZPKaWTlHIvgJTyNSnlEDSzXE9gYS15JAPeQghTZd4FSKolbYMx9vg/QzO/PAva0mFgC/AK4Cel9AS+pfbvrSYZaIq4rvpLRlOM1Ihv0nOY8Atagz+G8+/+b8AoqpuFoHHvJ2gjp7uB/wkhwptJXqtBKQIrQghhL4TojTaE9UezwVZO/MYAw0ySbwIeEdqSukC0H8YHJnkFoQ2D/6irPGPvao+xvF01VmO4ofXgco3D8rpsuwB/ow3fuxjnGhablJGCZppYKYRwN9ppewgh6hrGbwGmCiEmCSFsjc/2NJr93JQHhBDBRtmeBDYbwz8D/iGE6COEcK5Fbje03nmpEGIYcKtJXAbaqCPEJGw9sFgI0ReqJr5vNn4eahxh2KMpw1JAX/OBpJQJwF7gJSGEoxBiAJpt/uOaaS+RZcBcoS2rdECztWcAOiHE1WhzGfViNEt9ASwx9rj7oC1KqORboKcQ4lahLTWehWau+aaZnmMvminzdoyKQEqZY3yW26muCBrzfmLM61O0d+V7IUSP+tK3JZQisA5mCSEK0cw4X6NNOA6RUiabpHmL6jblt4BtwFHgGNok3lsm8bcCG42214uxEa2Xt6lG+GrACchEUyZ1LimVUu5Ca4iPAAe5sGG4E62BOo5m5vocbTKytryi0Oz9L6GZYX5Hsx0/XyPpJ2gKJs7493/G+/9nlP1HtInGH2vcdz+wVAhRgNaL/syk7GI0W/NvRlPQCCnlVmA5EGk0QRxDm/MAzRz3jvGZzqJ9b6/UUU23oI10ktGW7z5nrLcmI6U8imY7X2g0Pz1kfK4ctPfg60Zk9yCa+SYVrWOxwaScLLRJ6EfRnnURcK2UMrPpT1FV/wfRFNkxk6hf0UyJpoqgwe9njTI2AkuBHy+ygqnNIdTBNG0D45D/L2BCbROStaQ9DIwxTs62KYQQ8cA9UsrvLS2LQmENtOodkoqGY+zZ92lE2l7mlUihUFgLyjSkUCgU7RxlGlIoFIp2jhoRKBQKRTvH6uYIOnXqJLt163ZJ9xYVFeHi4lJ/QkUVqs4ah6qvxqHqq3E0pb4OHjyYKaX0qS3O6hRBt27dOHDgwCXdu3v3bsaNG9e8ArVxVJ01DlVfjUPVV+NoSn0JIWru+q5CmYYUCoWinaMUgUKhULRzlCJQKBSKdo7VzRHURkVFBYmJiZSWll40nYeHBydOnGghqdoGF6szR0dHgoODsbev6QxUoVBYE21CESQmJuLm5ka3bt2o7m6/OgUFBbi51fTWrLgYddWZlJKsrCwSExPp3r27BSRTKBTNRZswDZWWltKxY8eLKgFF8yKEoGPHjvWOwhQKRdPJ27aNmCsn4DvvPmKunEDetm3Nmn+bGBEASglYAFXnCoX5ydu2jZRnnkWWliIAXXIyKc88C4DHtGkXv7mBtIkRgUKhULRV0letRtYYecvSUtJXrW62MpQiaEZ27NhBeHg4oaGhLFu2zNLiKBSKNoAupXav8nWFXwptxjTUGL78K4kVO6NJzi0h0NOJhZPDuW5QUP03XgS9Xs8DDzzArl27CA4OZujQoUyfPp0+fRrkGVqhUCiqIaUk78uv6oy3C6j17KZLot2NCL78K4nFXxwlKbcECSTllrD4i6N8+VfTjlXdt28foaGhhISE4ODgwOzZs/nqq7q/RIVCoagLXU4OSQseJmXxYuy7dUN06FAtXjg64vvwgmYrr82NCJ7fFsXx5Pxa4/R6PUeSCijXG6qFl1ToWfT5ET7dd67W+/oEuvPctLrOUtdISkqic+fzZ3oHBwfz559/NlJ6hULR3inc8xspixejy83F97FH8b77bvK//Zb0VaupSE7GPjAQ34cXNNtEMbRBRVAfNZVAfeENpbZzHdSqGoVC0VAMpaWkv/oqOZs+xKFHD7q/tR7HPn2QUpIeMIy0+9ej65RkFid9bU4RXKznXlBQwJR1+0nKLbkgLsjTic33jrzkcoODg0lISKi6TkxMJDAw8JLzUygU7YfSkydJXriQsphYvG6/Hd/HHsXG0RGA/dvj2f/NGXy7utHR0zwHibW7OYKFk8NxsretFuZkb8vCyeFNynfo0KHExMRw5swZysvLiYyMZPr06U3KU6FQtG2kwUDWe+9x5uaZ6HJz6fzOO/g//RQ4dKC0qAKA8OH+jL45jBsfj8DGzjxWhjY3IqiPytVBzb1qyM7OjrVr1zJ58mT0ej1z5syhb9+LzysoFIr2S0VyMslPLKZ43z7cJk7Ef+nz2Hl5kZFQwO6PTuLk7sDU+wfg4ePEwAmd68+wCbQ7RQCaMmhqw18b11xzDddcc02z56tQKNoWedu+IXXpUtDrCfjPf/C44Xp0FQb2bonl7x8ScHS157KJXVpMnnapCBQKhcIS6PPzSX1+Kfnbt+M0aBCBLy/HoXNnMhML+d/6I+RnltJndCAjr++Bo0vLefVVikChUChagKI//iR58WJ0GRn4zH+Ijv/6F9hq85WuXh1w9XLkyjt7E9TTq8VlM+tksRBiihAiWggRK4R4opZ4DyHENiHEYSFElBDibnPKo1AoFC2NobyctBUrOHf33dh06EC3Tz+h47x5RO/P4Os1f6PXG3B0sef6RwdbRAmAGUcEQghbYB0wEUgE9gshvpZSHjdJ9gBwXEo5TQjhA0QLIT6WUpabSy6FQqFoKUpPnSJ50eOUnTyJ5+xZ+C1aRH4h7Fz9N0nROQT08KCsSIezu4NF5TSnaWgYECuljAMQQkQCMwBTRSABN6HtvHIFsgGdGWVSKBQKsyMNBnI++oj0V1Zi4+pK8Jtv4DxmLH/tOsf+7fHY2grG3hpO39GBCBvLbzwVte2IbZaMhbgJmCKlvMd4fQcwXEr5oEkaN+BroBfgBsySUm6vJa+5wFwAPz+/IZGRkdXiPTw8CA0NrVcmvV6Pra1tvekU56mvzmJjY8nLy2tBiVo3hYWFuLq6WloMq6Et1pdNTi7umzbR4cQJyvr3J/+O2zG4u2PQS+K+k3RwB//BAnunxiuAptTX+PHjD0opI2qNlFKa5Q+4GXjX5PoO4PUaaW4CVgECCAXOAO4Xy3fIkCGyJsePH78grDby8/MblO5SuPvuu6WPj4/s27ev2cqwBPXVWUPrvr3w008/WVoEq6Kt1Vfe/3bIk8OGyxOXDZLZkZtlaXG5/H1rrCwtrpBSSllaVN6k/JtSX8ABWUe7as7J4kTAdBdEMJBcI83dwBdGOWONiqCXGWXSOPIZrOoHSzy1/0c+a3KW//jHP9ixY0czCKdQKKwNfWEhyU8sJmnBAhy6dKH7F1vI7TWeyKX7OLjzLAnHswHo4NxyS0IbgznnCPYDYUKI7kASMBu4tUaac8AE4FchhB8QDsSZUSat0d/2EFQY/Q3lJWjXAANmXnK2Y8aMIT4+vunyKRQKq6L44EGSFz1ORUoKne6/H6db5/DTljhO/3WGjkEuTJ7bD//uHpYW86KYTRFIKXVCiAeBnYAt8L6UMkoIMc8Yvx54AfhACHEUzTz0uJQys0kF/+8JSD1aa5STXgcpf4G+rHpERQl89SAc3Fh7nv794Wp14phCoTiPLC8nY90bZL3zDvZBQXT9+COcBw1ix1tHiT+WxYjrQrhsYhdsbVu/SzezbiiTUn4LfFsjbL3J52RgkjlluICaSqC+cIVCoahBWVwcyQsXURoVhceNN2B/9wL03m4AXH5TKCOu74Gnr7OFpWw4bW9n8UV67iUFBbi9O1IzB9XEozPcfcGCJYVCoahCSkluZCRpy1/GxtER/9Wvcaq8B4dWHaPHYF8m/bMv7h2dLC1mo2n9Y5bmZsKzYF/ji7J30sIVCoWiDnQZGSTMm0fq80txjoigw5pP+PZ3Nw58G09ohC9XzAyztIiXTPtTBANmwrTXtBEAQvs/7bUmTRQD3HLLLYwcOZLo6GiCg4N57733mkdehUJhcQp++IG46TMo/uNP/J5+muJ7lvLNhjMY9AamPTSQiXf3xcnNsruDm0LbMw01hAEzm9zw1+TTTz9t1vwUCoXlMRQVkbZsGbn//RyH3r3xW/oSHv3DcS6qIGJqNwZP6op9B+vfpNo+FYFCoVDUQ8nff5P0+ONUnEvA8R/3c8x5NEVf5XFzH81J3PBpIZYWsdlof6YhhUKhuAhSpyNj7Trib7sdQ4WOwkXv8F1yf5JP59N7ZACaa7S2hRoRKBQKhZHys2dJWrSI0sNHcJh2Mwc9riHjz2K69uvImFt6WuWKoIagFIFCoWj3SCnJ/fxz0l5ahrCzI+jVlbhMmsKxtYeZdE93Qof4tsmRQCVKESgUinaNLjublGeepfCHHyi6/HrOhk6j2/ih2NrZMGPBIEuL1yIoRaBQKNothT//TPJTT1NapOPcTS9zNtMFzzJBUW4ZHZzaT/OoJoubiTlz5uDr60u/fv2qwrKzs5k4cSJhYWFMnDiRnJwcC0qoUCgqMZSUkLp0KefunUdq0Cj2jX2RhBxXIqZ2Y9bTQ/EOcLG0iC1Ku1QE2+O2M+nzSQzYOIBJn09ie1zTXUvU5oZ62bJlTJgwgZiYGCZMmMCyZcpxnUJhaUqORXHmhhvJ+eRTvO/6B7kjZ+Id5M6sp4YxfFoIdvbWvy+gsbSfsY+R7XHbWbJ3CaX6UgBSilJYsncJAFNDpl5yvrW5of7qq6/YvXs3AHfddRfjxo1j+fLll1yGQqG4dKReT9a775G2dh1JYdcy8PWn8Js4isnFFTg42rWKIyMtRZtTBMv3Ledk9sla4/R6PVHZUZQbyquFl+pLefa3Z/n81Oe13tfLuxePD3u80bKkpaUREBAAQEBAAOnp6Y3OQ6FQNJ3yxESSH3+C1FNZnBr9PAXSnU6GYPxovYfFtCRtThHUR00lUF+4QqGwXqSU5H31FUn/WcHp4CkkDLkcFw9Hrp7dk5DLfCwtXquhzSmCi/XcCwoKuHHnjaQUpVwQF+ASwIYpG5pVFj8/P1JSUggICCAlJQVfX99mzV+hUNSNPjeXlCXPU7BjB+cun0eCQ3/6jwtmxIwQHNrRiqCG0O4mi+cPno+jrWO1MEdbR+YPnt/sZU2fPp2NG7VTzzZu3MiMGTOavQyFQnEhhb/9xvEbbiN17zF8Hn2E8WvmceOiIYyZ3dMqlUDlApd/n/13sy1wMcX6aqSJVE4Irzm0htSiVPxd/Jk/eH6TJopBc0O9e/duMjMzCQ4O5vnnn+eJJ55g5syZvPfee3Tp0oX//ve/zfEICoWiDgxlZaSvfJVj38dxOvRBvAJcGXLPaIQQ+Lt1sLR4l4S5FriY0u4UAWiV11wVWEldbqh/+OGHZi1HoVDUTunJk0QvfomjjqPJ6zmcoDAPxt3e2+pdQ6w5tKZKCVRSqi9lzaE1ShEoFAoFgDQYyN7wAafe/5q/+t6Pg6MdE27pTfgIf6tXAhnFGbXOaQKkFqU2WzlmVQRCiCnAGsAWeFdKuaxG/ELgNhNZegM+Uspsc8qlUCjaBhUpKZx94lkq/txD0FUTMYwKYvC1YVZ9WhhAhaGCT098yhuH36gzjb+Lf7OVZzZFIISwBdYBE4FEYL8Q4msp5fHKNFLKFcAKY/ppwMNKCSgUioaQvnU7v310lCyPa5i+5Gp8Zl1PFysfAQDsS9nHS/teIjY3ltFBoxkeMJx1f62rZh5q7gUu5hwRDANipZRxAEKISGAGcLyO9LcA6rxHhUJxUXR5eRx89l2OFPSgolME/Ud643XDQKs3A6UWpbLywEp2xO8gyDWI18a/xrjO4xBC4OPkw5pDa0gpSiHAJaBZFriYIqSUzZZZtYyFuAmYIqW8x3h9BzBcSvlgLWmd0UYNobWNCIQQc4G5AH5+fkMiIyOrxXt4eBAaGlqvTHq9Hlvb9udHpCnUV2exsbHk5eW1oEStm8LCQlxdXS0thtXQ2PqyOR5D0h4d2e7hONvk4X+lO06drPs3rZM6fsr/iR15O5BIrnK/iqvcr8LB5kLzVlPer/Hjxx+UUkbUFmfOEUFt6rkurTMN+K0us5CU8m3gbYCIiAg5bty4avEnTpzAzc2tXoEKCgoalE5xnvrqzNHRkUGD2ofP9oawe/duar6firppaH0ZysvJWLOGrPc3kB7xb4aPcmHwreOwsbXurVB7k/ayct9K4vPjubLzlSwcupBgt+A605vr/TJnLSYCnU2ug4HkOtLOxsrNQo11Q/3SSy8RGhpKeHg4O3futITICoVVkLQnis3zIkn6+Eu8Zs7k+vV3EnHHcKtWAkmFSSz4aQH3fn8vEsmbV73JmivXXFQJmBNz1uR+IEwI0V0I4YDW2H9dM5EQwgMYC3xlRlmqkbdtGzFXTuBE7z7EXDmBvG3bmpxnY9xQHz9+nMjISKKiotixYwf3338/er2+yTIoFG2J8tIKflj6FV99mEyh8MTtiRcIeH4Jti7We1ZAmb6M9YfXM+PLGexN3sv8wfP5YvoXjA4abVG5zGYaklLqhBAPAjvRlo++L6WMEkLMM8avNya9HvhOSllkLllMydu2jZRnnkWWajPwuuRkUp55FgCPadMuOd/GuKH+6quvmD17Nh06dKB79+6Ehoayb98+Ro4cecnlKxRtiTO/nWb3pmMUCze6iDiuXHItLp2bb7mkJfg54WeW7VtGYmEik7pO4rGIxwhwDbC0WICZ9xFIKb8Fvq0Rtr7G9QfAB81VZuqLL1J2onY31Dq9nopjx5Dl1T2NytJSUp56mtzPancB0aF3L/yffLLRstTlhjopKYkRI0ZUpQsODiYpKanR+SsUbZH8nd9x+J1D4NKNCSP1hN/7T6teEZSQn8Dy/cv5OfFnQjxCeGfSO4wIGFH/jS1Iu9tZXFMJ1BduFhlqWallzS+6QtFUpJSc2B2P/HYzNts20rf/IAKfuhmXnj0sLdolU6Ir4b2j77Hh2AbsbOx4LOIxbu11K/a2re/8gzanCC7Wcy8oKCB1xnXoki+cs7YLDKTrh5uaVZa63FAHBweTkJBQlS4xMZHAwMBmLVuhsBZy04v5cf1+UpL1BKZUMPr+++h0330I+9bXYDYEKSU/nPuBl/e/TEpRClNDpvLIkEfwdW69buitd9r9EvF9eAHCsbobauHoiO/DC5q9rLrcUE+fPp3IyEjKyso4c+YMMTExDBs2rNnLVyhaK3nbthF95URKXtzKp0//Svq5Qvqk72DSs1Pxeeghq1UCZ/LOMO/7eTy8+2FcHVzZMHkDy65Y1qqVALTBEUF9VE4Ip69ajS4lBbuAAHwfXtCkiWJonBvqvn37MnPmTPr06YOdnR3r1q1TG90U7YbKBRuJ3hHEhczAJ+MvesZtoeuTj+AyZLClxbskiiuKeevIW2w6vgknWyeeGPYEs8JnYWdjHU2sdUjZzHhMm9bkhr8mjXVD/dRTT/HUU081qwwKRWunvFRH7NpPcC4tJSBlL44lWXTMOQFA5ptv4nXzTRaWsHFIKdkRv4NXDrxCenE614Vex/zB8+nk1MnSojWKdqkIFApFy3P6QDI/bzyGIfAmRp47io3UVykBAF1K7e6WWysxOTG8tO8l9qfup7d3b14d9yoDfQZaWqxLQikChUJhVgqzivnxtV9JSLPHpTCNvmc+x0ZeuIHSLqB1rKmvj4LyAt48/CafnPgEVwdXnhnxDDeG3YitjfWad5UiUCgUZkFKScr/fmXbl0UYsKFn8R+MuH88hqx7qm3qBPMt2GhOpJR8E/cNKw+sJLs0mxt73sj8QfPxdPS0tGhNRikChULR7OQfOkrOa69S9McfdBlwK/1uHELwzU8gbM4vVExftZqK5GTsAwObZcGGOTmZfZIX/3yRv9L/YkCnAaybsI6+nfpaWqxmQykChULRbBTHxbP31Z2cLglkRFwanZ98kt6zZyEcqrtUrlyw0dq9teaV5bH2r7V8duozPBw8WHr5UmaEzsBGtK2V90oRKBSKJqPLzubEqo85cLYTxc5hdOmYQ4//fIyrv5elRbskDNLAl7FfsvrgavLK85gVPosHLnsAjw4elhbNLChF0AxkZWUxYcIEAFJTU7G1tcXHxweAffv24eBgmfNTv//+e9auXcuXX35pkfIVbR9DcTFZH2zkt+9zSfIdjrNrCVff1pWQUVdaWrRLJioziv/8+R+OZh5lsO9gnhz+JOHe4ZYWy6woRdAMdOzYkb///huAJUuW4OrqymOPPVYtjZQSKSU2Nm1rSKlon0idjtwtX5C5di26jAzsxz9C/6HujLxtLPYdrHP1TE5pDmsOreGLmC/o6NSRF0e/yLUh17YLP2CqVTIjsbGx9OvXj3nz5jF48GASEhLw9Dy/wiAyMpJ77rkH0DyV3nDDDURERDBs2DD++OOPC/KLiIggOjq66nr06NEcPnyYP/74g5EjRzJo0CBGjRpFTEzMBfc+/fTTrF69uuq6V69eJCYmApr7i2HDhnHZZZdx//33YzAY0Ol03HHHHYwYMYJ+/frx2muvNVu9KKwXKSX5u3YRdd3tfPffZAq7DqLrJx9zzRv3MGZOhFUqAb1Bz+aTm7l267V8Gfsld/S5g23XbWNaj2ntQglAGx0RbF156IKw0CG+dBviQUW5nm9eP3xBfK+RAfS+PICSwnJ2vHWsWtz1j38PYK0AACAASURBVF76tvfjx4+zYcMG1q9fj06nqzPdQw89xKJFixgxYgTx8fFce+21HDtWXY5Zs2bx2Wef8cwzz5CYmEhWVhYDBw4kLy+PPXv2YGtry44dO3j66afZvHlzg+Q7duwYW7duZe/evdjZ2TF37lwiIyPp0aMHmZmZ/PHHH7i5uZGbm3vJdaBoGxQfOkTaipXEZHgQF/IPhL09LnfNxnmw9Z4T8Hf637z454ucyD7BMP9hLB62mFCv+s8/b2u0SUXQmujRowdDhw6tN933339frbefk5NDSUkJTk5OVWEzZ85k2rRpPPPMM2zevJmZM2cCkJuby5133snp06cbLd/333/P/v37iYjQzrQuKSmhc+fOTJ48mejoaBYtWsR1113HpEmTGp23om1Qdvo06a+uIuXPaKL73kl+aBBd+noz9tZw3Ds61Z9BKySzJJNVB1fx9emv8XX2ZcXYFUzuOrndjABq0iYVQV09+IKCAuwdbC/aw3dydWjSCKAmLibH6tnY2FQ7i6DUZEONlLLeieWuXbvi6urK8ePH2bx5Mx988AGg+S2aPHky999/P7GxsUyZMuWCe+3s7DAYDBeULaVkzpw5vPDCCxfcc+TIEb744gtee+01tmzZwttvv93wB1dYPRVp6WSuXUvuli3YODlRNm0hFUWBTJrVk9AhvlbZaOoMOjZHb2bdX+so0Zfwz37/ZO6AuTjbO1taNIui5ghaEBsbG7y8vIiJicFgMLB169aquKuuuop169ZVXVdOPtdk1qxZvPTSS5SVldGnTx8A8vLyCAoKAqhSDjXp1q0bBw8eBLSVTJXnIVx11VV89tlnZGZmAtoKqHPnzpGRkYGUkuuvv57nn3+eQ4cuNLcp2ib6wkLSV6/m9OTJnN59grIbHqDHru+4/JmbuXXJCMIi/KxSCRxIPcDMb2aybN8y+vv054vpX7BgyIJ2rwSgjY4IWjPLly9nypQpdOnShT59+lBWVgbAunXruO+++9iwYQM6nY7x48dXUwyV3HzzzTzyyCMsXbq0Kuzxxx9nzpw5vPzyy4wfP77Wcm+++WY++ugjBg0axLBhwwgJCQGgf//+PPfcc1x11VUYDAbs7e1Zv349tra2/POf/0Sv12Nra8vy5cvNUBuK1oQsLycncjOZb75JSWEF8WMeJlEfTJC3J5d5ewNg62J9fcf04nRWHljJt2e+JdAlkNXjVnNllyutUpmZC1HbsYmtmYiICHngwIFqYSdOnKB379713ltQUICbm5u5RGuT1FdnDa379kJr3ylbG9JgIP/b/5GxejXliYlkX34rJ91GU6GDwVO6MmRKV+zszbMayJz1VaGv4OMTH/Pm4TfRGXTM6T+HOf3m4GRnnfMa0LT6EkIclFJG1BZn1hGBEGIKsAawBd6VUi6rJc04YDVgD2RKKceaUyaFQnGeot9/J/2VlZRGRdEhPBzbpes5/KMe/yB3xt0eTsdAV0uLeEn8nvw7L+17iTN5ZxgXPI5FQxfR2b2zpcVqtZhNEQghbIF1wEQgEdgvhPhaSnncJI0n8AYwRUp5TgjRus9zUyjaCKUnT5L+ykqK9uzBJjAIh8Uv0/2OqSAEDuGZdOvfCWFjfaaTlMIUVhxYwa6zu+js1pl1E9YxJniMpcVq9ZhzRDAMiJVSxgEIISKBGcBxkzS3Al9IKc8BSCnTL7UwKaWy+bUw1mZWVEBFUhIZr71G3tfbsHF3x/b+pzmY04P8g6Xccb0OZ3cHug/0sbSYjaZcX87GqI28c/QdpJQ8eNmD/KPfP+hg28HSolkFZpsjEELchNbTv8d4fQcwXEr5oEmaSpNQX8ANWCOl3FRLXnOBuQB+fn5DIiMjq8W7urri5+eHh4fHRZVB5cSnouHUVWdSSvLy8khLS6OwsNACkrVOCgsLcXVtfeYUUVSEy/924Lx7NwAF4ycRFzyZ7DN22DlBwBCBe3DLd6Sao76iSqLYkr2FDF0GlzlfxvVe1+Nt591MErYumlJf48ePt8gcQW1vVU2tYwcMASYATsDvQog/pJSnqt0k5dvA26BNFtecLKmoqCAxMZGkpKSLClRaWoqjo2NjnqHdc7E6c3R0ZODAgdjb27ewVK2X1jZZbCgtJfvDD8l6+x0MhYV4XHcdHvfez+dvn6Uwvoz+44IZMSMEByfLLCBsSn0lFiTy8v6X+Sn9J7q5d+P/hv0flwdd3rwCtjLM9X6Z89tPBExnZ4KB5FrSZEopi4AiIcQvwEDgFI3A3t6e7t2715tu9+7dDBo0qDFZt3tUnVknUq8n78uvyHj9dXSpqbiMHYPXgw/j1r8XAP3H6gkM88Q/xPrcKpfqSnn/2Pu8f+x9bIQNDw95mDt634G9reqQXCrmVAT7gTAhRHcgCZiNNidgylfAWiGEHeAADAdWmVEmhaJNI6Wk8OefyVj5KmUxMTj270/AsmXElwfz3YbTXPvvAPy7ezB4cldLi9popJT8lPATL+9/maTCJK7udjWPRDyCv4v1+jpqLZhNEUgpdUKIB4GdaMtH35dSRgkh5hnj10spTwghdgBHAAPaEtNjdeeqUCjqouTwYdJfWUnx/v3Yd+1C0OpVVAwYzXcfR5NyOpqgcC+cXK2z13w2/yzL9i1jT9IeQj1DeX/y+wz1r9+Hl6JhmNUwKKX8Fvi2Rtj6GtcrgBXmlEOhaMuUx8eTvmo1BTt3Yuvtjd8zT+M1cyaHfkhi33/2Y+9oy4S7ehM+wt/qVtYVVxTz7tF3+SDqAzrYdmDR0EXM7jUbexvrVGitFeViQqGwUnSZmWS+8QY5n/0X4eBAp/vvx3vOHGxdzzs6DB3iy6ibwnB2t8wpeZeKlJLvzn7Hiv0rSCtOY3qP6Tw85GE6OXWytGhtEqUIFAorw1BURNaGD8h+/30MZWV43nwTPg88gM7Zk1++iKVL3470GOzL4MldrW4EABCXG8eL+17kz5Q/6eXdixVjVzDIVy1YMCdKESgUVoKsqCD388/JWPcG+sxM3CZNwmfBAhy6dyP2YDq/fvYnpYUVuPtovnSsTQkUVRSx/vB6Pjr+EU72Tjw1/Clu7nkztjZq74+5UYpAoWjlSCkp+G4XGatWUR4fj1PEEPzWvo7TZZeRn1XCrjeOcPZoFj5d3Jj24EB8urR+x4rb47az5tAaUopSCPg8gDHBY/jx3I9klmRyQ9gNPDT4Ibwd2+amsNaIUgQKRSumeP9+0l55hdLDR3AI7UHwG2/gOn5cVW8/LS6fpOgcRt0UyoDxwdjYtn430dvjtrNk7xJK9drhSClFKWyO3kyQSxAfX/Mx/X36W1jC9odSBApFK6QsJob0la9SuHs3dr6+BPzfC3hcdx3Czo6MhAJyUovoOdSf0AhfAnt64uJhHT51KgwVrDywskoJmGLAoJSAhVCKQKFoRVSkppLx+uvkbf0SG2dnfB55BO87bsfGyYmKcj37v4jl7+8TcPPuQI/Bvtja2rRaJZBdms2pnFNEZ0dzKucUp3JOcTr3NBWGilrTpxaltrCEikqUIlAoWgH6/Hyy3nmX7E2bwGDA+4476DjvXuy8vAA4dzyLnz+JJj+zlD6jAhh5Qyi2rcQMVGGo4EzeGa2xz9Ya/OicaDJLMqvS+Dj50NOrJyP7jGRrzFZyy3IvyEftELYcShEoFBbEUF5OziefkPXmevR5ebhPm4bP/IdwCA6uSpObVsy21w/j6evMdY8MIqinl8XkzSrJIjonmpicmKre/um80+gMOgDsbewJ9Qzl8sDLCfcKp6d3T3p69aw28RvuFV5tjgDA0daR+YPnt/jzKDSUIlAoLIA0GMjfvp2M1WuoSErC5fLL8X3sURz79NHipSQ9vgC/7u54+jlzzX0D6Nzby2xHRtakQl9BXF5clUmnstHPKs2qSuPr5EtP756MChqlNfpePenq0bXeXb9TQ6YCnF815BLA/MHzq8IVLU+9ikAI4Qe8CARKKa8WQvQBRkop3zO7dApFG6Rwz2+kr1xJ2YkTdOjTm85Ln8d11Kiq+Nz0Yn7+JJrE6BxmPTWUTsFudB9gvh21mSWZ1Uw6p3JOEZcXV9XLd7BxoIdnD64IvoKeXj2r/rwcL31kMjVkKlNDprY6t93tlYaMCD4ANgBPGa9PAZsBpQgUikZQEhVFxsqVFO39HfugIAJXvIz71KkIG83Wr9cb+HvXOfZvj8fWVjD2luY9M7iylx+dE82p7PONfnZpdlUaX2dfwr3CuSLoCsK9jb18967Y2SjjQVumId9uJynlZ0KIxVDlVVRvZrkUijZDeWIiGavXkP/NN9h6euK3+Ak8b7kFG4fz/n+kQbL1lUOkncmnxyAfrpjVExfPS1sNJKUkqzSrarVOZYN/JvcMOnm+lx/qFcrY4LH09OpJuHc4YZ5heDp6NsszK6yLhiiCIiFER4yniwkhRgB5ZpVKoWgD6HJyyHzzTXI+jUTY2tJx7lw6/usebN3O7/ytKNdjZ2+DsBH0vjyAwZO7EnJZw88MLteXa718k0Y/JiemWi/fz9mPcO9wxgaPrbLld3Hvonr5iioa8iY8AnwN9BBC/Ab4ADeZVSqFwsrI27aN9FWr8U1OJiYgAKeBAynaswdDcTEeN1yPz7//jb2fX7V7zhzO4JfIU4y6KYzQIb70vSKozvyllGSUZFywLv9M3hn0Uhugd7DtQKhnKOM6j6tmy/foYH2nkClalnoVgZTykBBiLBCOdg5xtJSy9h0hCkU7JG/bNlKeeRZZWooAdCkpFKSk0KF3b4JeXk6HsLBq6Yvyyvg18hSn/8rAO9AFN+/qZ0KX6cs4nXv6/Iod40RuTllOVZoAlwB6evVkfOfxVUs0u7p1VQ7aFJdEQ1YN3VkjaLAQAinlJjPJpFBYFemrViNLL3SZoM/Lu0AJRP+Zyi+Rp9BXGBg+I4SgUY6cyj/MtqPRVY1+fH58VS/f0daRUM9QruxyJWFeYYR7hRPmFaZ6+YpmpSGmIdPz4ByBCcAhQCkCRbtHSokuObnWOF1KStXnMn0ZsbmxHE4/Q7Gnjuh+u/k47yC5W8/vsA10CaSnV08mdJ1QZdbp4tZF9fIVZqchpqF/m14LITyAD80mkUJhJRiKikh5/vk64wu8HFn04+PoDnqQZkjhiP9ukODUw4kwxzAmBEw4v2LHKwx3B/eWE16hMOFSlg0UA2H1plIo2jBlMTGcmz+fivh4DvS2p39MBY668/GldhA5Igi/byJwLe5ISO8c7ho7jXDvcIJdg1UvX9GqaMgcwTaMS0cBG6AP8FlDMhdCTAHWALbAu1LKZTXixwFfAWeMQV9IKZc2SPJG8OVfSazYGU1SbglBf/zIwsnhXDeo7hUaCtj/9Vt0PrSCMTKD1N0+JAxeyNDp91paLItQri+vcqoWmxuL7c5fGfPpSUocJGtm2RDVTTIqSnDrbknHfEj3cmJPxPX0KB6FeydHxt4TTpc+HS39GK0K9X41DnPXV0NGBK+YfNYBZ6WUifXdJISwBdYBE4FEYL8Q4msp5fEaSX+VUl7bUIEby5d/JbH4i6OUVGiTb0m5JSz+4iiAUgZ1sP/rt+h38GmcRDkI8CcDj4NPsx/a9I9Vb9CTVJhETE4MMbkxxOTEEJsby9n8s+ilHvsKyT27YPxhPak9O3LusZu4r/sgnv7pcX7rW8RvfbV8/AqCmB41nJNBv/Lqoqex76B6/6a01/frUmmJ+hJSyvpTXUrGQowElkgpJxuvK3cmv2SSZhzwWGMUQUREhDxw4ECD5Ri17EeScksuCHewtWFQF7WLsiauhnyWp95DJ5F/QVyR7MB+5yssIFXzIpHk2+hJtisjxa5c+2+vfa4Q2u9BSOiotydA50CgrgOhmTYM25mNU3Y5hQM6UTTIB2y0U8KyxH5ed++CT35vTvr9AYB3qRsP5SXSUQ6tU472ytDiX3ERZReEt5X3q7mpq75S8cF/SWyD8xFCHJRSRtQWV+eIQAhRwHmTULUoQEop65vZCgISTK4TgeG1pBsphDgMJKMphahaZJkLzAXw8/Nj9+7d9RR9ntqUAEC53kBu7oU+0dsNUuIrM+kh4wk1xNPDEE8PGY+vzNK+4VpwpoxepX+3rJxNpNAG4u1tOONgQ7y9MP63ocD2/EN66yTdKgwMLTbQrcJA93JJlwoDTsa3v/ysDUX7bBE24DJWh1dgMpRrK4WkFKQVXcHMM7dRLiRnvY7gKfKZn3+Ga4qKSbOxrvpqCZy5sFGrDLe296slqKu+fGVmo9rCi1GnIpBSNvUE7Nqak5qK5RDQVUpZKIS4BviSWiaipZRvA2+DNiJojLfCoD9qHxEEeTqx8/ErG5yPVaPXQeYpSD0CqUch5bD2v+pwEAGdwsB/PAQMIHvXSrxr8SKSJnzwf67hPZCWpExfxpm8MxeYdUxPvXKxdyHUM5TJXmGEeobS06snoZ6hdXrRNJSXk75sOUV7P8HpsssIWvUq9gEBVfEZCQXs/ugk6akFdO1wgDHub7MgKaMqPrUV15clSV0Sij8ZF4S35vfLktRVX+miU7N5bm3wqiEhhC/aPgIApJTn6rklEehsch2M1uuvQkqZb/L5WyHEG0KITlLKTJqJhZPDq80RADjZ27JwcnhzFdG6KC+CtCit0U85ov1POw56Y6/CzhF8+0Df68C/P/gPBL8+4OBSlcXpLEecKm2SRkqkAwlDFmLpM6T0Bj0JBQnE5sZWa/TPFZzDIA2AdjhKiEcIQ/yGEOYZRphXGGGeYfi7+Fcd+l4f5QkJJC14mNKoKLzvvhvfRx5G2J/3s19WomPrykPYOdjSY1AKY5JW4GzT+uqrNZIweCEerfT9ao20RH01ZNXQdGAlEAikA12BE0Dfem7dD4QJIboDScBs4NYaefsDaVJKKYQYhrYqKeuCnJpA5YRw1aohT6e2s2qoKPN8776y4c+KpWrg5egJAQNg2L/Af4D2uWMY2F78ax86/V72A50PrcBXZpIuOpEwpGVXdUgpSS9Ov6DBj8uLo8yo1ASCzm6dtV5+t8mEeoXS07Mnnd0713s4ysUo+P57khc/CUIQvG4tbhMmVMWln83Hp4sbHZzsmPyvfvh1c8fRZTT7vy60aH1ZE63h/bImWqK+6p0sNtrvrwS+l1IOEkKMB26RUs6tN3PN3LMabfno+1LK/wgh5gFIKdcLIR4E7kNbjVQCPCKl3HuxPBs7WWyK1R6CISXkxFdv8FOPQoHJAMujs9bY+/fXGnz/AeARDA3sAddFS9RZXlkesbmxxObEVjPr5Jefn7D2cfIhzGjSqezhd/fojrO9c7PJISsqSF/5KtkffIBjv34ErV5VdWRkSUE5ez6P4dSfaVxzX3+6D6zdQ6jVvmMWQtVX42hKfV3SZLEJFVLKLCGEjRDCRkr5kxBieUMKllJ+C3xbI2y9yee1wNqG5NVu0FdARnT1Bj/1KJQZbfbCFjr1hO5XGE07xsbf2fvi+bYCSnWlxOXFXdDLTy9Or0rjau9KmFcYk7tNPt/wt4Cf/IrkZJIefoSSw4fxuv12fBctxMbBASkl0X+m8tt/Yykv1RFxTTc692n9da1QNIaGKIJcIYQr8CvwsRAiHa0Hr2gqZQWaPT/lCKQaTTzpJ0BvtAXaOYF/P+h/43nTjm8fsHeyrNz1oDPoSChIqOrZVzb8pnZ8BxsHQjxDGO4/nFCv0Cpbvp+zX4Pt+M1F4c8/k7zocaROR9DqVbhPmVIVt+v948TsTyOghwfjbuuFd6DLRXJSKKyTiy0fXQt8CsxAM9ssAG4DPIBm3/3b5ilMr97gpxyB7Diq7PnOHbXGfvg8CBiofe7YAyzkimB73Pbzh4t/Xvvh4lJK0orTqhr8yv+nc09TbtCUmUDQxb0LYZ5hTOk+hTDPMEK9QuniZvmDUaROR8Zrr5P19tt06NWL4NWrcOjWDb3egABsbG3oMdiHwDBP+o4ORNi0rIJSKFqKi/0SY9B2FQegnVH8qZRyY4tIZc0YDJBzxmSppnHlTmHa+TSeXTVzzsDZ50077oFNtuc3F9vjtrNk7xJK9Zpr5ZSiFJ7b+xxxuXH4OPucb/hzYygoL6i6z9fZlzDPMIb3Hl5lyw/xCMHRzrGuoixGRVo6yY8+SvGBA3jefDN+Tz2JjaMjqWfy2P1RNOHD/Rk0qQs9BvlaWlSFwuxcbB/BGmCNEKIr2oqfDUIIR+ATYLOU8lQLydh60ZVDxgkTW/4RSD0GlY2jjR349IIeV5437fj1A6fWvaN5zaE1VUqgkjJ9GW8ffRsANwc3wjzDuKb7NVUNfqhnqNX4yC/6/XeSHluIobiYwOXL8Jgxg/JSHX9sPsXR3Ym4eHTAy7/5JqEVitZOQ9xQnwWWA8uFEIOA94ElaCuB2g+l+ecnbisncjNOgsF4WJu9i2bPHzjbuGqnP/j0BvvW1xuuCykl+1L3kVKUUmeaXTftsogdvzmQej2Z69eTuXYdDiEhdN34AR1CQ0k8mc0PG09QmFtG/7HBjJgRgoOTOs9X0X5oyD4Ce2AK2qhgAvAzULcTdmtHSihINfbuTTZl5cSfT+Pio/Xww64ymnYGgHcI2NhYTOymUGGoYGf8TjZFbeJE9glssMGA4YJ0AS4B+LtY55YfXVYWyQsXUbR3Lx4zpuP/3HPYOGu9fjsHWzq42DP5X/3wD7GOUY1C0ZxcbLJ4InALMBXYB0QCc6WURS0kW/Nx5DP4YSlj8xLhr2CY8CwMmKnZ87NPV2/wU49Ckcl2bu8QbfJ20B3GSdz+4GadjWFNCsoL2HJqCx+d+Ii04jS6e3Rnycgl2NrY8p8//lPNPORo68j8wfMtKO2lU3zgAEmPPIo+Lw//F5biccONnNibQn5WCiOv64F/iAeznhyqJoMV7ZaLjQieRJsPeExKmd1C8jQ/Rz6DbQ9BRYnm/CgvAb6cB7uXaT3/CqNes7EH314QNvm8acevHzi2vVOjUgpT+OjER2yJ2UJRRRHD/Ifx7MhnGR00GhuhjWrsbezPrxpyqX3VUGtHGgxkvfceGavX4BAcTOe336LYvTNfvvoXKafzCO7lhUFvwMbWRikBRbvmYpPF41tSELPxw1KoqOF0zqCHvESIuPv8qh2fXmDnYBkZW4iozCg2Rm3ku7PfATC522Tu6nsXfTr2uSDt1JCpTA2ZarU7P3U5OaQ8sZjCn3/GbcoUfJc8z1+/ZnJoxz7sHW2ZcFdvwkc03PeQQtGWafszYnl1nKGjL4erG7RB2qoxSAO/JP7CxqiNHEg7gIu9C7f3vp3bet9GgGtA/RlYISWHD5P48MPoMjLxe+ZpvG69lYKsUv7+7hyhQ3wZdVMYzu5tW+krFI2h7SsCj2DNHFRbeBumVFfKtrhtbIraRHx+PP4u/jwW8Rg3ht2Iq4OrpcUzC1JKcj78kLQVr2Dv60vA+x9yrrAjXoB7JydufX4Ebt7Ws4pLoWgp2r4imPBs1RxBFfZOWngbJLs0m80nNxMZHUl2aTa9vXuz/IrlTOw2sUkeOVs7+oICUp58ioJdu3AZfyUltzzGF/9NorQwm8AwTzoFuyoloFDUQdtXBANmav9/WIrMS0R4mKwaakOcyTvDpuOb2HZ6G2X6MsYGj+WuvncR4RfR5u3gJVFRJC14mIrkZFzmL+aIfiBnPzmDTxc3pj04kE7BbXMEpFA0F21fEYDW6A+Yyc9WOvFZF1JKDqQdYFPUJnYn7sbBxoFpPaZxZ587CfEMsbR4ZkdKSe7mzaS9+BK23t503riJrV+VUZyfy+ibw+g/LggbW+vc26FQtCTtQxG0MXQGHbvO7mJj1EaisqLw6uDFvIHzmB0+m45OHS0tXougLywi9bnnyN++HcMVUwl56UkcOnlzpXsObh0dce/Yuj20KhStCaUIrIiiiiK2nNrCxyc+JrkomW7u3XhmxDNM7zG9VTp2Mxel0adIWrCAkoRk0ma+QHSmN7rDRQyc4E1Qz9rPH1YoFHWjFIEVkFqUyscnPubzU59TWFHIEL8hPDHsCcZ2Hlu1Aay9kLvlC1JfeIEc/4GcmrKQwnRJn1EBhI9oG7u9FQpLoBRBK+ZE1gk2Ht/IzjM7kUgmdp3IXX3vol+nfpYWrcUxlJSQuvQF8rZuJWHUXGLsB+Lp4sR1c8PVKEChaCJKEbQyDNLAnqQ9bIraxJ+pf+Js58zsXrO5vc/tBLkGWVo8i1AWF0fi/AWUno7H9/778Lz6djxO5DLk6q7Y2bcvJ7gKhTlQiqCVUKYvY3vcdjZFbeJ03ml8nX15ZMgj3NjzRtwd2p6/o4aSt+0bTr/4GtEhN+NzTxh9HxoHQECYOjdYoWguzKoIhBBTgDVoZxe8K6VcVke6ocAfwCwp5efmlKm1kVuay+bozXxy8hOyS7MJ9wrnxdEvMqXbFOxt2+4GsPowlJWR8uIyjvyeRfyAx7B1dKB/RJilxVIo2iRmUwRCCFtgHTARSAT2CyG+llIeryXdcmCnuWRpjZzNP8uHxz/kq9ivKNWXMjpoNHf1vYvh/sPb/Aaw+ig/d44Tj77AXx3GUBRyBSEDOzLmll64eHawtGgKRZvEnCOCYUCslDIOQAgRCcwAjtdI929gCzDUjLK0CqSU/J3xNx8c+4CfEn7CzsaOa0Ou5c4+dxLqFWpp8VoF+d99R8qTT2Fw6oTN8CCuvr0/IZf5WFoshaJNI6SU5slYiJuAKVLKe4zXdwDDpZQPmqQJQjvz4ErgPeCb2kxDQoi5wFwAPz+/IZGRkZckU2FhIa6uLe9uQC/1HCk+wo/5PxJfHo+zjTNXuF3BGLcxuNu2bvt/i9WZTkfFlt8pTJKElP9O3r/uQe/tbXWjI0u9Y9aKqq/G0ZT6Gj9+/EEpZURtceYcEdT2C66pdVYDj0sp9Rf7wUsp3wbeBoiIiJCX6iaipX3rF1cUszV2Kx8e/5CkwiQ6u3XmqUFPMb3HdJztreNw9Jaos7yTZ/n+5V2kOozCvWsxyTLlsgAAFGdJREFUPZc+ipOXi1nLNBfWen6DpVD11TjMVV/mVASJQGeT62AguUaaCCDSqAQ6AdcIIXRSyi/NKJfZSStK45OTn/DfU/+loLyAQb6DWBixkHGdx2Fro5Y7ViINkkPv/cSBP0sw2HXmsj56RjxwDbbKP5BC0aKYUxHsB8KEEN2BJGA2cKtpAill98rPQogP0ExDVqsEorOj2XR8E9+e+RaDNDChywTu6nsXA30GWlq0VofU6Uh+9XUOneyOh8xjwgOX4zNIrQpSKCyB2RSBlFInhHgQbTWQLfC+lDJKCDHPGL/eXGW3JFJK9ibvZWPURn5P+R0nOydm9pzJ7X1up7Nb5/ozaGfoKwwc+d//t3fn0VHWZxvHv3cme0IWyA4EoSAoGBFQ61aKthVEiih1wYraWmotCvpalxbbal1Q2wJVi3VpRSgi7looVhEUEVoWKQgECEQkJJAAIfs2mfv9IwONMUgSGJ5Z7s85njN5lvHidyZz5dlzSZz/CHWrVzNs7Hh6TrkdV3To3CvJGH/j0+sIVHUhsLDFtFYLQFVv8GWW462+sZ6F+Qt5cdOLbCvdRmpMKpMGTeIHJ/+AxKhEp+P5pcJtpSx+dh3l5UpOoZLz+GMkjhrldCxjQp5dWdxOZXVlvLL1FeZunktJTQl9kvvw4HkPMqLnCCJd9hzc1tRWNbDi9Tw2LS8iumYfQ6o+YuBzDxDVK/ifmWBMILAiaKNd5buYvXk2b+a9SY27hnOzzuXB8x7knKxzAu4UxxNt0VOfUri9nOxdizn9tDC6/eYxwmID46wpY0KBFcFRrCtex4ubXmTxF4sJkzAu6XkJ408dT9/OfZ2O5tfK99cQFRuBe8Naur33e7pWVtHnzh+TdMXlTkczxrRgRdCKRk8jS3YtYdbGWawrWUenyE7c2P9Gxp0yjrTYNKfj+TWPR9mwpICVb2+nZ8IBur92H8nZ2XR9YTrRfa08jfFHVgTNVDdU89b2t5i9aTa7KnbRNb4r95x1D2N6jwmYC8CcVLKrgqVzcineWUGaFpLy4UwShg8n44EHcMUH5gVixoQCKwJgX80+5m6ey/yt8ymrKyMnJYfJgyZzUfZFdgFYG23+pIglc3KJioKc3a+R8vkyMn55L0lXX23HUIzxcyFdBHmleczaNIsFOxbg9ri5MPtCru9/PQNTB9qXVxs1uj24wsPI7J3IN1LKyXz7YWJTEuj60kvEDOjvdDxjTBuERBEs2LGAGWtnUFRVROarmVzS8xJyS3NZvns50a5oLu9zOdedeh09Eno4HTVg1FTU8/Gr26irdjP82h5U3D+F7MWLif/ORWQ9/DCuBP++mZ4x5n+CvggW7FjAbz/5LbWNtQAUVRXx/GfPExcex61n3MqVJ19JUnSSwykDh6qy5d97WP5KHvW1bnLOiCF/7JW49xSRfu89JI8fb1tTxgSYoC+CGWtnHC6B5jpFdWJCzgQHEgWuytI6Fs/aREFuKRm9EhiUtJ26GQ9BSgonzZlNzMCBTkc0xnRA0BfBnqo9rU7fW7X3BCcJfBFRYVSW1nHB5SeRvPAJKv+6iLih3yJr6lTCk5OdjmeM6aCgv99vRlxGu6abL6ver/zr+Y00NnqIio1gzFWJxP3hFirfe4/UO+6g+8yZVgLGBLigL4JJgyYR7frynS2jXdFMGjTJoUSBob7WzUcvbyX/PaVw20HKims4+OqrfHHNNXiqq+nxwt9ImfATJCzoP0LGBL2g3zU0stdIgP+dNRSXyaRBkw5PN1+V/98SPpq3lcqDdXTuA2NuPI0Dj/6OsrfeIu7cc8h6/HHCu3RxOqYx5jgJ+iKApjIY2WukPRavDTyNHla+tYNwdw1n7ZxF3JLV5P8tHNxuUiZOJOVnNyMuu8jOmGBi2/UG9SiblhdSV+MmzBXG0H4lnPH+3cTvWN304Gm3G4mMJLJHtpWAMUHIiiDEHSis4o0/rGXJ7Fw2L296pHTVk48jNdVfWk7r6ymeNt2JiMYYHwuJXUPmq9wNjaxZtJO1i3YSEe3iwvGncPIZSZT86QncxcWtr1NUdIJTGmNOBCuCEPXx/G1sXFbIyWelc+4VvfGsWkb+pVNp2L0biYlBa2q+sk54ZqYDSY0xvmZFEEJqqxpodHuIS4xi0MU96DUwlfTYcvbeeStVy5YR1ac32bNm4S7eS9F9v0Zr/3dFtkRHk3b7ZAfTG2N8xadFICLDgRmAC3hOVae2mD8a+B3gAdzAZFX92JeZQpGqkremmGXzt5F+UgIjb8khPlapXTSLHS+8QFhUVNN9gsaNQyIiDq9XPG06DYWFRGRlkXb7ZHvQvDFBymdFICIu4Cngu0ABsEpE3lbVTc0WWwy8raoqIjnAfKCfrzKFoooDtXz40hZ2bthPanYnzrz0JMoXLWLv1Edx79lD4ujRpN35f4Snpn5pvcRRo0gcNcpOuTUmBPhyi+AsIE9VdwCIyDxgNHC4CFS1stnycYD6ME/IKcg9wIKZG0CV88b2pm92HcW/mcT+FSuJ6tePrn/8A7GDBjkd0xjjMFH1zXeviIwFhqvqTd6frwPOVtWJLZYbAzwCpAEjVXVFK+81AZgAkJ6ePnjevHkdylRZWUl8fHyH1g0k6lEkTGisV4rWKul96klesoDYDz5Ao6Oo/P73qbngAmjDNQGhMmbHi41X+9h4tc+xjNewYcPWqOqQ1ub5cougtZvSf6V1VPUN4A0R+RZNxwu+08oyzwDPAAwZMkQ7uqsi2HdzNNQ3suof+RTklnLF3YMJCxPK6/5B8WOP4y4pIXHsFaTdcQfhnTu3+T2DfcyONxuv9rHxah9fjZcvi6AA6N7s525A4ZEWVtWPROQbIpKiqvt8mCsofbFpPx/O3UL5vlpOOS+T6k1b2P/oQ9SsXkP0gAF0e+pJYnJynI5pjPFDviyCVUAfEekJ7AauBsY1X0BEegPbvQeLBwGRwH4fZgo69TVuPpy3ha3/3ktSeiyjfnoykQtfpGDqXFzx8WTcfz9JY6+wW0MYY47IZ0Wgqm4RmQi8S9Ppo39V1Y0icrN3/tPAFcB4EWkAaoCr1FcHLYKUKzKM0qJqhozoQe/69eyfeCdVBw6QdPVVpN52mz0rwBhzVD69jkBVFwILW0x7utnrR4FHfZkhGB0sruY/7+QzdFxfomLCGTkqhpKHplC8bh0xp59O+jN/IaZ/f6djGmMChF1ZHEAaGz2se+8LVi34HJdLKN6cQPjbz3Pw5fm4kpPJfPhhEi8bbQ+LMca0ixVBgNiTX8bSObns311Fr4Ep5MRtpeqWe2gsLyf5hz8k9daJuBISnI5pjAlAVgQBYvWCz6mtcnPR8E5Ezfkd5Z99RsyQwWTcdx/Rffs6Hc8YE8CsCPxY/vp9dMmKIyElhgsuzaDs2ZlU3TufhpQuZD3+GAmXXopIa5drGGNM21kR+KGqsjqWvbyV7WtLOG1oFgM8ayiZ8Sc81dV0vuEGUn5+Cy67GtMYc5xYEfgR9SgbPy5kxRvbaWzwMPjMaFLmT2Hv5o3EfvObZEz5FVG9ezsd0xgTZKwI/Min73/Bite3k9Urjv7F79L4+EtoRgZdp0+j08UX224gY4xPWBE4rLHBQ1V5HQldYjj17DR0/X+Im3c3jXV1dJkwgZSfTiAsLs7pmMaYIGZF4KDCbaUsmbMFV3gYI7+rlDz0ELHbthF7/vmk/+qXRPXs6XREY0wIsCJwQG1VAytez2PT8iI6JUVwysHlFNwwi4isLLo9+QTxF11ku4GMMSeMFcEJdqCwijenf0ptRT39Mg6S+Y/HCKuvocstt9DlJzcRFhPjdERjTIixIjhBPI0ewlxhJKbHkJXiJm3LbGKWrCR+2DDSf3kvkd27H/1NjDHGB6wIfMzjUTYsKeCzj3Yz+vquHJz+e3r8619EZGeT/vRMOtlDOYwxDrMi8KGSXRUsnZNL8c4KMjtVsf2qu4hqqCB10m10/tGPCIuKcjqiMcZYEfiCp9HDyrd2sO79XURFKDnFb9Nl6bskfO97pN99FxFduzod0RhjDrMi8AEJE/Zt30d3zw56LP4zcd3TSX/uOeLPP8/paMYY8xVWBMdJTUU9K97czqALM2l4fQ695zxPmCuM1Mk/o/P48UhkpNMRjTGmVVYEx0hV2fLvPSx/JY/66nrCX5tJau57JF5yCWl3/YKIjAynIxpjzNeyIjgGB4ur+XDuFgpyS0nWEnJW/YUuXeNInzWLuLPPcjqeMca0iRXBMVj3bj57tu6j7/bX6XZgNam3/ZzO116LREQ4Hc0YY9rMp0UgIsOBGYALeE5Vp7aYfy1wt/fHSuBnqvpfX2Y6VnvzyxEXRG9aQdoL00kqKSVtxFDS7lxIeGqq0/GMMabdfFYEIuICngK+CxQAq0TkbVXd1GyxfGCoqpaKyAjgGeBsX2U6FvW1bla+tYMNSwpI8+xmwEcPE92vHyc99hSxgwc7Hc8YYzrMl1sEZwF5qroDQETmAaOBw0Wgqp80W34l0M2HeTosf/0+Pvx7LlVldXQrXEbv4g9Iv28KyVddhYTb3jVjTGATVfXNG4uMBYar6k3en68DzlbViUdY/k6g36HlW8ybAEwASE9PHzxv3rwOZaqsrCS+nY94LN/lYddyiK3ZyymbZxN5WiYVo0ejCQkdyhBoOjJmoczGq31svNrnWMZr2LBha1R1SGvzfPnnbGv3UW61dURkGPBj4PzW5qvqMzTtNmLIkCH67Q7en2fp0qW0ZV31KBUHaoncv4ui5x4kpjCaXskHyXr2EWJOP71D/+9A1dYxM01svNrHxqt9fDVeviyCAqD5LTW7AYUtFxKRHOA5YISq7vdhnjY5UFTFklkbObjrAGctm0JUTDhD7riDpLFXIC6X0/GMMea482URrAL6iEhPYDdwNTCu+QIikg28Dlynqlt9mOWoGhs8rP5nPmv/+Tmuhhp6571GlzGXkjZ5EuHJyU5GM8YYn/JZEaiqW0QmAu/SdProX1V1o4jc7J3/NPBroAvwZ+8TudxH2oflSzUV9bz28ArKShtJ37uK06JyyX7qF8T073+ioxhjzAnn01NeVHUhsLDFtKebvb4J+MrB4RPF41G0vIzS6TOI2yj0qt3OqTdfRuJldyFhYU7FMsaYEyokz31UVfLW7OWTv29g4JppRO7byTnXXkvqrU/gCpGzgYwx5pCQKIKyd96heNp00goLWZ/dj7zTb6SoIp5OFQWE9ehJz2enEd23r9MxjTHGEUFfBGXvvEPRfb9Ga2sp6DqUHT1GowehT+E7nDnhQpK+/yTe4xPGGBOSgr4IiqdNR2trAaiOzSTpYB4nb5tHp6RIkkdPczidMcY4L+iLwF1UdPh1n7xXEG1EAPde2wowxhiAoD81Jjwz8/DrMG8JtJxujDGhLOiLIO32yUh09JemSXQ0abdPdiiRMcb4l6DfNZQ4ahTQdKygobCQiKws0m6ffHi6McaEuqAvAmgqg8RRo+wGV8YY04qg3zVkjDHm61kRGGNMiLMiMMaYEGdFYIwxIc6KwBhjQpzPnlnsKyJSAuzs4OopwL7jGCcU2Ji1j41X+9h4tc+xjFcPVU1tbUbAFcGxEJHVTjz4JpDZmLWPjVf72Hi1j6/Gy3YNGWNMiLMiMMaYEBdqRfCM0wECkI1Z+9h4tY+NV/v4ZLxC6hiBMcaYrwq1LQJjjDEtWBEYY0yIC5kiEJExIqIi0s/pLP5ORBpFZJ2I/FdE1orIuU5n8ncikiEi80Rku4hsEpGFInKy07n8UbPP10bvZ+wOEQmZ76KOaDZmh/6757i+f6gcIxCR+UAmsFhVf+twHL8mIpWqGu99fTHwS1Ud6nAsvyUiAnwCzFLVp73TBgKdVHWZo+H8UIvPVxowF1iuqr9xNpn/aj5mvhASLSwi8cB5wI+Bqx2OE2gSgFKnQ/i5YUDDoRIAUNV1VgJHp6rFwARgordQjQNC4sE0wGXAIlXdKiIHRGSQqq51OpQfixGRdUA0TVtRFzqcx98NANY4HSJQqeoO766hNGCv03n81KHfyUMeUdWXj9ebh0oRXANM976e5/3ZiuDIalR1IICInAO8KCIDNFT2Ixon2NbA1zv8O+kLQV8EItKFpr9oB4iIAi5AReQu+2I7OlVdISIpQCpQ7HQeP7URGOt0iEAlIr2ARuzz5ZhQOEYwFnhRVXuo6kmq2h3IB853OFdA8J5l5QL2O53Fj30ARInITw5NEJEzRcQOsB+FiKQCTwNP2h9mzgn6LQKadgNNbTHtNWAcYAfzWtd8f6QA16tqo5OB/JmqqoiMAaZ7T+urBT4HJjsazH8d+nxFAG5gNvBHZyP5vZbHCBap6nE7hTRkTh81xhjTulDYNWSMMeZrWBEYY0yIsyIwxpgQZ0VgjDEhzorAGGNCnBWBMV9DRH7lvUvmeu9dH88WkckiEtuGddu0nDFOs9NHjTkC7+01/gh8W1XrvFdYR9J0p9EhqrrvKOt/3pbljHGabREYc2SZwD5VrQPwfqGPBbKAJSKyBEBEZorIau+Ww/3eabe1stz3RGSF9xkPr3jvimuM42yLwJgj8H5RfwzEAu8DL6vqhy3/0heRzqp6QERcwGLgNlVd33w579bE68AIVa0SkbuBKFV9wIF/mjFfEgq3mDCmQ1S1UkQGAxfQ9MyBl4/wZKgrRWQCTb9PmcCpwPoWy3zTO32597b7kcAKX2U3pj2sCIz5Gt57LC0FlorIBuD65vNFpCdwJ3CmqpaKyAs0PcehJQHeU9VrfJvYmPazYwTGHIGI9BWRPs0mDQR2AhVAJ++0BKAKKBORdGBEs+WbL7cSOE9EenvfO9aeaWz8hW0RGHNk8cATIpJE010y82h6rOI1wD9FpEhVh4nIpzQ9k2AHsLzZ+s+0WO4G4CURifLOnwJsPUH/FmOOyA4WG2NMiLNdQ8YYE+KsCIwxJsRZERhjTIizIjDGmBBnRWCMMSHOisAYY0KcFYExxoS4/weckn4FdnY0/AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states = game.states[1:size+1]\n",
    "true_values = pd.Series([x*(1/(size+1)) for x in range(size+2)][1:size+1], \n",
    "                       index=states)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "\n",
    "for i, value_function in value_sets.items():\n",
    "    values = list(value_function.values())[1:size+1]\n",
    "    ax.plot(states, values, marker='o', label=str(i))\n",
    "ax.plot(true_values.index, true_values, linestyle='--', label='True values')\n",
    "plt.title('TD(0) Value Updates on Random Walk')\n",
    "ax.set_xlabel('State')\n",
    "ax.set_ylabel('Value')\n",
    "plt.legend()\n",
    "plt.savefig(\"random_walk_td0.pdf\")\n",
    "plt.grid()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chapter 7: n-step Bootstrapping"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## n-step TD\n",
    "\n",
    "<IMG SRC=\"tdlambda.png\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example 7.1 Random walk\n",
    "\n",
    "n-step TD Updates on Random Walk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "size = 19\n",
    "game = RandomWalkGame(size=size)\n",
    "\n",
    "# Set rewards for this exercise\n",
    "game.terminal_rewards = {'T1': -1.0, 'T2': 1.0}\n",
    "\n",
    "# Initialization of parameters\n",
    "gamma = 1.0\n",
    "learning_rate = 0.1\n",
    "initial_value = 0.0\n",
    "n = 5\n",
    "\n",
    "# Initialise value function\n",
    "value_function = {\n",
    "    s: 0.0 if s in game.terminal_states else initial_value\n",
    "    for s in game.states\n",
    "}\n",
    "\n",
    "episodes_to_record = [0, 1, 10, 100]\n",
    "value_sets = {}\n",
    "\n",
    "np.random.seed(5)\n",
    "\n",
    "n_iters = 101\n",
    "for iteration in range(n_iters):\n",
    "    \n",
    "    if iteration in episodes_to_record:\n",
    "        value_sets[iteration] = value_function.copy()\n",
    "\n",
    "    past_states = [game.state]\n",
    "    past_rewards = [None]\n",
    "    \n",
    "    T = 999999\n",
    "    t = 0\n",
    "    while True:\n",
    "\n",
    "        # Behaviour policy\n",
    "        if t < T:\n",
    "            move = np.random.choice(game.available_moves())\n",
    "            game.make_move([1, move])\n",
    "            past_states.append(game.state)\n",
    "            if not game.game_over:\n",
    "                reward = game.get_rewards()[1]\n",
    "            else:\n",
    "                reward = game.get_terminal_rewards()[1]\n",
    "            past_rewards.append(reward)\n",
    "            if game.game_over:\n",
    "                T = t + 1\n",
    "\n",
    "        #import pdb; pdb.set_trace()\n",
    "\n",
    "        # State to be updated\n",
    "        tau = t - n + 1\n",
    "        \n",
    "        # TD(λ) update\n",
    "        if tau >= 0:\n",
    "            g = 0\n",
    "            for i in range(tau + 1, min(tau + n, T) + 1):\n",
    "                g += gamma**(i - tau - 1)*past_rewards[i]\n",
    "            if tau + n < T:\n",
    "                g += gamma**n*value_function[past_states[tau + n]]\n",
    "\n",
    "            value_function[past_states[tau]] += \\\n",
    "                learning_rate*(g - value_function[past_states[tau]])\n",
    "        \n",
    "        t += 1\n",
    "        if tau == T - 1:\n",
    "            break\n",
    "\n",
    "    game.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEWCAYAAABIVsEJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOydd3hUxdrAf5NNLyQkIR0IIRg6AiGIBUEQsAAWimLns+sV9coVFQG76L0XLFyxixWQoiAKClIURJpICS3U9EZ63935/jibsNlsyCbZZJMwv+c5T/ZMeeedZTnvmXln3hFSShQKhUKhqC9OjlZAoVAoFK0TZUAUCoVC0SCUAVEoFApFg1AGRKFQKBQNQhkQhUKhUDQIZUAUCoVC0SCUAVEghHATQsQLIUJsLHtYCBHUhPpsEkLc21Ty7YkQQgohoh2tR2tECHG3EOJ3R+tRX8x/n621D/ZCGZBWjBCi0OwyCiFKzO5vE0LMEUJUCCEKTNdRIcS7QohQC1H3A1uklGkmuZX1zOVHAUgpy4BPgKdr0elWIcQpIYSwSHcWQmQIIa63/zdhG0KIz4QQL1ukRZqMgHMTt90s7TQWk45Fpn/zZCHEf4UQOkfr1RiEEOuEEP8yuw839dNaWp0vUYpzKAPSipFSeldewBlgrFnaV6ZiS6SUPoA/cCMQAuy2MCIPAF9YiF9iLl9KecIs72vgLiGEmxW1VgJ+wJUW6WMACaxtSF8VzUo/02/qSmAyMNXB+jSWLVT/PQ4FDltJO1b5EqWwDWVALhCklBVSyoNoD4RM4J8AQohOQFfgz3rISgJygEus5JUCS4E7LbLuBL6SUuqFEO2FED8IITKFEDmmzxHW2jKNhr40u6/2Ji+E8BVCfCyESDW9Mb/cmDdm0+jpGdOUXo4Q4lMhhLtZ/nRTWylCiKkWda8TQvwlhMgXQiQKIeaYZW8x/c01vd0PMdWZKoQ4ZGprnRCisyldCCHmmUZteUKIfUKI3rXoHCaEWCWEOCuESBBC3Gfx/S0VQnxuGoUeFELE2vJdSCkTgK3AxWby7jHpWyCEOCGEeMAsb5gQIkkI8U+T3qlCiHvM8gNMeuYLIXag/e7M+3GpEGKnqb87hRCXmuVtMv3bbjN9f6tN8r4yydsphIispStbgMuEEJXPuyuA+UCsRdoWU1s2/z4tEUK8KYT4XQjha0v51o4yIBcYUkoD8D3afxiAPsAJKaXeouhY0wPpoBDiISuiDgH9amlmETBBCOEB2kMeGAt8bsp3Aj4FOgOdgBLg3QZ2aRGgB6KB/sAooLH+k9uA0WgPuIuAmQBCiDHAU8DVQDdgpEW9IjRD6QdcBzwkhLjBlDfU9NfPNKL7w5T3LHAT0AH4DfjGVG6Uqc5FJnmTgexa9P0GSALCgAnAq0KIEWb544DFJjmrsPG7FkJ0R/udJJglZwDXA+2Ae4B5QogBZvkhgC8QDvwfsEAI0d6UtwAoBULRRjVVBlgI4Q+sAd4GAoD/AmuEEAFmsm8B7jDJ7gr8gfY78kf7Pc6upSs7ADfO/V6HAr+Y+mWeVmnk6/37FEI4CSE+BPoCo6SUeecr31ZQBuTCJAXtPx1oD5UCi/ylQA+0h9p9wCwhxK0WZQpMdWsgpdwKpKNNmQFMAo5KKfea8rOllMullMVSygLgFWpOedWJECIYuAZ4XEpZJKXMAOahPWgaw7tSykQp5VmTbpV9nwR8KqU8IKUsAuaYV5JSbpJS7pdSGqWU+9Ae7Ofr1wPAa1LKQyYD/ipwsWkUUgH4AN0BYSqTailACNERuBx4WkpZavqOP0J70Fbyu5TyR9PLwxfUbvgr2SOEKEJ7KG8C/mfWxzVSyuNSYzPwM+deRjDp/aJpxPsjUAjEmEaFNwOzTP9WB9CMfyXXoU0hfSGl1Espv0GbZhprVuZTU9t5wE/AcSnletN39y3aC0QNTH67P4GhJkPlZ5qS/c0srSew2VS+vr9PF7R/a3+0aeTi85RtUygDcmESDpw1fc5Be1BVIaWMl1KmSCkNUsptwFtob7bm+AC552njc85NY92B2cNCCOEphHhfCHFaCJGP9ubn14Cpp85o/3lThRC5Qohc4H2gthVielN5c1wAo+mqJNHs82m0N3tMfy3zqhBCDBZCbDRNfeQBDwKBdej/lpnuZwEBhEspf0V7610ApAshPhBCtLMiIww4a3rQmesVbnZvPq9fDLiL8zvzBwDeaKOewYCXWR+vEUJsN41Oc4FrLfqYbTGaLTbJ6gA4U/v3F2Zxb60f6WafS6zce5+nT1vQRhlXAJWrpn43S0uUUp429bG+v89oYDzwgpSy/Dw6tDmUAbnAMM35jkV7+wLYB0TV8UCRaA82c3oAf5+nzufACNNc/yVojvdK/gnEAIOllO04N71j2QZo00KeZvfmq2QSgTIgUErpZ7raSSl71aLTGSDSIq0L2sPD3IB0NPvcCW3EBpBqJc+cr9GmiDpKKX2BhWZ9shb2OhF4wEx3Pymlh8loI6V8W0o5EOiFNpU13YqMFMBfCGH+EtAJSLZS1mZMI4ylaNNEs0Bbwg0sB/4NBEsp/YAfsf7vZkkmmgGv7ftLQTOoWOQ3qh9mbEEzFEM599vfClxG9ekrqN/vE7SR2j3AT0KIGDvp2ypQBuQCQQjhIoTogTbUDkGbY650iB8D4szKjjc5EoUQIg54DM1vUpkfjjZc315be6a3ud9N7f1isbrFB+2NMdc0fVDb3DXAXrRphk4mX8ozZm2kok2h/EcI0c40D91VCFHbdMNy4DohxCghhE4IEYbm31hsUe4RIUSESbdngSWm9KXA3UKInkIITyt6+6CNBkpN39sUs7xMtFFOlFnaQuAZIUQvqFoQMNH0eZBpROOCZkRLAYNlh6SUicA24DUhhLsQoi+a7+Ery7IN5HXgfqEtb3VF8yVkAnohxDVovpo6MU2frQDmmN7wewJ3mRX5EbhICDFFaEu+J6NNK/1gp35sQ5tyvR2TAZFS5pj6cjvVDUh9fp+YZH2D9ltZL4ToWlf5toIyIG2fyUKIQrTpplVojtiBUsoUszLvU33O/BY0B2MB2khirpTSfL56CrDINLd8PhahvVV+bpE+H/AAstCMUK1Le6WUv6A9wPcBu6n5QLkT7cEWjzYdtwzNSWtN1kE0f8ZraNNFf6DNjb9gUfRrNMN0wnS9bKr/k0n3X9G+n18t6j0MvCiEKEB7a19q1nYx2lz6VtOU1SVSypXAXGCxaarkAJpPBzQn9YemPp1G+3f7dy1f061oI6sUtGXUs03fW6ORUu5H8w1MN02TPWbqVw7a72BVPcQ9ijbNlAZ8huaormwnG805/0+0vv4LuF5KmdX4XlR9/7vRDOABs6zf0KY8zQ2Izb9PizYWAS8Cv55nRVibQkh1oNQFj2lq4i9ghDVHrZWyfwNDTU7rNoUQ4hRwr5RyvaN1UShaOi16V6yieTCNJHrWo2z3ptVIoVC0BtQUlkKhUCgahJrCUigUCkWDUCMQhUKhUDSIC8oHEhgYKCMjIxtUt6ioCC8vr7oLKhkXnIyWoIOSoWQ0pYzdu3dnSSk71MiQUl4w18CBA2VD2bhxY4PrKhltW0ZL0EHJUDKaUgawS1p5pqopLIVCoVA0CGVAFAqFQtEglAFRKBQKRYO4oJzo1qioqCApKYnS0tLzlvP19eXQoUONakvJ0HB3dyciwqbzeRQKRQvmgjcgSUlJ+Pj4EBkZiRC1BxUtKCjAx8en1nxbUDK0RRvZ2dkkJSU1qn2FQuF4LvgprNLSUgICAs5rPBT2QwhBQEBAnSM+hULR8rngDQigjEczo75vhaL5yFu9mmNXjSDowYc4dtUI8lavtpvsC34KS6FQKNoqeatXc/qFuRjLK3AD9CkppD4/CwDfsWPPX9kG1AikhbB27VpiYmKIjo7m9ddfd7Q6CoWilSOlZO9H69nedzpHu00+l15aSsa8+XZpQ41A6sl3fyXz5rojpOSWEObnwfTRMdzQP7zuiufBYDDwyCOP8MsvvxAREcGgQYMYN24cPXvaFGFdoVAoaiAlnPbqh0dJJl1OVT+HTZ963mN/bEYZkHrw3V/JPLNiPyUV2smiybklPLNiP0CjjMiOHTuIjo4mKko77fSWW27h+++/VwZEoVDUC6PByL6NSVw0KIjirz+l3/5PcKkoQlA96rpzqNVDO+uNMiBmvLD6IPEp+VbzDAYD+5ILKDcYq6WXVBj417J9fLPjjNV6PcPaMXtsr/O2m5ycTMeOHavuIyIi+PPPP+upvUKhuJDJOJ3Pxi8Pk5VYSO7ixQT//int+ven7NAhpNmqR+HuTtATj9ulTWVA6oGl8agr3VaklTNZ1EolhUJhC+WlenasPsm+XxNxdxf0O7OEgOQ/CX35JXxvvpn8H34gY958KlJScAkLI+iJx+3iQAdlQKpxvpFCQUEBYxbsJDm3pEZeuJ8HSx4Y0uB2IyIiSExMrLpPSkoiLCyswfIUCsWFw7YVxzm4JZmuATmEr3oVz4hgwpcuwT0mBtBWW/mOHcumTZsYNmyYXdtWq7DqwfTRMXi46KqlebjomD46plFyBw0axLFjxzh58iTl5eUsXryYcePGNUqmQqFouxTnl5Ofrb3M9r+kHZdVrKPz8pn4jxlBl2XfVhmPpkaNQOpBpaPc3quwnJ2deffddxk9ejQGg4GpU6fSq9f5/SYKheLCQ0rJoW2pbFueQFBkO0bElZE+/Snc8wsIMU1ZNef0tzIg9eSG/uGNNhjWuPbaa7n22mvtLlehULQNctOL2fTVYZKP5hLa1Zde+l2cmToP18hIOn30Me4xFzW7TsqAKBQKRQunME2yeNkOnF2dGHpDOO2WzqV42x+0GzeW0NmzcWrkcbcNRRkQhUKhaKHoyw04u+rwCICel4XSI/gsObPvpSS/gNBXXsb3ppscumJTGRCFQqFoYZSV6Nn+3XHO7Epk0N43CE1OROfjQ1ZBAa5RUQ6bsrJEGRCFQqFoQZzYm8mWb45QlFdGx9Tf0aemowOMBQWg0+E/9Z4WYTzAwct4hRBjhBBHhBAJQogZVvKnCyH2mq4DQgiDEMLflHdKCLHflLer+bVXKBQK+1FWouenhfv5aeF+3L1dGXzmM7odWYLOWH6ukMFA1v/ec5ySFjhsBCKE0AELgKuBJGCnEGKVlDK+soyU8k3gTVP5scATUsqzZmKGSymzmlFthUKhaBJc3HSUFlUw5Mau9BvZkaO9rL8X2ysQoj1w5AgkDkiQUp6QUpYDi4Hx5yl/K/BNs2jWzEydOpWgoCB69+7taFUUCkUzkp1SyJoFf1NSWI6Tk+CGJ/rT/6owMl99tdY69gqEaA+EtThMzdKwEBOAMVLKe033dwCDpZSPWinriTZKia4cgQghTgI5gATel1J+UEs79wP3AwQHBw9cvHhxtXxfX1+io6Pr1NdgMKDT6XA+tBK3315HFKQgfcIou2IG+h432tTnShmWbN26FS8vLx544IE6gyjWJqM+tAQZCQkJJCcn4+3t3Sg9CgsLHS6jJeigZLQuGUaDJPOgJPswODlDx8sFXkECUViI34cf4nrkKKW9euF69ChOFRXn6rm6UnDbbZQOjmvWvgwfPny3lDK2RoaU0iEXMBH4yOz+DuCdWspOBlZbpIWZ/gYBfwND62pz4MCB0pL4+PgaadbIz8+X8u8lUr4cLOXsdueul4O1dFtl1MLJkydlr169GiXDVlqCjPj4eLlx48ZG69ESZLQEHZSM1iMj6chZ+eWsP+S7D2yQv3xyUBbnl0kppSw5fEQeu2qEPNSnr8xZuVJKKWXuqlXy6PCr5MGY7vLo8Ktk7qpVdtOjPgC7pJVnqiNXYSUBHc3uI4CUWsregsX0lZQyxfQ3QwixEm1KbEujNPppBqTtt5rlYdBD6l9gKKueUVEC3z8KuxdZlxnSB65RJwwqFAqN/ZuSMBqMjH2sH516BgCQv+5nUp55Bp23N52//AKPvn2Bpg2EaA8caUB2At2EEF2AZDQjMcWykBDCF7gSuN0szQtwklIWmD6PAl5sco0tjUdd6QqF4oJHSsmxXekERnjTPsSLYVO6o3NxwsVNhzQayXp3AVn/+x8e/foR/s7buAQFOVplm3GYAZFS6oUQjwLrAB3wiZTyoBDiQVP+QlPRG4GfpZRFZtWDgZWmHZjOwNdSyrWNVuo8I4WSggJ8PhoCeYk1M307wj1rGt28QqFoW+Rnl3DmN0l8ykF6XxnOlbfG4O7tAoChsIiUGU9TuH4DvjfeSMic2Ti5uTlY4/rh0I2EUsofgR8t0hZa3H8GfGaRdgLo18Tq1WTELFj9mDZtVYmLh5auUCgUJoxGyf6NSWxfdQKDHi6bEE3f4RFV+eVnzpD0yCOUnThJ8LPP0P6OO1rlIXLqPJD60HcSjH1bG3EgtL9j39bSG8Gtt97KkCFDOHLkCBEREXz88cf20VehUDiEfb8m8vu3xwiL9iP6GsHFIzvhpNMet0V//MGpiZOoyMik04cf4H/nna3SeIAKZVJ/+k5qtMGw5Jtv2uT2FoXigqKi3EBRThl+wZ70uiIcH393ovp3YPPmzYDmC8n54gvS576BW1QXIhYswLVTJwdr3TiUAVEoFIpGkhh/lk1fH8ZJ58Sts+JwcdPRdcA5Z7ixvJy0OS+Qt2IF3iNGEDZ3Ljpvx4RgtyfKgCgUCkUDKSksZ+u3CRz5Mw2/YE+GTYmpmqrKW72ajHnzCUpJ4aiLC7KigsCHHybw0UcQTm3De6AMiEKhUDSAnLQiVry5h/ISPbHXRjLwms44u2jRGfJWryb1+VnI0lIEICsqEC4uuHaJbDPGA5QBUSgUihqYjx6OhYUR9MTj+I4dC4ChwojOxQnfIE+iBwbR89IgfMozKd6wnvKTJyg7cZL8tWvBLAQJaEYkY978KjltAWVAFAqFwgzL0YM+JYXU52dRUVDE8dJOxP9dxIjg/YjTRwk/eZKsl5PIMhqr6juHhtYwHpW0pEi69kAZEIVCoTAjY958ZGlptbQ8lyD+/LGcQu9SAjP3k/P7CnwiAnHv1RPf66/HNSoKt6guuEZG4uTpybGrRqBPqRmZqSVF0rUHyoC0AKZOncoPP/xAUFAQBw4cAODs2bNMnjyZU6dOERkZydKlS2nfvr2DNVUo2i7SaKR4x85qD36JIKHrTSRGDMO1PJ9hVzjR7cpJuIROQ5wnGnXQE49XjWIqEe7uBD3xeJP2oblpO96cZmLNiTWMWjaKvov6MmrZKNacaHwIk7vvvpu1a6tHYnn99dcZMWIEx44dY8SIEbz+ugrIqFA0BRXpGWS9/wHHx1zDmbvvBrNNfQJJmZsf4Sm/c1niR/S6bRiuERHnNR6gBUEMfelFnMPCkIBzWBihL73YpvwfoEYg9WLNiTXM2TaHUoP2VpFalMqcbXMAuC7qugbLHTp0KKdOnaqW9v3337Np0yYA7rrrLoYNG8bcuXMb3IZCoTiH1Osp3PIbucuWUbh5MxgMeMbF0eEfj1JUqOePZQl0PvEDXiUZ9Ir/BCd3N0Jfql+81pYeSdceKANixtwdczl89rDVPIPBwMGzByk3P58YKDWUMmvrLJYdXWa1Xnf/7jwd93S9dUlPTyfUNF8aGhpKRkZGvWUoFIrqlJ85Q+7yFeStWIE+MxNdYCABU+/B7+abcencmUPbUtm2MYGK4HYE6pPwPPIzLmGh1VZhKc6hDEg9sDQedaUrFIrmx3IJbuCjj+Lk6krusmUUb98OTk54Dx2K34Sb8b7ySoSLCzlpRWz671+kHMslrJsfw26LoX3IiDY9erAHyoCYcb6RQkFBATevu5nUoprL8EK9Qvl0zKd21SU4OJjU1FRCQ0NJTU0lqBWdEaBQOAprS3DTnn0WAJeICDo8Pg3fG2/EJTi4Wr34ralkJxcy/Pbu9Lg0FOHUOoMbNjfKiV4Ppg2YhrvOvVqau86daQOm2b2tcePGsWiRdsrhokWLGD9+vN3bUCjaGtaW4ALoAgLo+vM6Ah98sMp4pB7PI+1EHgBx13fh1tmD6Xl5mDIe9UCNQOpBpaP8rT1vkVaURohXCNMGTGuUAx20cO6bNm0iKyuLiIgIXnjhBWbMmMGkSZP4+OOP6dSpE99++609uqBQtGn0qdZPxTacPVsVQqSsRM/2lcc58FsyETHtGf94f1zcdLi4nX9llaImyoDUk+uirmu0wbCktnDuGzZssGs7CkVbRUpJ7pKlIK3nV27gO/FXJlsWH6E4v5x+wzsSN65LM2rZ9lAGRKFQtGqMZWWkvfQSecuW4xoTQ8Xp01Y38J3cl8VP7+8nIMKbax7qS3BkOwdq3TZwqA9ECDFGCHFECJEghJhhJX+YECJPCLHXdM2yta5CoWj7VKSmcvr2O8hbtpyAhx4kasXyahv4dGFheD79Er5jx9K5dwBX3dmdic/EKuNhJxw2AhFC6IAFwNVAErBTCLFKShlvUfQ3KeX1DayrUCjaKEV/7iD5iSeQZWVEvPsOPiNHAuc28K39fiNFR3zI2VXM7eMqcPdyocelYQ7Wum3hyBFIHJAgpTwhpSwHFgO2LjVqTF2FQtGKkVKS/dlnnJk6FZ2fH5HfLq0yHgD6CgN/rjrBiXWS3PQSLp/YDTdPNVvfFAgpa/E6NXXDQkwAxkgp7zXd3wEMllI+alZmGLAcbZSRAjwlpTxoS10zGfcD9wMEBwcPXLx4cbV8X19foqOj69TXYDCgqyP+jZJhOwkJCSQnJ+Pt7d0oPQoLCx0uoyXocMHIKCuj3Zdf4rFzF6X9Lyb/rruQ7ueW1hvKJSd+kZQXgFe4nohBLji7N3xZbov/PppJxvDhw3dLKWNrZEgpHXIBE4GPzO7vAN6xKNMO8DZ9vhY4Zmtda9fAgQOlJfHx8TXSrJGfn29TOSXDNuLj4+XGjRsbrUdLkNESdLgQZJSdPi2Pjx0n47v3kJnvfyCNRmNVnl5vqPr829Kj8szB7Bbdl9YmA9glrTxTHTmFlQR0NLuPQBtlVCGlzJdSFpo+/wi4CCECbanbmpg6dSpBQUH07t27Ku3s2bNcffXVdOvWjauvvpqcnJyqvNdee43o6GhiYmJYt26dI1RWKJqVwi1bODlhIvr0dDp++CGB99+HEAIpJcd2pfPl83+Qk1YEwOUTu9Gxp7+DNW45VEYQ/8fpf9gtgngljjQgO4FuQoguQghX4BZglXkBIUSIEFpsZSFEHJq+2bbUbSryVq/m2FUjONSjJ8euGkHe6tWNllmfcO6HDx9m8eLFHDx4kLVr1/Lwww9jMBgarYNC0RKRRiOZ//sfiQ88iEt4OJHLl+F9+WUA5GeXsGbBPn7+6CCePq5IYx3CWiGNefhLKVl2dBmzt82uCsFUGUHcXkbEYZ4lKaVeCPEosA7QAZ9Izb/xoCl/ITABeEgIoQdKgFtMwymrdZtaZ/M4O3DuqEugUZE66xPOfc2aNdxyyy24ubnRpUsXoqOj2bFjB0OGDGlw+wpFS6FaIMSQEHT+/pTFx9Nu3FhCX3gBJw8PAPZtTOSPlcdBCC6f2I0+wyNwamMhSGo7PqK4opiBIQPJLsnWrlKLv2afrQV6LTWU8taet+yyIdqhSxNM01I/WqQtNPv8LvCurXUbS9qrr1J2yHo4d73BQMWBA8jy6v8gsrSU1OdmkrvUeqgRtx7dCTEFc6sPtYVzT0lJYejQoVXlIiIiSE5Orrd8haKlUSMQYloa+rQ02o0fT9jrryHMDnoqOFtGeEx7rrw1Bh9/99qFtlIMRgP/3vXvKuNRSamhlBe31zyXRCd0+Lv7E+ARQIB7AFF+UQS4B/DpQetBXtOK0uyip1rbVg8sjUdd6U2ig5VVc+b/sRSK1kptgRCLd+5EX2Fk5w8n6dQrgIiY9gy5IQrhJNrMb98ojRzNOcqO1B3sTNvJ7vTdFFQU1Fr+9SterzIWAR4B+Ln54SRqeiTWnlprNYJ4iFeIXfRWBsSM840UCgoKSBt/Q7XzkitxDguj8xef21WX2sK5h4eHk5iYWFUuKSmJsDC1OUrR+tGn1nzQAaSX+rL9xT/JzyrF2VVHREx7nHQtP5D4mhNreGvPW6QWpRK6LLRa4FWjNJKQm8DOtJ3sTNvJrvRd5JVpkYE7+XRiVOQoNpzZQG5Zbg25oV6hNk8/TRswrdo0GNg3grgyIPUg6InHq/lA4FycHXtTGc59xowZ1cK5X3vttdx33308+eSTpKSkcOzYMeLi4uzevkLRnJSfOgU6Hej159JcvDnW9WbSQ+Lw0zlxw5P9Cb+oveOUrAfW/Bezt83mj5Q/KNYXsyttFzll2srKcO9whnccTlxIHINCBlWNDgaFDGr0w988gnhqUSqhXqF2iSBeiTIg9aDSUZ4xbz761FScQ+1z1GV9wrn36NGDSZMm0bNnT5ydnVmwYEGjNwUqFI4kb/UPpM2ejXBxASenqinhjA79yQgaQJ/oci6ddiXOLq3nd/7Wnrdq+C/KDGV8f/x7QrxCuCLiCmKDY4kLjSPcO9yqDHs9/CsjiDfF6YrKgNSTyjg79qS+4dyfe+45nnvuObvqoFA0N8aSEtJeeYW8ZcvxGDiQ8H+/Sfrm3SR++R2+CVvpzAm6Dy+n463X1y2shWHN7wAgEPx88882+26a8uFvD5QBUSgUzU7ZsWMkP/kkZQnHCXjwAfwfeph9m1LZsdUbr4unEvHkbQy/arij1aw3aUVp/GfXf2rND/EKaTOOf1AGRKFQNCNSSvJWrCDtpZdx8vKi40cfUhTeh2Vv7iU7qZCoiztwxeSL2PX3H45WtV6UG8r5PP5zPtj3AUZpZETHEWxN2dpkzuuWgjIgCoWiWTAUFpH2wgvkr16N5yWXEP7mG+RXeLL8pT/xbOfKNQ/0Iap/B0erWW+2JG1h7o65nCk4w1Udr2L6oOlE+ERUX4VlZ+d1S0EZEIVC0eSUHjpE8uNPUJ6YSIdpj+F84x04d/DGHxh2e3e6DgjCzaN1PY4S8xOZu3Mum5M2E9kukoUjF3JZ+GVV+S3df2EPWte/mEKhaFVIKcn55hsyXp+Lzs+PwP99zO5j3px4eSe3zIyjfYgXPS9rXfuYiiuK+Wj/R3x28DNcnFx4cuCT3N7jdqhMmI4AACAASURBVFx0Lo5WrdlRBkShUDQJhvx8Umc+T8HPP+M59Aryb57O96tSqSjPZNC1kbQL9HC0ivVCSsm60+v4985/k16cznVR1/HkwCcJ8gxytGoOQxkQB5Odnc2IESMASEtLQ6fT0aGDNg+8Y8cOXF1dHaLX+vXreffdd/nuu+8c0r6i9WEeCPFoYCBSr8dYWEjgU0+xtaA/ySsTCevmx7DbYmgf4uVodc+L5S7yyTGT2ZayjR1pO+ju3503hr7BgOABjlbT4SgD4mACAgLYu3cvAHPmzMHb25unnnqqWpmqw1ucWn74BsWFiWUgRENWFlIIAh9+iA73/h/ha05yUVwIPS4NbfHLWK3tIp+/Zz7uOndmDp7JhIsmoHNqPZsamxL1RGqhJCQk0Lt3bx588EEGDBhAYmIifn5+VfmLFy/m3nvvBbTIvTfddBOxsbHExcWxffv2GvJiY2M5cuRI1f3ll1/O33//zfbt2xkyZAj9+/fnsssu49ixYzXqzpw5k/nz51fdd+/evSoC8KJFi4iLi+Piiy/m4Ycfxmg0otfrueOOO+jTpw+9e/fm7bffttv3omiZWAZCzGvXhR2xz3Ji7V8ADLquCz0vC2vxxgOs7yIH8HXzZXL3ycp4mKFGIBas/M+eGmnRA4OIHOhLRbmBH975u0Z+9yGh9Lg0lJLCcta+f6Ba3o3/bPgwNz4+nk8//ZSFCxeiN4sRZMljjz3Gv/71Ly655BJOnTrF9ddfz4ED1fWYPHkyS5cu5fnnnyc5OZns7Gz69etHXl4ev//+OzqdjrVr1zJz5kyWLFlik34HDhxg5cqVbNu2DWdnZ+6//34WL15M165dycrKYv/+/QDk5tYMCKdoW1QGGdXr3DkeNZ7k8KG4lZ7FcPasgzWrP7WFOs8ozmhmTVo+yoC0YLp27cqgQYPqLLd+/fpqo4ucnBxKSkrw8DjnpJw0aRJjx47l+eefZ8WKFUyaNAnQHu533nknx48fr7d+69evZ+fOncTGxgJQUlJCx44dGT16NEeOHGHatGlce+21jBo1qt6yFa0DWV5OxltvAZAZ0IcjF91CuWs7Oib+SpdTP+AeHOBgDW2n3FDOx/s/RlLzyASwXwj0toQyIBbUNmIoKCjAxVV33hGFh7dro0Yclnh5nXM0Ojk5VTsLpNRsukBKWafDvXPnznh7exMfH8/y5cv54osvAC2u1ujRo3n44YdJSEhgzJgxNeo6OztjNJ47L7SybSklU6dO5aWXXqpRZ9++ffz000+8/fbbLF++nA8++KAePVe0BspPnyb5n09ReuAAHoMHU5bmjWtFIX0PvE+7gjNNFqm6KdiVtosXt7/IybyT9A3sy5GcI5QZyqry2+IucnugfCCtBCcnJ9q3b09CQgJGo5GVK1dW5Y0cOZIFCxZU3Vc65S2ZPHkyr732GuXl5fTs2ROAvLw8wsO1aKCfffaZ1XqRkZHs3r0b0FaGVZ5HMnLkSJYuXUpWVhagrSg7c+YMmZmZSCmZOHEiL7zwAnv21JwWVLRepJTkfvcdJ268meMl4ZRPf5vIRZ8R+8hoLkn5Ap+CMziHhRH60ot2Dzxqb/LK8pi9bTb3rLuHckM57418j6+u+4oXLn2BUC/TiaBeocy5dE6b20VuDxw6AhFCjAHeQjvX/CMp5esW+bcBT5tuC4GHpJR/m/JOAQWAAdBLKWObS29HMXfuXG666SYiIyPp2bMnZWXaG9KCBQt46KGH+PTTT9Hr9QwfPryaQalk4sSJPPnkk9Ui+T799NNMnTqVN954g+HDrQevmzhxIl9++SX9+/cnLi6OqKgoAPr06cPs2bMZOXIkRqMRFxcXFi5ciE6n4//+7/+QUiKEYO7cuU3wbSgcgaGwkLQ5L5CycQ9HBz5Fri4Io9T2QbQfN5b248a2ip3XUkp+PPkjb+x8g7yyPO7pfQ8P9XsID2dt2vdC2EVuDxxmQIQQOmABcDWQBOwUQqySUsabFTsJXCmlzBFCXAN8AAw2yx8upcxqNqWbmDlz5lR9jo6OrjGSmDx5Mtdeey0+Pj7V0jt06MCyZcvqlB8WFobBYKCg4NxRmZdffjlHjx6tun/55ZcBbXQxcuRIQJtKW79+fTVZlTKmTJnClClTarT1119/1amPonVRsncvZ6bP4JhLP84MehY3b1dGTuzGRXHBjlatXiTmJ/Lyny+zLWUbfQL78MHVHxDjH+NotVoljhyBxAEJUsoTAEKIxcB4oMqASCm3mZXfDkQ0q4YKhQJpMJD94UdkvvMOeV2GcDpiNDGXhHDZhGg8vB2z0bUhVBgrWHRwEQv/XoizkzPPxD3D5Bi1LLcxCHPHbLM2LMQEYIyU8l7T/R3AYCnlo7WUfwroblb+JJADSOB9KaVVL60Q4n7gfoDg4OCBixcvrpbv6+tLdHR0nfoaDIZGn/ynZJwjISGB5ORkvL29G6VHYWGhw2W0BB2aSoZTTi6en31NcRb4dHElf8oUSso8cG9f+36OltiXk2UnWZy9mJSKFPp69GWC/wTaO9d9PG5L7IsjZAwfPny3VTdB5S7n5r6AiWh+j8r7O4B3aik7HDgEBJilhZn+BgF/A0PranPgwIHSkvj4eGk0GmukW5Kfn19nGSXDNoxGo4yPj5cbN25stB4tQUZL0KEpZOT9sl5uGn2ffH/qd/K9h9bL4vwyh+hRX344/oO8+turZe/PessRS0fIe9feK/t81keOWDpCbji9odn0aEsygF3SyjPVkVNYSUBHs/sIIMWykBCiL/ARcI2UMrsyXUqZYvqbIYRYiTYltqW+Sri7u5OdnU1AQECr2CXb2pFSkp2djbu7u6NVUZhhHsfqWGgoFZE92VcYQ3bkLQSGuHLV//XDw6flT1dZhiFJL04nvTidS8Mu5b/D/ouXS8uOwdXacKQB2Ql0E0J0AZKBW4Bq3lghRCdgBXCHlPKoWboX4CSlLDB9HgW82BAlIiIiSEpKIjMz87zlSktLG/3QUzI03N3diYiI4PTp043SQWEfLONYlWTm8kfXMUh/Fy67qQt9R0bi5NQ6Xq7m7Z5nNQzJybyTyng0AQ4zIFJKvRDiUWAd2jLeT6SUB4UQD5ryFwKzgADgf6bRQeVy3WBgpSnNGfhaSrm2IXq4uLjQpUuXOstt2rSJ/v37N6QJJUPRoqmMY1Xq1h73shxc9CV0O/YtgW759B21sm4BDqaoooiNiRtZd3Id6cXpVsvUFp5E0Tgcug9ESvkj8KNF2kKzz/cC91qpdwLo1+QKKhQXAGXpWZyMupHEjsPp9/e7+OceJTR9B7TgKd3iimK2JG9h3cl1/Jb8G2WGMoI9g/Fy9qJIX1SjvApD0jSoUCYKxQXMmfhsdgyeRYlre8JSfsenMLEqzzk01IGa1aRUX8rW5K2sPbWWzUmbKdGXEOgRyM3dbmZMlzH069CPn07+VM0HAioMSVOiDIhCcYGy+ZsjHNicTDs/H7pvnUf7vISqPEfFsbI8yOmRix/B182XtafWsilxE0UVRbR3a8/YqLGM6TKGAUEDqu3jqAw3UiXDK5RpA6apMCRNhDIgCsUFhDTt+xJCEBjhTex1kXQ6tprcvAR0HQLRZ2bhEhZG0BOPN3scK2sHOc3cOhOAdq7tGB05mtGRo4kLicPZqfZHlwpD0nwoA6JQXCDkZhSz+esjVScD9roiHH1WFgmzP8fnmjFEzJvn0Ifu/N3zra6gau/eng0TNuCic3GAVorzoQyIQtHGMRiM7P3lDDvXnEKnE8QMPudQzlr4PrKsjA6PPeYw/QrKC1h8eDFpxdZXSuWW5irj0UJRBkShaMNknM7n188Pk51cSFT/DgydfBFefm4AlCclk7NkCX433YibDUvZ7c3Z0rN8Gf8l3xz+hsKKQtx0btXO4KhEraBquSgDolC0YYrzyiktquCaB/sQdXGHanlZ776r+UIeeaRZdUorSmPRwUUsO7qMMkMZIzuP5L4+93Ei74RaQdXKUAZEoWhjnNqXRX52KX2HRxDZN5Dbu7fH2bV64MuyhATyVq3C/847cQlpnjf8M/ln+OTAJ3x//HuQmrN7ap+pRPlq58v0COgBqBVUrQllQBSKNkJFiWTtBwc4vieDDp186D00DCedUw3jAZD51ls4eXgQ8MD9Ta7X0ZyjfLTvI9adXoezcGZCtwnc0/sewrzDapRVK6haF8qAKBStHGmUHNqWSsKPEiGzGDwuiv6jOuGks35idcm+fRT8sp7ARx/FuX3dIc1txXIPxw3RN3Ao+xCbkjbh6ezJXb3u4s6edxLoEWi3NhWORRkQhaKVk5tRzKavj+ARCDc+EodfsOd5y2fMm4eufXv8777bbjpY28Px3t/v4aHz4OGLH2ZK9yn4uvnarT1Fy0AZEIWiFWKoMHL6YDZRF3egfYgXN/9rIPEnd9dpPIr++IPiP7YT/MwMdN7notNajh6s+R4MRgM5ZTmkF6eTWZxJRnFG1fXjyR+trqDydfPloX4P2afTihZHnQZECBEMvIp2gNM1QoiewBAp5cdNrp1CoahBSkIum748TE5aMbc8H0dAuDfBke04dOr8wQ+llGTMm49zaCh+t9xSlW51B/jvM/nu2Hd4uniSWZJJenE62SXZGKShmkwn4USge6BV4wHUGh1X0TawZQTyGfAp8Jzp/iiwBFAGRKFoRsqKK/jjuxMc3JKMj7871z/aj4Bwb5tGDwDZ69ZQum8fOU/extZj35BYkMiZgjPsTNtZwzDopZ4/0/6kq19XgjyDiPKNIsgzqMbl7+6Ps5Mzo5aNIrUotUabag9H28YWAxIopVwqhHgGqs7xMNRVSaFQ2A+jUbL8jd3kphfTb2RH4q7vgqu7s9XRw6xts9ibsZdAj0ASCxJJLEgkKe8MT7+bhgiAp1wWY9wt8HXzpZNPpxrGw5yV4207D2TagGlqD8cFiC0GpEgIEQBIACHEJUBek2qlUCgAKMorw7OdK05OgrixUbQLdCeoczuM0siJvBO8vuP1GvGjyg3lLD6yGIAgjyAifCK47UwEHbPSyHz2Lr6+diwRPhFVTm17jB5UFNwLE1sMyJPAKqCrEGIr0AGY0KRaKRQXCLVNPxmNkgObk9n+3XEunxSNe+8yDreP52D6QeLj4zl89jDF+uJa5QoEf972Jx7OHhjLyznx5jXoevXiijueRlgcFGWv0YPaw3HhUacBkVLuEUJcCcQAAjgipaxocs0UijaOtemnOdvmkHYmB+PmYMpTdeR1SOYfh18jIyEZADedGzH+MYzrOo6eAT15+6+3ySrJqiE7xCsED2cPAHKXLKUiJYWQF1+sYTxAjR4UDUdUng9QawEh7rSWLqX8vNGNCzEGeAvtTPSPpJSvW+QLU/61QDFwt5Ryjy11rREbGyt37dpVLx2/+yuZN9cdITm3hHA/D6aPjuGG/uEXrIydq96n4543CZKZZIgOJA6YzqBxD1yQMhpb39rUUZ/UK7nk9HjKdSXsjFqNS0wJPQN70jNAu6J8o6qdhbHmxBpm/TaTcvRVaa448+IVL2sjmaIiEkaNxq1rVzot+syqAbFHX5SMti1DCLFbShlrmW7LFNYgs8/uwAhgD9AoAyKE0AELgKuBJGCnEGKVlDLerNg1QDfTNRh4DxhsY91G891fyTyzYj8lFZqTMTm3hGdW7Aew+cHblmTsXPU+vXfPxEOUg4AQMvHdPZNd0kDs9baFxNj1wwf03jO71ctobP0KQ0V14yEBAXnumSQE7uKB+8fzeMg7NQ9OkoDhnLEI3n+amelZvOfvTZqzjhC9gYfO5hK8/zR01nN20SIM2dkEvfMWwmjdWd4Svk8lo3ll7IR6GxFr1DkCqVFBCF/gCynluEY1LMQQYI6UcrTpvnKV12tmZd4HNkkpvzHdHwGGAZF11bVGfUcgl73+K8m5JTXSXXVO9O/kZ5OMv87kUm4wtgkZ/025nXBRc7pEYTtZTk58286bpT4+ZDnrcKvwZMjp8RS65bCr41oAQiv0/JyU0ui2DGWChB+C8Qwqo+MVOY2Wp2g7pNGBkDkJdRc00ZgRiCXFaCOCxhIOJJrdJ6GNMuoqE25jXQCEEPcD9wMEBwezadMmmxW0ZjwAyg1GcnNzbZJh7aHdGmV0MGYRhnXjISUscplkkx53VSzF2ixKa5NR3/rpLkXs8UnliGc2BiGJLPZlXHIohozbcdN7sifiFwDcjUYeO5vLZ86N78emI7GEViSzrf+llDh71SzUwL4oGa1fRpDMqtezsDZs8YGsxrSEF3ACegJLpZQzGtWwEBOB0VLKe033dwBxUsp/mJVZA7wmpfzddL8B+BcQVVdda9hrBBLu58HWGVddGDKMBtjxIfz6EsbyQqyF56vP20zanGhCyGz1Mmyprzfq2XBmA18d+oq/Mv7Cw9mD8V3Hc2PIRI6vLubMwbO4uh1nXdclHG2XRojewLScXAYWeTW6HyklHchf643P6FGEv/FGo/vSUD2UjNYvA2ofgVgP11mdfwP/MV2vAUMbazxMJAEdze4jAMtxe21lbKnbaKaPjsHDpXoobA8XHdNHx1wYMtIOwMdXw9qnoeNgDvR6mhLpWq1IiXQlccB0m/VIHDC9Tcg4X/3c0lw+2v8RY5aP4anNT5FRnMH02OlsmLiB5y55jg66UNJP5tOpZwZTfJ9j2dmd7DuVyM9JKVxVqLdLP7LSByANBjr847zvVHX2pbF6KBmtX8b5qLcPxF4IIZzRwqKMAJKBncAUKeVBszLXAY+ircIaDLwtpYyzpa411CosG2VUlMDmubDtHXD3gzGvQ58JIITZio4sMkRgI1eFtG4Zb3/7JN/nrSPTWdBBL7nC51LoGM4PJ36gzFDG4JDB3NbjNoZGDOVsUjGnD2QTe20kAOWlelzdnZukH0md78PrP1/SftJEQmbNapbvQslo2zJqG4EgpbR6AQVAvpWrAMivrV59LjTDcBQ4DjxnSnsQeND0WaCttjoO7Adiz1e3rmvgwIGyoWzcuLHBdVuVjOMbpZzfT8rZ7aRc+ZCURdmO0aOFy/jh+A8y9otY2fuz3tWufov6ydlbZ8ujZ49KKaUsL9XL35cdkwse+lV+Mv03WVJQbjcdapOR9M+n5KF+F8vy9PQGy7CHHkpG25EB7JJWnqm1OtGllD71MlENQEr5I/CjRdpCs88SsHpgs7W6ikZQfBbWPQd/fw3+UXDnKoi60tFatTjKDeXsy9zHy9tfrhFCBCDQI5A5l84B4MzBbDZ9fYSC7FJ6XhHGkBu64u7l0qT6lR45Qv6aNQTcey8uQUFN2pZCYfMqLCFEENo+EACklGeaRCNF8yIl7P8W1s6A0jy44p8wdDq4eDhasxZBhaGCA9kH2JG6g51pO9mbubfW0OUAGcUZAJQWVbDuwwN4+rpx4z/7E9bNfif/nY/MefNx8vEh4N7/a5b2FBc2tpwHMg7NgR4GZACdgUNAr6ZVrYWwbylseJEr85LgrwgYMQv62raEzq4y7IGlHpc8BAnr4fivED4Qxr4NIb2bXy8HYS0O1ejI0RzMPsjOtJ3sSN3B3sy9lOi11Wsx7WOYeNFE4kLieOXPV2qedSGhX8llSClx93Jh3LT+BEZ4o3OxZa1Kw8lbvZqMefMJSkmhEPAZMwadrzr9T9H02DICeQm4BFgvpewvhBgO3Nq0arUQ9i2F1Y9BRQkCIC9RuwfbDYA9ZNgDa3qsexZ0bnDNGzDoXnDS1SWlzWAtDtWzvz/L878/T4Up1Fu0XzQ3Rt9IXEgcA4MH4ud+bsNlsb64Wv12JYEMO3krYXnRnPw7i6iLOxDcpV2T9yNv9WpSn5+FLC2lcrl/4aZN5K1eje/YsU3evuLCxhYDUiGlzBZCOAkhnKSUG4UQc5tcs5bAhhe1FUnmVJTA2mfA3cY3vLXPWJex4cXmNSDW+gLg6Q+DGx/SoLVgMBo4nHOYV/98tYYPwyiNuDu789plrxEbHEuAR0CtcioDDb696x2CE3oQmzQGZ2dnhk6JoUvfwCbtgzkZ8+YjS6v3Q5aWkjFvvjIgiibHFgOSK4TwBn4DvhJCZIBZ5La2TF6S9fTiLPi6kQ//2mQ3FbW1V5DWvHo0MwajgaM5R9mRtoNdabvYnb6bgoqCWsuX6EsYHTnaJtnXRV0HP0Vw6kw2Xft34IrJF+Hl52Yv1W1Cn1rzHI/zpSsU9qRWAyKEeBf4BhgPlACPA7cBvsCLzaKdo/GN0KZ6LPEOglsX2ybjm1ugMKNmuns70JeBcxM/cCpKYfuC2vN9I5q2/Sai1nM0pJGjOUc1H0baDs1glGsGo3O7zoyKHEVcSBz/2f2fKoe3ObYcolReqsdJJ3B20dH3qo4YfM8y5vY+du9jXRT+vhWE0BZCWOAcGtrs+iguPM43AjmGtgs9FO0M9G+klIuaRauWwohZVX6DKlw8YNQrmtPZFka9UlOGcNJWPC0YDKNehu7XYTVgTWOQEuK/g59nQd4ZCLkYsg6D3my6w8VD62Mrw5r/YubWmSw6uIjkwmTyy/MB6OTTiVGdRxEbEsug4EEEewVXyZDIBh2idHJfFlu+OUL3S0MZPDaKjj38OZ5u53+7OjAUFpHx5pvkLlmCLigIY24usry8Kl+4uxP0xOPNqpPiwuR8+0DeAt4SQnQGbgE+FUK4A18DS6SUR5tJR8dR6aPY8CIyLwnh24AVVLXJ8PTX9l0suQ0ir4Axr0GInd5iU/7SfC9n/oCgXnDn9xA1rGoVVoP7YidqGz3URnFFMRnFGWQUZ5BenM5rf75Ww3+hN+o5mnOU8dHjiQ2OZVDIoPOOJup7iFJRXhm/LTnK8T2Z+Id50blX7f6RpqRoxw5Sn32OiuRk/KdOpcO0xyj4+Wcy5s2nIiUFl7Awgp54XPk/FM2CLScSngbmAnOFEP2BT4A5aAc5tX36ToK+k9jcmGM6a5PRZRjs/hQ2vgoLr4ABd8JVM7UpsoZQkAYbXoK9X4FnAFw/X5NZubrKHn1pJNZGD7O2zeJQ9iE6tetUzVBU/q2cgqoLozTywqUv2KyLrUewntibyYZFhzBUGBk8Por+ozqh0zXt0lxLjKWlZM6bx9nPv8ClY0c6f/kFngO1UbDv2LH4jh2rjpJVNDu27ANxAcagjUJGAJsB2/+XKmpH5wxx92lxpja/ATs+gAMrYOhT2h4NW/0jFSXwxwL47b9gKIdL/6HJsHWlWDMyf/f8GqOHckM5i+K12VGBINAjkGDPYDr5dCI2OJZgr2CCPYMJ8gwi2DOY+36+j7Tims5/W/wX9UFKiRCCdoHuBEf6MPSWGPyCPe3ahi2U7N1LyoxnKD91iva33UbQP5/EybP59VAoLDmfE/1qtP0e1wE7gMXA/VLKombS7cLBo702hRU7FX6eCetnayOTq1+CHmNr949ICQdXwi+zNT9H9+vh6hchoGvz6m8DqYWpfH34a6sP/kp+mfALgR6BNU/hs+DxgY83yH9hK4YKI7vXnqIor5zht3cnMMKHcdP620V2fTCWl5P17gKyP/oI55BgOn36CV5DhjS7HgpFbZzvf+qzaP6Op6SUZ5tJnwubwG4wZQkkbND8I0vvgM6Xw5hXIfNI9V3kA+7UyiVuh+A+cMNq6DLU0T2owd+Zf/NF/BesP70eicRd5241hlSoV6jNI4j6+i/qQ8qxXDZ9dZictGK6DQrGaDDi1MzTVQCl8fGkzHiGsqNH8Z1wM8EzZqDz9m52PRSK83E+J/rw5lREYUb0COhyJez5DH59Bd4fCkIH0nBuF/nGV8DVRws/0v/2FrWLXG/Us/7Mer6I/4J9mfvwdvHm9h63M6XHFP7K+Msuowdb/Re2UlZcwbaVx4n/LQUff3euf7QfnXs3v6NcVlSQ9cEHZL23EOf27YlY+B4+yq+haKE05EhbRXOgc9bCi/SeAPN6gzVHsns7GHhX8+tWC/nl+aw4uoKvD39NalEqHX06MiNuBjdE34CXi3akaph3GNA0o4fGUFFm5PjuDC4e2ZG4sVG4uDWPQTaPY3WsQwdwdUWfnEy7668nZOZz6PxsO69eoXAEyoC0dDz8oLzQel6+3Q9hrBNrS3D7BPbhy0Nf8l3Cd5ToS4gNjmVG3AyujLgSnZWRkb1HDw2lMKeUg7+lIH0k3u3duOPlIbh5Nm24dXMs41jpM7WjR/3uuIPQ555tNj0UioaiDEhroLYd8c28i7y2AIRGacTZyZlrIq/h9p630zOgZ7PqVV+MRsmBzcls/+440ijpPEJLb07jAZDx5r9rxLECKNywAZQBUbQClAFpDdS2I76Zd5G/tectqwEIvV28WXXDKjp4dmhWfRpCVlIhm746bDqT3J8rp8Sw58Cfzda+sbSUgp9/JvfbZegzrIS4QcWxUrQeHGJAhBD+aOFRIoFTwCQpZY5FmY7A50AIYAQ+MO2ORwgxB7gPyDQVf9Z0QmHbxB474u1AWpH1JbhFFUWtwngYjZK17++nvFTP1VN70m1QMMLeIWRqoTQ+ntxly8lbvRpjQQEunTrh5OODsaCmb0vFsVK0Fhw1ApkBbJBSvi6EmGG6f9qijB74p5RyjxDCB9gthPhFShlvyp8npfx3M+rsWBy8izyvLA8XJxfKjeU18uy9gc/epBzLISiyHc4uOkbf1xsff3fcvZt+uspQUED+Dz+Q++0ySuPjEa6u+Iwejd+ECXgOiiV/zZoqH0glKo6VojXhKAMyHhhm+rwI2ISFAZFSpgKpps8FQohDQDgQj6JZOZF7gn/8+g/0Rj0uTi5UGCuq8uy5gc/elBZWsHX5MQ7/kcalN0XTf1QnOnTysZv8aiuoTDGo2l1/PSW7d5P77TLy161Dlpbi1r07wTNn4jv2+monBVbGq1JxrBStFUcZkGCTgUBKmWo6b71WhBCRQH/AfLL6USHEncAutJFKjpWqikayJWkLT295GledK4uu0aLdtrQluJZIKTm6I53fi39zqAAAIABJREFUvz1GebGegWM602dYuF3bqLGCKiWFlGeeJX3uGxiysnDy8sL3hvH4TZiIe6+etU6VqThWitaMkFbOErCLYCHWo/kvLHkOWCSl9DMrmyOlbF+LHG+0+FuvSClXmNKCgSxAoh25GyqlnFpL/fuB+wGCg4MHLl5s4zkeFhQWFuLdyJ3ArUmGlJJf83/l+9zvCXcJ576g+/B39m92PRoiI22vkezD4OEPYXECdz/rD2/3P3fg/f33OJ09i9Hfn8Lx4ykdHFd7Y1IiSkpwysmh/fy30FnxX0hnZ/Jvm0LpgAHgZvtZLy35+1QylIzhw4fvllLG1siQUjb7BRxBe+iDdt7IkVrKuQDrgCfPIysSOGBLuwMHDpQNZePGjQ2u29pklOpL5bO/PSt7f9ZbPrHxCVlUXuQQPeojw6A3yLKSCimllJmJ+fLvX89Ig8FYa73cVavkoX4Xy/iY7lXXoX79ZMZ7C2X+r7/Ks998I9PnzZPJT8+Qp+6+WyaMuUYe6j+gWnmrV/cejepHY1AylIymkgHsklaeqY6awloF3AW8bvr7vWUBoY35PwYOSSn/a5EXKk1TYMCNwIGmVffCIaski2kbp7Evcx8P93uYB/6/vTuPj7q8Fj/+OdlDQhIIEBL2HQFlCYK7gKIItaAVW+raS4t663qlVSv3FrD+LtdqgUoVNypeW9BqFbhaKSARFyqyyy4ga0ICIRAmCdnm/P6YCYRksswksxDO+/WaF7N8n/M9ScicPN/nmefpfx9hEvi1oLyRsz+flW/vILldPNff24dW7ZvTqn3tYx2e9xIv5tisWWefCA8nok0bIlNSiO7Vi/hrriGibVsi26Zw5Nn/R/mxY9Xi2gwqcyEJVgGZAbwrIhOBA8B4ABFJA15X1dHAlcBdwLcistHdrmK67nMiMgDXJax9wH0Bzr9J2pa7jYc/fZj8knxeuPYFbuh8Q7BTqlVpcTlHNjjZ9s5aYptHkT6qVb3b1vZZi87vLCSibVsikpORCM+/IlpWZjOozAUvKAVEVXNx7S1S9flMYLT7/heAx4vXqnqXXxO8AC3dt5QpX0whKSaJt256i94tewc7pVrl7M/nk1e3cCoX+l6dxuW3dPPqk+RhcXE4HdWXiIlISyO2f/8629sMKmPsk+gXPKc6eXnTy8zdNJcBrQcwc/hMWsXW/y/5YIlLjKZZQhTJ/YsZdrt3xc6xapWreISHQ3n5mee97UHYDCpzoQvti9vGrwpLC3k843HmbprLuO7jeOPGN0K2eKgq27/K4h9zv0WdSlxSND/6dTpxbbz7JHnpkSNk/voJonv1ou0z04lIS0Nx9TxSn5luPQhjvGA9kAtI5ZV027zbhjAJI6coh8mDJ3N3n7sDtqyHt07kFJLxl50c3plHavdEiovKiImL9DpfLSvj8OOTcZaU0G7mTKK7dqHFrbdaD8IYH1kBuUBUXUk3p8i1kN+9fe7lnr6hs6dIZeXlTjYuO8A3H+0jPCKMYXf0os+VaUiYb4Xu6B9fpGjdOtJ+/xzRXbs0crbGXHjsElYdPtr7ETe8dwMP7X+IG967gY/2fhTslHziaSVdgKX7lwYhm/opL3WydVUmnS9O5qdTh9L36nY+Fw/H55+T++qrJI2/zS5TGdNIrAdSC0/7X0z9aipAyC3fUZeaVtKt6flgKTldxsblBxl0Y0eiYiIY/5vBxMZHNShmaXa2a9yjZ09Snn66kTI1xlgPpBae/mo/XX6a2etnBykj33x28DPE84zokFpJ9/tNR1kw7Wu++eh7Du1wLW3W0OLhGvd4HGdxMe1mzSQsJqYxUjXGYD2QWtX013lWQRaPrXyMPsl9ztxaxHhcyiuoCksLee6b53j/u/dp26wtx4uPU1J+djn2UFlJt+BkMZ8v3MWeDUdJbhfHjZP60bZLYt0N6+Hoi3MoWlsx7tG1UWIaY1ysgNSibVxbsgqqf2I5JjyGnXk7WX5g+ZnnUuNSzxSTvsl9zykqnvYR9/clsI05G3nq86c47DjMz/r9jAcHPMiy/ctCciXdFfO3k7nrBJeN68qAkR0JD2+cjrHj8y/IfeUVEm/7kY17GOMHVkBq8cigR84ZAwFX8Zh6xVTGdB1Dfkk+23O3sy1325nbigMrzhybGpdKi+gW7MrbRZmWAf4fRyktL+WlTS8xb8s8UuNSmXfjPAa3HXzmfGO6jgmJaavHswooO+1aCfrq23sgIiSlNGu0+K5xj18T3aMHbW3cwxi/sAJSi4o3+Jr+ak+ISmBo6lCGpg490ya/JJ8duTvYmruVbbnbWLZ/GeVafk7cinGUxi4gu/N289QXT7Hj+A5u6X4Lv77018RHNWwJ6MZWXupk3Sf7WPfJfhI7K4yCFm3jGvUcWlZG5uOTXeMes2cRFhvbqPGNMS5WQOrg7V/tCVEJDEkdwpBU174Sl8y/xONxWQVZzFw3k3Hdx9ElsWGfSXCqk7e3vc3s9bOJj4pn9vDZjOg4okEx/SHzuxNk/GUHeUcK6XFpCmHtcvxynqNz5lC4di1p/zPDxj2M8SMrIH5W0zhKdHg087fOZ96WefRv3Z+x3ccyqvMomkd5t+VqliOLKV9OYc2RNQzrMIypl08lOTa5sdJvNFs/P0zGX3bSPDmGHzzUn059k8nIONro53F88SW5r7xK4o9uJXHs2EaPb4w5y6bx+tkjgx4hJvzcqaMx4TFMu2Iay8cv5/H0x3GUOJi+ejoj3h3Bk58/yerM1TjVWWtcVWXJniXcuvhWthzbwrQrpvHH4X8MqeKhqpScdo39dL64FYNu7MSE/xpKp77+ybE0O8c17tG9G22nTPHLOYwxZ1kPxM/qGke5t59rKZGtuVv5cPeHfPz9x3y09yNS41L5YbcfMrbbWDokdDhnJlfK31JoHduaLblbGNRmEL+76nd0aN4hmF9mNaeOn2bVgp2UnC5n3GMDiUuK5vJbuvntfFpWRubkyTiLimg3y8Y9jAkEKyABUNc4iojQr1U/+rXqx68u/RUrD6zkwz0f8tq3r/HK5lfonNCZw47DlDpLAcguzCa7MJtRnUYx45oZhIeFB/grqpnTqXybcYivF+1FVRlyc1eUGjZ2aUTHXnqJwm++IXXGfxPdzX+FyhhzlhWQEBMdHs2oLqMY1WUU2QXZLNm7hDkb5lSbyQWw6dimkCoe+blFLH1tKzn78unYtyXXTuhFQiv/9wQcX37JsZfnknjrrSSNG+f38xljXGwMJISlxKXw84t/XuN4SKitY1Wx7MjIiX34wYP9A1I8wk6eJPNX7nGP/7RxD2MCKSg9EBFpCbwDdMa1p/ntqprn4bh9wCmgHChT1cHetG8qaprJFQrrWB3ccZyNyw5y0/39iIwO57Yn0gOyr8jJJUvI+cNMWmVlUS5Cy5/9zMY9jAmwYPVAngRWqGoPYIX7cU2Gq+qAiuLhQ/vzXk0zuYK5jlVZsbLizW0snrWRkzmFOI4XAwSseGT9539RlpXlGltR5dif/sTJJUv8fm5jzFnBGgMZCwxz358PZABPBLD9eaWumVyBpKrsWpPN7o8VLc0mfVQnBo/uTERU4MZicmbOQk+fu0qynj5NzsxZtuaVMQEkqhr4k4qcUNWkSo/zVLXacrYi8j2QByjwiqq+6k1792uTgEkAKSkp6QsXLvQpZ4fDQXx8w5YFaQox1Kl8v1wpd5bT4bIIYpJ873H4mkeb+x/wOKtLgZy5Lwcsj8ZqbzEsRqjHGD58+LoqV4FcVNUvN2A5sMXDbSxwosqxeTXESHP/2wbYBFzjflyv9lVv6enp6quVK1f63PZ8j1FeVq4blx/QwvxiVVUtOFmsn674NOB5qKqe/Phj3dart8fbruEjApZHY7a3GBYj1GMAa9XDe6rfLmGp6vU1vSYi2SKSqqpZIpIKeFwUSVUz3f/miMgHwBBgFVCv9qbhcvbns/LtHRw76ACg/3UdaJYQ5fPWsg2Rv/SfHJ78KyI7d6bsyJFzLmNJTAxtHns04DkZcyEL1iD6YuAe9/17gEVVDxCROBFpXnEfuAFXD6Ze7U3DlBaX88V73/HejLUU5pcwalI/LhnRPmj5nFqxgsOPP07sxRfT5b33SH1mOhFpaSgQkZZG6jPTbfzDmAAL1iD6DOBdEZkIHADGA4hIGvC6qo4GUoAP3LN6IoC/quontbU3jeerv+9my2eH6XtNOy4f15XoZpFBy+XUypUcevQxYvr2ocPrrxEeH0fizTeTePPNIbG3iTEXqqAUEFXNBa7z8HwmMNp9fy/Q35v2pmEK80soKyknoVUsg0d3pselKaR1T6q7oR85Vq3i8MOPENOrFx1fe43wBg4kGmMajy1lYlBVdqzO4sv3dtO6Y3PGPjqQuMRo4hKjg5qX44svOfTgQ0T16E7HN14nPCEhqPkYY85lBeQCdyK7kIy/7uDwzhOkdk/k6h/3DHZKABSsXs2hX/6SqK5d6fjGG4QnJgY7JWNMFVZALmAHdxznozmbCY8MY9gdvehzZVpQZldVVfCvrzn4wL8T1akTHf88j4gWHj/iY4wJMisgFyBnmevDo227JNLnylTSR3cO+uWqCoXffMPBBx4gsn07Kx7GhDhbjfcCUlJUxqoFO9mzVCkrLScyOpxrJvQKneKxfj0H7rufyNRUOr35JhHJobO7ojGmOuuBXCD2bjzKqoW7KDhZTMvuUMeOuQFXuGEDB3/+CyLbtKHjm38molWrYKdkjKmDFZAmrqSojBVvbWfvhqMkt4vjpvsuZvv+9URGh85GVEWbN3PwF5MIb9WKjvPfJLJNm2CnZIypBysgTVxEdDjFBaVcNq4rA0Z2JDw8jO37g53VWUVbtnJg4s8Jb9GCTvPfJDIlJdgpGWPqyQpIE5Sb6eBfH+xhxN0XEds8irGPDgyJ2VUVTi5ZQs7MWbTJzGSfCGGJia7ikZoa7NSMMV6wAtKElJWWs+4f+1m/dD+RMeEczyqgXfPgLHxYk4rNoPT06TObQWlREYXr1pGYlhbs9IwxXrAC0kRkfpfHyrd3ciK7kJ5DUrjyth40S4gKdlrV5Pz++eqbQRUX22ZQxpyHrIA0EZtXHqa8zMnND/WnY9/Qmv6qqhR+vYa8v7xNWY7nlffLsqrv+W6MCW1WQM5TqsrudTm0ah9Pi7ZxDPtpL8Ijw0JqdpWzqIiTS5aQ979vU/zdd4QnJREWH4/T4ah2bISNfxhz3rECch46dfw0qxbsZN+3ufS9ph3DftqLmPjgLbdeVenhw+QtWEDe397DefIk0b17k/rssySMGc2pZcvOjIFUsM2gjDk/WQE5jzidSu5O5a8ffA2qXHlbdy4ZHrxNnipTVQrXfEPe229zasUKEKH59dfT8q47iU1Px72vy5lxjpyZsyjNzCQyLY02jz1q4x/GnIesgJxHvl15iCMblI59E7l2Qi8SWsUGPIfKU3C/S0uj1S//HZxO12WqXbsIT0oieeJEWkz4CZE1zKqyzaCMaRqsgIS4spJyHHnFJKU0o89VaezL3M0P7ux/5i/6QKo6BbcsM5MjT08BcF+m+h0JY8YQFhMT8NyMMYFnBSSEHdx+nIy/7kQEfvrboURGh5PYQYJSPABy/jCz2hRcgPDkZLp88Peg5WWMCY6grMYrIi1FZJmIfOf+t9qa3SLSS0Q2Vrrli8ij7temisjhSq+NDvxX4T9FjhKWv7mNxbM3IgLD7uhNWHhwFk5WVYo2b+bI9GdqnGpbfvy4FQ9jLkDB6oE8CaxQ1Rki8qT78ROVD1DVncAAABEJBw4DH1Q6ZKaqPh+gfAMm70gBf39+PSWFZQwe3Zn0mzoRERn4qbmlhw9zcskSTi5aTMn33yPR0UhMjMceiE3BNebCFKwCMhYY5r4/H8igSgGp4jpgj6qG0DKAjau81El4ZBiJbZrRfVAb+l3bjuR28YHNweHg1NKlnFy0mMI1awBoNngwyRP/jeY33ogjI8Om4BpjzhBVDfxJRU6oalKlx3mqWuPWcyIyD1ivqnPcj6cC9wL5wFrgcVXNq6HtJGASQEpKSvrChQt9ytnhcBAf37A3dE8x1Knk7oTcXUq3G4WImNovBTUkj5iv1xC/aBFhx4/jbNkSx9ixnB6cTtSOHcT861/EbNyElJZS1qYNpy8bStGQoThbJdcdY+gQn/Lx1/c00DFCIQeLYTH8GWP48OHrVHVwtRdU1S83YDmwxcNtLHCiyrF5tcSJAo4BKZWeSwHCcY3hPAvMq09O6enp6quVK1f63LamGNn7TurC332tc+5boR+9tEkdJ077LY8Tixfr9v4DdFuv3mdvffrq9vTBuq1Xb90xZKhmTZumhRs2qNPp9FseTTFGKORgMSyGP2MAa9XDe6rfLmGp6vU1vSYi2SKSqqpZIpIKeF4gyeUmXL2P7Eqxz9wXkdeA/2uMnAPF6VS+en83mz89SGxCFDfddzFdB7b26zk9zqAqL4eyMtq9+Efir72WsKjQW3zRGBO6gjUGshi4B5jh/ndRLcdOABZUfqKi+Lgf3oKrZ3PeCAsTCk4U0/fqdlx2SzeiYxv/x1Cak0PR+g0UbVhP4foNNc6g0uJiEkaObPTzG2OavmAVkBnAuyIyETgAjAcQkTTgdVUd7X7cDBgJ3Fel/XMiMgBQYJ+H10NOYX4Jh1Y7Od6rgJapcYyc2JcwL/bpqPoJ8MrLf2h5OcW7d1O03lUsijZsoPTQIQAkOprYiy+2RQyNMY0uKAVEVXNxzayq+nwmMLrS40Kg2trkqnqXXxNsRKrK9q+y+Or93ZSchqP782mZGud18aj6CfCsp6eQv3w5WlBI0caNZ4pDeOtWNBs4iBZ33kGzgQOJuegiJCrqnBgVbAaVMaYh7JPofnQiu5CMv+zg8K4TpHZPJK5nPr0u8/4v/pyZs6pvwlRSgmPpP4nu2ZOEH4yh2aBBxA4cSGT79h4/1GeLGBpjGpsVED/a9mUmxw45GH5nby66IpXPVn3mdQxnURFlmZmeXxSh6+Laho/OZYsYGmMakxWQRnZk70lUIbVbIpeO6UL/6zoQlxjtdRwtKeHE++9z7KWXazzGxi+MMcEUnAWWmqCSojJWLdjJ+79fx9eL9wAQGR3udfHQ8nJOLlrEntFjODJtOpEdO5L8wP1IlRVubfzCGBNs1gNpBHs3HmXVwl0UnCzmkmHtGTq2q9cxVJVTy5dzdPZsSnbvIbrPRXR47VXirroKESG6a1cbvzDGhBQrIA2079tj/GPutyS3i+em+y4mpUuCV+1VlYKvvuLozFmc3rKFqC5daDdrFs1vGImEne0g2viFMSbUWAHxgTqVEzmFtGgbR8e+yYy4uzc9h7Yl3Msl1ws3bODozFkUrllDRFoqqc8+S+LYHyIR9mMxxoQ+e6fyUm6mg4y3d5KXXcCd0y4nJj6Si67wvHVrhaofAkz6yY85vX4DjowMwpOTSXn6aZJ+fLstJWKMOa9YAaknZ7ny9eK9rF+6n6iYCK4a34PouLq/fZ4+BHjsDzMhJobWjz1Gy7vuJKxZM/9/AcYY08isgNRDcWEpez5RSk7to9fQtlx5W3dim9evt5Dz/AueN2FKSqLVfZMaO1VjjAkYKyD1EN0skuZpcMWN/enYp9rKKudQp5PT27bjyMjA8dlnlGVnezyupueNMeZ8YQWkntoODKuxeJQ7CihY/ZWraKxaRfnRYyBC7CWXENa8Oc5Tp6q1sQ8BGmPOd1ZA6lDTKrgl+/fj+OwzHBkZFHyzFkpLCWvenLirrqT5sGHEXX01ES1b2iKGxpgmywpILTwNgGc++RTZ//Mc5ceOARDVrRst77qL+GHX0mzgQCQy8pwYtoihMaapsgJSC0+r4FJejvPUKVKmTCF+2LVEtW9fZxz7EKAxpimyAlKLGnfxKymh5Z13BDgbY4wJLbaYYi1qGui2AXBjjLECUqs2jz1qq+AaY0wNglJARGS8iGwVEaeIDK7luFEislNEdovIk5Webykiy0TkO/e/LfyRZ+LNN5P6zHQi0tJQICItjdRnptsAuDHGELweyBbgVmBVTQeISDjwJ+AmoA8wQUT6uF9+Elihqj2AFe7HfpF48830+HQFOXNfpsenK6x4GGOMW1AKiKpuV9WddRw2BNitqntVtQRYCIx1vzYWmO++Px8Y559MjTHG1ERUNXgnF8kAJqvqWg+v3QaMUtWfux/fBQxV1QdF5ISqJlU6Nk9VPV7GEpFJwCSAlJSU9IULF/qUq8PhID4+3qe2FqNpxwiFHCyGxfBnjOHDh69T1erDDarqlxuwHNelqqq3sZWOyQAG19B+PPB6pcd3AS+675+ocmxefXJKT09XX61cudLnthajaccIhRwshsXwZwxgrXp4T/Xb50BU9foGhjgEdKj0uD2Q6b6fLSKpqpolIqlATgPPZYwxxkuhPI33G6CHiHQRkSjgJ8Bi92uLgXvc9+8BFgUhP2OMuaAFZQxERG4BXgRaAyeAjap6o4ik4bpsNdp93GhgFhAOzFPVZ93PJwPvAh2BA8B4VT1ej/MeBfb7mHYr4JiPbS1G044RCjlYDIvhzxidVLV11SeDOoh+PhGRteppEMliXPAxQiEHi2ExAhGjqlC+hGWMMSaEWQExxhjjEysg9feqxbAYIZyDxbAYgYhxDhsDMcYY4xPrgRhjjPGJFRBjjDE+sQJSDyJyi4ioiPT2sX25iGwUkU0isl5ErvAhRlsRWSgie0Rkm4h8LCI9fchhqzuP/xARr3/+leJU3LxeCdlDjM5etk8Rkb+KyF4RWSciq92fLfImhqPK43tFZI43MWqKFegYlduKyGj3NgcdA3V+d3sVkf+t9DhCRI6KyP95GeOFSo8ni8hUH3JpLyKL3N+HPSIy2/1hZG9iVPwf3SIifxORZg3IYa+IzBGR6AbksEREkupu5THO0+7f+83ueEN9ieOJFZD6mQB8gevT8L4oUtUBqtofeAr4b28ai4gAHwAZqtpNVfsAvwFSfMihLzASGA381ps8qsSpuM1ohBj76tvQ/b34EFilql1VNR3Xz6XuzembOBG5DtcHdEep6oEAn74A6Ccise7HI4HDXsYoBm4VkVa+JuH+//F34EN1bffQE4gHnvUyVMX/0X5ACXB/A3LoAcQCzzUgh+PAL71sj4hcDvwAGKSqlwDXAwe9jVMTKyB1EJF44EpgIr4XkMoSgDwv2wwHSlV1bsUTqrpRVT/3JQFVzcG1QvGD7v/s55MRQEmV78V+VX0xiDkFnYhcDbwGjFHVPUFK4x/AGPf9CcACL9uX4Zop9FgDchgBnFbVPwOoark73r9524uo5HOgeyPkcLf7/cQXq4F2PrRLBY6parE7l2OqmllHm3qzAlK3ccAnqroLOC4ig3yIEevuOu4AXgee8bJ9P2CdD+etkaruxfXzb+Nl04qvpeL2Yx9OXznGB1627Qus9+GcteWwEZjeCDGDJRrXenDjVHVHEPNYCPxERGKAS4CvfYjxJ+AOEUn0MYe+VPldUdV8XEseeVMEANelOFyb2n3bCDns8zGHcOA6zq4F6I1/Ah1EZJeIvCQi1/oQo0Z+W423CZmAaz0ucP2CTMD7N7AiVR0AZ7qUb4lIPw3+HGpfeh9nvpYGaIwYAIjIn4CrcPVKLvU1BxG5F2jUZR4CqBT4Clcv+ZFgJaGqm93jWROAj32MkS8ibwEPA0U+hBDA0+9VTc/XJNb9hwW4eiBvNFIO3qjIoTOugrTMy/aoqkNE0oGrcV3JeEdEnlTVN72N5Yn1QGohrkUbRwCvi8g+4FfAjxty2UdVV+Na1KzawmS12Aqk+3pOT0SkK1DO+bcU/lbgTC9QVX+J668zb76fTY0TuB24VER+E+RcFgPP4/3lq8pm4SqGcT603UqVPwREJAHX1hDeXNqrPE73kLp2RW1oDilAXTuxVssB6ARE4cMYCLguoalqhqr+FngQ+JEvcTyxAlK724C3VLWTqnZW1Q7A97j+4vWJuGZyhQO5XjT7FIgWkV9UinOpr91REWkNzAXmhEAvyFufAjEi8kCl53y9tt1kqGohrsHSO0RkYhBTmQdMV1VvLvmcw72y9ru4ioi3VgDNRORuOHP55wXgTff3KBBqymGOqnrdq1LVk7h6ZJNFJNKbtiLSS0R6VHpqAL6vSF6NFZDaTcA1+6my94GfehnnzPV24B3gHvfAWr243+RvAUa6pyVuBaZydoMtb3LYimu3yH8C07xoXzVOxc2XWVg+c38vxgHXisj3IrIGmA88Ecg8GpP7OntxQ+O433hHAVNEZKyXzZuJyKFKt//wMYdDqjrbl7ZVvICrp+7t+St+V8aLyHfALuA0rlmLAVEph9vcOeQCzortKHyMuQHYhPcTeeKB+eKa+r8Z6IPrvaNR2FImxgSZiPQHXlPVIcHOxTQ+cX3uawFwq6o26mSYYLMCYkwQicj9uC5PPKqq/wx2PsZ4wwqIMcYYn9gYiDHGGJ9YATHGGOMTKyDGGGN8YgXEGD/xtAqqiDxanzWZ6nucMcFkg+jG+IF7yZo/AMNUtdi9wmwUriVHBqvqsTra76vPccYEk/VAjPGPaqug4lrZIA1YKSIrAUTkZRFZ6+6pTHM/97CH424Q174n68W1P4Wvq7oa02isB2KMH7jf4L/AtczKcuAdVf2sas9CRFqq6nH3chcrgIfdixKeOc7de/k7cJOqFojIE0C0qp7PKwibJsBW4zXGD2paBdXDobeLyCRcv4upuJaa2FzlmMvcz3/pXsczCtf+EMYElRUQY/zEvd5ZBpAhIt8C91R+XUS6AJOBS1U1T0TeBGI8hBJgmapO8G/GxnjHxkCM8YNaVkE9BTR3P5eAayvYkyKSgmvjogqVj/sXcKWIdHfHbiYiPf2ZvzH1YT0QY/wjHnhRRJJwbdW6G9c2whOAf4hIlqoOF5ENuPaP2At8Wan9q1WOuxdYICLR7ten4Fpp1pigsUF0Y4yi/deEAAAAN0lEQVQxPrFLWMYYY3xiBcQYY4xPrIAYY4zxiRUQY4wxPrECYowxxidWQIwxxvjECogxxhif/H+X88RroJLuaAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "states = game.states[1:size+1]\n",
    "true_values = pd.Series([(2*x*(1/(size+1)) - 1)\n",
    "                         for x in range(size+2)][1:size+1], \n",
    "                        index=states)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "\n",
    "for i, value_function in value_sets.items():\n",
    "    values = list(value_function.values())[1:size+1]\n",
    "    ax.plot(states, values, marker='o', label=str(i))\n",
    "ax.plot(true_values.index, true_values, linestyle='--', \n",
    "        label='True values')\n",
    "plt.title('TD(%d) Value Updates on Random Walk' % n)\n",
    "ax.set_xlabel('State')\n",
    "ax.set_ylabel('Value')\n",
    "plt.legend()\n",
    "plt.savefig(\"random_walk_td%d_19.pdf\" % n)\n",
    "plt.grid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>10</th>\n",
       "      <th>100</th>\n",
       "      <th>True Values</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.10</td>\n",
       "      <td>-0.368335</td>\n",
       "      <td>-0.943730</td>\n",
       "      <td>-0.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>B</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.10</td>\n",
       "      <td>-0.399838</td>\n",
       "      <td>-0.897351</td>\n",
       "      <td>-0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.19</td>\n",
       "      <td>-0.459379</td>\n",
       "      <td>-0.814759</td>\n",
       "      <td>-0.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.10</td>\n",
       "      <td>-0.350100</td>\n",
       "      <td>-0.778533</td>\n",
       "      <td>-0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.220020</td>\n",
       "      <td>-0.720796</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.150461</td>\n",
       "      <td>-0.653122</td>\n",
       "      <td>-0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>G</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.133647</td>\n",
       "      <td>-0.492940</td>\n",
       "      <td>-0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>H</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.094026</td>\n",
       "      <td>-0.282326</td>\n",
       "      <td>-0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.054104</td>\n",
       "      <td>-0.265055</td>\n",
       "      <td>-0.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>J</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.013983</td>\n",
       "      <td>-0.203500</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>K</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.045428</td>\n",
       "      <td>-0.096588</td>\n",
       "      <td>0.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>L</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.067295</td>\n",
       "      <td>0.202457</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.083581</td>\n",
       "      <td>0.316446</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>N</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.158966</td>\n",
       "      <td>0.433538</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.212566</td>\n",
       "      <td>0.558365</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.311715</td>\n",
       "      <td>0.620013</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.404926</td>\n",
       "      <td>0.644650</td>\n",
       "      <td>0.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.444872</td>\n",
       "      <td>0.773039</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.459113</td>\n",
       "      <td>0.891250</td>\n",
       "      <td>0.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0     1        10       100  True Values\n",
       "A   0.0 -0.10 -0.368335 -0.943730         -0.9\n",
       "B   0.0 -0.10 -0.399838 -0.897351         -0.8\n",
       "C   0.0 -0.19 -0.459379 -0.814759         -0.7\n",
       "D   0.0 -0.10 -0.350100 -0.778533         -0.6\n",
       "E   0.0  0.00 -0.220020 -0.720796         -0.5\n",
       "F   0.0  0.00 -0.150461 -0.653122         -0.4\n",
       "G   0.0  0.00 -0.133647 -0.492940         -0.3\n",
       "H   0.0  0.00 -0.094026 -0.282326         -0.2\n",
       "I   0.0  0.00 -0.054104 -0.265055         -0.1\n",
       "J   0.0  0.00  0.013983 -0.203500          0.0\n",
       "K   0.0  0.00  0.045428 -0.096588          0.1\n",
       "L   0.0  0.00  0.067295  0.202457          0.2\n",
       "M   0.0  0.00  0.083581  0.316446          0.3\n",
       "N   0.0  0.00  0.158966  0.433538          0.4\n",
       "O   0.0  0.00  0.212566  0.558365          0.5\n",
       "P   0.0  0.00  0.311715  0.620013          0.6\n",
       "Q   0.0  0.00  0.404926  0.644650          0.7\n",
       "R   0.0  0.00  0.444872  0.773039          0.8\n",
       "S   0.0  0.00  0.459113  0.891250          0.9\n",
       "T1  0.0  0.00  0.000000  0.000000          NaN\n",
       "T2  0.0  0.00  0.000000  0.000000          NaN"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat([pd.DataFrame(value_sets),\n",
    "           pd.Series(true_values, name='True Values')], axis=1, sort=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RMS error metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1317165375145056"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# RMS error metric\n",
    "def rms_error(x, true_values):\n",
    "    \"\"\"Root-mean-squared error in values\n",
    "    \"\"\"\n",
    "    \n",
    "    return np.sqrt(np.sum((x - true_values)**2)/len(x))\n",
    "\n",
    "# Use array version here\n",
    "true_values = np.array(\n",
    "    [(game.terminal_rewards['T1']*(1 - x) + game.terminal_rewards['T2']*x)\n",
    "     for x in np.linspace(0.0, 1.0, game.size + 2)]\n",
    ")[1:game.size+1]\n",
    "\n",
    "values = np.array(list(value_sets[100].values()))[1:size+1]\n",
    "rms_error(values, true_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "from importlib import reload\n",
    "\n",
    "import randomwalk\n",
    "reload(randomwalk)\n",
    "\n",
    "RandomWalkGame = randomwalk.RandomWalkGame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### n-step TD - effect of variation in n (and alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "128 [0.5067675989278085, 0.5346220655366686, 0.5882351606556061]\n"
     ]
    }
   ],
   "source": [
    "size = 19\n",
    "terminal_rewards = {'T1': -1.0, 'T2': 1.0}\n",
    "\n",
    "game = RandomWalkGame(size=size, terminal_rewards=terminal_rewards)\n",
    "\n",
    "# Initialization of parameters\n",
    "initial_value = 0.0\n",
    "gamma = 1.0\n",
    "\n",
    "# Iterate over learning_rates and n_values\n",
    "learning_rates = np.linspace(0.01, 1.0, 100)\n",
    "n_values = [2**i for i in range(8)]\n",
    "\n",
    "rms_error_sets = {}\n",
    "\n",
    "# TODO: Should be repeating the test 100 times for each\n",
    "# pair of n, alpha values and averaging the results.\n",
    "for n in n_values:\n",
    "\n",
    "    # OVERRIDE HERE\n",
    "    #n = 10\n",
    "\n",
    "    rms_results = []\n",
    "\n",
    "    for learning_rate in learning_rates:\n",
    "\n",
    "        # Initialise value function\n",
    "        value_function = {\n",
    "            s: 0.0 if s in game.terminal_states else initial_value\n",
    "            for s in game.states\n",
    "        }\n",
    "        \n",
    "        rng = random.Random(5)\n",
    "\n",
    "        n_iters = 10\n",
    "        for iteration in range(n_iters):\n",
    "\n",
    "            past_states = [game.state]\n",
    "            past_rewards = [None]\n",
    "\n",
    "            T = 999999\n",
    "            t = 0\n",
    "            while True:\n",
    "\n",
    "                # Behaviour policy\n",
    "                if t < T:\n",
    "                    move = rng.choice(game.available_moves())\n",
    "                    game.make_move([1, move])\n",
    "                    past_states.append(game.state)\n",
    "                    if not game.game_over:\n",
    "                        reward = game.get_rewards()[1]\n",
    "                    else:\n",
    "                        reward = game.get_terminal_rewards()[1]\n",
    "                    past_rewards.append(reward)\n",
    "                    if game.game_over:\n",
    "                        T = t + 1\n",
    "\n",
    "                # State to be updated\n",
    "                tau = t - n + 1\n",
    "\n",
    "                # TD(λ) update\n",
    "                if tau >= 0:\n",
    "                    \n",
    "                    g = 0\n",
    "                    for i in range(tau + 1, min(tau + n, T) + 1):\n",
    "                        g += gamma**(i - tau - 1)*past_rewards[i]\n",
    "                    if tau + n < T:\n",
    "                        g += gamma**n*value_function[past_states[tau + n]]\n",
    "\n",
    "                    value_function[past_states[tau]] += \\\n",
    "                                learning_rate*(g - value_function[past_states[tau]])\n",
    "\n",
    "                t += 1\n",
    "                if tau == T - 1:\n",
    "                    break\n",
    "\n",
    "            game.reset()\n",
    "            #break\n",
    "\n",
    "        values = np.array(list(value_function.values())[1:size+1])\n",
    "        rms_results.append(rms_error(values, true_values))\n",
    "        #break\n",
    "\n",
    "    rms_error_sets[n] = rms_results\n",
    "    \n",
    "    # OVERRIDE\n",
    "    #break\n",
    "\n",
    "print(n, rms_results[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "T1    0.0\n",
       "A    -1.0\n",
       "B    -1.0\n",
       "C    -1.0\n",
       "D    -1.0\n",
       "E    -1.0\n",
       "F    -1.0\n",
       "G    -1.0\n",
       "H    -1.0\n",
       "I    -1.0\n",
       "J    -1.0\n",
       "K    -1.0\n",
       "L    -1.0\n",
       "M    -1.0\n",
       "N    -1.0\n",
       "O    -1.0\n",
       "P    -1.0\n",
       "Q    -1.0\n",
       "R    -1.0\n",
       "S    -1.0\n",
       "T2    0.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(value_function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOydd3hURduH79nd9N4IpNMCJPQWekdFRcSKvYsVBbu+vBaQV/0UQWzYBREL2EBApITeew2BQHrvdbNlvj/OBkJISEKy2SS793Xlyu45c2Z+c/ac85yZeeYZIaXEhg0bNmxYLypLC7Bhw4YNG5bFZghs2LBhw8qxGQIbNmzYsHJshsCGDRs2rBybIbBhw4YNK8dmCGzYsGHDyrEZAhv1QggRLYR42NI6bIAQ4pwQYpylddQFIcQoIURSpe/10i6EWCqEuNE86poXQgh/IcQJIYRDU5VpMwSNgBDiDSHED2bKu6jSn1EIUVrp+12msnVCiELT3ykhxMdCiHbm0GPDRlMjhOgJ9AL+NH1vJ4T4SwiRIoSQQoiwKukDhRB/CiFyhBBJQojHrrDc+hqr74QQs6+krMpIKdOBjcCjDc2rrtgMQTNHSula8QckABMrbVtiSvazlNIN8AYmA22Bfa3VGAghNHXZVt88zElTl9fKmAoskRdmvxqBNcDNNaT/ATgL+APXAXOEEKPNrrJxWYJS76ZBSmm1f8A54HngMJAP/Aw4Xib9S0AyUAjEAGOBa4ByQAcUAYdMaT2Ar4FU0zGzAbVp3/3ANmCBqdyTwNg66h1XZdsbwA9VtqmBQ8D71eThAOQB3Stt8wNKgTaAF7ASyARyTZ+DKqWNBh6urmwgDJCAprZzUI0uFfAycAbIBn4BvKvk+xCKMdxc3TZT2huAY6Y6RgPdqpy/l0y/t7ZCZxUdQ4A9pt9lDzCkSt1nmX67QmAt4FtDfUYBSaby0oDFdTy3NeYP3APEm87Pa5WvB9PvOg9IMf3NAxyqaHkRyDD9HjcC1wKngBzg1Rrq0d50LlWm718BGZX2/wA8a/r8AHDCpD0OmFr1fFR3LQNdUR7cU2rQEAcMq2a7xnQNhFXa5mra5ldp2xfA4hry9jX9Dnmm87AF5VpcjGJwSlHu6xdN6X81/Z75KNdhpGn7oyjPgHJT+hWm7QHActNvfhaYVqnsgcBeoABIB+ZWqVsJENokz8KmKKS5/pkuxt2mH8vbdBE/VkPaLkAiEGD6HgZ0NH1+g0sfxn8ACwEXlAfs7oobA8UQ6IHpgB1wu+nC8q6D3loNgWn7W8CuGvL5Bni70vcngTWmzz4ob1rOgJvpwv+jUtpo6m4IajwH1Wh6FtgJBKE81BYCS6vku8iUl1MN28KBYmC86by+CJwG7Cudv4NAMOBUjQZvlAf0PaYb8Q7Td59KdT9jKsfJ9P2dGuozyvQbv2uqj1Mdz221+QMRKA+YEab85pryr3iYvmU6f21QDPt2YFYVLf81nZdHUB5MP5p0RAJlQIca6pIA9DN9jkF5MHertK+P6fN1QEdAACNRHmR9K2m4xBAAfU15XF9D2S5UebBX2ledIXAzbWtTaduXwIEa8v8f8LnpvNgBwwFxmfvtQVMZFYb3YKV93wGzK31XAftM590e6GA6d1eb9u8A7jF9dgUGVSnrMHBDkzwLm6KQ5vpn+qHvrvT9PeDzGtJ2QnmbGgfYVdn3Bhc/EP1R3jidKm27A9ho+nw/ylubqLR/d8VFUYveuhqCx4DYGvIZB8RV+r4NuLeGtL2B3Erfo6mDIajtHFRTzgkqtYqAdihvWJpK+XaopqzK22YCv1T6rkJpiYyqdP4evMz5vQfYXWXbDuD+SnX/T6V9T2AyoNXkNQrl7fByLczqzm21+aM8TH6qtM/FlH+FITgDXFtp/9XAuUpaSrnQIq14WEZVSr8PuLEGnYuBGShdjjEo98ljVGktVHPcH8AzlTRUNQRvorRURl/mHAWatF5yHqnGEJi2b0VpbTuiGJocIKaG/N9CGXvoVJf7rcp+T1P5Hqbv33GxIYgCEqoc8wrwrenzZtM5qKlVWeN92dh/tjECpZlXQQmKZUYIsbryoKyU8jTKW+sbQIYQ4ichREANeYaivF2kCiHyhBB5KG+4bSqlSZamX9tEPErLpLEIRLkBqmMD4CSEiBJChKI8kH4HEEI4CyEWCiHihRAFKBerpxBCXc/y63IOqqb/vVLaE4ABxaBUkFjNcZW3BaCcRwCklEbT/sBa8qj2eBPxVY6v9nqpgUwpZVnFlzqe25ryD6isXUpZjNJFVJP2qtdTtpTSYPpcavqfXml/6WXqsgnlQT7CpDka5Y1/JLDFdJ4RQkwQQuw0DdLmoXQ9+daQJyjGZLuUcuNl0uSZ/rtdJk1V7kIxUonAZyj97Uk1pP0/lFbjWiFEnBDi5ZoyFUKohRDvCCHOmH6/c6ZdNdUxFAiouKZN5+RVLlzTD6G0/k4KIfYIIa6vcrwbF+pvVmyGoAaklBNklUFZKeWPUsphKD+wRGn2Y/pcmUSUt2FfKaWn6c9dShlZKU2gEEJU+h6C0kpoMEIIFTARpb/zEkw37i8ob+h3AiullIWm3c+hdINFSSndUW5+UJr7VSlG6eaooG2lz3U5B1RJP6FSWk8ppaOUMrmy9OqqU+lzCspvowhWzm8wSqvgcnlUe7yJkCrH14eqZdXn3FYlFaUuygFCOKN0NVVQVXujXU8ohmA4ijHYhPLGPRTFEGwy6XFA6Qt/H/CXUnoCq7h83R4DQoQQH9aUwGTwKrrL6oSUMl5Keb2U0k9KGYVynnbXkLZQSvmclLIDyj0zQwgxtmJ3leR3ApNQWtQeKK1SuFDH6p4DZ6tc025SymtNZcdKKe9AeTl6F1gmhHCB884FnVDG+syOzRDUESFEFyHEGNMFX4byBlXxhpUOhJkewEgpU1EG+j4QQrgLIVRCiI5CiJGVsmwDTBNC2AkhbgW6odw4DdFoJ4ToBixFeSjPvUzyH1HGJu4yfa7AzVS3PCGEN/D6ZfI4CIwQQoQIITxQmr1Anc9BZT4H3ja1UBBC+AkhJl2uvtXwC3CdEGKsEMIO5cGrRekvrwurgHAhxJ1CCI0Q4naUvvmV9dRRE/U5t1VZBlwvhBgmhLBH6dKofP8uBf5jOm++KF1JjeLSLKWMNem+G2VQvmJw82ZMhgClD9wBZexBL4SYAFxVS9aFKM4WI4QQ71wm3SoUo3MeIYSjqTwAB9P3in3dhBBuQgh7IcTdJh3V3gtCiOuFEJ1MLw0FKPd05fu6Q6XkbijXUzbKC9CcKtlVTb8bKBBCvCSEcDK1KLoLIQaYyr5bCOFnejGrePOvKHsgStde1RaqWbAZgrrjALwDZKE039ugNPNAGfQDyBZC7Dd9vhfl5jiOMuC4DKXfu4JdQGdTfm8Dt0gpKzf168PtQogilIvpL5QLtZ+UssY3QinlLpQ3+gBgdaVd81AGKrNQBh/XXCaPf1E8rQ6j9DFXfWDWdg4qM9+kfa0QotBUdlRNZdegJwblYbXApH8iirtteR2PzwauRzEg2SiDzddLKbPqo+My1PncVqPtGMqg/o8orYNcLu7umI3igXIYOALsN21rLDahdC8lVPougAMmfYXANBRjnIvy9vxXbZlKKfNQBvcnCCFm1ZDsC+CuKi3oCm8eULzuSivtuxplUDYXpdVxjZQys4a8OwPrTHntAD6VUkab9v0PxbjmCSGeR3FMiEdpIR5H+Q0r8zUQYUr/h6krbiJK1+tZlN/9K5TWBChG8Jjp3p2P4jVV0ZV4F8rLUZNQMTpuowkRQtyPMuA6zNJabNhoCQghfkRxBPjD0lrMjRCiDYqh7VN5jMmc2Ca52LBho9kjpbzT0hqaCillBkpXcZNh6xqyYcOGDSvH1jVkw4YNG1aOrUVgw4YNG1ZOixsj8PX1lWFhYXVOX1xcjIuLS7Xb8/Pz8ff3R62u71ypC+gyShAqgcbX6YrzaCxykhMxGo34BofWWO/WRkFWKWUlejz8nNAbtVZR56qY87eWWi3lSUnI0jLU3l7YtW0LKsu/P9a1zoXlhSQUJuDl4EWAa2PO12xi0o6AgxvF9n5X/Fvv27cvS0rpV+3Oppi+3Jh//fr1k/Vh48aN1W5fvXq1nDVrljQYDPXKrzJGvVEmvrZF5v595orzaCwSjh2W7992nTy4dpWUsuZ6tyb2/H1Wfjx1vdy35pyU0jrqXB3mqLfRaJS5v/4qT/TuI2OiBsmCdesavYyGUJc6JxYkysE/Dpa3/nWrLNOXmV+UuUjYJeXr7lIe/rVBvzWwV9bwXG1xLYLGIicnB29vb1QNeLvRZ5WAXmLf7nJRBpqGvSt+w8nNnYiRYywtpUk4dziLXSviCB/oT5+rQiwtp1VhKCgg9fXXKVy9BudBgwh49x3s/P1rP7AZoTVoeW7TcyDhg1Ef4KBusjVeGp+YVaDSQKdxkH3QLEVYvo3XREgpOZh4IWxHdnY2Pj4+lzmidnQpxQDYtbNsd0RuajJx+/fQ66rrsLNvwRd8HclNK+bfb47hF+zG6Lu7cvE8IxsNofTwYc5OvonCtf/iN2MGIV9/1eKMAMB7u9/jePZxZg+bTbBbcO0HNGdiVkPoUHDyNFsRVmMIft2bxI2fbGP5viQMBgO5ubl4e3s3KM/y1CLQCDR+lh0fOPDPSlRqDb2vutaiOpoCbameVZ8dQW2nYsJjPdDYX/n4jo0LSKOR7G++5dydd4GUhC35Ad9HH0E0YPzMUqw4s4JfTv3CA90fYExIC28hZ5+BzJPQxbz3ttV0Dd3QO4A/DyXzwrJDUF6M0WhslBaBnb8LQm05e1peWsKx6HV0GTwMF08vi+loCqRRsv674+RnljLp2d64eTvWfpCNWjHk5ZHy8isURUfjNn4c7WbPRu3hUfuBzZDTuaeZtXMW/fz7Ma3PNEvLaTinTFFIulxj1mKspkXgqFHx9WgD/UK9WLByLwC+vpeLkHt5pJToUost3i10bNN6yktL6TNhokV1NAX718Zz9lAWQ2/uRGB46zZ6TUXpoUPE3XQTRdu24f+f/xD40Uct1ggU64qZsWkGzhpn/m/E/6FRtYL33JjV0CYSvMLMWozVGAIOLMZx8bV8N14Q6VaKTqo4U2x/xdkZC8sxFuuwD7DcQLE0Gjnwz9+07RROu05dLKajKUg8nsOuP+Po3L8NPccEWVpOi0dKSc6ixZy7+x6EUBH24494331Xix1vkVLy5vY3iS+I570R7+HnXL2XZIuiOBvit5u9NQDWZAgibwJnX5y3vUdHhyKK7L2ZuuQAm0/VFJTw8pQ3g4Hi+MMHyE1Jou81rbs1UJhTxtqvj+HVzoVRtsHhBmMsLibluedInzMH1+HDaf/bcpx6dLe0rAbxU8xPrD63mqf7PM3AdgMtLadxOP47SANETjZ7UdZjCBxcYdiz5MTtpyA/j8kj+9HB14VHFu1la2z9owzrUpUIuJY0BAf+WYmLpxfhg1tvEFOD3siaL45iMBiZMLUH9o6toLlvQbRnznD2ttspWPMPfs/NIOjjBS22K6iCI5lHeG/Pe4wIGsGD3R+0tJzG48hy8OsK/uY30tZjCAD6P8Rpe+Wk9ozowo+PDKK9rwsPL9pTb2OgSy1G7e2IykIPprz0NOIO7KXnuGtQa+wsoqEp2L78NBnnChh7bzc8/Z1rP8BGjRSsXcu5W2/DkJtLyDff4PvII4hmMEu4IeSV5fHcpudo49SGOcPmoBItuz7nyUuEhO3Q4xZoghZwKzlrdcTemdMeQ/EmF+/8Y3i72LPk4SjCfFx46Ps9RMdk1DkrXUox9hZsDRxetxohBD3GXm0xDebmzP4MDm9MoueYIDr2rWmpYxu1IQ0GMuZ+SPK0Z7Dv3In2vy3HZVC91vxplhilkVe2vkJWaRYfjPoAD4eW3bK5iKPLlf/db26S4qzGEGxJ2sINy2/gbK6RjnaZEP0/kBIfVweWPjKITm1ceXTRPtYdT681L6PWgD671GLdQnqdjqMb/6VjvyjcvK/c86k5k5dRwoZFJ/Bv786QmzpZWk6LxZCfT+LUx8j+4gs8b72V0MWLlXhBrYCvj3zN1uStvDTgJbr7tuwxjks4ugwC+4N3h9rTNgJWYwg8HTwpyixCr9fTqecgiN8GcdEAeLnY8+PDg+jWzo3HftjH6iOpl81Ll1YMEuws5DEUu3s7pYUF9Bo/wSLlmxuDzsg/Xx5FqAVXP9IdtcZqLtNGRRsby9lbb6N41y7avvkm7Wa9hcr+yj3lmhMxpTF8fPBjrm1/Lbd1uc3SchqXzBglyFyPW5usSKu5wyJ8IggtD0UKSdjYB8AjGNa/Bab1GDyc7Vj8cBS9gj158sf9LN+XVGNeulSTx1CAZVoEh/9djad/O0J79LZI+eZm+2+nyUosYux9EbZJY1dI4fr1nLt9CsaSEkK//w6v21vPwzKjJIPvs74nzD2M1we/3vq8yI4sA6FqEm+hCqzGEKhVaoK1weQ55WHn5AKjXoaU/XDywnrr7o52LH5oIIM7+vDcr4dYvONctXnpUosQThrUHk0f1yc7KYGkE0fpOe6aFj/QVx1xBzM5vDGJXmOCad+zdXZ7mRMpJVmfLyTpyaew79iR9suX4dy3r6VlNRo6o44XNr2AVmqZO2ouznatzIFASjjyK7QfAW5NF+Op9T1JaqCgoABVsYokhyRicmKg5xTwDYcNs8FoOJ/O2V7D1/cNYFy3Nsz88xifRp++JK+KgWJLvIkc+nc1ao2GyFHjmrxsc1OYU8aGRSfwC3Fj8OSOlpbT4jCWlZHy/AtkzpuH+8SJhC5e1CIDxl2OBfsXsD9jP3f43EFHz1Z4jSTuhtyzTdotBGY2BEKIa4QQMUKI00KIl6vZP0oIkS+EOGj6+6+5tJw5cwaANOc0tqVsA7UGRr+mBHQ6/MtFaR3t1Hx2dz9u6BXAe2ti+N/qE0hTF5I0SnRplgktodOWcXzzBjpHDcXZvRV5SABGg5F/vzmG0Si56uFI1HZW847SKOgyMoi/514K/v4bv+nTCXjvXVSOratbbX3Cer499i23d7md/i79LS3HPOz/HuxdIeLGJi3WbHebEEINfAJMACKAO4QQEdUk3SKl7G36e8tcenr06MH9999PgH8A25K3KRsjJkG73hA9B/TlF6W3U6uYd3tv7h4UwsJNcbzy2xEMRok+qxSpM1pkoPjUzm1oS4rpOc78U86bmn1r4kk9nc/IO7rg2aaVNffNjCYpiXO3T0F7+jRBHy/Ad+qjra7fPLEgkZlbZxLpE8mLA160tBzzUJYPx35XXEYdmvb5Ys7XroHAaSllnJSyHPgJmGTG8i6LPjYWxx+WMMJrAAczDlJUXqRM1Bj7X8hLgL3fXHKMSiWYNak7T4/pxE97EnlyyX5KkgoBy8woPrJhLV7tAgjq1rpc5dLi8tnz9znCo/zpEtU6XBubisKNG/H6v/fBaCRsyQ+4jWt9XYZl+jJmbJoBQllkxl7dOjyfLuHIMtCVQL/7mrxoc06LDQQSK31PAqqbxTJYCHEISAGel1Ieq5pACPEo8CiAv78/0dHRdRZRVFREdHQ09keO4PXDD/h5347eRc/X676ml3MvkCp6efbEZf3b7CoKxqC59AHfzx7u6GrP0mNp9D2TxzXCju0n98KpOstoMGV5OSSfPEbgoOFs2rSp1vQV9W7uGHSSM2skGidQBWUQHX1lsZ+g5dS5sXDauBG3X36lPDCQ7KeeJDkjAzLqPimypfBj9o+cLDrJVL+pxO6NJZbYVvlb99v7CcIljL2nCiA2uto0Zqt3TWtYNvQPuBX4qtL3e4AFVdK4A66mz9cCsbXle6VrFpenpMjjXbrKzMWL5MAfBso3t795IVHyfmVN0HVvVp+Jid/3J8llL22Q217bLNMLSuulo6FsWvKt/GDKRFmUm1On9C1l/d613xyVnzy+QaaeyWtwXi2lzg3FqNfLtDlz5PEuXWXCE0/KjWvWWFqS2fg99nfZ/bvucv6++Rdtb3W/dcpB5Rm0c+Flk5lrzWJzdg0lAZXXiAtCeeuvbIQKpJRFps+rADshhFl8BjVt26Jyd0cXe4aodlFsT9l+fgCYgD7KKP2OT6EgpcY8buwTSD8nB04Yddz06XbiMovMIfUSDHo9x6LX0aHvgFa1+Ezs3nRO7UpnwHVhtO3Quga/zYWxtJSkZ54h5/tFeN93L0EfzQeH1rk8aUxODLN3zmZg24E80fsJS8sxL/u+B7UD9Gxab6EKzGkI9gCdhRDthRD2wBTgr8oJhBBthWlUSwgx0KQn2xxihBA4hoejjYlhWOAwkouSOVtw9kKCMf9RQr5unFNjHobCcjSlBoYNCaG03MDNn21nf0KuOeRexNkDeynJz6P76KvMXlZTUZSrZdOPMfi3d6ffNaGWltMi0OfkEH///RSt34D/q6/i/8orLXIpybpQWF7IjOgZuNu78+6Id1vHIjM1UV6izB2ImAROlnnRM5shkFLqgaeAf4ATwC9SymNCiMeEEI+Zkt0CHDWNEXwETJHnX9MbH4euXdGeOsXQdkMAJf7QebzCYMAjcHAJpB+v9viKGcWh3XxZ/vgQ3J3suPPLnXWKT9QQjmxci4uXNx36tA6XOWmUbFh0HIPeyLj7I1BZcKnPlkJ5QgLn7rgD7ckYAj+aj/e991haktmQUvLfbf8luSiZ/xv5f/g6tfKJhUd+AW0B9LvfYhLMegdKKVdJKcOllB2llG+btn0upfzc9PljKWWklLKXlHKQlHK7OfU4dAnHWFKCX76ks1dnNidtvjjBiOfBwQ3+nVnt8eUpSleQfTsXwnxdWP74EML93Xh08V4W74w3i+ai3BzOHthL5IgxqFrJ29/h6CQST+Qy9JbOttDSdaD0yFHOTbkDY14+Id9+i/v48ZaWZFYWHV/EuoR1TO83nX7+/Swtx7xICbsWgn8PCB1iMRlW9Srm2EVZzrHs5ElGBo1kf/p+CsoLLiRw9oYRL8LpdRC77pLjdanFqD0dUDkr8f99XR346dFBjO7Shpl/HOWd1ScxGhu3QXNiy0ak0UjkqNZx8+emFbPj9zOE9vAhcniApeU0e4q2bSP+vvtQOTkRunQpzn37WFqSWdmfvp8P933IuJBx3Btxr6XlmJ+zmyHjOERNbZJ1B2rCqgyBQ6dOIATamFOMDBqJXurZnlylETLwEfBqD2v/Awb9Rbt0KUWXTCRzttew8J5+3BUVwuebzvDMzwcp0xloDKSUHNu0nnbhXfEOCGyUPC2J0WBk/fcn0NirGG1bcrJW8lf+TeJjj2MfHEzo0h9x6NDe0pLMSlZpFs9vep4gtyBmDZ1lHdfHroXg7NPkISWqYjWGIOF4NsvmHcPYPgJtTAw9fHvg6eDJpqQqPvkaB7hqFmSeUKZ7mzCWG9BnVb8GgUatYvaN3Xnpmq6sOJTCPV/vIqe4/JJ09SU97jTZSQlEjhjb4LyaAwf+TSD9bAEjp3TBxQIB+1oSOYsWk/L88zj37k3oD4uxa9O6F+bRG/W8uPlFCssLmTtqLq72lgnx3qTknIWYVcrYgJ1lw4FYjSEQCDITCtF16EnZqRjUKjXDA4ezJXkLBmOVN/iu10PoUMWDqCwfuLAGgX0NoaeFEDw+qiOf3NmXQ0n53PTpNs5mFTdI87FN61Hb2dFlyPAG5dMcyE4uYveKs3Ts24ZO/Vv3Q60hSCnJ/GgB6XPm4DZ+HMFffYnazc3SsszOxwc+Zk/aHmYOnkm4V7il5TQNe74ClRoGPGxpJdZjCNz9FItb3rYjuoREjMXFjAgeQb42n8NZhy9OLARc/TaUZMPm94FKaxC0u/ybynU927H0kUEUlOmZ/Ok2dsVdmTesQa/j5PbNdOw/CEeXlv12ZDAYWffdcRycNYy8M9w6mvxXgDQaSZ81m6xPP8XjlpsJ/PBDVK10jkBlNiZs5OujX3Nr+K3c0PEGS8tpGrRFsH+x4jLqbvmxMqsxBK7ejgiVoMytLUiJ9vRphgQMQSM0bEqsJmRDQB/ofRfs/AyyTqNLLUY4qlF71X5j9gv14vcnhuDjYs/dX++67CI3NRG3fw9lhQVEjhxT72ObG/vXxJOVWMSoO7vi5NpK48Q0EKnTkfLiS+T++CPeDz1Iu1mzEJpW7DtvIrEgkde2vkaETwQvDXzJ0nKajgOLQZsPUY/VnrYJsBpDoFarcPN2oETlDkDZyRjc7d3p69/30nGCCsb+FzSO8M+rykBxPdYgCPVx4bfHhzKwvTfP/XqI9/+JqZdH0bFNG3Dx9CKsZ8teVCQ7uYi9q87RuX8bOvTxs7ScZolRqyXpmWcpWLkSv+nT8X/hBatoNZXpy5gePR0hBHNHzcVB3fpbPwAYdLD9YwgZAsEDLa0GsCJDAODu60RRMahcXNDGxAAwMmgkp/NOk1yUfOkBbv4w8kXkqX/RpRRgX0u3UFU8nO347oGB3DEwmI83nuappfspLa/do6ikIJ+zB/bQddioFj13wGgwsmHRCRycNQyfYiX9vvXEWFxM4mOPUbRhA/4z/4Pv1EctLalJkFIye+dsTuWe4p3h7xDo2vK94urMkWVQkATDpltayXmszhAUZJfhEB5O2SmTIQgeCUB0YnT1B0U9ht5jCFIvsGtX/5F9O7WKOZN78J/rurH6aBq3LdxBWn7ZZY+J2b4Zo8FA5IiW3S10cF0iGfGFDL893NYlVA2GwkISHn6Ekl27affO//C+6y5LS2oylscu588zfzK111SGB7V8Z4g6YzTCtnnQJhI6N5+5QVZjCKTOgLeDirJCHarO3dDGnEJKSah7KB09OrIhYUP1B2rsKe8yAwD77JXVp6kFIQQPD+/A1/f1Jy6ziEmfbOVQYl6N6U9sjcYvJAy/0JbrN56bVszuFWfp0MePTv1sXkJV0efmknD/A5QeOULghx/ieWPTrkhlSY5lHWPOrjkMCRjCYz2bRx95k3FqjbIq4rBnLTqBrCpWYwiS/jmIz750XChBH9wVY2Eh+tRUAMaEjGFf+j7yyqp/OJfrQhGqcjT7ZkFB6hVrGNPVn+VPDMFOreK2hTv469ClkU5z01JIjY2h67BRV1yOpZFGycYfTqKxV1mi72AAACAASURBVDFiis1LqCr67GwS7rsf7alTBC34CPerW08wwdrIK8tjRvQMfJx8eGf4O6hVLbfrs95ICVs/BM8QiLzJ0mouwmoMgcFR6Zt3UWnR+oQAUHbiBABjQ8ZikIYaB43Lk4uwC3RFGMuVGccNoGtbd/58cii9gjyZtvTAJYPIJ7duAiHoOnRkg8qxJMe2ppB6Op+ht3SyTRyrgj4zk/h776M8IYGgzz/DbfRoS0tqMgxGAy9vfZnM0kzmjpyLl2PrCaleJ+K3Q9JuGDJNWTO9GWE1hsC1g7/yX2Wg1NEXVCrKjimLoUX4RNDWpS3rE9ZfcpzUG9GlFGHf3k9pzh1dBme3XJKuPvi4OvDDw1FMGaAMIk/9YR9FWj1SSk5s3Uhwt+64+7ZMD5uiXC07fjtNYBcvug5uZ2k5zQpdegbx996HLjWV4IULcR061NKSmpQvDn/BtuRtvDzwZXr49bC0nKZn0zvg0kZxS29mWI0h8Ajwp1RfhJtGTWGeHoeOHSg1GQIhBGOCx7A9ZTslupKLjtOlFYNBYh/kpozye4bAqucVF7AGYK9R8b+bevDGxAg2nMzgpk+3cXD/YXJTU+g2vGW+JUop2fxTDEaDZPTdXWxdQpXQpaWRcO+96NPTCfnyC1yimofbYFOxJWkLnx36jIkdJnJruGXj6liEc9uUAHPDngX75hdx12oMgZ2jI8WyADeNPQVZpThGRFJ27Pj5VcrGhIxBa9CyI2XHRceVJ5tCTwe6gp0TXPOOMtiz6/MGaxJCcP/Q9ix6cCDpBVo++epXhFpD5yjLhaNtCHEHMzl7KIsBE9vj4df8LnZLoUtPJ/6++9BnZRH89Vc492vloZWrkFSYxMtbXqazV2dmDp5pnS8IFa2Bfg9YWkm1WI0hACi30+KqciI/qxTHyEgMWVnoTYt99/Pvh4eDxyXdQ+WJhaicNai9Ta6jXa6FzlfDxv9Bfv1nDFfH0E6+/PH4IDoUniLWMYQf9mdgxvV5zEJ5qZ4tP53CN9iV3mODaz/AStClpxN/770YsrIJ+fornPu07jDSVdEatMyInoGUknmj5uGkcbK0pKanmbcGwMoMgdFVYq9yQJtThn1EBABlx5TVyDQqDSODRhKdFI3OeKHbR5dUhF2Q24W3GCHg2vdAGmHNy42mTaTE4qArwbnrAGb/fYLnfjnUaOGsm4Kdf8VRXFDOqDu72lYcM6FLzyDh3vswZGUT/NWXOPXubWlJTc6cXXM4kXOCOcPnEOxupS8Izbw1AFZmCNQ+igeLk1Fi8G8PQpwfMAale6iwvJC9aXsBJfS0LqNY6RaqjFcYjHwRTqyAmDWNou3Etk04OLvwv2duZ/q4cH47kMytn+8gOa+0UfI3J+nnCjgSnUSPkUH4t3e3tJxmgT4zk4T77kOfmUnwl19aXUsAYPmp5fwW+xuP9HiEUcGjLC3HMrSA1gBYmSGw91fC+bqqdBSWCOw7dLjIEAwJGIKTxol18crqZLrUYjCiDBRXZfBT4NcVVr2gLD7dAHTlWk7v2UGngYOxd7DnmXGd+fLe/pzLKmbigq3sOHNlEUybAqPBSPSSk7i42xM1qYOl5TQL9NnZxD/wALqMDIK//KLVrypWHUezjvL2rrcZEjCEJ3s/aWk5lkFKWP8WuLZt1q0BsDJD4Brsi0EacFXplQHjyIiLDIGTxokRQSNYl7AOvVFPeWIhAPbB1cQY0tjDdXMhPwE2vdsgXWcP7KW8tPSiuQPjI/z546mheDnbcffXu/h669lmOW5wJDqZrMQiht0WjoNT8/KNtgT63FwSHngQXVIywZ9/ZnUDwwC5ZbnMiJ6Bn5Mf7w5/17omjVXm1D+QuFPpPWjGrQGwMkPg7u9PkS4XVzXkZ5biFBmJPjMTnWnAGODqsKvJKcthX/o+dMlFqNzsUbvXMCkqbCj0uRu2L4C0o1es6+S2TTh7eBIS2fOi7R39XPnjyaGM7dqGWSuP8+zPB+sUtK6pKMrVsuuvOEIifejYt2XOe2hMDAUFJD70MOXx8QR/9ikuA63LRRSUSWMvbX6J7NJs5o6ai6ejp6UlWQajUWkNeLWHvs1/7WXrMgS+bSjU5eCmsTO1CCIBLmoVDAschpPGibXn1lKeVIh9UC0RR8fPAicvWDENqq50Vge0JSXE7d9D+KBh1UYadXO04/O7+/HC1V3461AKkz/dRnx2w1Y+ayy2LY/FaJCMmNLZOl0CK2EsLibx0amUxcYStOAjXAYPtrQki/DJwU/YkbqDV6NeJdI30tJyLMfRZZBxDMb8B9R2llZTK1ZlCBycnSmlEBeVAwUZJTh262YaMD5+Pk1F99COuG3oM0urHx+ojLM3THgXkvfB7i/rren0nh0YdLrLhpRQqQRPju7Edw8MJDW/jOsXbGX9ifR6l9WYJB7P4fTeDPpNCLX6OQPGsjISn3hSCSA39wNcR4ywtCSLsCFhA18e+ZKbO9/MzeE3W1qO5dCXw4bZ0LZHs4spVBNWZQgA9M5GVEKFPluLysUF+/btL2oRAFwVehVhOUpICodOdWjadr8ZOo1XmoJ5ifXSE7N9M+5+bQgI71pr2pHhfqx8ehgh3s489P1e5q6NwVCPxW4aC4POyKafYvDwc6LPVSFNXn5zQpaXkzRtGiW7dxPwzju4j28+oYWbknP553ht62tE+kTyStQrlpZjWfZ9B3nxMPZ1ULWMR2zLUNmICE+lmWan1aMt1eMYGXmJIRgeNJyBJT3QavS1twhAmVtw3QeAhL+fU7wF6kBJQT7xRw7SZciIOnetBHs7s/zxIdzSL4iPNpzm/m93k1NcXqdjG4sD6xLIzyhlxJRwNHZWOhAISIOB5BdfonjzFtq++QYeE6+3tCSLUKIrYXr0dOxUdnw46kPrWWmsOsrylXkDYcOh0zhLq6kzVmcI7Noo3RguKijIVDyH9BkZ6DMzz6dxVDsSVdKTQ64nMQpj3TL2CoUxMyH2H2UFojoQu2sbRoOBrkPq15XgaKfm/27pyZzJPdgVl8PEBZdf36AxKcwpY9+qc3To40dIpE+TlNkckVKS+vrrFK5ZQ5sXX8TrttssLckiSCmZuW0mcflxvDviXdq5WnmgwS0fQEk2XDW7Wa03UBtWZwjc2vmiNZTgqjKSl16Ck2nAuPToBa8ffUYJblondjgeYl/6vrpnHjUVggbA6hehKLPW5Ce3b8Y7IOiKFqARQnBnVAjLHlcGJW/9fAeLd8ab3cV027LTAAy9pZNZy2nOSCnJePc98pctx+fxx/B5sHn7iJuTRccXsTZ+Lc/0fYbBAdY5QH6e3HjY+Rn0nAIBLWsWufUZAj+T55DaSG5aMY4REUpI6iNHzqcpO6W8XR/ziGP1udV1z1ylhhs+hvIiWPPSZZMW5WSTdOJYvbqFqqNnkCcrnx7GkE4+zPzjKNN/PkixVn/F+V2OxBM5nNmvDBC7+1hhzBgT2V98Sc533+F11134TZtmaTkWY1fqLubum8v40PE8EGm9xvA8698CoYKxMy2tpN5YnSHw8POnQJeDq0ZDbloJKhcXHMLDKT146HyasthcNH5O9OzUl3/j/0VXn5DTbbrCiBfg6HI4uarGZKd2bQMp6TKk4eu1ernY8819A3j+qnD+OpTCjZ9sI6Wojl1adcSgN7Ll51O4+zrSe7z1DhDn/vwLmR9+iPv11+P/2qtW6zabVpzGi5tfJNQ9lFlDZ1nteThP0j7FZXTwU+ARZGk19cbqDEHFXAJHoaEgVQkx7dSrF6VHjiCNRqTOgDYuH8dwL65tfy352ny2pWyrXyHDpoN/d1g5HUpzq01ycvtm/ELb4xPYOIG4VCrBU2M6s/ihKHKKy3lzRyl/HEhulLwBjkQnkZtWwvDbrHeAuGDNP6S98QYuI4YT8L85iBbiEdLYaA1apm+cjtagZd7oebjYuVhakmWRUglA6dJGiSnUArG6K9nBxYVSoUzIMmaVYTRKnHr1wlhYSPnZs2jPFoDeiENnLwYHDMbTwZNVcTW/2VeL2g4mfQLFmbDmUle6gswMUk+dpMvghrcGqjK0ky+rnhlOqLuKZ38+yCu/HWlwFNOSgnL2rDxLSKQPYT19G0lpy6J4505SXngBp969CZo/H2HX/CcJmQMpJXN2zeFo9lHeHvY2HTxs8aU4/IuyBOW418GhDl6GzRCzGgIhxDVCiBghxGkhRI0xm4UQA4QQBiHELebUYyoLrbsWPXpCNILC7FKcevcCoPTgIcpO5YJa4NDBAzuVHVeHXc3GxI2XrFxWKwG9YfgMOLQUYi4eZ4jZoSx12aWe3kJ1xd/dkZcGOPLYyI4s3Z3A5E+3czbrymcj7/rzDPpyI8Nutc4B4rLjx0l68insw0IJ/uxTVE7WOz6yLHbZ+YiiY0PGWlqO5dEWwr//hYC+0OtOS6u5YsxmCIQQauATYAIQAdwhhIioId27wD/m0lIVZz8vkvSnCbQT5J7Oxz4sDJW7O6WHDlEWm4tDew9U9kr3x7Xtr6XMUMaGxA31L2jEi9AmElY8e1EXUcyOLbTt2BlP/7aNVaVLUKsEL0/oytf39Sc1v5SJC7ay4lBKvfPJiC/g+PZUeowJwqut9XUBlCcmkvDoVFTu7gR/+SVqTyuNnQMcyjzEnF1zGBo41HojilZlywdQlAYT3msxk8eqw5zKBwKnpZRxUspy4CdgUjXpngaWAxnV7DMLHn7+xOTuBKD8QAZCpcKpZ0/KTmWiTy/BsbPX+bS92/SmnUu7+ncPgRKh9MZPlS6i1YoXUW5aCulxp83SLVQdY7v58/e04YT7u/L00gO89nvdu4qklGz9JRYnVzsGXFd/F9eWjj4nh4SHHwadjpCvvsSurfkMd3MnqzSLGRtn4O/sb90RRSuTfQZ2fKK4iwYPsLSaBmHOuMGBQOV4C0lAVOUEQohAYDIwBqjxTAohHgUeBfD39yc6OrrOIoqKii5Jn5lfQF5ROqleevwT8tm0LhpXjzb4uUShczRyoPw0xujT59NHqiPZkLyBFetX4Kaufx9gWMjNhB3+maOGDhxJULblCE296lFfqtb7ia6SZSo7luxKYPPxJJ7s7UBbl8u/B+THS1LPSAIGCHbs2mo2rY1Fdb/1FaPV4v3hPDQpqeROf5aUxERIrF/4kKaiUetdDXqpZ0H6AnLLc5nRdgYHdhwwW1l1xdx1rgvdj8zGS6rY5XIV5U2kxVz1NqchqM6frOpsp3nAS1JKw+Xcz6SUXwBfAPTv31+OGjWqziKio6Opmv6Ukx1JOzaR6aIlUGtHX7uOFPmr0Gfo8B7iQPvxgy5K3y6nHetWrKM4sJiJXSfWuezzDBsCX52k+9kv2Zc+noAuEVx1/Q31z6ceVFfvcWNgw8l0nvvlELN2lfP25B7c2Cew2uP15QaW/LMT32A7Jj0wAJWq+bsHVlfnK0Hq9SQ9+RRFCQkELfiIiLHNuy+8sepdE3N2zSFOG8e7w9/l2g7Xmq2c+mDuOtfKyVUQvQfGz2LI0KYLsGeuepuzaygJqOwbGQRU7aTuD/wkhDgH3AJ8KoS40YyaAPBqGwBAuWs+OUZJ/ppzGHIEZYd+QHfuyCXpw73C6eTZiRVxK66sQI09TF5IVr6OrKQkujZRt1B1jOnqz6pnhhMR4M6zPx/khV8PUVJ+6QS0g+sSKMrVMuzWzi3CCDQWUkrS3nyLok2baPvfmbg1cyNgbv468xdLTy7l3oh7m40RsDjlJUpXr183GPS4pdU0CuY0BHuAzkKI9kIIe2AK8FflBFLK9lLKMCllGLAMeEJK+YcZNQHgFRCEUKmQhixOlRhAgsugdqjUaZQeOnRJeiEEN3S8gcOZhzmbf/bKCvWPIMZ7EgJJuHtWA2vQMNp5OLH0kUE8PaYTy/YnMXHBVk6kFpzfX5ynZd8/CXTo40dguNdlcmp9ZC/8grxff8Vn6lS8pkyxtByLciz7GG9uf5OBbQcyvd90S8tpPmz5QFmZ8Lr3W8RaA3XBbIZASqkHnkLxBjoB/CKlPCaEeEwI8Zi5yq0LGjs7vNoFUl6SQbpewg0d8ZzYUZlYduhQtfF6ru9wPSqhYsWZK2sVSCmJSdQS7C1x2fx6vcNVNzYatYrnrurCDw9FUVCmZ9In287HKtr5VxxGg5EhN3W0qMamJn/FCjLnzcP9hon4PfuMpeVYlJyyHJ7d+Cw+Tj7838j/Q6OyLUMKQNZp2P6RMkAcNszSahoNs/o7SSlXSSnDpZQdpZRvm7Z9LqX8vJq090sp6xa2sxHwDQ6lKEfpqcrTS4Ra4NS7F4asLHTJl7pZ+jn7MThgMCviVmCU9Q/fkHEujtzUFLpMuBOkAf54XFnOzsIM7eTL6meGM7iDKVbR57s5uSOVXqODrWrBmeJdu0l59TWcBw4kYPZsqw6ZoDfqeX7T8+SW5TJv9Dy8Hb0tLal5ICWseg40TnDVLEuraVRaruNrA/ENDqUgKx2NnYHcVGWylVOviollB6s9ZlLHSaQVp7E7bXe9y4vZvhmVWk3n0TfANe/AuS2w85Mrr0Aj4uvqwLf3D+C1a7vieKyAMiGREe6WltVkaOPiSHr6aexDQgha8BHC3t7SkizKB3s/YE/aHl4f/DoRPpdM/bFeDv8CcdFKUDnXNpZW06hYryEICQUpcfEsJjdNMQQO4eGonJ0p3b+/2mNGB4/Gzc6t3t1DUkpidmwhtEdvnNzclQXvu16vRCtswKL3jYlKJRjn6UGwXs0xb8E9i/Yw999T6A2Wb7WYE31ODomPTkXY2RG8cCFqDw9LS7IoK86s4IcTP3BXt7uY2PEKPORaKyU58M8rENgf+j9kaTWNjvUaguBQAOzs88hNU8JHCI0Gp969KdlX/RoEjhpHrgq7in/j/61XyInU2BgKMjMuhJQQAibOB0dP+O1R0JU1rDKNgMFgZPvy03i1dWbeq8OY3CeIj9bHMuWLnSTl1jO8RgvBqNWS9MST6DMzCf7sU+yDqneltRaOZx/nzR1v0t+/P8/1f87ScpoX/85UVh+bOL9FzyCuidZXozri4d8WjZ09yByKcrWUlykulE79+6E9dQpDfn61x03qNIlSfSn/xv9b57Jitm9GrdHQaUCl+Qkuvsqs44xjsO6NhlSlUTi+JYW89BIG39QJd2d7PritF/On9CYmrZAJ87fw1xWEp2jOSKOR1FdeofTgQQLeew+nnj0tLcmiVAwOezl68f7I97FTtQ5vmEbh3FY48IMSYrptd0urMQtWawhUKjXeQcGUl6QDkJeuvPU69+sPUlJSQ/dQb7/ehLiF8Pvp3+tUjtFo4OT2zbTvMwAH5yqxejqPh4FTYddnELvuyivTQLSlenavPEtguCdhPS4sPzmpdyCrnhlOuL8b05YeYMYvByky06I3TU3Wx59QsGo1bZ5/Dverr7K0HIuiM+p4ftPzZJdmM2/0PHycrHcJ0kvQlSmxwjxDYeTlF5tqyVitIQDwCwmjMFt5081Orhgw7gl2dpTW0D0khGBy58nsS9/HufxztZaRcPQwJfl5dBs2svoE49+CNhGKF1Edlrc0B/v/iaesSMfQWzpf4i0T7O3Mz48OYtrYzvxxIJlr529hf0L1ayy0FPJXrCDr00/xuPkmvB9qff299aVicPiNIW8Q6RNpaTnNi83vQXYsXP8h2LdeLzqrNgQ+waGUFuSi1mjJTlIWqVE5OuIUGUnJ3prXKp7UcRJqoa5Tq+Dk1k3YOznTvm8NoZTsHOHmr5X+xz+fUFzUmpCi3DIOrU8kfKA/fiHVx1HSqFXMGB/OL1MHY5SSWz/fwfx1sS1yILnkwAFSX/sPzgMG0O71163aTRTgj9N/sOTEEu7udrdtcLgqqYdh6zzofRd0at0zzK3aEFQMGLt6FpOVVHh+u3P/fpQeO4axrPpBXD9nP4YHDefP03+iM9a8jKWuXEvs7m10HjgEO3uHmoX4Ryh+ybFrYdfCK6vMFbJrxVmklETdUPsCI/3DvFn1zHAm9mzHh+tOcdvCHSRkt5yBZF1yMklPPY2mbVsCP5pv9W6iR7OOMmvHLKLaRtkGh6ti0MNfT4GzD1w129JqzI7NEAD2jvlkJRWdn1Hs1K8f6HSUHjpc47E3d76Z7LJsNidtrjHN2f17KC8tpduwUbWLGfgohE9QvBNSLw1zYQ6yk4uI2ZFKz1FBuPvWbbEVd0c75k3pw/wpvYnNKGLC/M38ujex2tnYzQljcTGJTzyJLC8n+PPP0HhZV+iMqmSVZvHMxmfwdfK1zRyujh0fK/fhde+Dc+ufUGfVhsDV2wcHFxcwZqMt0VOYo7QAnPv2BSEo2be3xmOHBQ7Dz8mP32J/qzHNia2bcPH0Irh7j9rFCKEsb+nsA8seBG1RvetTX3b8fgZ7Jw39JoTV+9hJvQNZ8+wIugd68MKywzyxZD85xeWNL7IRkEYjKS+/jDY2lsC5c3HoYN3LK+oMOmZEz6BAW8D8MfPxcrRuo3gJmadg4xxlrk9EdUuotD6s2hAIIfANDqWsWPEcykpUHr5qDw8cOnem9DLjBBqVhhs73cjW5K2kFaddsr+sqIizB/bQZcgIVHVdxMPFB276UlnwYvWL9a9QPUiKySX+aDZ9rwnF0eXKXAUDPZ348ZFBvDKhK+tOpHP1vM1ExzTZ+kJ1JnPBAgr/XYf/yy/hOrz1xIe5Uv63+38cyDjArKGz6Ord1dJymhdGgzJWZ+8M1821tJomw6oNAZhCTWQkAZKspAtv4c79+1F68CBSX7O75OROkzFKI3+e/vOSfbG7t2PQ6+k2tAZvoZpoPxxGPA8Hl8Chn+t3bB2RUrLjt9O4ejnQc3RQg/JSqwRTR3bkzyeH4e1sz/3f7uE/fxypNrS1JShYtYrszz7H45ab8brnHkvLsTi/nvqVX0/9yoPdH+Sa9tdYWk7zY8cnkLQHJvwfuPlbWk2TYfWGwCc4FG1JMe4+BrISLwwYO/Xrh7GkhLITJ2s8Ntg9mKh2USyPXY7BePHyj8e3bMCrXQD+HTvXX9TIlyFkCKycrkQ7bGTO7M8kI76QqBs6oLFrnCUHIwLc+fOpoTwyvD1LdiVw3UdbOWBhN9Oy48dJefU1nPr2pd1//2v1HkL70/efX3N4Wp9plpbT/Mg8BRtmQ5froMctllbTpFi9IfALDgPAybWgSougPwAle2seJwC4Lfw2UotT2Zp8YSnH3LQUko4fJXLkuCt7+Kg1cPNXoHGAX+9v1BAUBoORnX+ewTvAhfCoxl2D19FOzWvXRfDjw4Mo1xu5+bPtfLA2hnJ907uZ6rOzSXzqKdSengTZPIRIK05jevR0AlwCbGsOV0dFl5CdkzJnwMpeGqzeELTp0BGVWo00JFOYXYa2RHEHtfP3xz40lJJduy57/OiQ0fg5+fFzzIVunGPR6xFCRcSIMVcuzCMQJi+E9CNKsKtG4sS2VPIzShl8Y0ezrTw2uKMPq58dzuQ+QSzYcJrJn27jVHph7Qc2ErK8nKRnnsGQnUPQxx+j8fVtsrKbI2X6Mp7Z+Axl+jI+GvMRHg7WHVivWrbNV7qErrWuLqEKrN4Q2Ds60a5zFwqzlS6Yi1oFgwZRsmfPZccJ7FR23NT5JrYmbyWpMAmj0cCxTesI69UHN58GPoDCr4Ih02DvN3B0ecPyAnRaA3tWnqVdJw9Ce5g3jIC7ox0f3NaLhff0Iy2/jOsXbOWLzWcwGM3vZpr+zjuU7t1Hu9mzcepu3TNlpZS8ueNNjmcf553h79DR07oWG6oT6ccUL6FuN0CPWy2txiJYvSEACI7sRV5qPNJYdt5zCMBlUBTG4mLKjl4+VPQt4bcghGB57HISDh+kKCeb7qPHN464sf+FoIHw17QGjxccWp9ISUE5gyd3arL+8qsj2/LP9BGMCvdjzqqT3L5wB+eyis1WXt7y5eT+uBTvBx/EY+L1ZiunpbDo+CJWxq3kyd5PMjpktKXlND/05fDbVHDytMouoQpshgAI6d4TKY3Y2adfPMM4KgqA4p2X7x5q69KWkUEj+S32Nw5vXIujmzsd+kU1jji1Hdz6Lajt4Zd7QVd6RdmUFek4sDaesJ6+tOvYtF0Dvq4OLLynH3Nv60VMuhLNdPGOcxgbuXWgOXuWtDfexGXIYNrMsK2xuy15G3P3zWV86Hge7fmopeU0Tza/p3S/TpyvRAS2UmyGAGjXuSsaewc0mpSLuoY03t44dOlC8c6dteZxe5fbKS7I4/SeHUQMG4XGrhHD+HoEKfMLMo7BqheuKIv9/8RTrjUwaJJlJlMJIbipbxBrp4+gf5gXM/88xj3f7Gq0tQ70mZl4LvwCTZs2BHzwAUJj3TNlEwoSeGHzC3Ty7MTsobNRCdutfgmJe2DLXOh1J3S9ztJqLIrt6kBZzD6wawTlJefISSnGUMnLxWVQFKX792PUai+bx+CAwfTNDkQajESOGtf4IjuPg+HPw4HFcGBJvQ4tytVyODqJLlFt8Ql0bXxt9aCdhxOLHhzInMk9OJiQxzXztrB0d0KDQlRInY6k6dNRFRcT9MnHVh8+oqi8iGkbpqEWauaPno+zXeuNmnnFaIvgt0fAPRAmvGNpNRbnsoZACKEWQlhFGzs4sicl+akYdEXnl64EZcBYlpdTeqD6dYwrEAgikjzJcteS426mUAujXoGw4fD3DEg7UufD9qw6izRKBl7f3jy66okQgjujQljz7Ah6BHrwym9HuO/bPaTkXVm3V8b771O6dx8Fd9+NY1frnilrlEZe2fIK5wrO8cHIDwhya9iEwVbLP69C7jmY/Dk42ryoLmsIpJQGwCqCbYR0V1aoMuoTyUyoNE4wYACo1RTv3HHZ488d2o8hs4AzHbUsOVG/N/Y6o9bALd8oS1z+cq8SuroW8tJLOLEtlcgRgXUOLNdUBHs7s+ThKGZNimTP2Ryu/nAzP++pX+sgf+Xf5Hy/CK977qEsaqAZ1bYMPj7wMdFJ0bw0dpxCrAAAIABJREFU8CUGtrOdj2o5uQr2fw9Dp0HYUEuraRbUpWtomxDiYyHEcCFE34o/sytrYvzbd8LeyRlkEulnC85vV7u64tS9OyW1DBjvXfEbrl7e9Bw+jlVxq8gpyzGPUNc2cNv3kJcAf9S+fsGuFXGo7VT0v4LAck2BSiW4Z3AY/zw7gogAd15afoR7v9lNch1aB2Uxp0idOROn/v3wf/HKxk5aE2vOreHLI19yc+ebmdJliqXlNE+KMuCvp8G/B4x+zdJqmg11MQRDgEjgLeAD09/75hRlCVRqNUER3cGYRFpcwUX7nAcNovTIEQxF1UcEzTgXR8LRQ/SZcAN3Rt5NubGc5aca7vdfIyGDlJXNTq5UJsLUQGZCIaf3ZtBrTBDO7s17Zm2IjzNLHxnErEmR7IvP5eoPN7NkV3yNrQNDYSFJ055G7epK0IcfIhpzcL4FciL7BDO3zqRPmz68FvWa1YfTqBYplZen8iK4+Utl5r4NAGp1rZBSWo3zcWj3XsTt201WYgrlpXrsnZTT4zIoiuyFCynZuxe3UaMuOW7fyt+xc3Si57hrcHRxZUjAEH6K+Yn7u99vvkXABz0Bibth/ZsQ0Ac6XBrcbtdfcTg4a+gzPsQ8GhqZitbBqC5teGn5YV77/Sh/H07lnZt6EuJzYcBTSknKK6+gS0omdNH3aPz8LKja8hQYCnh749t4Onoyd9Rc7NStzyjqdDqSkpIoMy0W5eHhwYkTJ+qXibYQOj4C3V+EbCC7nsc3A+pSb0dHR4KCgrCrx8tRrYZACOEBvA6MMG3aBLwlpay9g7qFEdy9FwCG8jjSzxUQ3E1ZkMKpTx+EvT0lO3ZcYggKs7M4uX0zva+6DkcXxSPnrm538eT6J1kfv958ER6FgEkfQ8YJWPYATN2suJmaKM6UxB/NZvDkjjg4t6wHQ8XYwdLdicxZdYKr523mxWu6cN//s3fe4U2VXxz/vEnTvTelhUJZpWwQZC9ZsodMEURFFBQZCgj+EEVAFGWLIgoCgggiQwXZyN6UJXuVAh3QPdIk7++PCwi0dCZNCvk8Tx6S3HvPPZekOfd933O+p24wKpXgzoIFJG3egu/oUTjWrGlud81Khj6DBdELiNPFsajNIrwdns5c+IiICFxcXAgODkYIQWJiIi4uWbdWzZKMVIg+C3ZlwLN0kS0cy+m6pZTExsYSERFBqVK5Tw7JzdTQD0Ai0P3eIwH4MddnKEJ4B5XEN7gM+vTD3Lzwn3Kmyt4ex+eeI2nX7kzHHN2wDmmQ1HjxvzX1BsUbEOQSxJIzS0zrsJ0L9FyqVEeueAV0SoqrlJKo4xJHV1sqF1Bm2lzczyzaOKwRtUt5MmHdaV76di/nNu4g6quvcWndGs9+/cztplmRUvLZ/s+4lH6JT+t/SkWviuZ2yWSkpaXh5eWVvykvg0HJEFKpwb1EkQ0CuUEIgZeX14ORU27JTSAIkVKOl1JeuveYADyVLZ6EENTp8hLSEM+Fg49mCTk1bID24kUybtx48F7c7Vsc3bCe8vUa4ub7n1CVSqjoE9qH49HHORaVfdppgfEuC52/gRuHHxSbXTt9h5QYqPViMBrboq0yWdzdgYWvPse0l6oSc/UGMR+8T7JPMXw++eSZnwdf9u8yVp1fRUvXls9Eb4F8f94JN0CXBu4llUr9p5z8/D/lJhCkCiEetHUSQtQH8pfwXQQo89zz2Dn5cPvSVgz6/wrLnBs2BHgwKpBSsvn7OajUKhr1eTWTnc5lOuNi68JPp38yvdOh7aHBcDiyCHnwR/b9fhGNE1RsEGD6cxcCQgi6VPXn+6u/46xPZ0RYTzr/eIyTN5662clcs+/mPqYenEqTwCa0dX+2q2KzJTUOUmLAyRfsXc3tjcWSm0AwCJgjhLgihLgCzAbeNKlXZkSlUlO2TmsMGVGc/ue/lFHb0qWxCShG0j9Ks/ozu7ZzNfwoDXr1y1Jl1FHjSI/yPdhybQvXE66b3vFm4yCkORd/W0XM9SR8KwnUNk9P4Xj0jJnojh6h5GefMObttkQnpdNxzm4m/3WGtAx9zgaeIq4nXGfE9hGUcivF5IaTrfIRT0KnVdKsNQ7gWszc3mTJgAED8PX1pVKlSmb1I6fKYhVQXkpZFagCVJFSVpdShufGuBCitRDirBDighBidBbbOwohwoUQx4QQhx4eeZiTGm1agXDi0Lr/UkCFEDg3aEjK3n2kxMawfdF8ipUtT9UWbZ5op1eFXqiEisVnFpveaZUaQ6f57E/ug6dtJD7FzNsdzJgkbt9O7Pz5uL/0Em4dO9K6UjE2D2tMtxqBfLvjEq2m72TPxRhzu1koJGmTeGfrOwghmNl0Js625pUMsVikhLirgASPYLDQYNm/f382bNhgbjeyzxqSUhqEEEOAFVLKhOz2fRwhhBqYA7QAIoCDQoi1UsrTD+22BVgrpZRCiCrACsDsGgHegW7YuzxHbMR2rp44RsnK1QBlnSD211/Z+PXnpKck02LgO9k2pvd19OXFUi/y+4XfGVxtsMkbgpw9kU6c1o82Xl/hdToKXmhb5HOlMyIjiRw1GrvQUPzG/VcA5Oao4fNuVehYLYAxq0/Qe/5+GgXaUL12Bm5FLEsqt+gNekb9M4orCVf4tsW3BLkGmdslszBh3SlOXL+LWp3N+pdeqzxs7EB1NEebFQNcGd/+yb0rrly5Qps2bWjQoAF79uyhePHirFmzBgeHglXrN2rUiCtXrhTIhjHITZjcJIQYKYQIEkJ43n/k4rjawIV7C8xaYDmPyVVIKZPkfxVDToDpu5bkAqESBIY2RK1xZdVn/2Prwm9JT0nmjrsr/5QP4tL5M9Tr/jI+JYJztNUvrB+pulRWnF1hUp/1GQYOrL+Mb0kXSvUcgFvCWWXxuABibuZGarXcGDYcdDoCp3+Nyi5zUKtXxpsNQxvxZqPS7Lqho/lXO1gfHlkgETtLZcbRGeyM2Mno2qOpU8xIMudPI1KvBAGVDRixjuf8+fMMHjyYU6dO4e7uzqpVmYtGly5dSrVq1TI9unUreA/k5HQdBhN9r3Oj1Tvg3r+DH3pPknPmUHHg4cnxCCDTt1cI0RmYDPgCWa56CSEGAgMB/Pz82L59ey7cVkhKSsrT/vdJt5XYOL6MS8Bujv61jvCtm9Cnp+Gg0VAzQUuqu0+u7Vawr8DC8IUExwajEaa5W409J0m6I/GqomVHrCfF/TtQ9sgiziU6EFn8ydNXlozzypU4HT9O3Buvc/PyZbh8+Yn71nUE96qSXy5lMOTno8z3CeeVirZ4OVjmlEBeOZB0gMWxi2ng3AD/W/5sv7X9wbb8fseLEm5ubiQmKhpgw5uUQK8vnuWIQBh0OKZcR+JOilNQnqaE7tvPiqSkJEqWLElISAiJiYlUqlSJs2fPZjqmQ4cOdOjQIU/2k5KSMBgM2Z7fICXXEg0424BK5Nz2NS0tLU/fiWwDwb01gpellJkT6HMmqxymTOFMSrkaWC2EaAR8CmTScJZSfgd8B1CrVi3ZJIvq3iexfft28rL/fa773mHtyWM06z4CjV0s//y8EL+QspSLT+XujJmUqVgRja9vrmzZ37Tnjb/f4G7AXbqX755nX3JCm6ZjyR97KV7eibY9qiOEYLvUU9YljXIXv6dc/Q5FTlwrcetWIjZvwaN3b0JHjMjdQdu3s/mlRizcc4Vpf5/jo71ahrcoR/96wdioi25ACI8O55cNv1DLrxYzW87MVK2e3+94UeLMmTOPFFJlWVglJdy5CNIAPmVx0RhPZNHZ2RkHB4cH53R0dCQpKSmTD0uXLuWLL77IdHyZMmVYuXLlE22rVKpsC8USUjOQMhknTfb73cfe3p7q1avnuN99clIfNZB/XaEI4OFJzEAgMptz7QRChBAWURrpW8oVISDyYhz+IWV56aPPaNS7P+73/uCSd+/Jta06/nWo5FWJH0/+iM7w5P7H+SV863VSEzN4vmPIfznEQg1dv1cWylb0VbIniggZN24QOXoM9mFh+I4eladjbdQqXm9Ymk3DG/F8aS8m/nGGznP3cCKiaKaa3kq+xdBtQ/Fx9FHkI0wlWfI0kHRbkZFwK65kCpmBPn36cOzYsUyPJwWB3JKQmoFaJbA3Ub+l3Nwm/S2E6CryXqVwECgrhCglhLAFegJrH95BCFHmvt17iqa2KCogZsfOwQafkq5EnHlURdSuQgXUPt4k7diRa1tCCF6v/DoRSRFsurrJqH4qLSivUaqqN/6lH1uMtneDXstBr4NlvUFrul7BxkJmZHBjxEjQ6yn+9VeobPMnlhfo4ciCfrWY07sGtxLS6DhnF5+sO01SuvEDsalI1aXy7tZ3SclIYXaz2XjYP9sNd7IlPQkSbyoS7Y4WcS+ZK3r16kXdunU5e/YsgYGBLFiwINM+UkoS0nS42GlMVkSZm/gyHGUhVyeESEOZ8pFSymyrM6SUunsZRxsBNfCDlPKUEGLQve3zgK7AK0KIDJQitR7Sglb5gkI9OLLxGumpOuzuCdAJIXBp0oSEP//CoNXm+oeqaYmmlHIrxYITC2gd3NpoH+j9FpR1Ojxhyca7LHRbAD93h9WD4KVFoLLcaZLomTNJPXaM4l9/hW2JgonlCSFoW6UYDcp688XGf/lxz2X+OnmT8e3DaBXmZ9GVyVJK/rf7f/x7519mNptJGY8y5nbJctFnKBISajuTSUgEBwdz8uTJB69HjhxpFLvLli3LcZ8UrR6dwYCrgw3oTVMzk+MvgpTSRUqpklLaSild773OVYmelPJPKWU5KWWIlPKze+/NuxcEkFJ+LqUMk1JWk1LWlVLuKtjlGJegUE+kQXLj7KM5+c5Nm2FITiblwMFc21IJFQMqDeDs3bPsumGcy3zQgrJ2Di0oy7ZQZKvPrIWdmecvLYWknTuJnf897j164NrGeAvcbg4aJnaqzKq36uHmoGHQksO88dMho/VLNgXfhn/LhisbGFpjKE2CmpjbHcvlfr2AQQeewYqe0FNGQmqGcgNqqnkhsgkEQoiXH3pe/7FtQ0zmkQXhX9oNGzs11x+bHnKqVxdhb0/S1q15ste2VFv8nfz5/sT3RvHvQQvK9rlQGaw7BKr2gu2T4PQao5zfmGTcjlLqBcqVw29MptpDo1CjhAfr3mnAhy9WYPeFWFp8tZN5Oy6S8ZCUiCWw6eom5hybQ/vS7RlQaUDOBzzLPFgXCISnsDezlJL4tAyc7WxQm3Akn53l4Q89n/XYtmfi26m2UVG8rHumQKCyt8epfn0St23LU766Rq2hf1h/jkQd4eCt3I8msuJBC8oGAblrQSkEtJsOgbXhtzfh5vECnd+YSL2eyA8+wJCWpqwL2Nub7FwatYqBjULYPKIx9ct4M+Wvf2k3cxcHr5ioo1weORN7hrG7xlLFpwrj64236Okrs5OWoKwLOHiAo5e5vTEJ6ToDWp0BVxOOBiD7QCCe8Dyr108tQaGexEelkhD7qM6eS7Om6G7eJD2PzTG6lu2Kl70X34Z/WyC/9q2514KybR4a0mvsFdlqRy9Y1gsSbxXIB2MR+913pOzfj/+4sdiFhBTKOYu7O/B9v1rMf6UWSek6Xpq3l/d/PU5sUnqhnD8rYlJjeHfbu7jZuTGj6Qzs1EW7KtyUCEOGMiVkYw9uQU+ttHRCagYArg6mzRbLLhDIJzzP6vVTS2CokqkRceaxdYImTUAIErduy5M9ext7Xq30Kvtv7udoVM6l71lx+3ICF49EUe2FoLy3oHT2hV7LIPUuLO+jNOwwIymHDhE9azaubdvi1qVLoZ+/RUU/Ng1vxFtNQlh99AbNpu1g6f6rGAyF+xVP16czdNtQ4tPjmdl05lPbYMYoSAP2qbeVegGPUk/lusB9EtIycLS1QWPiOpjsrFe4Jwh34qHn91+XN6lXFoRnMScc3WwzTQ/ZeHnhUK1antcJAF4q9xKe9p58ezzvowIpJXt/v4CDi4bqL+Qzq6ZYFegyH24cgjWDzSZDoY+L48b7H6ApXhz/CR+bbRrE0daGUa0r8NfQhoQWc2Hs6pN0nrub8Ii4Qjm/lJKP93xMeHQ4kxpMItQrtFDOW2RJiMTGkKZkCGlMN41obtJ1elK0epNPC0H2gSAUaA+0e+j5/ddPbyukxxBCEFTBk4h/7yIfu0t0ad6MtNOnybh5M082HTWO9Avrx+7I3YRH50rI9QHXT9/hxtk4ar0Y/KCncr4IbQfNx8PJVbDj8/zbySdSSm5+9D900dEU/2oaamfzq2iW9XNh2RvPM71HNW7EpdFxzm7Grj5BXIrWpOddcHIB6y+tZ0i1IbxQMlNhvZWHSbkDydFoNW7K2kAR5vr16zRt2pTQ0FDCwsKYMWPGI9vvJGsRCDyc8ldLkxeeGAiklFeze5jcMwsiKNSDtOQMYiKSHnnfuVkzABK35W16CKBn+Z6427kz7/i8XB8jDZI9qy/i6m1PWMPieT5nJhoMu5dJNBlOFKzyMa/ErfiVxE2b8B32Hg6VKxfqubNDCEGn6sXZOrIx/esFs+zANZp+uZ1lB66ZZLpo67WtzDwykzbBbRhYZaDR7T9VZKRC/HXQOJFuV/SnzmxsbJg2bRpnzpxh3759zJkzh9OnFXFmg0FyJ1mLq4Ppp4UgdwVlzzyB95rYXzsdi0+J/3Q+bEuVwrZkSZI2b8azd+882XTUOPJKxVeYeXQm4dHhVPGpkuMxZw/cIjYiiRavVTRO0xkhoP0MpRjn97eVoXZQ7YLbzYH0Cxe4PXkyTvXq4flq5u5uloCrvYbx7cPoXiuI/605yZjfTrD8wDU+6ViJqkHuRjnH2TtnGf3PaMK8wvikvrX1ZrZIA9y5rIjIHZyPQ2Q4qI348+VfGdpMeeJmU8hQFytWjGLFlIY5Li4uhIaGcuPGDSpWrEhcagZ6g8SrEEYDkDuJiWceJzc7fEu6cPFI9CPvCyFwadWK5P0H0N3NeyOY3qG98bDzYM6xOTnum6HVs+/3S/iWdKFsTb8c9881NnbQY6nSwWlZL7hr2sGeIT2dGyNGonJ0JODzKQgLrnIGCC3myoo36zK9RzUi49PoNHc3o1eFFzi7KCY1hiFbh+Bi68LMZjOxt3l657oLjJTKlJA+/V6TGfMsDptShvrKlSscPXqUOnUUgeY7yVrsbNQ42RXOvXquzyKE0ACVgBtSyijTuWSZlKnpx57fLhAfnYKbz3+FK66tWxH73XckbtqER/e8KYs6aZwYUGkA0w5P4/Dtw9T0q/nEfY9tukZyXDotXw9DqIx85+jkBb1/hQUvwM894LWNik6RCYj6chrpZ88S9N232Pj4mOQcxub+dFHzUF9mbjnPj7uv8OeJmwxvUY6Xny+ZZ2XT+xlCcWlxLGqzCB/HovH/YDZ2Twd1RXAtB3Yu0GYKqVmpj5qYUqVKUa2a0qSqZs2aWTaU6dOnD3369MmT3aSkJLp27cr06dNxdXUlRasjRasjwN2h0EaJ2VUWzxNChN177gYcB34CjgohehWKdxZESE3lj/XC4UdjoF1oKJqSJUjMZ7u5HhV64O3gzayjs55YnJYcn86Rv69RuroPAWWMMy2RCZ9y0H0xxJ6HX/sr+i1GJnH7du4uXoxnv1dwbtTI6PZNjYu9hrFtK7LhvYZUCXTn43WnaTtzV57aZN7XEAqPDmdSw0lU9Hpm8i7yx8WtsOUTpWrYKXey76bC7qHGSGq1Gp0us4BhXkcEGRkZdO3alT59+tDlXvr0nSQtKiHwKMROe9ndyjSUUp669/xV4JyUsjJQE/jA5J5ZGK5eDviVcuX8oUcDgRAC11atlemhO3mvTnWwceCNym9w+PZh9t3cl+U+B9ZewqAzULeziYutSjdWqo8vboU/Rxo1rTQjKoqbYz7ErkIFfHLbX8BCKePrwuLXajPv5Zoka3X0nr+ft5cezpV20fwT8/nz8p+8U/0dWpRsUQjeFmHuXoWVr4F3eXD0LBJFY3mRoZZS8tprrxEaGsrw4YqQg05vIC41A3dHjUklJR4nuzM9nDPXAvgdQEppGeWoZqBsLT9iI5K4e+tROWfXNq1Brydx0+Z82e1Wrhv+Tv7MPjY706ggJiKR03tuUrlxIO6+haClUqMvNBgOhxfCnplGMSkNBm6O+RBDairFv/wi39LSloQQgtaV/Nk8vDHDXijH1n+jaD5tB19tOkeqNmuFyL+v/M2so7NoW7otb1R+o5A9LmJoU+CXlxUxuZ5LLbb5fEHYvXs3ixcvZuvWrQ9GDst/W4OUEm/nwq0qz+5/N04I0U4IUR2oD2wAEELYAObp+mBmQmr4gshieqhCBWV6aGP+pods1ba8WeVNwqPD2X59+4P3pZTsXH4OeycNtdoGF8DzPNLsIwjrApv+B6d+L7C5O4t+Inn3bvxGj8auzNMlp2yvUTP0hbJsHdGEVmH+zNxynmbTtrPm2I1Hgvqp2FOM3TWWqj5VmVBvgjVDKDukhHVD4dYJpbmSV+HIjmRHVjLUH3/8cYFsNmjQACkl4eHhHDt2jIOHj1CtXjPcHW2x1xTugnh2geBNYAjwI/DeQyOB5sAfpnbMEnH2sKNYiFvW00Ot2+R7egigU5lOBLsGM/PoTPQG5Y7y3P5b3LwQT93OIdg7FWJnKpUKOn0DQXVg9Ztw/UC+TaWdOUP0V1/h3Lw57j2M36bTUghwd2Bmr+r8OqguXs62DF1+jG7z9nL8ehy3k2/z7pZ38bD3YHrT6VYNoZzYNxdOrICmY6FcK3N7U2hEJaQhAT/Xwv9+ZFdQdk5K2fper4CFD72/UUpZtCd5C0DZWn7cvZlM7I1Hi8tcW7cq0PSQjcqGd6q/w4W4C6y7tI70VB27f7uIb7AroXWLGcP1vKGxh57LwDUAlvWE2It5NmFITeXGyPdRu7tTbOKnz8Rd8HPBnqwZ3ICpXatwNTaFjnO30XnV6yRmJDG7+WyrhlBOXNoBf38EFdpBw2fnZyZdp+dOcgaejrbY2hR+emx2WUMzs3sUppOWROnqPggB5w/efuR9uwoVsC1ZkoS//sq37RYlW1DJqxJzjs1h39rzpCZqadyrnPHTRXOLkxf0WakM1Ze+BMl56yJ6e+pUtBcvUmzKZGw8irYcQF5QqwTdnwtiy4iGVKz6BwmGqyRd68mfhyFFW3RaZRY6d68oGWteZaDzPIvupGdsohLSEQJ8zTAagOynhgYBDVAazh8CDj/2eCZxcrOjZCUvTu+ORPfQoqAQAte2bUnZv5+MW/lbTxdC8F7N99BGqTi5PZKKDQLwLZmrZnCmwytE6XscHwHLe0NGWq4OS9y6jbhly/Hs3x/n+vVzPuApZOGZeVxP38/AsKE0LdGY6ZvP0+zLHaw6HFHo6qYWT3qSooYr9Yo6rl3h1giYk1StjrspWrycbQtFTiIrsjtrMeA7oBXQF9AAa6WUi6SUiwrDOUulWosSpCZm8O++R3/w3Tp2ACmJX7cu37Zr+T5Hu2tvkGaTTKUXjVhBXBBK1IEu38H1/bB6IBiy7+ili47m5tix2IWG4jN8WCE5aVmsubCG7098T9eyXXmn1gDm9KnBr4Pq4udqx4hfj9Nhzi72XszbCOupRUr4/S2IOg3dfrCIxeHCQkpJRFwqNioVPoWcKfQw2a0RxN7rL9wU6A+4A6eEEH0LyzlLJaCsO74lXTi2+VEhMtuSJXGoVo34NWvy1LnsYY5uvIZjnCc7S61g8aUfjeVywQnrBC0nKm0u/x73xN2kwUDkmA8xpKQ8NamieeXw7cN8vPdj6vjXYezzYx+sjTwX7Mnqt+szvUc17iRp6TV/H68vOsSFqKQcLD7l7PxC6afd4hMo82ypr8YmaUnV6glwt89zhboxyfHMQogawHvAy8BfPMPTQvcRQlCtRQnio1K5Ev5oValbp45oL1wk7Z6KYF6IvZHEwT8uU6aWL5Vql2TJ6SXcSLphLLcLTt3BUGcQ7JsD+77Jcpe7S5aSvGsXfqNHFVq3MUviesJ13tv2HoHOgUxrMg2N6tFsL5XqvrppE95vVZ59l2JpNX0n434/QXSi+bqjmY0z62DbZ1Clh9JX+xlCqzNwKyENR42KpvXr0K5dO7P5kt1i8QQhxGGU3sU7gFpSyteklHn/hXsKCanug6u3PUf/vvbI+66tWyM0GhLWrs2TPb3ewJZFZ7BztKFRj3K8U/0d1ELNjCMzcj64sBACWk1SMjo2jMlUY5B29hxRX36Jc9OmuPfsaSYnzUd8ejxvb3kbiWRO8zm42T1Zr8leo2Zw0zJsf78JvWoHsezAdZp8sY1ZW84/OwvKt04q/bOL14T2M4tE5bCxkFISGad0B/ztp+8IDTVvM6LsROc+Ai4BVe89Jt0b4gpASilz1k1+ilGpVVRtHsQ/v5zn5sV4ioUof/Rqd3ecmzQhfv0f+I4cidDkLv//wNpLRF9LpNUblXBwscUBf14Je4Xvwr+jb2hfKvtYiGa/Sq0U+fzUEX4bqLS+LFkPQ1oakSNHonJ1fWZSRR8mw5DBiO0jiEiKYH6L+ZRwzV33OG9nOyZ2qsyr9UsxdcO/TNt0jsX7rjKsRTleqhlo1ukCk5Ico6jd2rsq6rd56DT2+YHPORV9CrXaeGmWFTwrMKr2qCduN7YM9d2UDBLSMpBJsWzc8Bdjx47lq6++yq/7BSa7b1kplOKxdvceD3coa2961yyf0HoB2DnZsH/tpUfWBNw6dUQfG0vS7t25snP+0G2ObLxGxYYBlKn5n7DWgEoD8LL34otDX+R7zcEkaByUTCL3EkqNQdS/RE37ivTz5wmYPAkbLy9ze1ioSCmZuG8i+2/tZ0K9CdTyr5VnGyE+znzbtxYrB9UlyNORMb+doNX0nWw4ecuyPntjoNPCilcgOUqRj3A1Q51MPjCWDHVahp7IuFSc7Wz4dNwopk6disrMqbJPHBE8qQuZEEIN9ASeqS5lWaGxU/N8xxB2/HyWkztuULlJIADODRsFtvDjAAAgAElEQVSidncnfs0aXJo0ydZG7I0kti7+F//SrjTqXu6RbU4aJ4ZUH8KEvRP4++rftAq2oCpLR094eRUsaEHSpC7c3SDxeKUvzg0bmtuzQmfhqYX8dv43BlYZSIeQDgWyVSvYk5WD6vL36dtM3fAvg5YcpnoJd0a1rsDzpZ+CACsl/DEMru6GrguUaaE8Mqr2KBKLqAy13iC5FpuCSgjC92zF19eXmjVrsn37dhN5nTuyWyNwFUKMEULMFkK0FArvoEwXPb1aAXkkrGEAJcI82bPqwgMxOmFri2u7diRt3pJtw5q05Az+nHcCWzs1rQdWRq3J/HF0LtOZ8h7lmXZoGqm6VJNdR77wKInuxe+J3K7HzkuF79uvmdujQmfz1c18ffhrWgW3YnC1wUaxKYSgVZg/G99rxOddK3MzLo2e3+3jlR8OcPJGvFHOYTb2zoGjS6DR+1A5+2YtloYxZKgj41JJ0+kJ8nRg/769rF27luDgYHr27MnWrVt5+eWXC+16Hia78chioDxwAngd+BvoBnSUUnYsBN+KBEIImvUNRW2rYvPCMxj0So69e/eXkBkZxK/OWrQtJUHLmulHSbqTRuuBlXByzzqHWK1SM7r2aG4m32ThyYWmuox8IaUkcsZSDHpbAurEoFo9INcFZ08DJ2NOMuafMVT2qczE+hNRGVkh00atosdzJdj+fhM+fLEC4RFxtJu1i8FLj3AxugimnJ7bqKQeh3aAJh+a2xuTkJ0MdWxyOndTtPi62ONir2Hy5MlERERw5coVli9fTrNmzViyZIlZ/M7um1taStlfSvkt0AuoBbSTUh4rHNeKDk7udjTuVZ6oKwns/f0S0iCxL1cOh+rVifvll0xzvPHRqaz64jBxt1N48e0qFMuh2Uwt/1q0Cm7FgpMLiEyKNOWl5Im7S38mecdOfD8Yhf2rc+DqLvjtDTBkLcP8NBGZFMmQLUPwcvBiZlPTtpq016gZ2CiEnR805d3mZdl+NooWX+1g5K/HuX4n5x4IFsHtU0pvgWJVnjn5CIDEtAwi76bhYq8xi6hcTmT3aTxoUSWl1AOXpZSJpnepaFK2lh8VGwRwbNM1/pgbTmqiFo+ePdBevUrK/v2Acgd9/fQdfvviMOkpGXQcVp2SYbmb9x1RcwQCwbRD00x5Gbkm7ew5oqZOxalxIzxe7qMM81tNUgqD/vrAqE1tLI1EbSKDtwxGq9cyp/kcvBwKZ+7e1V7D8Bbl2PlBUwbUL8Xa45E0m7adRafSuRlvYdOGD5MUBT/3BDtnRcjQ1sncHuWZgshQp2bouRabgp1GRQnPrNtPNmnShPXr1xvL3TyTXfpoVSFEwr3nAnC49/p++qiZRXAsjyZ9yuMd6MzulRdYPvEAdV6sTkJAVa4t+wONpjRHNl4l+loiLp72dBxWHc9iuf+DKOZcjAGVBzD32FwO3DxA7WK1TXgl2WNIT3+QKhowadJ/X+y6gyHxltLQxtkPGj99jezup4leib/C3BfmEuJe+EVzXs52jGtXkdcblmbW1vMsP3CNxlO307tOCd5uEoKvq+lGJ3kmI1XRqEqJgVf/BLfi5vaoUMnQGbgSk4xKJQj2cirUrmN5IbusocLXQi3iCCGo3CSQYmXc2Dj/FNuWXYByAyEdmH8SNx8HmvQpT/nn/bHJR+OJV8NeZc2FNUzaP4lf2/+KRl2IPQoeImrqF6SfP680oH88VfSFCZAcrVSLOnlDrQFm8dEUSCn5bN9n7L25l0/qfULdgLpm9cffzZ7POlemml00h1O9WbzvKssOXKN3nRK81dgCAoKUsGYwRByEHksgoLp5/SlkMvQGLsUkozdIQnycsLWxzCAAuZCYKAhCiNZCiLNCiAtCiNFZbO8jhAi/99gjhKhqSn8KC+9AF3p+VJse42rTprsfFU//SMOQm/Se8DxhDYvnKwgA2NvYM6b2GC7GX2TxmcVG9jp3JG7dxt2lS5/cgF6lgg6zoGxL+GMEnM5bhbUl88PJH1h1fhVvVH6DzmU7m9udB/g4qpjStQrbRjShY7UAftp7lYZTt/Hx2lPcijfj4v22SXByFbzwMYQ+W6VHGXoDl6KTydAbKOXthINtdpMv5sdkgeBevcEcoA1QEeglhKj42G6Xgcb3qpQ/RVE7fSpQ26jwDnSmdLMwSpcUOG74ESGzV+3MDY2DGtM0qCnzjs/jZtJNI3iaezJuR3Hzww8VVdHsGtCrNfDSQiheC1a9Bpd3FpqPpmLD5Q1MPzKdNsFtGFLdMjVxSng5MrVb1QcBYcm+qzSauo2xq08QcbeQF5WP/Qw7p0L1vlD/vcI9t5nJ0Bu4fC8IBHs74WRn2UEATDsiqA1ckFJeklJqgeXAI2mnUso9Usr7ifb7gEAT+mM2PPq+TMaNG/nuXvY4o2uPRkrJ1INTjWIvN0iDgcjRozCkp1N82pc5q4raOkHvX8AzBJb1hsiim2x25PYRxu4aSw3fGnza4FOjp4kamwcBYWQTutUKZMWh6zT5YjsjVhwvHKXTy//A2nehVGNo9/UzpSGUrtNzKToJrd5AsJcTzkUgCAAIU5WvCyG6Aa2llK/fe90XqCOlzPJ2SggxEqhwf//Htg0EBgL4+fnVXL58ea79SEpKwtnZOR9XYEQMBrzGf4x0cuLOqA+M8ofxd/zfrItbxyDfQYQ5hGXabuzrdtywAZff15Dwch9SGzTI9XG26bHUODIalSGdo9Unk+pousVCU3zWURlRTLs1DWeVM8P9h+OktryMl5yuOzbVwIYrGey4riPDADX91LxYWkNpN+MvAzomR1D96Ci0th4crT4FncY4n4ebmxtlypR58Fqv1xtVa8gYpOslt5OVUb+vowp7G+MHwNxe94ULF4iPf7T4sGnTpoellFnrn0gpTfIAXgK+f+h1X2DWE/ZtCpwBvHKyW7NmTZkXtm3blqf9TUXs0qXydPkKMvnQIaPY0+q0ssPqDrLlry1lsjY503ZjXnfy4SPydMUwef2996TBYMi7gejzUn5eWsqvwqSMizCaX49j7M86JiVGtlnVRjZc1lBei79mVNvGJLfXHZOYJr/Y8K+sPH6DLDlqvez57V65/WxU/j7TrEi8LeXXlaScGiLlncvGsXmP06dPP/I6ISHBqPYLSnyKVp6IiJNnIuNlqlaX6+O++uorWbFiRRkWFiZ79uwpU1NTs90/t9f9+P+XlFICh+QTfldNOcaNAIIeeh2I0vbyEYQQVYDvUSqWn9qWTe6dO6N2dyf2B+M0m9GoNYyvO57I5EhmH5ttFJtZoY+L48bIEWgCAij2ySf5UxX1LqPoEqXFw+JOee59bA5Sdam8s/UdolKimNlsJkGuQTkfZOF4OdsxslV59oxpztgXQ7kUk0S/Hw7Qevo/rDh0nXRdAQoBtSmKAGFyjDIl6BFsNL8tGSkltxLSuBKbjJ2NihBfZ+xzmQxy48YNZs6cyaFDhzh58iR6vZ68zHYYE1NOYB0EygohSgE3UITqej+8gxCiBPAb0FdKec6EvpgdlYMDHr17EfPNPNIvX8auVKkC26zhV4Pu5bqz9MxS2pZqS5h35imigiClJHLcOHTRMQT/vBR1QUS+AqopiqVLusDSrtBvncX2pdUb9IzaOYqTMSf5usnXVPOtZm6XjIqznQ1vNCpNv3rBrD0eyff/XOKDleF8sfEsfZ8vSe86JfDOS9tEg16pKL9xBHr+nC8hubxwa9Ikkk+e4o4Rp4bsQivg/+GTZS+ykqFe9dtqYtKUqmEPR1uKuzugUuXtRkmn05GamopGoyElJYWAgICCXkq+MNmIQEqpA4YAG1GmfVZIKU8JIQYJIQbd2+1/gBcwVwhxTAhxyFT+WAIevXsjNBruLDJey+f3ar6Hl70XH+/9mAxDRs4H5IG7S5aStHkLvsOH41DZCP0QguvDS4vgZriiRZ9hedWwUkqmHJjCtuvbGFV7FM1LNje3SybD1kZFt5qB/DW0IYtfq01YgCtfbTpHvSlbGfnr8dwJ3EkJG0bDv+uhzedQ4UXTO24mHpahdnZxZe7Cn0lK11Hc3YFADyUI5EaG+j7Fixdn5MiRlChRgmLFiuHm5kbLli3NcGWYbo3AVI+iukZwn8hxH8kzlatI7a1bRrO56comWWlhJTk/fP6D9wp63SnHjsnTlSrLa4PeMt4c8n2Or5ByvJuUS3tIqdMazawxPusFJxbISgsryS8OfFFwhwoJY37Hz99OlGNXh8sK4/6SJUetl53m7JK/Hbn+5HnvXdOlHO8q5caxRvMhK8y9RnD58mVZpkwZqdMbZMTdFDl0zHg59INxMjk9I98279y5I5s2bSqjoqKkVquVHTt2lIsXL872mKK4RmAlC7zeHIiUkph584xms3mJ5rxQ4gXmHpvLxbiLBbanu3uXiPeGofH1JWDKZON3G6vyErT9Es79Bb+/BYaC11cYg/WX1vP14a9pHdya4bWGm9sds1DG15mJnSqzf2xzxrevSHxKBsN+Oc7zk7fw6frTj6afnlgJm/4HYV3ghU/M53QhobG15XxUIrFJ6bg62OFmr8LxsUKxvIwINm/eTKlSpfDx8UGj0dClSxf27NlTWJfzCEUjyfUpwjYwEPduXYn7dSVeAwZgG1TwRUghBGOfH8uhNYf4aPdH/NTmp3zbkgYDkaNGoY+JoeTPP6N2e3Lf3QLx3OuQlgBbJoCts9nzzfdG7uWj3R/xnP9zfNbgM4uvFTA1rvYaXq1fin51g9l7KZafD1zjp71XWLDrMjVLejA4OJKmBwchSjZ46tVE03V6Iu6koNUZEAhK+zjj6qAhKSk90745NaZ5mBIlSrBv3z5SUlJwcHBgy5Yt1KqV9+52xuDp/fQsGO9BbyHUamJmzzGeTQdvPqzzISdiTvDT6fwHgph580je+Q9+H47BoXIlo/mXJQ2HQ4NhcPhHRafeTIqlZ2LPMGz7MIJdg5nedDq26hyK5Z4hVCpB/TLezOldg71jmjOmTQU8E8/y3L4hXND7Mdp2FNsuxqPTW8aozphk6A3cuJvKudtJJGt1aNQqyvo5G61IrE6dOnTr1o0aNWpQuXJlDAYDAwcONIrtvGIdEZgBjZ8vHr17c2fRIrwGvoFdiHEULFsHt2bjlY3MOTqH9/3fz/Pxidu2ETNrNq7t2+Pes6dRfMqR5uNBmwx7ZytZRE0ySVKZlOuJ13lr81u42LrwzQvf4GprFdV9Et7OdrxZxYaBB6eQ4ezOqhIz+evfNJafOIiXky1tKvvzYuVi1CnlhTqP2TOWhFZnIDYpndhkLVKCh5OGCrUqceb0qQf7jBw50ijnmjBhAhMmTDCKrYJgDQRmwuuN14n75ReiZ84icMZ0o9gUQjDu+XF0XtOZxTGL6WroikaVO4XS9EuXiXz/A+xCK1DskwnGXxd4EkJA68+VYLB9siJNUe+dQjl1bGosgzYNQid1/PDCD/g7+RfKeYssybGwpCtCl4rtgI2M9g1lmE7PjrPRrDkWycrDESzZdw1vZztaVPSlRUU/6oV45zqv3pxIKUnV6olJ0hKfqmTfuTtq8HW1w87G8v0vKNZAYCZsPD3x7N+PmLnfkHLoEI5Gmhv0dvDmf3X/x/Dtw5l3fB7vVM/5R1WflETEkCEIjYagWbNQOTgYxZdcc1+xNCNFmSKysYfab5j0lMkZyby95W2iUqKY33I+pd1Lm/R8RZ70JPj5JYi/Dn1Xg28oAHY2alqG+dMyzJ8UrY5t/0bz54mbrD0WybID13HQqKkX4kWjcj40KudDsJdj4d1k5IIMvYG4FC13kzNI0+lRC4G3iy1eTnYWLRttbKyBwIx4vf46cb//zq0JEyi1ahUiJyG3XNKiZAvqONXh+xPf06B4A6r7PlkHXur1RI4YifbqVUr88AOa4mZqHKJSQ5f5oEuHP0eCxgGqm6aRt1avZejWoZy9c5YZTWc8dQVjRkenhRV9IfIo9FgKJetluZujrQ1tqxSjbZVipOv07Lt0h82nb7PjXDRb/o0CIMDNnjqlvahdypPngj0o7e2c5yKsgiClJE1nIDE1g4Q0HSla3QPfi3s44O6gsdjmMabEGgjMiMrREf9x44h4ezCxixbh/Ybx7oK7enYlQkQw5p8xrGy/EmfbzOJfUkpuT5pM0o4d+P3vI5zqmK/rGfCffPWyXrBmiDIyqJw57a4g3K8a3n9rP5MaTKJxUGOj2n/qMBiUFN+LW6HD7FwXjNnZqGlczofG5XwAuBKTzM7z0ey/dId/zkez+ugNQKlyrlzcjUrFXSnn50J5fxdCfJyNJt2s0xtIy9CTmmEgOV354dcZlKQER1s1fq72uDtosCsC01emxBoIzIxLs2Y4v9CcmDlzcW3zIraBxrkjd1A5MKnBJF7d+CqTD0zmswafZdrn7k8/KU1m+vfHs3fvLKyYARs7pZvVz93ht4HKSCHMOE1gpJR8uu9TNl/bzAfPfUD7kGerWUqekVLpP31ypdJcpkbffJsK9nYi2NuJV+oGI6XkYnQyx67Hcfx6HMcj4li09ypa3X+ZR15OtgR5OlLc3QFvZ1teLKEs4AohUAlIz5AYUjNASiSgN0j0UqI3SDJ0BrR6iVZnQPdQjYqtjQoXew1Odja42NmgeYamfnLCGggsAP8PP+Riu/bc/vRTAud9Y7Q51Bp+NRhYZSDzjs+jTrE6dAjp8GBbwqZN3J7yOS4tWuD7Qd4zjEyKreM9XaKusOp1UGkgtF2BTEop+frw1w86jPWtmP8ftWeGbZPg4HyoO8SozWWEEJTxdaaMrzPdaiotSPQGydXYZM7eSuRSTDIRd1O4dieFMzcTiElKp4GPDzfiHpMkSUnO0ratWqBRq3C1t8FOo8Zeo8Jeo0ajtv7wPwlrILAANAEB+AwZQtTUqcSt+BWPHt2NZntQlUEcunWIifsmUsm7EqXdSpO0azeRw0dgX6UyAVM/R1jinKidM/T5VRGp+7W/Mkoo3zrf5uafmM+Pp36kR/keuVpAf+bZO+e/DmMtJ5q82E+tUgq1Svtk3b/g9OnTlC3mikFKpISk5GScHB0BgRDK8WqhPLekxeicGDBgAOvXr8fX15eTJ08+sm3WrFnMnj0bGxsb2rZty9SppmtEZYG/AM8mnv1ewalBA25PnEjqiZM5H5BL1Co1UxpOwV5tz/s73ufunl1EDB6MbUgIJb79tvAzhPKCvasiX+1fSVmsPPd3vswsPbOUWUdn0b50ez6s82GR+qEwC0eXwMYPIbQDtJ9hER3GhFDu8u1s1Nhr1NipBQ62NjjYqh/c7atUosh9tv3792fDhg2Z3t+2bRtr1qwhPDycU6dOGa1u4UlYRwQWglCrCfhiKpe7duXG0KEEr1qJjYeHUWz7OfnxWYPPmL7oLa6veAunoJKU+GEBand3o9g3KfZuSrriTx3hlz6KzHHZFrk+fPX51Uw5MIXmJZrzSf1PnnnpiBw5uQrWvgOlm0LX75U1GgvjnxXnuH0l3qgdyryDnGnYvdwTt2clQ71mzRocCngj1ahRI65cuZLp/W+++YbRo0djZ6fIgfv6+hboPDlh/auwIGw8PAicMQNddDSRH4xC6gvQKOQxqp3X8fEKQYyjjlMfdcPG09Notk2Ogwf0/R18KsDyPnA+d72f111cx/g946kfUJ+pjaZio7Le92TL2b+UBfqgOkrAtclDT4JngIdlqN3d3Vm1alWmffIiOpcd586d459//qFOnTo0btyYgwcPGusyssT6l2FhOFSujN/Ysdz6+GMi3/+AgM+nIDS5qw5+EneXLePWpxNxCq3Agl4u7Dw3k1Kla1LZxwg9BgoLR094ZY0yMljeS8lnL/dk7fYNlzcwbvc4avvXtuoH5YaL22BFP/CvrHQYs3U0t0dPpGH3ciQmJuJSkEZJ+aBUqVJUq6bUnNSsWTPLO/m8iM5lh06n4+7du+zbt4+DBw/SvXt3Ll26VGC7T8I6IrBAPHr2wHfkCBL+/JOId97FkJaWLzsGrZbbk6dwa8InODdqRPDixXzcbjq+jr4M2z6M2FTLbxn5CPeDgW+oMk109q8sd9t0dROj/xlNNZ9qzGw2E3sb+0J2tIhxZZdSu+EVAi//pkzHWcnE/WkaALVajU6ny7SPsUYEgYGBdOnSBSEEtWvXRqVSERMTU+BreBLWQGCheL3+Ov4fjydpxw6uvzkI3Z07eTre5upVrnTtyp1Fi/Do04fA2bNQOTribu/O9KbTiU+PZ/j24Wj1WhNdgYm4Hwz8wuCXvnBm/SObN17ZyPs73qeyd2XmvjAXR43l3tlaBNf2w9Lu4F4CXlmr/P9ayTd9+vTh2LFjmR4rV67Mk51OnTqxdetWQJkm0mq1eHt7m8JlwBoILBqPnj0JmPo5KYcPc7FVa+4sWoTMyL4dZcbNm9z+fCqen09Fn5BI0Hff4v/ROITNf7OAFTwr8En9TzgSdYSP93yMNJP8c765v2ZQrCr82g9O/gbA4eTDjNo5iqo+VZnXYh5OGiczO2rhRByGpd3AxR/6rQVnH3N79MzRq1cv6taty9mzZwkMDGTBggWAklZ66dIlKlWqRM+ePVm0aJFJM6KsawQWjlv79thXrMjtSZO5PXkKd1f8imvbF3GsXh37ylWQGVp0t2+jvXqN+LVrSdq2DaQkrW5dyk//GrVr1rLKbUq14WrCVeYcm0MJ1xIMqjooy/0sFgd3JZvo5+6w6jXWRR1iUcx6avjVYG5z60ggRyIOw+LOSlDtt04JBlaeSHBw8CN5/sZK51y2bFmW79va2rJkyRKjnCM3WANBEcAuJISg7+eTtG0b0bNnEzNrdpZNXNSenni9/jru3buz58L5JwaB+7xZ5U2uJ15nzrE5BLkE0bZ0W1Ndgmm4V2ewfFlbPru+jkrCyxoEcsP9IODoAf3Wg5uZhAatWAzWQFBEEELg0qwZLs2aoU9IIPV4OGmnTqJycMDGvxgafz/sQkNR3VcwvXA+VzbH1x1PZFIk43aPw83OjQbFG5j4SozL92eXMYNYmghnvrx8FLuDCwqtn0GR5PEg4F7wVqlWij7WNYIiiNrVFeeGDfAeNAjPfv1wbdUSh6pV/wsCecBWbcuMZjMo416GYduGceT2ERN4bHwM0sCXB79kxpEZtC3dlq96bibeu77Sz2DrRLO1vbRoru1T0m+tQcDKY1gDgRVcbV2Z98I8/J38GbxlMKdjT5vbpWxJ16czaucoFp1eRK8KvZjUYBIaWydOVxyhaOPs/EJRzTQ8fX10883lnbC4C7j4wat/WYOAlUewBgIrAHg5eDG/5XxcbF14c9ObFhsM4tPjeXPTm2y4soERNUcwpvaY/2QjhFrpdFZ3CBz4DlYPVJqqPOuc3wxLX1JSRPv/Ca4B5vbIioVhDQRWHuDv5M+ClgtwtHHktY2vcTTqqLldeoSLcRfp82cfwqPDmdpoKv0r9c+cUieEopbZfDyc+BWW9VDaLD6rnPwNlvUE77LQ/w9lRGDFymNYA4GVRwhyDWJRm0V4O3jz5qY32Ru519wuAbDt2jb6/NmHJG0S37f8njal2jx5ZyGg4XDoOAcu7YBF7SEpuvCctRQOL4SVAyCwlrIm4ORlbo+sPERaWhq1a9ematWqhIWFMX78+Afb3n//fSpUqECVKlXo3LkzcXFxJvXFGgisZMLfyZ8fW/9IkEsQb295m9/O/2Y2X3QGHXOOzeHdbe8S7BrM8nbLqeFXI3cHV38Zei6FqDOw4AWIuWBaZy0FKWHX17BuKJR5QZGNcCgCSrPPGHZ2dmzdupXjx49z7NgxNmzYwL59+wBo0aIFJ0+eJDw8nHLlyjF58mST+mJNH7WSJd4O3vzQ6gfe3/E+4/eM5+yds7z/3PuFquAZmRTJ6H9GczTqKB1DOjLu+XF51w0q3wb6r4efeyjBoNdyKPG8aRy2BAx62DBaWSOp1A06fQM2T5fg3raF33Hz4nnUNsaTofYtWZqm/Qc+cbspZKiFEDg7K414MjIyyMjIeDDV2bLlf4KKzz//fJ4lKvKKdURg5Ym42bkx94W59K3Yl5///ZlBmwYRlRJl8vNKKVl/aT3d1nbj3N1zTGk4hYkNJuZfPC6wFry+CRy9YFEHOGHaPyqzkZEKK15RgkC9d6DL/KcuCJgTU8hQ6/V6qlWrhq+vLy1atKBOnTqZ9vnhhx9o0yabqVAjYB0RWMkWG5UNHzz3AeU9yjNx30Q6renEmNpjaFe6nUm0T67EX2Hi/onsv7mfKj5VmNJwCkEuRkh19CwNr22CX16GVa9B9L/Q5EOwxDad+SEpGpb3hoiD0HoKPP+WuT0yGU37D3xqZKjVajXHjh0jLi6Ozp07c/LkSSpVqvRg+2effYaNjY1RpK2zw6R/BUKI1kKIs0KIC0KI0VlsryCE2CuESBdCmLYXm5UC0bFMR1Z2WEmIWwgf7vqQd7e9y9WEq0azH5cWx4wjM+iytgunYk4xts5Yfmr9k3GCwH0cPRWxuvu1Br++AtrMDdCLGk5JV2B+U7h1AroveqqDgDkxpQy1u7s7TZo0eaRt5aJFi1i/fj1Lly41eQtOk40IhBBqYA7QAogADgoh1kopH05QvwO8C3QylR9WjEdJ15IsbL2QJWeWMPvobDr+3pEOIR14s+qbFHfOn15NbGosP53+ieX/LidVl8qLpV9kZK2ReDuYSHLXxlapNfANVaqQv38BeixRtPiLImf/ovrRUUq18Kt/QvFcLqRbMQl5GRFER0ej0Whwd3cnNTWVzZs3M2rUKAA2bNjA559/zo4dO3B0NL12limnhmoDF6SUlwCEEMuBjsCDQCCljAKihBBFTO3s2UWtUtMvrB9tS7dlwYkFrDi7gnUX11E3oC5tSrWhaVBTnG2ds7WRnJHM1mtb+evyX+yN3Ite6mldqjUDKw+kjEcZ01+EEFB3sBIMVr4G3zWBzvOgQhH6Ghr0sONz2DGVFJcQXHpd1TsAABD5SURBVN9YZy0UK2LcvHmTfv36odfrMRgMdO/enXbt2gEwZMgQ0tPTadFC6c/9/PPPM2/ePJP5IkylRS+E6Aa0llK+fu91X6COlHJIFvt+DCRJKb98gq2BwEAAPz+/msuXL8+1H0lJSQ9W5p8lCuu67+rusiNxB0eSj3BXfxcbbPDX+OOv8cdP44eNsEGPHp3UEZURxQ3tDaJ0UUgkHmoPajjVoK5zXfw0BS90ys8126VFEXbqc1wTL3AtqBOXS72MVBWsNaipsclIIPTM13jdOcItv2YcCXgZR7enu0bAzc2NMmX+u0nQ6/VGbV5fVMjtdV+4cIH4+PhH3mvatOlhKWWtrPY35Yggq0mtfEUdKeV3wHcAtWrVkk2aNMn1sdu3bycv+z8tFOZ1d6YzBmkgPDqcbde3cf7ueS7GXeRQ/KFH9gtwCqBisYp08exC3YC6VPWp+p88hBHI9zU37wAbP6TEoQWU0F2Bbj9Y7lTR1b3w21hIugXtvsa/5qs47tjx1H/Hz5w588jisDkWiy2B3F63vb091atXz7VdUwaCCODhlb5AINKE57NiRlRCRTXfalTzrfbgvXR9OgZpwEbYoBIq1CoLvYPT2EO7ryCkKawZAvMaQutJUKOfMo1kCegzlKmgf6YpmkEDNkDxmub2yspTgimzhg4CZYUQpYQQtkBPYK0Jz2fFwrBT2+Fg44BGrbHcIPAwoe3hrd3Kguu6oYpk890r5vYKbp+CBS2VTKeqvWDQLmsQsGJUTBYIpJQ6YAiwETgDrJBSnhJCDBJCDAIQQvgLISKA4cA4IUSEECL7tlpWrJgSt0CliXu7r+HGEZhbF3bPNI+KqTYFNo2HbxtB3FV4aSF0mgt2z96UiBXTYtKCMinln8Cfj70376Hnt1CmjKxYsRxUKqg1AMq2hD9GwKaP4PCPiqpp+RdNP11kMMDp1bB5ghIAqvWBFp9aReOsmIynpKzSihUT4BYIvX+BPqtApVEqd39oDec3maYDmpRwYQvMb6Kohto6KaqhneZag4AVk2INBFas5ETZF5S1gxe/hPgIWNoNvm0Ix5cbpzJZmwKHFymL1Eu6QOpd6PydshZQqmHB7VuxWOLi4ujWrRsVKlQgNDSUvXsflX3/8ssvEUIQExNjUj+sWkNWrOQGtQZqv6FkEp34VZF5Xv0mrB+uLDKHdVJUTR08cmcvPREuboWzG+DsH5AWD36VoP1MqNoTbOxytmGlyDN06FBat27NypUr0Wq1pKSkPNh2/fp1Nm3aRIkSJUzuhzUQWLGSF2xsoXofJXvn+j5lVHDqdwhfDgjwC4OA6uBaHFyLgb076LWKMmjqHYj6F6JOK6J3eq2yvWwrqNkfStaznHRVCyZu3UVSr8eTqjbez5dtgBPu7Z9cO2IKGeqEhAR27tzJwoULFR9sbbG1/U8tdtiwYUydOpWOHTvm+xy5xRoIrFjJDyqV8sNdsh60mQo3DsHVPXB1N5z/G5KiyLJ+0rU4+FRQahbKtoSg58GIP2hWTMf58+dZtmwZ8+fPp3v37qxatYqXX375kX2WLl3KF198kenYMmXKZOopcOnSJXx8fHj11Vc5fvw4NWvWZMaMGTg5ObF27VqKFy9O1apVTXpN97F+A61YKSgaewhuoDzuo8+AxFuQngA29spUj50r2FuzowuKe/sQ1E+BDLVOp+PIkSPMmjWLOnXqMHToUKZMmcKYMWP47LPP+Pvvv43pfrZYA4EVK6ZArQF3I0poWzE7j8tQp6amZtonLyOCwMBAAgMDHzSj6datG1OmTOHixYtcvnz5wWggIiKCGjVqcODAAZycnIx5SQ+wBgIrVqxYMRJ5GRH4+/sTFBTE2bNnKV++PFu2bKHi/9u7/xgp6jOO4++PB3hFREREqQceWBEJ4KlgoSiRFixn2igpqVKsgq2GVKrUYu2v2F+xlqZJTbX+qiK0QYyIUqWFalC0/kA9/AGHSKXWKmoDUrAHooA+/WO+d47HnjsHO7u3O88r2fCdme/OPM/eMs/OzO53Bg9m6NChbNr00Z0Aa2traWhooFevXjQ1NaUStxcC55wrkeuuu44pU6awa9cuBgwYwO23316SOLwQOOdcHrW1tTQ2NrZMz5pVmBsq1tXV0dDQ8Il9cl2LKDT/QZlzzmWcFwLnnMs4LwTOOZdxXgiccy7jvBA451zGeSFwzrmM80LgnHMlcuGFF9K7d2+GDBnysflXXHEFgwYNYtiwYUycOJFt27YBsHv3bi644AKGDh3K8ccfzzXXXFOQOLwQOOdciUydOpVly5btNX/8+PE0NjayevVqBg4c2LLDv/fee3n//fdZs2YNq1at4uabby7I7wz8B2XOubKydOlS3njjDaqqqgq2ziOPPJL6+vo2l6cxDDXAmDFjcu7IzzjjjJb2yJEjW8YpksSOHTvYs2cPO3fupEuXLnTvvv8DGfoRgXPOJfDyyy9zySWXsHbtWnr06MGiRYv26jN//nzq6ur2ekyaNGmftztnzpyWInX22Wdz0EEH0adPH/r168esWbPo2bPnPq+7mR8ROOfKSn19PU0VMAx1EldffTWdOnVqWeeqVauoqqrizTffZOvWrZx22mmMGzeOAQMG7Nd2vBA451wChR6GOp958+axZMkSli9fjsKd6+666y4mTJhA586d6d27N6NHj6ahocELgXPOdRSFOiJYtmwZs2fP5pFHHqFr164t8/v27ctDDz3Eeeedx7vvvsvKlSuZOXPmfm/PrxE451yJTJ48mVGjRrF+/Xpqamq47bbbAJgxYwZNTU2MHz+euro6pk+fDsBFF13E9u3bGTJkCCNGjGDatGkMGzZsv+PwIwLnnMsjrWGoFyxYkHP+hg0bcs7v1q0bCxcuLMi24/yIwDnnMs4LgXPOZZwXAudcWTCzUodQFvbldfJC4Jzr8Kqrq9myZYsXgzzMjC1btlBdXd2u5/nFYudch1dTU8PGjRvZvHkzAO+99167d3aVIEne1dXV1NTUtGu9Xgiccx1e586d6d+/f8v0ihUrOPHEE0sYUWmklXeqp4YkTZC0XtIGSd/PsVySfheWr5Z0UprxOOec21tqhUBSFfB7oB4YDEyWNLhVt3rg2PC4GLgxrXicc87lluYRwSnABjN7xcx2AXcCZ7XqcxbwR4usBHpI6pNiTM4551pJ8xrBUcDrsemNwGcT9DkKeCveSdLFREcMANslrW9HHL2At9vRv1JkMe8s5gzZzDuLOcP+5X10WwvSLATKMa/1d7+S9MHMbgFu2acgpAYzG74vzy1nWcw7izlDNvPOYs6QXt5pnhraCPSNTdcAb+5DH+eccylKsxA8Axwrqb+kLsC5wH2t+twHnB++PTQSeMfM3mq9Iuecc+lJ7dSQme2RNAP4G1AFzDGztZKmh+U3AX8FzgQ2AO8C01IIZZ9OKVWALOadxZwhm3lnMWdIKW/5T7adcy7bfKwh55zLOC8EzjmXcRVTCLI4nEWCnKeEXFdLekLSCaWIs9Dy5R3rN0LSB5ImFTO+NCTJWdLpkp6XtFbSI8WOMQ0J3uOHSLpf0gsh7zSuMxaVpDmSNklqbGN54fdlZlb2D6KL0f8EBgBdgBeAwa36nAksJfrtwkjgqVLHXYScPwccGtr15Z5z0rxj/R4i+kLCpFLHXYS/dQ/gRaBfmO5d6riLlPcPgdmhfTjwX6BLqWPfz7zHACcBjW0sL/i+rFKOCLI4nEXenM3sCTPbGiZXEv1Oo9wl+VsDfBtYBGwqZnApSZLz14B7zOw1ADPLSt4GHCxJQDeiQrCnuGEWlpk9SpRHWwq+L6uUQtDWUBXt7VNO2pvPN4g+RZS7vHlLOgqYCNxUxLjSlORvPRA4VNIKSasknV+06NKTJO/rgeOJfoi6BrjMzD4sTnglU/B9WaXcj6Bgw1mUkcT5SBpLVAhOTTWi4kiS97XAlWb2QfRBsewlybkTcDLwBeBTwJOSVprZP9IOLkVJ8v4i8DzweeAY4EFJfzez/6UdXAkVfF9WKYUgi8NZJMpH0jDgVqDezLYUKbY0Jcl7OHBnKAK9gDMl7TGzxcUJseCSvr/fNrMdwA5JjwInAOVcCJLkPQ34lUUnzzdI+hcwCHi6OCGWRMH3ZZVyaiiLw1nkzVlSP+Ae4Otl/skwLm/eZtbfzGrNrBa4G/hWGRcBSPb+/jNwmqROkroSjfS7rshxFlqSvF8jOgpC0hHAccArRY2y+Aq+L6uIIwLrOMNZFE3CnK8CDgNuCJ+O91iZj9iYMO+KkiRnM1snaRmwGvgQuNXMcn79sFwk/Fv/ApgraQ3RKZMrzaysh6eWtAA4HeglaSPwE6AzpLcv8yEmnHMu4yrl1JBzzrl95IXAOecyzguBc85lnBcC55zLOC8EzjmXcV4IXElJ2l7k7d0qaXCRtzkzfLe/vc+7VtKYPH3m5htdNUmfVv27SHpUUkV8vdzl54XAVZR8Oy8z+6aZvVjgbUrSJ/1fmgm0qxBI6gmMDAOQFVUY4G05cE6xt+1KwwuB63AkHS5pkaRnwmN0mH9KuK/Cc+Hf48L8qZIWSrofeCCMy79C0t2SXpI0P4xOSZg/PLS3S7o6jGW/MvwyFUnHhOlnJP0811GLpFpJ6yTdADwL9JV0o6QGRePi/yz0uxT4NPCwpIfDvDMkPSnp2RB3txwvwyRgWWx7V4V4GiXd0pxPq5helTRb0tPh8ZnY4jHhNXul+ehAUjdJy0McayTFR/ZcDExJ9hdzZa/UY2/7I9sPYHuOeXcAp4Z2P2BdaHcHOoX2OGBRaE8lGn+lZ5g+HXiHaAyWA4AnY+tbAQwPbQO+HNq/Bn4c2kuAyaE9vY0Ya4l+wTsyNq95+1VhO8PC9KtAr9DuBTwKHBSmrwSuyrH+ec2xxdcd2n+KxT2XcL+FsJ0fhfb5wJJYn4XhtRhMNLQzRCMLdI/FtYGPfmRaBWwu9fvDH8V5+DlA1xGNAwbHPvR2l3QwcAgwT9KxRDvxzrHnPGhm8THcnzazjQCSnifacT/Waju7iHb6AKuA8aE9Cjg7tO8AftNGnP+2aDz4Zl+VdDHRDrYP0U53davnjAzzHw/5dSEqVK31ATbHpsdK+h7RKaaewFrg/hzPWxD797ex+YstGp75xeYjH6IhGX4ZrkN8SDSU8RHAfywauXWXpIPNrKmN/F2F8ELgOqIDgFFmtjM+U9J1wMNmNlFSLdGn7mY7Wq3j/Vj7A3K/13ebmeXp80latimpPzALGGFmWyXNBapzPEdERWtynnXvbH6+pGrgBqIjmdcl/bSNdcPHhyOOt+OvR3OFnUJ0V6+TzWy3pFdbrfdA4L08cboK4NcIXEf0ADCjeUJSXWgeArwR2lNT3P5K4CuhfW7C53QnKgzvhE/c9bFlTcDBsXWPbj5/L6mrpIE51rcOaD7H37xzfjtcT/ikbwCdE/s315FG3CHAplAExgJHNy+QdBjRqaHdedbhKoAXAldqXSVtjD0uBy4Fhiu6MfeLROfpITqPf42kx4nOYadlJnC5pKeJTtG8k+8JZvYC8BzRKZs5wOOxxbcASyU9bGabiYrYAkmriQrDoByr/AvRtQ7MbBvwB6I7cC0mGp65LQdKegq4DPhOnrDnE73ODURHBy/Flo0lGuXSZYCPPupcK+E7/zvNzCSdS3ThONd9kdOO4zHgS6EQJOn/KtHpo/0ehlnSPcAPzGz9/q7LdXx+jcC5vZ0MXB++orkNuLBEcXyX6FtTiQpBoSi6CcxiLwLZ4UcEzjmXcX6NwDnnMs4LgXPOZZwXAuecyzgvBM45l3FeCJxzLuP+DxHscpOQ0eYPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot results\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "\n",
    "for n, rms_errors in rms_error_sets.items():\n",
    "    ax.plot(learning_rates, rms_errors, label='n = %d' % n)\n",
    "plt.title('n-step TD value error on random walk (%d states)' % game.size)\n",
    "ax.set_xlabel('Learning rate (alpha)')\n",
    "ax.set_ylabel('RMS Error')\n",
    "ax.set_ylim(0.0, 0.55)\n",
    "plt.legend()\n",
    "plt.grid()\n",
    "plt.savefig(\"random_walk_tdl_%d.pdf\" % game.size)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "Reference image (seed = 5): WHEN USING np.random.RandomState()\n",
    "\n",
    "<img src=\"random_walk_tdl_19_copy.png\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note, although the figure is similar, the absolute EMS error is lower here\n",
    "# (less than 0.1 in a few cases) than in the Figure 7.2 in Sutton & Barto"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8.7 Real-Time Dynamic Programming\n",
    "\n",
    "### Exercise 8.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class RandomStatesExperiment():\n",
    "\n",
    "    name = 'Random States Experiment'\n",
    "    roles = [1]\n",
    "    possible_n_players = [1]\n",
    "    help_text = {\n",
    "        'Move format': \"0 or 1\",\n",
    "        'Move not available': \"That action is not available.\",\n",
    "        'Number of players': \"This game is for 1 player.\"\n",
    "    }\n",
    "\n",
    "    def __init__(self, b, size=1000, moves=None, \n",
    "                 terminal_reward=0.0, seed=1):\n",
    "        \"\"\"Experiment described in section 8.6 of Sutton\n",
    "        and Barto book to test Trajectory Sampling\n",
    "        methods.\n",
    "\n",
    "        Args:\n",
    "            b (int): Branching factor.\n",
    "            size (int): Number of states.\n",
    "            moves (list): Optional. Provide a list of completed\n",
    "                moves. Each move should be a tuple of length 2\n",
    "                where the first item is the player role (1) and\n",
    "                the second is the action (0 or 1).\n",
    "            terminal_reward (float): Final reward when terminal\n",
    "                state is reached.\n",
    "            seed (int): Change this to produce a different\n",
    "                random environment.\n",
    "        \"\"\"\n",
    "\n",
    "        self.b = b\n",
    "        self.size = size\n",
    "        self.seed = seed\n",
    "\n",
    "        self.states = np.arange(0, size + 1)\n",
    "\n",
    "        # Last state is terminal state\n",
    "        self.terminal_states = [self.states[-1]]\n",
    "\n",
    "        # Initialize a random number generator for\n",
    "        # generating random state transitions and\n",
    "        # rewards\n",
    "        self.rng = np.random.RandomState(seed)\n",
    "\n",
    "        # First, create random state transitions\n",
    "        self.transitions = {}\n",
    "        for s in self.states:\n",
    "            actions = self.available_moves(state)\n",
    "            for a in actions:\n",
    "                # Add b possible next states\n",
    "                next_states = list(zip(\n",
    "                    [0.9/b]*b, \n",
    "                    zip(self.rng.choice(self.states[:-1], size=b),\n",
    "                        self.rng.normal(size=self.b))\n",
    "                ))\n",
    "                # Add the possibility of going to terminal state\n",
    "                next_states.append((0.1, (self.terminal_states[0], \n",
    "                                          terminal_reward)))\n",
    "                self.transitions[(s, a)] = next_states\n",
    "\n",
    "        self.start_state = self.rng.choice(self.states)\n",
    "        self.terminal_reward = terminal_reward\n",
    "        \n",
    "        self.n_players = 1\n",
    "        self.turn = 1\n",
    "        self.winner = None\n",
    "        self.game_over = False\n",
    "        self.past_states = None\n",
    "        self.moves = None\n",
    "        self.reset()\n",
    "        if moves is not None:\n",
    "            for move in moves:\n",
    "                self.make_move(move)\n",
    "\n",
    "    def reset(self):\n",
    "\n",
    "        self.past_states = []\n",
    "        self.moves = []\n",
    "        self.state = self.start_state\n",
    "        self.winner = None\n",
    "        self.game_over = False\n",
    "\n",
    "    def show_state(self):\n",
    "\n",
    "        print(self.state)\n",
    "\n",
    "    def available_moves(self, state=None):\n",
    "\n",
    "        return (0, 1)\n",
    "\n",
    "    def next_state(self, state, move):\n",
    "\n",
    "        role, action = move\n",
    "\n",
    "        next_states = [\n",
    "            s2_and_r[0] for p, s2_and_r\n",
    "            in self.transitions[(state, action)]\n",
    "        ]\n",
    "        probs = [\n",
    "            p for p, s2_and_r\n",
    "            in self.transitions[(state, action)]\n",
    "        ]\n",
    "\n",
    "        next_state = np.random.choice(next_states, p=probs)\n",
    "\n",
    "        return next_state\n",
    "\n",
    "    def update_state(self, move):\n",
    "\n",
    "        self.state = self.next_state(self.state, move)\n",
    "\n",
    "    def make_move(self, move, show=False):\n",
    "        \n",
    "        assert self.game_over is False, \"Game is over\"\n",
    "\n",
    "        self.past_states.append(self.state)\n",
    "        self.update_state(move)\n",
    "        self.moves.append(move)\n",
    "\n",
    "        if show:\n",
    "            role, action = move\n",
    "            print(\"Player %s made move %s\" % (str(role), str(action)))\n",
    "\n",
    "        self.check_if_game_over()\n",
    "\n",
    "    def get_rewards(self):\n",
    "        \"\"\"Returns the reward at the current time step.  In\n",
    "        this experiment, the reward of each transition is\n",
    "        different and selected from a Gaussian distribution.\n",
    "        Rather than store all the fixed rewards in memory,\n",
    "        we regenerate them deterministically when needed using\n",
    "        a seeded random number generator.\n",
    "        \"\"\"\n",
    "\n",
    "        # Use previous state + seed to set RNG state\n",
    "        # so that rewards are deterministic for any\n",
    "        # state transition\n",
    "        self.rng.seed(self.states[-1] + self.seed)\n",
    "        last_action = self.moves[-1][1]\n",
    "\n",
    "        # Return the reward for the last action taken\n",
    "        reward = self.rng.normal(size=self.b)[last_action]\n",
    "\n",
    "        return {1: reward}\n",
    "\n",
    "    def get_terminal_rewards(self):\n",
    "        \"\"\"Returns the reward after the terminal state was\n",
    "        reached.\"\"\"\n",
    "\n",
    "        assert self.game_over\n",
    "\n",
    "        return {1: 0.0}   \n",
    "\n",
    "    def check_if_game_over(self):\n",
    "\n",
    "        if self.state in self.terminal_states:\n",
    "            self.game_over, self.winner = True, 1\n",
    "\n",
    "        return self.game_over\n",
    "\n",
    "    def generate_state_key(self, state, role):\n",
    "\n",
    "        return self.state\n",
    "\n",
    "    def __repr__(self):\n",
    "\n",
    "        return \"RandomWalkGame(%d)\" % self.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "game = RandomStatesExperiment(b=5, size=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Game has 1000 states + a terminal state\n",
    "len(game.states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start state is chosen randomly, terminal state is 1000\n",
    "game.start_state, game.terminal_states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# From each state, there are two possible actions (0, 1)\n",
    "# Each action leads to b possible next states with equal\n",
    "# probability, or to the terminal state with probability 0.1\n",
    "\n",
    "# Transitions from current state with action 0\n",
    "game.transitions[(game.state, 0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transitions from current state with action 1\n",
    "game.transitions[(game.state, 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "game.available_moves()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test next_state function by finding all possible \n",
    "# next states if action 0 taken\n",
    "next_states = set()\n",
    "\n",
    "# iterate many times to make sure we catch them all\n",
    "for i in range(100):\n",
    "    next_states.add(game.next_state(game.state, (1, 0)))\n",
    "next_states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find all possible next states if action 1 taken\n",
    "next_states = set()\n",
    "\n",
    "# iterate many times to make sure we catch them all\n",
    "for i in range(100):\n",
    "    next_states.add(game.next_state(game.state, (1, 1)))\n",
    "next_states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "game.roles, game.turn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "game.make_move([1, 0], show=True)\n",
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "game.moves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gamelearner import RandomPlayer, TDLearner, GameController\n",
    "td_player = TDLearner(off_policy_rate=0.0)\n",
    "td_player.learning_rate, td_player.off_policy_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "game.reset()\n",
    "ctrl = GameController(game, [td_player])\n",
    "ctrl.play(show=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(td_player.value_function.items())[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "td_player.value_function[game.start_state]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "game.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "values = {s: 0.0 for s in game.states}\n",
    "len(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(values.items())[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_policy(game, state, action, values=None):\n",
    "    \n",
    "    actions = game.available_moves(state=state)\n",
    "\n",
    "    return 1.0/len(actions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_values(values, decimals=1):\n",
    "\n",
    "    v_array = np.full((len(values)//10 + 1)*10, np.nan)\n",
    "    for i, v in enumerate(values.values()):\n",
    "        v_array[i] = v\n",
    "    print(v_array)\n",
    "\n",
    "show_values(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "values = evaluate_policy(random_policy, game, values, max_iter=100, \n",
    "                         show=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_values(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.Series(values).hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "values[game.start_state]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (tf)",
   "language": "python",
   "name": "tf"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}