SARSA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from amalearn.reward import GaussianReward, RewardBase\n",
    "from amalearn.agent import AgentBase\n",
    "from amalearn.environment import EnvironmentBase\n",
    "import numpy as np\n",
    "from math import *\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.stats as stats\n",
    "import gym\n",
    "import timeit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import pprint\n",
    "import io\n",
    "import numpy as np\n",
    "import sys\n",
    "from gym.envs.toy_text import discrete\n",
    "from colorama import Fore, Back, Style\n",
    "\n",
    "UP = 0\n",
    "RIGHT = 1\n",
    "DOWN = 2\n",
    "LEFT = 3\n",
    "\n",
    "class GridworldEnv(discrete.DiscreteEnv):\n",
    "  metadata = {'render.modes': ['human', 'ansi']}\n",
    "\n",
    "  def __init__(self, shape=[10,10]):\n",
    "    if not isinstance(shape, (list, tuple)) or not len(shape) == 2:\n",
    "        raise ValueError('shape argument must be a list/tuple of length 2')\n",
    "        \n",
    "    self.shape = shape\n",
    "\n",
    "    nS = np.prod(shape)\n",
    "    nA = 4\n",
    "\n",
    "    MAX_Y = shape[0]\n",
    "    MAX_X = shape[1]\n",
    "\n",
    "    P = {}\n",
    "    grid = np.arange(nS).reshape(shape)\n",
    "    it = np.nditer(grid, flags=['multi_index'])\n",
    "\n",
    "    while not it.finished:\n",
    "      s = it.iterindex\n",
    "      y, x = it.multi_index\n",
    "\n",
    "      P[s] = {a : [] for a in range(nA)}\n",
    "\n",
    "      #Reaching target\n",
    "      is_done = lambda s: s == 0\n",
    "\n",
    "      #Due to existance of walls, these moves are not available\n",
    "      no_up = lambda s: s in [44,73,74]\n",
    "      no_down = lambda s: s in [23,24,54]\n",
    "      no_left = lambda s: s in [44,35,54,65]\n",
    "      no_right = lambda s: s in [32,42,52,62]\n",
    "\n",
    "      #Barries in the path\n",
    "      pit = lambda s: s in [13,72,41]\n",
    "      wall = lambda s: s in [43,33,53,34,64,63]\n",
    "\n",
    "      #Rewards of each state\n",
    "      reward = 15.0 if is_done(s) else -1.0\n",
    "      reward = -15.0 if pit(s) else reward\n",
    "      \n",
    "      ns_up = s if y == 0 else s - MAX_X\n",
    "      ns_right = s if x == (MAX_X - 1) else s + 1\n",
    "      ns_down = s if y == (MAX_Y - 1) else s + MAX_X\n",
    "      ns_left = s if x == 0 else s - 1\n",
    "      \n",
    "      P[s][UP] = [(1, ns_up, reward, is_done(ns_up))]\n",
    "      P[s][RIGHT] = [(1.0, ns_right, reward, is_done(ns_right))]\n",
    "      P[s][DOWN] = [(1, ns_down, reward, is_done(ns_down))]\n",
    "      P[s][LEFT] = [(1, ns_left, reward, is_done(ns_left))]\n",
    "\n",
    "      #Terminal state\n",
    "      if is_done(s):\n",
    "        P[s][UP] = [(1.0, s, reward, True)]\n",
    "        P[s][RIGHT] = [(1.0, s, reward, True)]\n",
    "        P[s][DOWN] = [(1.0, s, reward, True)]\n",
    "        P[s][LEFT] = [(1.0, s, reward, True)]\n",
    "\n",
    "      #Encountering walls\n",
    "      if no_up(s):\n",
    "        P[s][UP] = [(1.0, s, -5.0 , False)]                \n",
    "      if no_down(s):\n",
    "        P[s][DOWN] = [(1.0, s, -5.0 , False)]                \n",
    "      if no_right(s):\n",
    "        P[s][RIGHT] = [(1.0, s, -5.0 , False)]  \n",
    "      if no_left(s):\n",
    "        P[s][LEFT] = [(1.0, s, -5.0 , False)] \n",
    "\n",
    "      it.iternext() \n",
    "\n",
    "    # Initial state distribution is uniform\n",
    "    isd = np.zeros(100)\n",
    "    isd[44]=1\n",
    "    self.P = P\n",
    "\n",
    "    super(GridworldEnv, self).__init__(nS, nA, P, isd)\n",
    "\n",
    "  def _render(self, mode='human', close=False):\n",
    "    if close:\n",
    "      return\n",
    "\n",
    "    outfile = io.StringIO() if mode == 'ansi' else sys.stdout\n",
    "\n",
    "    grid = np.arange(self.nS).reshape(self.shape)\n",
    "    it = np.nditer(grid, flags=['multi_index'])\n",
    "\n",
    "    while not it.finished:\n",
    "      s = it.iterindex\n",
    "      y, x = it.multi_index\n",
    "\n",
    "      if self.s == s and not s==0:\n",
    "        output =  Back.CYAN + Style.BRIGHT + Fore.WHITE +  \"  웃 \" + Style.RESET_ALL if s not in [13,72,41] else Back.BLUE + Style.BRIGHT + Fore.WHITE + \"  웃 \" + Style.RESET_ALL\n",
    "      elif s == 0:\n",
    "          output = Back.GREEN + Style.BRIGHT + Fore.WHITE + \"     \" + Style.RESET_ALL if not self.s==0 else Back.GREEN + Style.BRIGHT + Fore.WHITE +  \"  웃 \" + Style.RESET_ALL\n",
    "      elif s in [13,72,41]:\n",
    "          output = Back.BLUE + \"     \" + Style.RESET_ALL\n",
    "      elif s in [43,33,53,34,64,63]:\n",
    "          output = Back.CYAN + Style.BRIGHT + Fore.BLACK +   \"  █  \"  + Style.RESET_ALL\n",
    "      else:\n",
    "          output = Back.CYAN + Style.BRIGHT + Fore.BLACK +  \"  .  \" + Style.RESET_ALL\n",
    "\n",
    "      if x == 0:\n",
    "          output = output.lstrip()\n",
    "      if x == self.shape[1] - 1:\n",
    "          output = output.rstrip()\n",
    "\n",
    "      outfile.write(output)\n",
    "\n",
    "      if x == self.shape[1] - 1:\n",
    "          outfile.write(\"\\n\")\n",
    "\n",
    "      it.iternext()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent_SARSA(AgentBase):\n",
    "    def __init__(self, id, environment, action_count, state_count, alpha, epsilon):\n",
    "        super(Agent_SARSA, self).__init__(id, environment)\n",
    "        self.alpha = alpha\n",
    "        self.policy = np.random.choice([i for i in range(action_count)],state_count)\n",
    "        self.Q_sa = np.zeros((action_count,state_count))\n",
    "        self.action_count = action_count\n",
    "        self.state_count = state_count\n",
    "        self.env = environment\n",
    "        self.R_avg = []\n",
    "        self.R_sum = []\n",
    "        if epsilon == 0:\n",
    "            self.change_epsilon = True\n",
    "        else:\n",
    "            self.change_epsilon = False\n",
    "            self.epsilon = epsilon\n",
    "        if alpha == 0:\n",
    "            self.change_alpha = True\n",
    "        else:\n",
    "            self.change_alpha = False\n",
    "            self.alpha = alpha\n",
    "    \n",
    "    def main_loop(self):\n",
    "        gama = 0.9\n",
    "        for k in range(1000):\n",
    "            if self.change_epsilon:\n",
    "                self.epsilon = 100/(100+k)\n",
    "            if self.change_alpha:\n",
    "                self.alpha = 100/(1000+k)\n",
    "            state = np.random.choice([i for i in range(self.state_count)])\n",
    "            p = random.random()\n",
    "            R = []\n",
    "            #[np.argmax([self.Q_sa[i][j] for i in range(self.action_count)]) for j in range(self.state_count)]\n",
    "            if p< 1-self.epsilon:\n",
    "                action = np.argmax([self.Q_sa[i][state] for i in range(self.action_count)])\n",
    "            else:\n",
    "                action = np.random.choice([i for i in range(self.action_count)])\n",
    "            flag = False\n",
    "            length = 0\n",
    "            while True:\n",
    "                length = length + 1\n",
    "                if length > 20000:\n",
    "                    break\n",
    "                _, state_p, reward, done = self.env.P[state][action][0]\n",
    "                R.append(reward)\n",
    "                p = random.random()\n",
    "                if p< 1-self.epsilon:\n",
    "                    action_p = np.argmax([self.Q_sa[i][state_p] for i in range(self.action_count)])\n",
    "                else:\n",
    "                    action_p = np.random.choice([i for i in range(self.action_count)])\n",
    "                self.Q_sa[action][state] = self.Q_sa[action][state] + self.alpha*(reward+gama*self.Q_sa[action_p][state_p]- self.Q_sa[action][state])\n",
    "                state = state_p\n",
    "                action = action_p\n",
    "                if done:\n",
    "                    if flag:\n",
    "                        break\n",
    "                    else:\n",
    "                        flag = True\n",
    "            self.R_avg.append(np.mean(R))\n",
    "            self.R_sum.append(np.sum(R))\n",
    "    \n",
    "    def take_action(self) -> (object, float, bool, object):\n",
    "        index_selected_arm = 0\n",
    "        obs, Ri, d, i = self.environment.step(index_selected_arm)\n",
    "        #self.environment.update_selected_arm(index_selected_arm, Ri)\n",
    "        #print(obs, r, d, i)\n",
    "        self.environment.render()\n",
    "        return obs, Ri, d, i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "constatnt time = 8.60210148999613\n",
      "dynamic time = 158.72152425799868\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEWCAYAAACaBstRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXwU9f348dd7NydJSIBwB004REG5QVAQ8QJP8MZ64IlarVRrrVattJVWre3XUq+fbbHeaL2gilLxrBaLqIAcIiAo4TLcEMj9/v0xs8km2d1Mjs3meD8fj31k9zPXZ3Y2857PMZ8RVcUYY4zxwhfrDBhjjGk+LGgYY4zxzIKGMcYYzyxoGGOM8cyChjHGGM8saBhjjPHMgoYxMSIil4vIx0Gf94tIz1jmqa5E5AMRuTrW+TDRZ0HDNDkiMlpE/isie0Rkp4h8IiLDq8xzvIioiNxWJT3bTd/vvjaIyO0Ntf5oUtVUVf22sbYXLVWDoWlZLGiYJkVE2gJvAH8B2gPdgV8DhVVmnQLsdP+GkqGqqcB5wN0icnIDr7/ZEZG4WOfBNH8WNExTcxiAqr6gqqWqelBV/62qywIziEgbnGBwA9BHRIaFW5mqLgZWAIOisX63RJIrIr8Uke1uyebioOnpIvK0iOSJyHcicpeIhPy/c0s2vd33ySLyR3eZPSLysZv2poj8pMpyy0RkUoj1BUpdV4nI98B7bvpIt6S1W0SWisjxQctcLiLfisg+EVkf2BcRmS4iz4ZYd1yVbR4BPA6Mckt6u8N9d6Z5sqBhmppvgFIReUpEThWRdiHmORfYD/wTmA9cFm5lIjISOBJYG431u7oAmTillinAEyLS1532FyAd6AmMddd1RQ3rA3gQGAocg1Miug0oA54CLgnav4HududFWNdY4AhgvIh0B94E7nXXeyvwioh0FJEUYCZwqqqmudte4iGv5VR1FXAdsNCtbsuozfKm6bOgYZoUVd0LjAYU+CuQJyJzRaRz0GxTgBdVtRR4HrhIROKrrGq7iBwEFgKPAq838PqrultVC1X1Q5yT8gUi4gcuBO5Q1X2qugH4I3BppBW5JZErgWmqusktEf1XVQuBOTilnz7u7Je6eS2KsMrpqpqvqgdxAs48VZ2nqmWq+g6wGDjNnbcMOFJEklV1i6quqGG/TStjQcM0Oaq6SlUvV9UsnFJCN+AhABHpAYwDnnNnnwMkAadXWU0mkIpzJX08UH7Sb6D1B9ulqvlBn79z15kJJLifg6d1j/wNkOluc13VCW7geAm4xA0uFwHP1LC+jUHvDwXOd6umdrvVR6OBru4+XIhTUtjiVoUdXsO6TStjQcM0aar6NfAPnJM7OFfWPuBfIrIV+BbnBFutCsm9Qv8jUAD8uKHXH6SdW7UTcAiwGdgOFOOcqIOnbYqwLtzlCoBeYaY/BVwMnAgcUNWFNawveCjrjcAzqpoR9EpR1fsAVHW+qp4MdAW+ximNAeQDbYLW08Xj9kwLY0HDNCkicriI/ExEstzPPXCupj91Z7kMp7fToKDXucDpItIhzGrvA24TkaQorR/g1yKSICJjgDOAf7rVWy8BM0QkTUQOBW4Bno2wHlS1DJgF/ElEuomIX0RGiUiiO30hTjXSH6m5lFHVs8CZIjLeXW+S25ifJSKdReQsNwAW4rTrlLrLLQGOE5FDRCQduCPCNrYBWSKSUMu8mWbAgoZpavYBRwP/E5F8nJP5cuBnbqN2NvCIqm4Nes3Faei+KMw63wR2AddEaf1b3fVvxqnWus4twQD8BOcq/VvgY5w2klkevodbga+Az3C6/t5P5f/Xp4GjqCEAVaWqG4GJwC+BPJySx8/ddfuAn7n7sROnAf3H7nLvAC8Cy4DPcboth/MeTo+1rSKyvTb5M02f2EOYjKk7t7vqs277SGNu9zJgqqqObsztGmMlDWOaGfc+kh8DT8Q6L6b1saBhTDMiIuNxqpW24VR1GdOorHrKGGOMZ1bSMMYY41mLHsAsMzNTs7OzY50NY4xpVj7//PPtqtox1LQWHTSys7NZvHhxrLNhjDHNioh8F26aVU8ZY4zxzIKGMcYYzyxoGGOM8cyChjHGGM8saBhjjPHMgoYxxhjPLGgYY4zxrNkFDRGZICKrRWStiNwelY0c2Akf3Adbl1efVrAHvnwOajv8ytavYP1/6pSdHfsL+dfSzXVatqEUl5bxwqLvKSopqzxh4yL4/tPQCwXkrYZv5jdYXrbsOciby7bAyjmwa0ODrbch7S8s4Z+LN1JtmJ61C5zvI4ZWbdnLp9/uiGkeoqmguJQXP/uesrKg737ZS7Bva8XnvVtg2T8bP3P1cXAXfPF07c89DaxZjT3lPnP5G+BkIBfnWQMXqerKUPMPGzZM63Rz38Fd6AM9ES1jYY9rOHrfO/gS0+CQo2Hj/2DrV5Qkteev3e9lXOLXHC4bYdhVkNkHPv8Hy3tfy7z/reBmfZr4Dtkw8nq4PxuAbYeeQefC71mafDQdfXtJ2/wJbRL87C0s46u4I9nV5RhGlH5B0u51tCvcjPYYzqsr97OhrDNju5dRVFjE8LIlSNF+5qWczUhZTmo8HMjfR35yNzokKqkHNsLJv4H//BHyt0NiW+h3FqU71vNt7iZ6lq7Hn30suuI1vk0dSkJaJl02v0N8aiYc2A6pnWB/HrTLhq4DKchbzzeb8kjUAvK7HcPaHYX0b1dCh5REuqx/1fnOBv4ICvfClmWUAWu0Oz1LviW+Yx/47mMAdqcdRkZCGdptMN/8cICcojUkdD8KchdD9yHOyTSzD/gTWL2/DV9syKOguJS+Q8cxvHQJyzZsZeOeIkqIY5isJtu3DYDFbY6je1kuhYmZJJQeoK0cJC/tCHLid8H2NRR1PJI1u0pok5pO8cH99Cpegz9/KyWHncbWNUvYnNqPIYe0I67PCbBxEWWF+1m+eQ+F2ScSt/G/HNU5Edm6jG8LUmnfqTsddi1F9+TyTdtRHCwsojg+nWGHppO/YTFFhYXsSe1F4v5cCg/spW3vUXRIawO5nzkXHPk/APCvw2Zw+oHXKd62GhLTSew1xvke8/PYtWs7c/b05rDkvXT17SE/fx/ZqaXkdziS7XGd6Z+0i7U/7KPz7i9Jyx4Mccmwbwuled9wsKiUvDa9aK+72ZHen+/9PRiZsJ6kH5ZCQgp0OoI5S7fQUzZz1MlTKPnqFfIOKB0z0li7L578YuVgXFtS+p/K4IP/g8Q0yM9D1y5gX1kiZdlj+OLbbRxf9im+9O7onlzWpQzhB01nyP4P+by0D207Z3MUayjt1J+vthXSMamMvUXQs2g1/n2b2Jl1EqVFBXTt0pX9332Jb+8mCnqNZ5Vmc2z+AsrSurJyl4/DS78hrmMfduVtJmX/BuI79qFk8xI041AS4uKh+2Ao3AfxKexd/SHvd7mCM9qsYP2WHciONazQbNq1TSVu7yZG+VdSQhzb+l/J7t176L/pRef7TukIcUnO+7ISED+07QapnVi/4wCHbv8Q31Hnkbt1G4npnejoy2dXWTL/WZPH8Lh1JB86lB/WL8ef3o21++LJap9MZmoinfd8xYH8fRwoLCa5TRuW+vpxWJt84vdtJDE+jqQeg6GkAJLSnd9FYlvYsgTN386a5AEctn0B9DmF/dvW4Ss5iN/nJ3F/0FN7+00CfwIUH4CtX5Hf9Wi+3bSNtIIt7Pen0ys5n+RLnoP2PWt//gNE5HNVHRZyWjMLGqOA6ao63v18B4Cq/j7U/HUOGgDT0+uYSyiURCgrJVFK6rT8AU2kjRTWefsNabcvg4yy3dXS87QthSSQ5T5j54C0oY0eiFo+SvHhp6x82x1lb4OtO0/TSZVCkikA4AfNIIN9JEhpDUtW2KHpdJA9Iaft8WWQHuI7DLZfUjnoS6Fj6bYat1VEPCXqi8lv5KAmkCxFEefZrm3JdI/PD5pBJ4m87/VVhuAL84TZaGx/l6ZSgq/abzBXMwFIpKjWv89CSSRRG/Z47ovrQNpd39Zp2UhBo7lVT3XHedJYQK6bVk5EporIYhFZnJeX16iZC0jUwjoHDIAnSk/niZLTGzBHkX1W/njs6m4ovTVk+o1J9/HTLs8wxP8yF3R5m9dTLohW9gDY4q94xtFVpXeWv/9pUchHf9fKNP+dzEuZVP55ctFdlNRyhJ1ri6aFTM/zdeIPGb+qcfnn067g8vS/e9rWyoQBvJtwfG2y12BuKb4+ZPqfis8rf39d0U8r3vvuqdf2vvUdWuM8W/3dwk67uOiX5e+vLbq50rSfFN1Ypzz9LfkKrg/ax4DRhTMZXTiTa4tuqfU6X006p055iSStJDpVkM0taEiItEqXGKr6hKoOU9VhHTuGHG+ryVtelsNfSs72NO/e1J5cVvSLem1veE77sNOO7J0TMv3F2y7g5euP4Yu7T+al60bxo5OODruOK4uqB569mlyrPGYdUpGPuXdfXP5+Nylhl7mkKNJjrCs8f9OpnHvisRXr1NRaX8XvJjVkese0RO69qOaH600ddziP/GiIp20NyunCmQO61ip/DeGHuC68VTai+oRz/sotv614HpQvrUv5+1dvqcfFz4//x8rxL9Q4W7cu4b+LPVrx+1iph1Satp10NpR1rnW2fn7aQF6+ObBfFaekzFTnkei/Ov+Y8AsffkbI5AmjR3B10c9qnZeqPiwdAIceW/OM9dDcgkYu0CPocxbO84ybpCJfUp2W26lpfHzXmZ7mLUlqz3rtUvOMdXTQnxZ6gj++8ucIdafvlVU/GaakZ9YuI2lBJ4bEijzt0dAna4CvykIHvGqSMiCt4mp1L21qlzcqn5yqSW4XOt1X+Tvs4J50ahRft99Vfa1N7A8IM9r/rvKE9j3B5y//+OSPT6mYlpxR9w12Ohx/m/AXNF62sSfoomKbVl7XHk3hG/cpvXvTD/eer/ikim3GV7/4aRPpt50d+gIiOaUti8sO856HMNZqd6edJoqaW9D4DOgjIjkikgBMBuZGY0MflR5V73V8V9Kh/P3qsixWlR0SYe4KB0giPdXbieFgRh+KtfaDFZckBp3IsseEna9vdvfqiWkhqgO6hP++fAKF6pwg96uzX/7DT/WW0fJtVg+M68s6h73CBygkvnpi96HV0xLTnEZ/V7WqqaCAeJDqx6VQ44hP7VAtHYDeJzpBKdy0gA69SU2MY4eGCdLB4mpXSmsoJd2cKu6dXaqc+KqcpFLauifnw06FuMS6baxTfwAS451TVGmkU1Wf8SGTF+hwCnEDcVI6x/at/LvdoymoW1LYmxEhaPSvUnUUl1xxIdDX/R2371U+OSUjQtAI81tI7NyHvRFKzTVySxebtT1IqAqZhtOshkZX1RIRuRGYD/iBWaq6Ihrburn4x3zuD11/S79JsPL1aslXF/2MvyX8sfzzd9qJPmziu7JOnFs0nVcSpgPwbulgTvR/GXbb+VT/R7up6AZm/HQq323fz5EvucXPK//NxgNdKFn+SeSd6XUirHuXj0v789uSS5l/ywns8HWk81/cK/Ext/LsD4dSsPQ1ro57i92aQobks0tT+dGoXjBwHfzB/aeY8q9K/yDlElLgpiWw/iP4102VJn38ixPYVbiMLm38TPjdWwzsmswj48+BI86ElE5s3ZNPl+dOcGa+8Fnn6u3Zcyuvf8hl8MlDFZ9vWcWZv/uEBKq3HV1UdCdbtD0pyclUax+9bC73/2Umv9h/P/mayPiiB/hYBDJ78+yRs/j757sqz3/lfMg4lGce/CmX+t6mNCENigoq73p8PG/97CQ48AWUFMJjowDYd9kC0g4ZCHEJMG0pFOXzbVE6OYXfICJw6DEVXYY79kWAgus/46DvIMkHt8GTEyrnJbMvbF/tXOmWOfv9WumxnO2vfPx33fgN7R72dtW6f8Dl+EbfTJs2KbD2HXi98m9+55SPuPSJ/3DpmMO5YPwJLDj5AHsOFvPRV0dxnP8rOHYatKvS7uDzw7RlTi88oPC6RZz454VMGdKOa4ZnQnoWHNwJTxxfPUM/XQ4FuyHVqTZKivMzrOAx+mW15+mrjmb2v94k7qsXOc//EXQ8HM55AroOdK7g3e8dgGlL+fSjnbBwC0XXLyIhMZlH23RnS+5/6fq0U330zPXHkzBvDmyDuMNOgVNucnpSlZXA/zuuYl1nPw6n3AuPj3byHZ/kBMObljgl4FPuhYRULvhgM49+sI52aSnlx5slz8PChyvWFSihiB+ueod1pR3p1aYA6XgYZeTy825P84dLxzrda2cOcua95j04sAuec/4nnhr2Ov9b9wPLth7k45+NhrJi6NCb3Wv+yw3dhsJ8t63m6DDnr3pqVkEDQFXnAfOivZ38EFeU5bKGVwoaj5ZOwqclrNasSrMVkMjhBU9Sip9i4hD3DLakrFfEoHHAvRovO+k3+BY4jaibtQNpnbI5shNw5p8hvg0ccjS6bgdFVFQNzDv2n5z2yfkVK7vmPaeP+rp3+V47sVoPgY596Rzca87n40fnXkBx5nfw4VukZQ+B7/5DerfezsktJejKqesgSGobOuPtc2DT55WSDo7/A90ykgHn6njBby7F7xPw+yDH+cfs0gk0KQMp2O1c1Ze6vXMSUqFov/M+vUel9dK2G/tpg5/qPZwWljlXqat/eTLMcBPPmwUHd0NiKjdPmQyP3M9BEskNes7M1rQjWa9ruWZMjlOmBegyABLacOEZp8K8t0lJSYUit4PFcbfBRw8gvjjSk+MhuXIwje9yhBMwoLwk45RZgn4nHftWWqZ7ef18leq+NpkV64pLcrqaAqeedjbMrxw02mV2hszDYPs3FYlxSU4XzypS+x4PnZy80aN6u1T7nIHMvqcfqYlxiAi9Ozklu6LBA2DZV9Chd7VlnExUBJLELn2ZP70XyfF+p9hZZXolGT0IroFOiPOxnXT2+dIguR0XnHcxxZ22wQcfQZ9TnIAB0LkfnDmz4oKlXTa/PPNQbp4wgIRE5zSXDCT37F++7p5dMqFdG9gGXdqlVZRCi6t8T3GJkO5W+xzcWVHSa+9edMU7JZhbT0njx+N60yYhDhLc73TMz+DADljqts0EV1VmDSX4F7PqNxOI97v/G8FVbl0HOUHEddnpx/OjMqW0TCG+4n8/4/CxlfPdY3jVb7dBNLfqqUZTEnQirqZKnfLyshzuK/kRe6vUax/UBApIpNiNzeVdRolczxsIWL7R08qLnb+c0KdihqGXw4CK3krFQbF/V9vKJyG6DQGfM728qA7VirA+n5Dod34OfvcfO+SPI0QdbiTJI6+p9Dkp3k+8v/qaJZAff2LFP2VyUB10XOj6/tIIxykxPuia6MhzYfhVACQkRt6H1MR46O92RHD3N4FiJ5+BPv1ZIyqqJnyh/40SkxqwGik+uaINJOgYJMWF+ReOq3LR4w9TTRQf9JsN0/aSlhRfcXxcCf7aVYGkJMbh89W+2iSw3cA1js8nJIbb56FTKn30+YSUxAjXxcG/5eD9C9dmFEgPsxs+n5BadXtt2jsllUT3Qitc+xaQnOAnLsT/Bj5/peMpIsT7fSTFRzhHRZEFjTAiBo0qdcql7g96D6mUBf2iCqh8oguUNPK0+j0gj5acVf5+/MDsignuj2xIVui6+54dUyoFjaGHVvlRipQHjar5qcYfuJJ19y9UgKjaAF4Tr/WrgbNCXEJ51Qtto9RDKMz3EQhmcX6Bs5+An6+ryH+J25sqwT3JxidVfF8S+rcivgb8p07OqMhLXFLFSSTc8ah6cgoTdEkIavRPCvwuo1snXhsRfz71rbuv7fEJfKeldehOH2jQD/x+EmrZfhH4XwwX/BtRs6ueajwRfpBBVyLPl4zj3bLB5Z93aFs6ujd5bdX2/Pqs/twzd4W7RufEuEvTeLD4fNLkANfGvQnAn0vOYV7pCP51ehkPHVuxPk7/k1O1kR1Uxxqkc9sk1vzuTPiN8/nwLm3h6vfgbydUzOT+cxSFahgONvxq2LsZxv0SPv6Tc5d7wNQPnfaKaPMnQKd+Tl35iKnwwyrY79xFzYT7oHuI+40mPuLULZcWOfXIz9WwjZRMOP4OLptfufH66jE57D5YxBXHZkNcHMQFVcuVukHj0FFO/fmIqc62oPYnH68unwc/rIR9W5w2ndluV+PEts6x8vlhwIWgyoGULH769H/okR7P3eBc3f73Yfj0EWeZQIA78R4QHyxw758IvjDw+Z2RBHLGwrbllXusRcPJv4V37nben/3/QjYSJ8U5322HlKCgF7jACHVj8qTHyttD6uXMmbD7O8gIqkab9Bh8+ihkhbznLbKLX4FlL0Lb7nDC3dD3tNot7/M7x67PKTXPG2UWNOoiqKRxV8lVlIUosP2rdCSzSifw5fAe5UEjcNdqEXEUj/4Z43wfwcI3eaV0DIUk8MurLkJ6V+l5kdYZxs+ouvpKpGr1SFaIHkLA2UOzOWbwqJDTAOeq89T7nPcnTa88rdsg5xUt5dVTCU51z8luFEwPqv8fWblhb+6Nx7Izvwj6dqqU/tfLtpHVLkLVkAgcfzv3dMsjI7kikKYkxnHPmf1DLxOo7+91IvQa57zfud5dX+WgkX/IOFK+fz/89r3KPtZ5BQTad5IznNcEdyCEoVNoA5x8zmGM7uP+ftp2gwm/qx40+p4KnY6Az/8Bu9ZXrp4CJ1hD3Y51cnsoPuh9/lE3VgSNgZNDznJE1zR+O+lIzjgqKIAd4v6Gc8ZWX2DQj7xvP5IqVV2A852ecm/d1pfZG05wb0o9LvQNszUaU/ubBqPBgkZdBJU0qgcMJzA8XDKJgyQR5xNOPLwT7379Q3nQCFQnHda5cvfKY6oGjIaiTlvKIR1SOCTbQ7/3em2rjsPSJLdzGvvEe43pgKzQbUMn9/N2pTn2sFr0Z88eDXdsgsSgasJACaNKSSNlystOj5aGVhgIGqHrxc8f1iNkOlBRjRU4PoEqwLre8xE4zMHH+2erqd5dLYIwbUHBRIRLR1ZpND90VPVjUVeRSi3NVZT3ydo06qK0mPGF93Fb8TVhZwm0ifh9wu/Pce5hEAmUNOIrF7ejLfDjiXL/7Xq59DWYcH/9bgaLtqonqTK351bVQOePq3WHAU8C1WERGlPDq3LsS92g5qtlG1UkcQl1vy+jthoiYJg6saBRF+17sloP4aXScWFnCQQNESHB7e3hc3tPFRPHlcfmBF0JNNAVQb+K8ZPoGZy3wPobIWh09zYURjXtsmHkdQ2aFcBp9D4kQpVcfQS6Io+YGp31VzXoIudv1e7HkQy5zPk77Ernb6BzwQj3gqdOAQjoN9H52yPEsCK1NeiS+q+jNroNgQS3lD/A7Z4e4ebUmAlV/ebFUe44YN0GR56vjqx6qpZGFfyFhZl9cEZory5wWi7RiiqLQK+cQPVUqfpCd62rj+lVRli9LOjmQ7d6qjZVP3XWoZeTl3qMEtygfhXF50YkplX/3qPp9D85bT2JHu4aDzjrL84LKgfl426te906wGGnNMy+N+b3FzA1qL2p/9kV3aubmil1HOziiDOj+r1aSSOCy4t+zvdlleu9S/BXf7BOCMHdYAMljUDvqSevdBtVy6uLolwCiEX11LUfOT1GTMMRqV3AMCYKLGhE8EHZYFZp5Ua4UnzM+2prmCUqAkPwWDlx7k1ND5c41UeH57jrdKtNXikNP/ZTw2rEoNF1IPQ5qfG2Z4xpFBY0aqkEHzc8/0WtlhERjujalqMm3eoUGwM9VtrnwPQ9/Lcs/PMsGkRzaAg3xjQL1qZRS6HuyQi2pKw3J/q/5PyRvTh+QMW4PG9Na6zSRChhGsKvWlD5jmBjjKmBBY1aijhEM/CT4p/QpySXyd26MrJnmOGyG1u4kkaUBjQzxrRcVj1VS5EGyAPnWRhLtXd5O0bT0Ihdbo0xLZoFjVqqqaQREFfLUUCjqjG73BpjWjQ7i9SS16DRPiX2o1GWs4ZwY0wDsTaNWvN24j22l/f2jD9PHkTurloM9FZbme6zODK8PW7WGGPCsaARBT3aJ9fqju+Jg0I8h7shDb8GOvcP+1B7Y4zxyqqnouCP50dxCPG68PksYBhjGoQFjSgYkRPl4ceNMSZGYhI0ROR8EVkhImUiMqzKtDtEZK2IrBaR8UHpE9y0tSJye+Pn2hhjTKxKGsuBc4BKzw8VkX7AZKA/MAF4VET8IuIHHgFOBfoBF7nzGmOMaUQxaQhX1VXgjMlUxURgtqoWAutFZC0QGLB/rap+6y432513ZePk2BhjDDS9No3uwMagz7luWrj0akRkqogsFpHFeXl5UcuoMca0RlEraYjIAqBLiEl3quqccIuFSFNCB7eQD7VQ1SeAJwCGDRvWgh78a4wxsRe1oKGqdXmYQi4Q/CzLLGCz+z5cujHGmEbS1Kqn5gKTRSRRRHKAPsAi4DOgj4jkiEgCTmN5HZ+FaIwxpq5i0hAuImcDfwE6Am+KyBJVHa+qK0TkJZwG7hLgBlUtdZe5EZgP+IFZqroiFnk3xpjWLFa9p14DXgszbQYwI0T6PGBelLNmjDEmgqZWPWWMMaYJs6BhjDHGMwsaxhhjPLOgYYwxxjMLGsYYYzyzoFGDp0tPjnUWjDGmybCgUYNPyo4iu+D5GudLqMWT+owxprmyM10DqT5grzHGtDwWNBqIz6KGMaYVsKDRQCxmGGNaAwsaDcRKGsaY1iAmY081R8+XnEBxajcoCD3dYoYxpjWwoOHRL0uupm9iGrAv5HQraRhjWoOwQUNEbom0oKr+qeGz07Rp6IcFAuD3WdAwxrR8kUoaae7fvsBwKh56dCbwUTQz1VSVRXh4rMUMY0xrEDZoqOqvAUTk38AQVd3nfp4O/LNRctfEqEZ65LhFDWNMy+el99QhQFHQ5yIgOyq5aeIixQwraRhjWgMvDeHPAItE5DVAgbOBp6KaqyYqUjnDGsKNMa1BjUFDVWeIyFvAGDfpClX9MrrZaprKIhQ1rKRhjGkNIgYNEfEBy1T1SOCLxslS0xUpLoiVNIwxrUDENg1VLQOWisghjZSfZstihjGmNfDSEN4VWCEi74rI3MCrPhsVkT+IyNciskxEXhORjKBpd4jIWhFZLSLjg9InuGlrReT2+my/HvkOO83u0zDGtAZeGsJ/HYXtvgPcoeN6KMkAACAASURBVKolInI/cAfwCxHpB0wG+gPdgAUicpi7zCPAyUAu8JmIzFXVlVHIW51YQ7gxpjXw0hD+YUNvVFX/HfTxU+A89/1EYLaqFgLrRWQtMMKdtlZVvwUQkdnuvE0maFjMMMa0BjVWT4nISBH5TET2i0iRiJSKyN4GzMOVwFvu++7AxqBpuW5auPRQ+Z0qIotFZHFeXl4DZjOyC4f1aLRtGWNMrHhp03gYuAhYAyQDV7tpEYnIAhFZHuI1MWieO4ES4LlAUohVaYT06omqT6jqMFUd1rFjx5qy2WAmD7e+AsaYls/TKLequlZE/KpaCjwpIv/1sMxJkaaLyBTgDOBErRifIxcIvmTPAja778OlN5qINVBWPWWMaQW8BI0DIpIALBGRB4AtQEp9NioiE4BfAGNV9UDQpLnA8yLyJ5yG8D7AIpxTch8RyQE24TSW/6g+eWho1qZhjGkNvFRPXerOdyOQj3PFf249t/swzii674jIEhF5HEBVVwAv4TRwvw3coKqlqlribn8+sAp4yZ23ybCYYYxpDbyUNHoBeaq6lwbqfquqvSNMmwHMCJE+D5jXENuvSeTRbKsTsS63xpjWwUvQuBx4XER2AP9xXx+r6q5oZiyWwsaMCHHBYoYxpjXwcp/GZQAi0g3nfopHcNobWuyjYmsbMwQQq6AyxrQCNZ74ReQSnBFujwK247RH/CfK+Yqp2lZPgZU0jDGtg5fSwkPAOuBx4H1V3RDVHDUBtQ0ZNsKtMaa1qLH3lKpm4ty1nQTMEJFFIvJM1HPWDHRumwi41VMWN4wxrYCXYUTa4jzy9VCcx7ymA2XRzVZshaudqlqiCO4xZW0axpjWwEv11MdBr4dVNTe6WYo99VhBFQgaIlbSMMa0Dl56Tw0AEJEUVc2PfpZiz2s7eHCgsJhhjGkNvFRPjRKRlTh3YiMiA0Xk0ajnrBkoL2kgdnOfMaZV8DKMyEPAeGAHgKouBY6LZqZizWtJo/xpfVY9ZYxpJbwEDVR1Y5Wk0ijkpckI16ZRNS5Uqp6yqGGMaQW8NIRvFJFjAHVHu70Jt6qqpfJc0iivnjLGmNbBS0njOuAGnCfl5QKD3M+tnrVjGGNam4glDRHxA5eq6sWNlJ8mIVJBo1t6Epv3FADg81V0uTXGmNYgYknDfVLfxEjztEThxp4Sgfd/fjxPXDoUgIp28Iqoce1xPaOeP2OMiRUvbRqfiMjDwIs4D2ECQFW/iFquYixSSSMxzk9SvB+o6D0VKGlsuO/0KOfMGGNiy0vQOMb9+5ugNAVOaPjsNA01NYQHJluPKWNMa+PljvBxjZGRJsVj7ymfxQxjTCvj6T6N1qamsaeqtnlY7DDGtBYWNEIIO8ptmPBg1VTGmNYiJkFDRH4rIstEZImI/Nt9lCzimCkia93pQ4KWmSIia9zXlFjkOyAQU+Lc+qmsdsmxy4wxxjQiLwMWthGRu0Xkr+7nPiJyRj23+wdVHaCqg4A3gF+56acCfdzXVOAxd5vtgXuAo4ERwD0i0q6eeQjL65P72iTE8ejFQ3j6qhHRyooxxjQpXkoaTwKFwCj3cy5wb302qqp7gz6mUHGengg8rY5PgQwR6YozYOI7qrpTVXcB7wAT6pOHGvJXwwwVb087qiud0pKilRVjjGlSvASNXqr6AFAMoKoHaYC2XxGZISIbgYupKGl0B4IHR8x108Klh1rvVBFZLCKL8/Ly6pS3cCGjatOFNWUYY1obL0GjSESScc+lItILp+QRkYgsEJHlIV4TAVT1TlXtATwH3BhYLMSqNEJ69UTVJ1R1mKoO69ixY817F0KbBD892odvp/D6ZD9jjGlpvASN6cDbQA8ReQ54F7itpoVU9SRVPTLEa06VWZ8HznXf5wI9gqZlAZsjpEdFm4Q4rj2uV43zWUHDGNPa1Bg0VPXfwDnA5cALwDBV/aA+GxWRPkEfzwK+dt/PBS5ze1GNBPao6hZgPnCKiLRzG8BPcdOiJlLVk9eh040xpqWp8Y5wEZmLEyzmNuAzwu8Tkb5AGfAdzvDrAPOA04C1wAHgCgBV3SkivwU+c+f7jarubKC8RDS6dyajenXgD/NXl6cFgobdn2GMaW28jD31R+BCnBP9IpyBC99Q1YK6blRVzw2TroR5VoeqzgJm1XWbtRW4ka9z2yT6dEp10qoECQsZxpjWxsvYUx8CH7rP1jgBuAbn5N02ynlrMqoGC6udMsa0Vl5KGri9p87EKXEMAZ6KZqaagkrP//YwjzHGtAZe2jRexLkT+23gEeADVS2LdsaakqrBocab/4wxpoXyUtJ4EviR+xS/ViM4ToQvUVhRwxjTuoQNGiJygqq+B7QBJlar11d9Ncp5i6lQgSKQZOUMY0xrFamkMRZ4D6ctoyoFWnTQCBZ+SPRGzogxxsRY2KChqve4b3+jquuDp4lITlRz1QRUChTV2jQaNy/GGNNUeBlG5JUQaS83dEaasrC9pxo1F8YYE3uR2jQOB/oD6SJyTtCktkDLHws8VJtGeZoVNYwxrVOkNo2+wBlABpXbNfbh3ODXaoQbLsTaNIwxrU2kNo05wBwRGaWqCxsxT01CRU8prVboSIhzavVSEj3dG2mMMS2Gl7PelyJyA05VVXm1lKpeGbVcNQHBpYuqJYrjD+vEz8f35dJRhzZyrowxJra8NIQ/A3TBeeTqhzjPstgXzUw1NVW73Pp8wg3jetM2KT5GOTLGmNjwEjR6q+rdQL6qPgWcDhwV3Ww1TdaGYYxp7bwEjWL3724RORJIB7KjlqMmyIKFMcY4vLRpPOE+Le9unCfrpQK/imqumhiLGcYY4/DyPI2/uW8/BHpGNztNlEUNY4wBIt/cd0ukBVX1Tw2fnSYo6D6+cGNQGWNMaxGppJHWaLlo4ixYGGOMI9LNfb9uzIw0ZdYQbowxDi9P7nuSEIMtNcTNfSJyK/AHoKOqbhfnjro/A6cBB4DLVfULd94pwF3uove63X8bRSBmWPAwxrR2XnpPvRH0Pgk4G9hc3w2LSA/gZOD7oORTgT7u62jgMeBoEWkP3AMMwwlgn4vIXFXdVd98eMxrY2zGGGOaPC+9pyoNjS4iLwALGmDb/wfcBswJSpsIPK3OQ7g/FZEMEekKHA+8o6o73Ty8A0wAXmiAfHhmocMY09p5ubmvqj7AIfXZqIicBWxS1aVVJnUHNgZ9znXTwqWHWvdUEVksIovz8vLqk82gdTbIaowxptnz0qaxD6dKSNy/W4FfeFhuAc6YVVXdCfwSOCXUYiHSNEJ69UTVJ4AnAIYNG1bnB19ImPfGGNOaeameqlPXW1U9KVS6iBwF5ABL3baCLOALERmBU4LoETR7Fk77SS5OFVVw+gd1yVddWEnDGGMcnh4IISIDcMabKp9fVV+tywZV9SugU9C6NwDD3N5Tc4EbRWQ2TkP4HlXdIiLzgd+5w5mAU0q5oy7brxeLHsaYVs5L9dQsYACwAihzkxWoU9CowTyc7rZrcbrcXgGgqjtF5LfAZ+58vwk0ijcOCxbGGAPeShojVbVftDKgqtlB7xW4Icx8s4BZ0cpHJFbAMMYYh5feUwtFJGpBozmwmGGMMQ4vJY2ncALHVqAQtxeVqg6Ias6akDp3wTLGmBbGS9CYBVwKfEVFm0arERwwrMRhjGntvASN71V1btRzYowxpsnzEjS+FpHngX/hVE8Bde9ya4wxpvnyEjSScYJF8B3c0epy22QE95hSa9QwxhjA2x3hVzRGRowxxjR9Xm7uywF+QvU7ws+KXraaJrtfwxjT2nmpnnod+DtOm0ar6z1ljDGmgpegUaCqM6OekyZKrUHDGGPKeQkafxaRe4B/U7n31BdRy1WTY4HDGGPAW9A4CufmvhOoPGDhCdHKVFNlTRrGmNbOS9A4G+ipqkXRzowxxpimzcuAhUuBjGhnxBhjTNPnpaTRGeeu8M+o3KbRarrcWlu4McY4vASNe6KeiyYuEDPEbtQwxrRyXu4I/7AxMtLUhIoPFjKMMa2dlzvC91FxsZ0AxAP5qto2mhkzxhjT9HgpaaQFfxaRScCIqOXIGGNMk+Wl91Qlqvo6reweDWsIN8YYh5fqqXOCPvqAYbTSW6StHdwY09p5KWmcGfQaD+wDJtZnoyIyXUQ2icgS93Va0LQ7RGStiKwWkfFB6RPctLUicnt9tm+MMaZuYvk8jf9T1QeDE0SkHzAZ6A90AxaIyGHu5EeAk4Fc4DMRmauqK6OUN2OMMSHUWNIQkadEJCPoczsRmRWl/EwEZqtqoaquB9biNLqPANaq6rfucCazqWdpx6tWWQ9njDFheKmeGqCquwMfVHUXMLgBtn2jiCwTkVki0s5N6w5sDJon100Ll16NiEwVkcUisjgvL68BsmnDoxtjTICXoOELOqkjIu3x1oC+QESWh3hNBB4DegGDgC3AHwOLhViVRkivnqj6hKoOU9VhHTt2rCmbtSJ2e58xppXzMozIH4H/isjLOCfqC4AZNS2kqid5yYCI/BV4w/2YC/QImpwFbHbfh0s3xhjTSGosaajq08C5wDYgDzhHVZ+pz0ZFpGvQx7OB5e77ucBkEUl0n03eB1gEfAb0EZEcEUnAaSyfW588GGOMqT0vJQ3cXkoN2VPpAREZhFNy2QBc625nhYi85G6rBLhBVUsBRORGYD7gB2ap6ooGzE81wVVR1qJhjDEOT0GjoanqpRGmzSBE9ZeqzgPmRTNfNbImDWNMK1frYURaG1Xo2TEFgPOHZsU4N8YYE1sxKWk0N53Skthw3+mxzoYxxsSclTSMMcZ4ZkHDGGOMZxY0jDHGeGZBwxhjjGcWNIwxxnhmQSMMe+CSMcZUZ0HDGGOMZxY0jDHGeGZBwxhjjGcWNIwxxnhmQcMYY4xnFjSMMcZ4ZkGjBvYsDWOMqWCj3BrTAhUXF5Obm0tBQUGss2KasKSkJLKysoiPj/e8jAUNY1qg3Nxc0tLSyM7ORuxOVROCqrJjxw5yc3PJycnxvJxVTxnTAhUUFNChQwcLGCYsEaFDhw61Lo1a0DCmhbKAYWpSl9+IBQ1jjDGexSxoiMhPRGS1iKwQkQeC0u8QkbXutPFB6RPctLUicntscm2MaUxz587lvvvuA2D69Ok8+OCDjZ6Hf/7zn/Tv3x+fz8fixYsrTfv9739P79696du3L/Pnzy9Pf/vtt+nbty+9e/cuz3/ACy+8wIwZM/j6668ZNWoUiYmJ1fYr3PLr16/n6KOPpk+fPlx44YUUFRV52ocNGzbw/PPP13bXQ4pJ0BCRccBEYICq9gcedNP7AZOB/sAE4FER8YuIH3gEOBXoB1zkzht1qtbp1phYOeuss7j99theIx555JG8+uqrHHfccZXSV65cyezZs1mxYgVvv/02P/7xjyktLaW0tJQbbriBt956i5UrV/LCCy+wcuXK8uXefvttJkyYQPv27Zk5cya33nprpfVGWv4Xv/gFN998M2vWrKFdu3b8/e9/r5bf448/ng0bNlRKa8igEaveU9cD96lqIYCq/uCmTwRmu+nrRWQtMMKdtlZVvwUQkdnuvCsxxkT063+tYOXmvQ26zn7d2nLPmf0jzvPss88yc+ZMioqKOProo3n00Ufx+/2kpqZy7bXX8v7779OuXTtmz55Nx44dmTlzJo8//jhxcXH069eP2bNn849//IPFixfz8MMPV1r3kiVLuO666zhw4AC9evVi1qxZtGvXjuOPP56jjz6a999/n927d/P3v/+dMWPG1GtfjzjiiJDpc+bMYfLkySQmJpKTk0Pv3r1ZtGgRAL1796Znz54ATJ48mTlz5tCvXz9UlSVLljBkyBBEhE6dOvHmm29WWu+iRYtCLn/EEUfw3nvvlZ/8p0yZwvTp07n++utr3Ifbb7+dVatWMWjQIKZMmcLNN99c5+8jVtVThwFjROR/IvKhiAx307sDG4Pmy3XTwqVXIyJTRWSxiCzOy8uLQtaNMTVZtWoVL774Ip988glLlizB7/fz3HPPAZCfn8+QIUP44osvGDt2LL/+9a8BuO+++/jyyy9ZtmwZjz/+eMT1X3bZZdx///0sW7aMo446qnwdACUlJSxatIiHHnqoUnrAvn37GDRoUMhXcImgJps2baJHjx7ln7Oysti0aVPYdIAvv/ySgQMHRmyADrf8jh07yMjIIC4urtp6a3LfffcxZswYlixZUq+AAVEsaYjIAqBLiEl3utttB4wEhgMviUhPINQ3qYQObiHrjVT1CeAJgGHDhlndkmn1aioRRMO7777L559/zvDhzvXgwYMH6dSpEwA+n48LL7wQgEsuuYRzzjkHgAEDBnDxxRczadIkJk2aFHbde/bsYffu3YwdOxZwrrjPP//88umB9Q0dOrRaNQ1AWloaS5Ysqfc+hqq6FhHKyspCpoNTNXXqqafWab3h0gGefPJJ/vznPwOwdu1aTjvtNBISEsjJyeG1116reWdqIWpBQ1VPCjdNRK4HXlXnW1gkImVAJk4JokfQrFnAZvd9uHRjTBOjqkyZMoXf//73Nc4bOPG9+eabfPTRR8ydO5ff/va3rFixok7bTkxMBMDv91NSUlJt+r59+8JWWT3//PP06+etuTQrK4uNGysqQHJzc+nWrRtA2PR///vfvPLKK3Vab2ZmJrt376akpIS4uLhK673iiiu44oorAKdN4x//+AfZ2dme9qO2YlU99TpwAoCIHAYkANuBucBkEUkUkRygD7AI+AzoIyI5IpKA01g+NyY5N8bU6MQTT+Tll1/mhx+c5sqdO3fy3XffAVBWVsbLL78MOCfp0aNHU1ZWxsaNGxk3bhwPPPAAu3fvZv/+/SHXnZ6eTrt27fjPf/4DwDPPPFNe6vAiUNII9fIaMMBppJ89ezaFhYWsX7+eNWvWMGLECIYPH86aNWtYv349RUVFzJ49m7POOos9e/ZQUlJChw4dIq433PIiwrhx48q/u6eeeoqJEyd63ud9+/Z53rdIYtUQPguYJSLLgSJgilvqWCEiL+E0cJcAN6hqKYCI3AjMB/zALFWt22WIMSbq+vXrx7333sspp5xCWVkZ8fHxPPLIIxx66KGkpKSwYsUKhg4dSnp6Oi+++CKlpaVccskl7NmzB1Xl5ptvJiMjI+z6n3rqqfKG8J49e/Lkk09GbV9ee+01fvKTn5CXl8fpp5/OoEGDmD9/Pv379+eCCy6gX79+xMXF8cgjj+D3+wF4+OGHGT9+PKWlpVx55ZX079+fl19+mZNOqqiA2bp1K8OGDWPv3r34fD4eeughVq5cSdu2bUMuD3D//fczefJk7rrrLgYPHsxVV13laR8GDBhAXFwcAwcO5PLLL69Xu4a05C6lw4YN06r9qr2au3QzN73wJWcM6MrDPxrSwDkzJrpWrVoVttdPrKWmpoYtRbRkV199NVdffTUjR46MdVYqCfVbEZHPVXVYqPltwEJjjGkEf/vb32KdhQZhw4gYYxpVayxltCQWNIwxxnhmQcMYY4xnFjSMMcZ4ZkHDGGOMZxY0atByOyQb03z97ne/q9fyr7/+eq3GmaqL0047jd27dwNON+OWwoKGMabZaQ5BY968eRFvUGyu7D4NY1q6t26HrV817Dq7HAWn3hdxlqeffpoHH3wQEWHAgAE888wzfPfdd1x55ZXk5eXRsWNHnnzySQ455BAuv/xy2rZty+LFi9m6dSsPPPAA5513Hlu2bOHCCy9k7969lJSU8Nhjj/Hmm29y8OBBBg0aRP/+/XnuueeYNGkSGzdupKCggGnTpjF16lTAucKfNm0ab7zxBsnJycyZM4d169Yxd+5cPvzwQ+69915eeeUVevXqVZ7vvLw8rrvuOr7//nsAHnroIY499limT5/OunXr2LRpExs3buS2227jmmuuCZnHMWPGkJ2dzeLFi8nMzCxft6py22238dZbbyEi3HXXXVx44YV88MEHTJ8+nczMTJYvX87QoUN59tlnm+Qjey1oGGMa3IoVK5gxYwaffPIJmZmZ7Ny5E4Abb7yRyy67jClTpjBr1ixuuukmXn/9dQC2bNnCxx9/zNdff81ZZ53Feeedx/PPP8/48eO58847KS0t5cCBA4wZM4aHH3640ki1s2bNon379hw8eJDhw4dz7rnn0qFDB/Lz8xk5ciQzZszgtttu469//St33XUXZ511FmeccQbnnXdetbxPmzaNm2++mdGjR/P9998zfvx4Vq1aBcCyZcv49NNPyc/PZ/DgwZx++um88MIL1fIYzquvvsqSJUtYunQp27dvZ/jw4eUPd/ryyy9ZsWIF3bp149hjj+WTTz5h9OjRDXZMGooFDWNauhpKBNHw3nvvcd5555VfZbdv3x6AhQsX8uqrrwJw6aWXctttt5UvM2nSJHw+H/369WPbtm2AM3jflVdeSXFxMZMmTWLQoEEhtzdz5szyIcA3btzImjVr6NChAwkJCZxxxhmAM1T6O++8U2PeFyxYUKnqau/eveWD/U2cOJHk5GSSk5MZN24cixYt8pxHgI8//piLLroIv99P586dGTt2LJ999hlt27ZlxIgRZGVlATBo0CA2bNjQJIOGtWkYYxqcqnqqWgmeJzCkeWB5gOOOO46PPvqI7t27c+mll/L0009XW8cHH3zAggULWLhwIUuXLmXw4MEUFBQAEB8fX76NcEOlV1VWVsbChQvLR77dtGkTaWlp1fIb+Owlj1X3K5Tg/fea11iwoGGMaXAnnngiL730Ejt27AAor5465phjmD17NgDPPfdcjVfS3333HZ06deKaa67hqquu4osvvgCcYFBcXAw4D2Vq164dbdq04euvv+bTTz+tMX+Rhgo/5ZRTKj1eNrgabM6cORQUFLBjxw4++OADhg8fHjaPoRx33HHlo/rm5eXx0UcfMWLEiLDzN0UWNMKI9zlXFIl++4qMqa3+/ftz5513MnbsWAYOHMgtt9wCONVITz75ZHnDeOBpc+F88MEHDBo0iMGDB/PKK68wbdo0AKZOnVr+pL8JEyZQUlLCgAEDuPvuuz2NIjt58mT+8Ic/MHjwYNatW1dp2syZM1m8eDEDBgygX79+lR49O2LECE4//XRGjhzJ3XffTbdu3cLmMZSzzz6bAQMGMHDgQE444QQeeOABunQJ9YDTpsuGRg+jpLSMB//9DdeN7UlGm4QGzpkx0dWUh0ZvrqZPn05qaiq33nprrLPSoGxo9AYS5/dx+6mHxzobxhjTpFjQMMYYD6ZPnx7rLDQJVmFvTAvVkqueTcOoy2/EgoYxLVBSUhI7duywwGHCUlV27NhBUlJSrZaz6iljWqCsrCxyc3PJy8uLdVZME5aUlFR+Q6FXMQkaIvIi0Nf9mAHsVtVB7rQ7gKuAUuAmVZ3vpk8A/gz4gb+pauPf5mpMMxEfH09OTk6ss2FaoJgEDVW9MPBeRP4I7HHf9wMmA/2BbsACETnMnfUR4GQgF/hMROaqanSHqTTGGFNJTKunxLkn/wLgBDdpIjBbVQuB9SKyFgjcLrlWVb91l5vtzmtBwxhjGlGsG8LHANtUdY37uTuwMWh6rpsWLt0YY0wjilpJQ0QWAKHuj79TVee47y8CXgheLMT8SujgFrJbiIhMBaa6H/eLyGpvOQ4pE9hej+WbI9vnlq+17S/YPtfWoeEmRC1oqOpJkaaLSBxwDjA0KDkX6BH0OQvY7L4Pl151u08AT9Q2v2HyuDjcrfQtle1zy9fa9hdsnxtSLKunTgK+VtXcoLS5wGQRSRSRHKAPsAj4DOgjIjkikoDTWD630XNsjDGtXCwbwidTuWoKVV0hIi/hNHCXADeoaimAiNwIzMfpcjtLVVc0cn6NMabVi1nQUNXLw6TPAGaESJ8HzItytqpqkGquZsb2ueVrbfsLts8NpkUPjW6MMaZhxbrLrTHGmGbEgoYxxhjPLGiEICITRGS1iKwVkdtjnZ+GIiI9ROR9EVklIitEZJqb3l5E3hGRNe7fdm66iMhM93tYJiJDYrsHdScifhH5UkTecD/niMj/3H1+0e2Vh9tz70V3n/8nItmxzHddiUiGiLwsIl+7x3tUSz/OInKz+7teLiIviEhSSzvOIjJLRH4QkeVBabU+riIyxZ1/jYhMqU0eLGhUISJ+nHGuTgX6ARe5Y2K1BCXAz1T1CGAkcIO7b7cD76pqH+Bd9zM430Ef9zUVeKzxs9xgpgGrgj7fD/yfu8+7cAbJxP27S1V7A//nztcc/Rl4W1UPBwbi7HuLPc4i0h24CRimqkfi9LKcTMs7zv8AJlRJq9VxFZH2wD3A0TjDNN0TCDSeqKq9gl7AKGB+0Oc7gDtina8o7escnEEgVwNd3bSuwGr3/f8DLgqav3y+5vTCuRn0XZwxzt7AGXlgOxBX9ZjjdOse5b6Pc+eTWO9DLfe3LbC+ar5b8nGmYqih9u5xewMY3xKPM5ANLK/rccUZieP/BaVXmq+ml5U0qmsV41y5xfHBwP+Azqq6BcD928mdraV8Fw8BtwFl7ucOOMPxl7ifg/erfJ/d6Xvc+ZuTnkAe8KRbJfc3EUmhBR9nVd0EPAh8D2zBOW6f07KPc0Btj2u9jrcFjerCjX/VYohIKvAK8FNV3Rtp1hBpzeq7EJEzgB9U9fPg5BCzqodpzUUcMAR4TFUHA/lUVFmE0uz32a1emQjk4DxWIQWneqaqlnScaxJuH+u17xY0qos0/lWzJyLxOAHjOVV91U3eJiJd3eldgR/c9JbwXRwLnCUiG4DZOFVUDwEZ7vhnUHm/yvfZnZ4O7GzMDDeAXCBXVf/nfn4ZJ4i05ON8ErBeVfNUtRh4FTiGln2cA2p7XOt1vC1oVNdix7kSEQH+DqxS1T8FTZoLBHpQTMFp6wikX+b2whgJ7AkUg5sLVb1DVbNUNRvnWL6nqhcD7wPnubNV3efAd3GeO3+zugJV1a3ARhEJPB3zRJyheVrsccaplhopIm3c33lgn1vscQ5S2+M6SU6UYQAAAd5JREFUHzhFRNq5JbRT3DRvYt2o0xRfwGnAN8A6nKHcY56nBtqv0TjF0GXAEvd1Gk5d7rvAGvdve3d+welJtg74CqdnSsz3ox77fzzwhvu+J85gmGuBfwKJbnqS+3mtO71nrPNdx30dBCx2j/XrQLuWfpyBXwNfA8uBZ4DElnacccbr2wIU45QYrqrLcQWudPd9LXBFbfJgw4gYY4zxzKqnjDHGeGZBwxhjjGcWNIwxxnhmQcMYY4xnFjSMMcZ4ZkHDmAYkIh1EZIn72ioim4I+J1SZd76IpNWwvlwRyYhuro3xzrrcGhMlIjId2K+qD1ZJF5z/vbKQC1aeNxc4UlV3RyeXxtSOlTSMaQQi0tt9zsPjwBdA1+BShIj8S0Q+d58HcXVsc2tMeHE1z2KMaSD9cO6+vQ7AKXCUm6KqO0WkDbBYRF5R1V2xyKQxkVhJw5jGs05VPwsz7WYRWQosxBlArlfjZcsY76ykYUzjyQ+VKCInAccBI1X1oIh8jDM2kjFNjpU0jIm9dGCnGzD6A8NjnSFjwrGgYUzsvQm0caunfoXzNEVjmiTrcmuMMcYzK2kYY4zxzIKGMcYYzyxoGGOM8cyChjHGGM8saBhjjPHMgoYxxhjPLGgYY4zx7P8Dl+NIQ0adYo8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "env =  GridworldEnv()\n",
    "env.reset()\n",
    "#print(env.P)\n",
    "agent_sarsa = Agent_SARSA('3', env, 4,100,0,0.5)\n",
    "t = timeit.default_timer()\n",
    "agent_sarsa.main_loop()\n",
    "t1_sarsa = timeit.default_timer() - t\n",
    "print(\"constatnt time = \"+str(t1_sarsa))\n",
    "agent_sarsa2 = Agent_SARSA('3', env, 4,100,0,0)\n",
    "t = timeit.default_timer()\n",
    "agent_sarsa2.main_loop()\n",
    "t2_sarsa = timeit.default_timer() - t\n",
    "print(\"dynamic time = \"+str(t2_sarsa))\n",
    "plt.plot([i for i in range(1000)],agent_sarsa2.R_sum, label=\"epsilon = 100/100+t\")\n",
    "plt.plot([i for i in range(1000)],agent_sarsa.R_sum, label=\"constant epsilon\")\n",
    "plt.xlabel(\"Trial\")\n",
    "plt.ylabel(\"cummulative reward\")\n",
    "plt.title(\"SARSA policy result\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}