DQLearning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from amalearn.reward import GaussianReward, RewardBase\n",
    "from amalearn.agent import AgentBase\n",
    "from amalearn.environment import EnvironmentBase\n",
    "import numpy as np\n",
    "from math import *\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.stats as stats\n",
    "import gym\n",
    "import timeit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import pprint\n",
    "import io\n",
    "import numpy as np\n",
    "import sys\n",
    "from gym.envs.toy_text import discrete\n",
    "from colorama import Fore, Back, Style\n",
    "\n",
    "UP = 0\n",
    "RIGHT = 1\n",
    "DOWN = 2\n",
    "LEFT = 3\n",
    "\n",
    "class GridworldEnv(discrete.DiscreteEnv):\n",
    "  metadata = {'render.modes': ['human', 'ansi']}\n",
    "\n",
    "  def __init__(self, shape=[10,10]):\n",
    "    if not isinstance(shape, (list, tuple)) or not len(shape) == 2:\n",
    "        raise ValueError('shape argument must be a list/tuple of length 2')\n",
    "        \n",
    "    self.shape = shape\n",
    "\n",
    "    nS = np.prod(shape)\n",
    "    nA = 4\n",
    "\n",
    "    MAX_Y = shape[0]\n",
    "    MAX_X = shape[1]\n",
    "\n",
    "    P = {}\n",
    "    grid = np.arange(nS).reshape(shape)\n",
    "    it = np.nditer(grid, flags=['multi_index'])\n",
    "\n",
    "    while not it.finished:\n",
    "      s = it.iterindex\n",
    "      y, x = it.multi_index\n",
    "\n",
    "      P[s] = {a : [] for a in range(nA)}\n",
    "\n",
    "      #Reaching target\n",
    "      is_done = lambda s: s == 0\n",
    "\n",
    "      #Due to existance of walls, these moves are not available\n",
    "      no_up = lambda s: s in [44,73,74]\n",
    "      no_down = lambda s: s in [23,24,54]\n",
    "      no_left = lambda s: s in [44,35,54,65]\n",
    "      no_right = lambda s: s in [32,42,52,62]\n",
    "\n",
    "      #Barries in the path\n",
    "      pit = lambda s: s in [13,72,41]\n",
    "      wall = lambda s: s in [43,33,53,34,64,63]\n",
    "\n",
    "      #Rewards of each state\n",
    "      reward = 15.0 if is_done(s) else -1.0\n",
    "      reward = -15.0 if pit(s) else reward\n",
    "      \n",
    "      ns_up = s if y == 0 else s - MAX_X\n",
    "      ns_right = s if x == (MAX_X - 1) else s + 1\n",
    "      ns_down = s if y == (MAX_Y - 1) else s + MAX_X\n",
    "      ns_left = s if x == 0 else s - 1\n",
    "      \n",
    "      P[s][UP] = [(1, ns_up, reward, is_done(ns_up))]\n",
    "      P[s][RIGHT] = [(1.0, ns_right, reward, is_done(ns_right))]\n",
    "      P[s][DOWN] = [(1, ns_down, reward, is_done(ns_down))]\n",
    "      P[s][LEFT] = [(1, ns_left, reward, is_done(ns_left))]\n",
    "\n",
    "      #Terminal state\n",
    "      if is_done(s):\n",
    "        P[s][UP] = [(1.0, s, reward, True)]\n",
    "        P[s][RIGHT] = [(1.0, s, reward, True)]\n",
    "        P[s][DOWN] = [(1.0, s, reward, True)]\n",
    "        P[s][LEFT] = [(1.0, s, reward, True)]\n",
    "\n",
    "      #Encountering walls\n",
    "      if no_up(s):\n",
    "        P[s][UP] = [(1.0, s, -5.0 , False)]                \n",
    "      if no_down(s):\n",
    "        P[s][DOWN] = [(1.0, s, -5.0 , False)]                \n",
    "      if no_right(s):\n",
    "        P[s][RIGHT] = [(1.0, s, -5.0 , False)]  \n",
    "      if no_left(s):\n",
    "        P[s][LEFT] = [(1.0, s, -5.0 , False)] \n",
    "\n",
    "      it.iternext() \n",
    "\n",
    "    # Initial state distribution is uniform\n",
    "    isd = np.zeros(100)\n",
    "    isd[44]=1\n",
    "    self.P = P\n",
    "\n",
    "    super(GridworldEnv, self).__init__(nS, nA, P, isd)\n",
    "\n",
    "  def _render(self, mode='human', close=False):\n",
    "    if close:\n",
    "      return\n",
    "\n",
    "    outfile = io.StringIO() if mode == 'ansi' else sys.stdout\n",
    "\n",
    "    grid = np.arange(self.nS).reshape(self.shape)\n",
    "    it = np.nditer(grid, flags=['multi_index'])\n",
    "\n",
    "    while not it.finished:\n",
    "      s = it.iterindex\n",
    "      y, x = it.multi_index\n",
    "\n",
    "      if self.s == s and not s==0:\n",
    "        output =  Back.CYAN + Style.BRIGHT + Fore.WHITE +  \"  웃 \" + Style.RESET_ALL if s not in [13,72,41] else Back.BLUE + Style.BRIGHT + Fore.WHITE + \"  웃 \" + Style.RESET_ALL\n",
    "      elif s == 0:\n",
    "          output = Back.GREEN + Style.BRIGHT + Fore.WHITE + \"     \" + Style.RESET_ALL if not self.s==0 else Back.GREEN + Style.BRIGHT + Fore.WHITE +  \"  웃 \" + Style.RESET_ALL\n",
    "      elif s in [13,72,41]:\n",
    "          output = Back.BLUE + \"     \" + Style.RESET_ALL\n",
    "      elif s in [43,33,53,34,64,63]:\n",
    "          output = Back.CYAN + Style.BRIGHT + Fore.BLACK +   \"  █  \"  + Style.RESET_ALL\n",
    "      else:\n",
    "          output = Back.CYAN + Style.BRIGHT + Fore.BLACK +  \"  .  \" + Style.RESET_ALL\n",
    "\n",
    "      if x == 0:\n",
    "          output = output.lstrip()\n",
    "      if x == self.shape[1] - 1:\n",
    "          output = output.rstrip()\n",
    "\n",
    "      outfile.write(output)\n",
    "\n",
    "      if x == self.shape[1] - 1:\n",
    "          outfile.write(\"\\n\")\n",
    "\n",
    "      it.iternext()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent_DQlearning(AgentBase):\n",
    "    def __init__(self, id, environment, action_count, state_count, alpha, epsilon):\n",
    "        super(Agent_DQlearning, self).__init__(id, environment)\n",
    "        self.alpha = alpha\n",
    "        #self.policy = np.random.choice([i for i in range(action_count)],state_count)\n",
    "        self.QA_sa = np.zeros((action_count,state_count))\n",
    "        self.QB_sa = np.zeros((action_count,state_count))\n",
    "        self.action_count = action_count\n",
    "        self.state_count = state_count\n",
    "        self.epsilon = epsilon\n",
    "        self.env = environment\n",
    "        self.R_avg = []\n",
    "        self.R_sum = []\n",
    "        if epsilon == 0:\n",
    "            self.change_epsilon = True\n",
    "        else:\n",
    "            self.change_epsilon = False\n",
    "            self.epsilon = epsilon\n",
    "        if alpha == 0:\n",
    "            self.change_alpha = True\n",
    "        else:\n",
    "            self.change_alpha = False\n",
    "            self.alpha = alpha\n",
    "    \n",
    "    def main_loop(self):\n",
    "        gama = 0.9\n",
    "        for k in range(1000):\n",
    "            #if k%10 == 0:\n",
    "            #    print(k)\n",
    "            if self.change_epsilon:\n",
    "                self.epsilon = 100/(100+k)\n",
    "            if self.change_alpha:\n",
    "                self.alpha = 100/(1000+k)\n",
    "            state = np.random.choice([i for i in range(self.state_count)])\n",
    "            flag = False\n",
    "            R = []\n",
    "            length = 0\n",
    "            while True:\n",
    "                length = length + 1\n",
    "                if length>20000:\n",
    "                    break\n",
    "                p = random.random()\n",
    "                #select action\n",
    "                if p< 1-self.epsilon:\n",
    "                    action = np.argmax([self.QA_sa[i][state]+self.QB_sa[i][state] for i in range(self.action_count)])\n",
    "                else:\n",
    "                    action = np.random.choice([i for i in range(self.action_count)])\n",
    "                _, state_p, reward, done = self.env.P[state][action][0]\n",
    "                R.append(reward)\n",
    "                p = random.random()\n",
    "                if p<0.5:\n",
    "                    A_star = np.argmax([self.QA_sa[i][state_p] for i in range(self.action_count)])\n",
    "                    self.QA_sa[action][state] = self.QA_sa[action][state] + self.alpha*(reward+gama*self.QB_sa[A_star][state_p] - self.QA_sa[action][state] )\n",
    "                else:\n",
    "                    A_star = np.argmax([self.QB_sa[i][state_p] for i in range(self.action_count)])\n",
    "                    self.QB_sa[action][state] = self.QB_sa[action][state] + self.alpha*(reward+gama*self.QA_sa[A_star][state_p] - self.QB_sa[action][state] )\n",
    "                state = state_p    \n",
    "                if done:\n",
    "                    if flag:\n",
    "                        break\n",
    "                    else:\n",
    "                        flag = True\n",
    "            self.R_avg.append(np.mean(R))\n",
    "            self.R_sum.append(np.sum(R))\n",
    "                \n",
    "    def take_action(self) -> (object, float, bool, object):\n",
    "        index_selected_arm = 0\n",
    "        obs, Ri, d, i = self.environment.step(index_selected_arm)\n",
    "        #self.environment.update_selected_arm(index_selected_arm, Ri)\n",
    "        #print(obs, r, d, i)\n",
    "        self.environment.render()\n",
    "        return obs, Ri, d, i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "constatnt time = 0.5003012350061908\n",
      "dynamic time = 0.4708940359996632\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEWCAYAAACaBstRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd5hU1fn4P+/M9l5Y6gJLE6Uj1QJiBREVDcYu0URi+8aYX6ImxkgKxphEDTHGWBNjQY0FYtcoGg2KqIgCKqAoTV2pS9k2c35/3JnZO+XO3C2zM8u+n+eZZ/eee+65723nPe973nOOGGNQFEVRFDd4Ui2AoiiK0nFQpaEoiqK4RpWGoiiK4hpVGoqiKIprVGkoiqIorlGloSiKorhGlYaitDEiMldE7g/830dEdouIN9VytQQRWS8ix6RaDiV9UKWhdGgCldo+EakRkR0i8j8RuUhEPBH5DhWRlwP5dorIIhE5MNnyGWO+MMYUGGN8yT5XsrErQ6XzokpD2R840RhTCPQFbgCuAu4O7hSRQ4AXgIVAT6AfsAJ4Q0Sq2lvYtkZEMlItg9J5UKWh7DcYY3YaYxYBpwOzRWRYYNeNwH3GmD8ZY2qMMduMMT8HlgLXxSpLRL4jIm+IyJ8DlslHInK0bX/PgLWyTUTWisiFDuVUiYgJVuwiUiYi94rIZhHZLiJPBtI/FJETbcdlisg3IjIqRplTRGSjiFwlIl8C9wbSZ4jIcpvFNcJ2zFUisilgaX0cvBYR+buI/Cay7BjnnAb8DDg94G57P+ZDUPZ7tIWi7HcYY5YGKr5JIvIpcCjwixhZHwF+EyM9yATgX0AX4FTgcRHpZ4zZBjwErMSyXA4EXhSRT40x/0kg3j+B3cDQwN9DA+n3AecA/w5sTwe2GGOWO5TTHSjDsq48InIwcA9wIrAsUNYiERkMVAGXAeOMMZsD1lWz+liMMc+JyPXAQGPMOc05Vtm/UEtD2V/ZjFWplmG951ti5NkCVMQp42vgFmNMgzHmYeBj4AQR6Q0cDlxljKkNVOx3AefGE0hEegDHAxcZY7YHyn01sPt+YLqIFAW2z8VSME74geuMMXXGmH3AhcDfjDFvGWN8xph/AHXARMAHZANDRCTTGLPeGLMunqyK4oQqDWV/pRewDdiOVcH2iJGnB1Adp4xNJnxGz8+xLIuewDZjTE3Evl4JZOodOG575A5jzGbgDeBbIlKCpVweiFNWtTGm1rbdF/h/AdfUDhHZEThfT2PMWuCHwFzgaxFZICI9E8iqKDFRpaHsd4jIOKwK/HVjzB5gCXBajKzfBl6NkR6kl4iIbbsPlgWzGSgTkcKIfZsSiLYhcFyJw/5/YLmVTgOWGGPilRc5PfUGYJ4xpsT2yzPGPARgjHnQGHM4lnIxwO8Cx+0B8mzldG/GOZVOiCoNZb9BRIpEZAawALjfGPNBYNfVWB3jPxCRQhEpDXT+TgZ+G6fIrsAPAp3SpwEHAc8YYzYA/wN+KyI5gQ7n7xLfMsAYswV4FrgtIEOmiEy2ZXkSOBi4HKuPozncCVwkIhPEIl9ETghc72AROUpEsoFaYB+WywpgOZZbrExEumNZJE58BVRFhjMrnQt9+Mr+wL9FpAartX0NcBNwfnCnMeZ1YCpWZ/YWLLfVbOAom2KJxVvAIOAbYB4wyxizNbDvTKwO5s3AE1j9Cy+6kPVcoAH4CKvPJFRJB/omHsMKCX7cRVkhjDHLsPo1bsVyya0FvhPYnY0VivwN8CWWMvxZYN8/gfeB9VhhyQ/HOc2jgb9bReTd5sin7D+ILsKkdDZEZCTwMnCWMeZ5hzzfAb4XcOm0p2y/AA7QCCUlXVFLQ+l0GGPeB2YCw9NpYJyIlGG5ue5ItSyK4oQqDaVTYoz5rzHmD8aYxlTLAhAYHLgBeNYY81qq5VEUJ9Q9pSiKorhGLQ1FURTFNWnjz00WXbp0MVVVVakWQ1EUpcPwzjvvfGOMiTlbwn6vNKqqqli2bFmqxVAURekwiMjnTvvUPaUoiqK4RpWGoiiK4hpVGoqiKIprVGkoiqIorlGloSiKorhGlYaiKIriGlUaiqIoimtUacRj7zZY+UTry9n9Nax8Mir5oy938danW8PSnnxvEzv3Nrgv+5s1sDbRstTRvPZJNZ9W78YYw6PLNrCv3pf4oER8uhiqP2l9OW3N+w/Dvh3NPuzrmlqee/8LeOfv4EuLKarahEafn4eWfkGDz59qUZKPMfDeA1DXtMji/9Z9w7rq3dbGzo2w+qkUCdcxUaURj0fOg0e/A7s2t66cJy+BR2fDji/Ckqfd8l9Ov+PN0PaKjTv44cPL+dVTq9yXfetYuP9U6+NoBufds5Sj/vgqyzfs4Cf/WsE1T8RbVsIl950MfxnX+nLaku3r4Yk58K8Lmn3o5Q8t581Hfg//vhze/Xubi5YqFry9gZ8+/gH/XOI4fmv/4Ys3YeEl8PzPQkln3fkWR/8xsGDj32fAw2dD/Z4UCdjx6HATForINOBPgBe4yxhzQ7z8Y8eONS0eEf6nkbB9PWsqjuP5bV1ZVTSJAwcPYcKaP7K1IYt6sijKzWD4AQP5y7t1jKhZzKsZh3FGr294r98cCjwNZL81nxlF68j/ahn7MkvILe9DTU531m/dS97OdbxvBjA4fy9Da9+lWsp5pWEY2ylgVsYbVEsZPTw7+cBfRV1BJdVFQ5i59S5WZo9imz+fjVkDOH/rH0Pi1pGJySnlTTOE+kZDTeEAcjI9jNvxLDsHnUrh2kV0kV1klFTyv80+vqKUQqmjH5t43wxgVLlhmz+Xz2oLMPV7GJy9jT7+jeR4DfsaheqsXtSQz9i9/+Wd/MkMqF3JVl8un2cOpCg3g7G7XgJgtxTQ4Ic1nn7sM5nkFpaR4RF21zUysGsBWfU7KTXb2f3153xcMB7/3h309VZT4NtJoW8H27wVePx1vCmj6JXno6L2c9b4ulFa1oWh5fDRduGbnXs5oGE1HzRWMiZnC3lSz2fFE8jcs5mG+lpyxEf34mxyvlkZuj97PQVUZ/Yiy9ThKalk+z4/e/J7U797G/3rP2ZnyVB27mvA4xH61X1M3b7d1JpM+nm+AuDzXieQv3kJtcX9qdm1g62mhJzCUrqZavbu28c6X1dG9y7Fu2EJDXipzh/EUP/H1GR1pbrWy87MCrbvqacsP4uBe99DjMF4vGR5hbfr+3KgfMFyBjMu6wvW+SogpxiPwKB977Oh6GAK93xBZe3HbMjsz6dU0i27Do+/kS6eXXwmvRm09z025wxibY2XYk8dI3O/wV9bw9ddxpFVvRK/J4PszAze2duVkbKOT+nFIYO649/8Hv+tG8QW04WTzCuslT4clL8bT/0uVmcNp9eeVXydO4B6ycbjEXbsrWek70NWZ49kcLdC3vliG8U5mVR1yWfTjn14d22gR2Em1VmV9Ni2lG1SSkNmIb19G/m6fDx+Y1i/dS+5mR4yTT2D/J/xOd0Z73uPN/OPZODe5XhEqMnqSi3ZZJdVUlWez859DWzdU0/XrW+xvetEimUv2zZ+whrvAMrys8jN9PL51r0UZfoY5FtLrsfPlxk9GbTvfQCe9B3KMblr2V3XwBv+oXTP2MthxlpL6rOySRTvWMXKrBEMK4fSTa/wbtFRGPGSk+llw7a9DOlZxFc7a9m0Yx+DuxfSY/syPAJbikfTo+YDVtOfPhXFrNiwjVEZn7M2ewgNPj/ZGV4yvUJVzTvUVh5O18JsVm3eSdWuZezxeWn05lKTW0mhfyeejGzK6jfxSe4o9tY3UpiTQcWO93m/sS/1ZDHKs5ZNmVWUlJTgEWHzjn0U5mRwYJmHnRtW8YHpT6bXQ4+SXN4oPpELzzm7RVWfiLxjjBkbc19HUhoi4gU+AY4FNgJvA2caYxyb5q1SGvMPhm3rQpt7TDa3NZ7MTzIfSXjo9+uv4CDP5/wwo1kLsLUJO00ePjyUye64+b42JXSVcLdNo/GQIc1zW3xpSsmhgRKH89WZTLaYMgAKZS/lUhMzXyQ7TD4lEt0C3GVyKZJ9zZLRDVtMGXUmkxypp7tsB2CrKQzJ6zeCR8K/l90mhwKpBeALfwW5UkeF7IpZ/np/NwBKpYZi2Rszz06TF9q33t+NMqmhSPZGXbP9vAA1JpfCwP5ak0mONDjm3WXyKHI4fyT2Y4PyVwWU6DemiN0mNyy/Fz+9PdWO5e0w+ewwBaHtYFnxWO/vRlV5Hp9v20tfsfLbn8tmU0a9yYxZZvB+bjRdqJRvEp6r2hSFPb/gNUdSIrtjvpt2vjYl7DXZAPSRr/GIYaspJLewlLzdX8Q9dqfJY7spJFfq6Bb4RjebMnrKtphyBa/5S1NKrcmip3zDM/4JzPz1s3HP40Q8pdHR3FPjgbXGmE+NMfVYa0GfnLSzRSyFnC915Ei9q0MzacRnknN760z8KcPuapzODY1nJizn/+r/LyrtM2+/ZstzSf3l/K/X+Y77l5sBTKm/mSn1N/O7xjNcl3t34/Ex0x/1TeEbU9RsORPxvfofM6X+Zs6rvzqUdn3D2bzgGwPAl5RGHbPId2jo/yPrb+IXDc73IXgP7m6c7pjn942nh+V/0Hc0APf6poXle8w3KWz7Ud8Rof8X+0eF7fu375Cw7ciy4rHQd1iYPFPqb+ZDfxUAv2k4J5QW/B1X/7u45d3WeFJYfjdMqb+Z3Rct4+iGW3jKNxGAeQ1NLehz638aVmaj7bu7q3E6VbUPcnjdfFb4o9/tnzecT1Xtgyz3DwDgJw0XRZ071u+vjScllPuHDZeE8r9fYj3HGxrP5LVpL/Cqb0TcY+9unM6U+pv5YcOlobRv1//CUa4gF9f/kCn1N7PCDGBsubu6qrl0NKXRC2uhmiAbA2lhiMgcEVkmIsuqq51bPQnxeKOSBHeWmWDYRX7Lzx2HejLj7jcIX5uShOV8TXSebVk9mi1PHZlkZuc477e1Au3/J8JH9P0Pni8Z1AbKtd87+7l2mejnad/vw+tKtkaH6wL4MmCRBQm+bXUmKyw98t2qoanFH/ns9xL+bPxGos672t87pjw15DnKGutaE72bdWTF3e/En19eQ1l+07E+W9UVae3YaUgwJ2vkNew2TfcqXuPMzXO2v+tej3XP/cbDX1/9NOGxwXfEXka1i2+63vYOFzVuS5i/JXQ0pRH9thNdixtj7jDGjDXGjK2oiDm7r8uzRd+e5iiNRC9sS7mj8YSEeapNdKs4kq2mMCptW0bXZstTRybZWc6VQb3tPjSn0mh0eD2bo3iC3NqY2CANVgQ7KIhKA9hDtGKMvB43lYmJ+RpbvOsfxMf+Sn7f8O24x+wy4ZW5/Z5st8kPsNsm92u+4fzb32R53N14PNWmmE2mS0x5auJUyLGu1UnRB4mnMOPxt1c/5euaOu5pnEaj8bDEPzS0L1Ipiu1WxbvX0KSMb26cRa3J5BPTpDxr47yrbr5t+/15r+fpNBoPr/uH8f6GxJF8QbntZdSRxXp/N37d4NxPEfzWvjKlZPprHfO1ho6mNDYC9iZRJdDK0KY4xFQaLg/FkE3rzcPv1/8wbPvexqks9R+U8Dg3lkasCnx3RmxlU9NrUsx0sD6uzAyrMnizONr1Yv9wax0q1Wd90VFXjQ4fZryPORajav/GH2xuHyeaWvPCe/6BUeeqj6GsIq+nNsIiuL4hsZswyJ8bZ7KNIqbW38hffDOj9tsri50Rloa9cmk04RXzvoBfHeC8hp/yqekZ2r6zcTrj6v7Kw74jY8rkyXV2AyZ6Di/6Do5KS1SJf+rvTlXtg1TVPhhz/7vmAAbW3c/XNlehXZlfPGUAXnH7lTY9v1f9Izmw7h9hFlwiWROX3XR/dpSOYmDd/XxFWZwjoolUzFPqb+Zun3OjsSGglH/VeC5vz0zOqsEdTWm8DQwSkX4ikgWcASxK1slMjJfPraXhwZBNM8ZbOLCb8JZeHVmhFyMeW0ns84/VUtyTEVvZSHa0VRIqx2SS4bVkEuJ3ojtZGrE+T0dLo5nuKbcWX6xy7S14fwwpI91GkWVEtoLj0ZyQlJ0RrjJ7Cz+ytR/vfQnKF8uKAtgjzi7Wllh8ycBve0+aozCg5e4yd2U33Z+8rJZZWM1tIAUbNgYPGd7WKT0nOpTSMMY0ApcBzwOrgUeMMSvjH9WK87XQlAZLubjtNI+HvZUI0EuqE1aar/lHhH1IzkS/VPsclEa8FmcdWXg8HocSw1ObU9E4XUMtWdxpc9HF8tHbcas07B9osAJvxBtqccaq1GvJ5AN/FRv8FVFlAOwx0ZWxk3IwMa43/NxN17knojGxxD8k9H+kknCy2KxychzlBNjniVYaQfkT9V+4t8vtZbs/5pHGI6LSPB4J80/Z73WssiMtQ4A1/l4JZXEjp73snAil8YhvStxjm/qyWt5AyvQmp3rvUEoDwBjzjDHmAGPMAGPMvGSeK1bL0u0r7ZG2sTQiFUQudXErwX/7JrLCDGjx+RoyYvuwMwpi+7whMD4kWHHHDOFO3IZuTiu7zmTyN9+JXFF/MRD7Odmpd7hfx9Td6JjPXqJEVdk2WcjixPrrmVT/JyDaZ7+P7KhjnKSNjLbzSOS5m+5SZGXysenDvY1TrXIiZPDF+cyDeZ0solpPdEe4+2o9+qkmstTdWvIAVzZ+P8qN5RUJewdjPUc7sRpgZ9f/LKEsbuS0l52bGf5MnvZPpC43djivVb6zfPGwv8MZHrU02hWf3+D/Zk2MPe5e6gPlC8Z7Pmq1HJEKolD2uXe3tMB94PdEV3IAGd74kSRBV14i91QGrZ+uJPghBe9DIqXhVM15op5l8z+yyFZ9ZAu0OR995HVIHFdLrHKD9zbSreem89kpT10MpZHONLeejHUf3bh/3ZXdZGm01D3VXPdZvVoaqePOF98nqzF6sJrbltD3Mp5ljCeW0mkeg3qEd0y/4hsVV2m86h8Z+t8eJRPJRodomb1ZsdM9VYeFbe8Jc5sJZf2tsQwMOCrq2A9t8fGxonR2mHzH6vqNjImh//cFzP1g6GHwPmww7iK+1vmbwok/9Xdna5yxHsGO+c2mnP/4rQ7dJ/yHR+WrNsVh29sjItIi94MVIQXwTuBvkEilcdqYSt4KBD28ZwbyScbg0L5vYpT7v0BE0Qp/uKX5kb9PVN7VEp7H6V4Yj3WPg+MjAJblTQasgWRO1HiKeNk/Oio9OMYjSHC8STAa7Cn/xLD9m0y54zli4fEIjD4ntB0MaAB41jchKn+s6w5aXU/4op93kA9s7/S7tnMAPBY4zl6B50YojYLsDGoP+lZTQv/wQIT3jFVmUKnFcsXFwl43JKtPIzkxofsBa7bEHj3amscwrvYvvJZ9Bbkx+jrOyL+LBRcfAX9oqkhuHreY+YeUwvymfH/zzeC8YXmwtintiLqbeHnuGezZu4d//e7tUPpPG77HzQ3fYje5DPF8zkNZljdvaO3dvHzlMXxUWAgBB9/pXRfywRdbOTGnkq/OX0q3e8eHytn57ScoPuAoHj72LX717w85sFQo2bmSu7P+iL9yPCvPnUp+dgY7e33CuPJu8N5PATig9h90k21sNBV8MPc4tu9p4Iw7ljBy5x004iWXenaTg0H4U+ZfANgx9c+8/swDzPBac3KZ0+5lY5GXT9av58KFX9FTvmGDscz6YMv6E1PJKbW/pG/3riw6uxdfSzlT/rCY1TnWfFP/vuxwTrz1dY6vv4F89jFjeA8e/mAH9WQyqvZvXDalPzcvDh+he4dvBk/I0VSTx8O+KTznG8dOCnjRN4Zasvjp1EHc/cLSKIW1i3xG1N5JFo3kSB0bYyi0t8xBjKy9g/930jjOX/Q2V2cs4KyMlzl5dCW3LYORlcXcetbB9CjOYeDbGxhZewc7KeDuM8ayu+ssMnIKeGBfBje90ou739mBH+Hq4w/khmdhfONdfG0Lxz2u7nd8bprcIJ/85nj2NfiY9edMNm2zRlT/6uShnDSyJ1P+mMXOPfvo3b0bf972ffp6vsYrMKz2LvaRza9nDuOE4T0ozJ7G9p3XcvTiL1nwdtOwqed+OImyvCz2eTeSk+HlGpMJ/p9DRjb1fmHHzp3cV9SFkb98IXTMlQ1zuK5hNg1ksPLq8ZyXWc5Nv7Ym4Hz4uKX8YlHsyR5+MnUwv3/+Y445qBsvrW4aAe71CMy4mTUjr2LWX19nZyD8+PFLDuXU2wwP+I5CgAmDK/nw47VhUVizxlRSnJvJ3a9/xvDauxyDAwBWmAGMrL0DsMayFLCPFTkXAnCN7/vMbfhOWB9VpHvqv1ceSXHusWw99AqyMjMoLCjkwGv+jUHIoT4kNwjDa+/i2lPGwBOrmTq0G3vqfLy+Nnb9ZLcYMzzJsQlUaTjg9baNiWqnmlK2UkQl0Q98e0Y3KAivYPIKivBmhr+4Bg/ZWeGPba/JwZudT1F2eKdlAxlswuqgXenvG0rfQy7dysI7vI0nm73k4PUKUto3bF9xFytE02TmsYdcGvOL8e20XG+enCLysy15isvDfbT1ZIYq+MKcTApzMulTnsebO60PwmQVUBuYXTdov2XmFrDN1lrPzMymskc5K77x46NJYQChaUu2mwJ2UUD3skKk4gC6AftsH3yPkpyQPPVkUljenXqsKSB2UIgUVLCHyEGgQl1GETQ2AhL6iIN/TVZ+mCx2QmGbcYzSnRSwt8Gwi4JQ/swM6z6W5GXRuywvLC9AdoaXgsA97pcPdfm92EMdAMN6WpaHNUVHk4twmykK69PIyvCQleHB580Ktai7FuZQkpfFbslnO5kMys0N7fN6hN2BAX6VpbmhAXal5V3pVhQ+3uDA7sFWu3Ws1Ua2QkyzgK450a4uH95Q+RklvcKGm+bkFTi6Z4JylOSFu5e8IuDx0qVLha3iheLcTEDYFUjzSRabCbd6uxflYAIPLd6gxiD28u2huuVFBWzaEV5/2JVGj+IcSgPyl3dp+uZrA/1fkddcQx7Z2da+rAxvgrlJm5q1mRo91b44dSI1p6MuFk6RPh4nZ6w3+qOp94QrEqfQVDuJBl0tXW+NHl3zVU1o9GqTcFZl1r/C+khG9U48BsSJ+samCq28oOnaVnst10tWeVVY/qBfP9btCbouguNWIudRC853FRmGOaIyXH4nf3O8j7Oq3KokuhbG7gNyQ+/SPI46sGtoNPae4gOi8tg//KjnYtvMy7auodEf3qfUiCdmR7j9nkSW6/ObUDRWr55NEy5EfhMt/RJGVka71g7sHu7WG1FZ7NhSPqhHEf27WPc/8l0MfkeRyiRS9oP7RrvWMrzC0J7RsjWX7MxoufOzm94xTzPDgiOJdfhyf/+otIwk9WmopeGAN8YUItB6peHU4RhZH/w76EP2xhg74MmD/3sX/mz52hMphHjn5cdrrbfw128B8Nk3e6M/1sC9GN+vjBevmEyG18Mv30p4ypjU2ZRGSW4WG9jHD44ayHcPvwn2/ICMLtEVJ8TuFF7sH82xdTeyxlQC4I94NMfV3cjrV4wLU8gv/WgyRbkREWmZ0a3CfQ2+sCddkpfJDts6JyMqi3npR5N5Y+1WrlsUHvU9sGsBa7+O7g+774LxnHfPUnqV5HLv+eMY1LWAow/qylc7DwL/eWzb0QVYGnbMSz86ghnzX6emrjGuj7ogYO35Dbx9zTEQmPzYjydmKK9dUURWqI0+P9c3nsU/fcdy8yFjuO3dN6xjIp5BSyc7ffDCiWzfW09uphe/gXqfn6Kcpqro9auOpDQvi/+uiT0F0KMXHUJBdgbP/3AyB3QrYNqw7lz75Ic8++GXoe9IRHjxiskce7M1wC2yor74iAFMHdqNY25qGgCX4RGmD+/Bw3MmkpXh4ZTb/hfz/D+bfiDXPxMe5PLg9ybA/db/wWdhpzCn6Z1L5DU6bUwlP55qNaLqG/1keIWlnzVNCdLos+77YQPLeWOttR7PWfU/p5TwiUAzNXqqfXHS0s19DLsj4t/djZ+wpgEAwBurJWugvKkj0020h6PSKKiA/CYzPdMr0ZWTTXEN6lbYqlA+u6VRHKi8+5TnU5yXDRWDHSOGnAZtBRUGgD+iEqshj9zyPmEV5MCuhWRnhN+LyCiTYKelvVLsUxburvB6hIFdC8nKiH6eka3mIP26NLkwDuhWiIiQk+mlb5cC6Bp7lH/f8nwGdbMsvHi33W4tVdisH+dGirOl0eAzNJLBZ6YHGR7hgMD5I/O1dILs/OwMKkvzKC/IpqIwm14luWGVamVpHvnZGY4t8mClPLi7dQ+7FGTTpSA76roGdWt6DpHvtCfw/Ox4A7X5hP7ljO7j3MnfuzTadWVviGTHeCfs34wkqEUKcjLoVpRDt6Icepfl0aM4PAx+b8Cla7eY95ITckUH0eipdsbJNG6upRH50X5oqmKXG/EehUJTY1gakbixNOLF6tvxesTRPRWSrRW+0gNsFWrQZM+JYc5H4qZPz25pDOxqVXRZGZ4oJRd5vsjrzQl89PbyfBFmTPCYWB9mdoaXqvIY4xsS3LbygJ87KHuQ4LsYbGHGIlbrFpyfu/1+Rl6/Xbl5REIVcZTSaKXVnYhIyzEeQdGcrB83I8XdNoaivo+ItFgNCXt9kug0/gQXvrfBqhucnnnonBo91b542+iGR1oBVzdcyFemlO9nPB2WHtn6yMBndXG6eNndTQLn7nqMsT6eQ2vn87+cHwQOjV/JNocbvzWCmaN6UVGYzZ//Y4Uk50S0/GeM6AERQTPxxiw0yd70sT36/UP4Ypu1ZkRkizUroqKPViqWPD5beZHfcZPSiJYrO9PD45ccxmPvbGTeM6sTyh1kWK9iHvzeBMZWhc9PFDxXpOKyk5fVPKURr0/j0iMH8vQHWwCr4glWPpH9bsleiife9UYSfD8iD1n84ymAu3fW7XsdK5/9XkRasot/PCWsPnHsvwyQ6LKD73lOZvzvXi2NdsapZdJ8SyP8Y64lOxR7byfyPfLGGSQX+bEmHtzmnrpGH16PsJkuVCdhzYr87AyOHdKNUb1LQv0bkR2H9imwg7jpPLS7p0rzsxgZ6CSN/MgjFVBkAyE7qDRsX29kCzYoT6wPMyfDS1l+Fkcf1PwZgw8d2CWqpRqstBvj1CaxWrcQz9JwVhpZGeH7vA5mXnMsgZbga4ZW8oSURvgxVZH7H6gAACAASURBVF3yqeqS7+g5sF+722ijWEqjrrFp0Gpko8Q6v01pJHiXI6/BiUTy6ojwdsbhG+T8jOebVU7kjKPgoBAiXqTtRPvFN0gg9DXwTpn8YMin+5djj4kf7dOjODdUqX7pCZQf4Z7yiDT11RR2d33uSLoEoqdK8yKURL5V2e4y+SFfvZv33+lbS3RsduAjD7qHgu6r3qVNvuTIDzlYccT6MIMVeEVEZFUwvVep83TjsagM5I9sWSbyjUPs+awgvqVhV6pekdA1RrpNymMo97YkkZvGTrCedqpwndybJqKh4e5c0ffdrpTKCqLL8YYpjfjlx7qGokCfT0VBdug9TzQOozUegXioe8qBRP5Ct+Tn5UDEtPYXHt43FCRzct2v8ONpcjB990WWPnUXf/r8eH4USHpg0C18umoZH5YeA/uafMkNF7zERX/8R8zz/umMUVy+YHlY2s5TH+LLrD48kRe1bhUv/egI7n/zcy450upgv3v2WLqWPAHb34W8cHdJl4JsLjjzTHbXdqNg1ClRZb037QmuW/ih800J8OuZw5gyuCvDekWEOR5+BbV53Tkq46jQvsjW2VkT+vDgW9aAvB8dewA3vfiJY4URy7UVjGQCK/zyZ9MPpDQvi5/8awUZHuHO88YyorKYCddbA82i3FNBSyOgCIb0KKJ6dx3VNXWhFmBhTia3n3Mw2zKeoqykhK6FOfzlrIM5ZEDzRjn/YsZQxvYtY1xV7M7ZKYOtDtB/fnd8zE5agBsqbuDqM44LbYdZGpGWV4RCCW5HWjrnH1bVLPdbc4k838NzJsa0QsFuacQuy17BPv2D6JHehw/swgnDEy9A9uhFh4QFcwBcd+IQhlcWU/f9N1ny/iqO7ts19G4GyfR6+OExg7jlpTUJLY1Ybrkpgyv4/awRnDiyJ6fdvsS6phiWxkMXTmTl5p1Ulee7cum2BFUaDkTGu7eU3JxopTG+bzEshXfyJ/F+rTVdwOjg8+09npf6FlL3edPqXuuKJnCPrysHeAuAplBOb2lvXvZHr1kAcNLInlFKg0HHMjg3dsf6wK4FzD2paWGbow8KWBk9Yq/oNn1ET+CcmPt2lw9nhUm8hndhTiYzR0crMDKyyBk/G/uKEpHv/7iq0tCHWRXouG2Ou2TyAU2RJjmZXuZMHsCbn1rhi4Jw7JDwgXuRCilY6QZdEX5j+PbYSv7yyrqwj3XasB5AU2V0wojmr4yYm+XlW2MqHfeP72cp9UmDnBcc+zD74LCIO3sjNLJF6o1wXXkdLI0Mr4cjB1fwysetWB0zDpHnm9DfWdmKg3sqSFBneISwsRjB3D+Z6hy5Z2dQ1wJWbwkPbQ0+m+weBzGlx0G8uCr2uufHDunGLS+tSXgeX0xHhHDa2N4BmS2pY7mnhlcWN7tR0lzUPeWAaSOlIbGin/yW/9Me9RTvNQq+JJHuiHjWZ6wXM1kjRKPP0/avVWTrzF43BN0nLR03EFVmjNtUlBNb2YZcN8aEnk9z13RoD5zcaxA7HDW0zyOhgXLJark60Zzon6boKYeyAlqjJNIVGsCtK8cKk46cjTj8WKcO/OA5ShwabkFys9x9P8FrKrSNcWmPd08tDQfiRDc2C0+Gs9Kwj9mI90EGPwSRyO3mvSDJmosmkgn9yrhq2oGcOLIHP338A86Z2DfxQQmIpzSa1l9uXpkL5kxkT11jU5lxpkC//ZwxPLViM795OtwdE3RP2SuKtNAZFy+BLcthgbUZZSlF9FvYiXRPXX/KcIb1KmZif+dV575zaFXrZY7gpJE9+firGv7mYk3t4PXEq7DnnTKMwwaETx0S+W05ccOpw6n3+SnOzWRU75LAXF+BqXQijnWydgZ3K+Rn0w+MbV0Dd543lr//7zOumnZgfGECeMR6NhP7l3HUH191dR1tgVoaDvicLI2i2A/cCU+MaUAwMZRGnDKCLehkTD+QDESEi6cMoLI0j39+dwJTh7a8szxI5Idp/yyDFpTbqJMgE/uXN7nhbIXGuk/di3P43qToqRoyPdFjOtpbZ8S87G5DYNRZoU2nkOHI/yHadVWan8WlRw6M20iZfIDzeistJcPr4YdHx54hIBKPJ/E7cPaEviFXZiSJLI0+ZXmcd0gVYL3fFx1hd/W5szREhDmTB9C1MPZEiMcO6cYD35sYNtAxFsFL9IjVtxec3sfNdbQFqjQccI7caN5DiWlpBJ66PRzXTYUeNBRaagS1VumkksgKy+6KClpQrb4+CS/PDZkZTS3ckKXSbsrZfd6okG630VPtUAnFw+2jCIrZ0jDgRO9OvLEVkYc2t/HSUmKJ3B7fuLqnHHBWGs17ITy2Oazuu2A8n1bvhmE9YOPbLPjyBNhqTZMeL3yyydUe9N27O/c10w9iaK8izrrTmigqxd9/q4j8Fuy3YGTvEs4/rIoLDuvXqnOMrypj9iF9+b6tFZmIoILx+U2L3YYtxc178KczRvGLhSv5w2kjw9LtMsaraFKtNNz66IPX0NJ+rUSVbbz7EPntJltpxHvP2uNxqdJwwNE91YoXYvIBFU1ROyfeQs3db0FwmvQ4DzuqT8Ol4rpwcrg7pb07MtuSqNtu2870CtedOJTWkuH18MuThzXrGHv0VDpy8qhenDwq2qVqj1UIulNC439sNzeR0gjmTNblu1VawWzNGUUe63jn/c4ZIo+NFf2UDGJJ1B7fuLqnHHCsBGo2J+V89kcdbC2FPsiQ26PjVvptjb1iS6XbLdw91b605rLtlXG88PJUR4K5feedphFxSyLlFG935PvXnEGJrSFV771aGg60VfQU026A2w+DyVdG7fr5CUOYeos1NXO85x98B2Nl+fbYSg4b2NQJOe+UYTGn5d7fsOt0N9/OORP7RK2j4YbZh/TlwB5N06ncfPpI3vq0aZpqr4sO2HTkx8cN5sVVXzG8VzF9A2uD/OXsg/nr4nWU5zeNZE9UmV4z/SD21DUmdWzACcN7cOrB8QNQTh/Xm/+s/qrFUVytck9F7Jo6rDsPL9vA6FasO+OGVOlzVRoO+P2+xJnyK2BPgoFN3YfB3J0xdw3uXsgD35vA2Xe9FeYXDbaagin2aAkroamMG2eF+6rPntD68Nb0xDhuuWlx/Wbm8BadNdJddcroSk4Z3TTQLtgSt/dptBetOV//igLWzJseljaxfzkTIwbQJWrpD+pWyKMXHdpyQVzwl7NjD2C106Ugm8cvOSxhPicSTSIY7x2LvEfFuZk8dnHy7knwsafK0kg795SIzBWRTSKyPPCbbtv3UxFZKyIfi8jUZMrhysQ0rXdeuosTDw+57Vht2uRgrzBT6Z4KtkAb022chtIsEvVppDogwE7QfZ2q9yztlEaAm40xowK/ZwBEZAhwBjAUmAbcJiJtv5B3AFfuhkR5eo1JXESEQoh3mkmDKsjwCLOTMJCqoxHep5E6OQqyMyjPz+JXJw9N+voSkahyajti9d3MO6XJykwnpRHEbuFcM/0g+lfEHoPS1qSr0ojFycACY0ydMeYzYC0wPlkncxWFkcjSOOGmhEX4XVgaQaXRrSibtddPb9Ua3fsLYYPpUlh7Zng9vHPtsWEuKzezzyrpRax3yO7qTUOdESbThZP78/L/m9I+522XszSfy0RkhYjcIyLBqT17ARtseTYG0qIQkTkiskxEllVXt2wyNXfuqQR5JPHtdRNX7k+xOZoORIfcqpNOaTsSR0+l38eXqsZJSpSGiLwkIh/G+J0M/BUYAIwCtgB/DB4Wo6iYNYcx5g5jzFhjzNiKCueZP+PhTzRhYV650+mbcKM0glnjuaeCebQFGyItVYbLeYyU9KMj9WkESZVIKYmeMsYc4yafiNwJPBXY3AjY5+muBJIzaIL4q6Qx9rsw4ya43nm6asCV0ghlbbNMSqqIM0mukua0JnoqZWhHuIWI2BccOAUIruazCDhDRLJFpB8wiNBSRm1PXLdRaGh2AmvEjdJoAy9YZ0TvidKWtGacRqrQkNsmbhSRD0RkBXAkcAWAMWYl8AiwCngOuNQY42IwRcuI36cROWDC4eG5UBrBiId4M8EeeaDlYhvWs9gxz/5OZWBFukMC4whG9i5p0YJG+xPpV411XBKNfE8nS6Np3JaOCAfAGHNunH3zgHntIUfcPo1IS+OyZXDnkVC3KyJfYqXRtzyflb+cGloLOxYzRvTkyMFdyW+jJWg7It2Lc1j1q6nkZnrZW+8jPzuD+WeM5o8RE/GlktYuAqWkjkT1bzstRdMsdER4muFzM3AvmCczB2Kt0OfyqbpRBp1ZYQTJy7LuQfBeWEuRJm2oTrNxu6CPkn4kcj+leg4uO01ju1Jz/jTUn+lB/IjboKURrCUcKq5mdIQr+w8a5dbxaEmfxgHdCmLkbE/UPZVWuHJPBfs0PN7YPbNp1ApWko86pzouiVrtsULin7jksLDlgtubThVy2xGIPyI8ok9DvMSuMrTF2Rlpb0+G9qW0nkSzCsSyNPKzM1LqNk7VTAjqP3HAXcht0NJwuI0tnNCwKjBVdTBiSOkYtHfdnUZu9g7LsF5FiTORZn0akbNetzNqaTjgc7Wghq1PI1aN0cKI4DPH96Z/RT4T+pW16Hilc5BMJfW/q4+iob2WoEshD3xvIhu3702YL52ip1I9NboqDQdMXCsh4mF5HNxTiaYicSpdJGpdAyX9af+1+5JHz5LcVIvQLhTnZlKcm3j8UzpZGqkmjfRnetHoxj0VxJMB5QOj82Vkta1QSoegvXzNXQutFfbKbCvtKckhnQb3BVFLI80wzVnnV7xw1qPw8TOw6DIr7fQHoKRPcoRTFODcQ6oozc/ixBE9Uy3Kfk+iualSQdoN7hORH8U70BiTeLGIDoy/OZ3YHi/kl8MB05rSDprR9kIpaU17d4R7PcLJo+Kvna3sfwSDdNLR0igM/B0MjMOaMBDgROC1ZAqVDsTtCA8+rJwSqN3RtK2D+RQ00FppH9LO0jDG/BJARF4ADjbG1AS25wKPtot0KSR+p2bgac15Bb54y5as1YWir4GSXJqip1Jzfjd9Gn2Aett2PVCVFGnSiPiD+wL7yvpbvyBaWyiK0m6kn3sqyD+BpSLyBFZteQrwj6RKlQbEnUbEcZ8qjc5McNRwOq69oChtRUKlYYyZJyLPApMCSecbY95Lrlipxx+3V9Nhn/ZpdGouP2YQPr/h22N7J86sKC0lxcOB4ioNEfEAK4wxw4B320ek9CC+0nBA3VOdmqKcTOaeNDTVYiidhFRVN3GbxsYaFv2+iHS6AQfxV+5zQC0NRVGSTKrnHXDTp9EDWCkiS4E9wURjzElJkyoNGFCRD1sddjpZIao0FGW/4razD+bt9dtSLUZMUuXXcKM0fpl0KdKQW88cDbc67XXS9eqeUpT9ienDezB9eOdeiz4SNx3hr7aHIPsFamkoipJkUr1+SsJaTkQmisjbIrJbROpFxCciu9pDuJSiHeGKoihRuGka3wqcCawBcoHvEcdx06lRS0NRlCQTbM6m9cp9xpi1gNcY4zPG3AtMac1JReQ0EVkpIn4RGRux76cislZEPhaRqbb0aYG0tSJydWvO74qta5z3OVohamkoitI+pHNH+F4RyQKWi8iNwBYgv5Xn/RA4FfibPVFEhgBnAEOBnsBLInJAYPdfgGOBjcDbIrLIGLOqlXI4s+CsODudoqdUaSiKsn/jxtI4N5DvMqyQ297At1pzUmPMamPMxzF2nQwsMMbUGWM+A9YC4wO/tcaYT40x9cCCQN70QpWGoij7OW4sjQFAtTFmF8kPv+0FvGnb3hhIA9gQkT7BqRARmQPMAejTp9ONS1QUZT8mxcFTriyN72C5ppaIyI0icqKIlCY6SEReEpEPY/ziWQixmuomTnpMjDF3GGPGGmPGVlRUJBJ1vycvy0uP4pxUi6EoShtw2phKAMoKUrOctJtxGucBiEhPYBZW30LPRMcaY45pgTwbsdxfQSqBzYH/ndLbn1Sr+mbywdypiTMpitIhuOyogVw0ZQCZ3tREayZUGiJyDtYMt8OBb7DCbf+bJHkWAQ+KyE1YimkQsBTL0hgkIv2ATVid5fF6qhUbOlW3ouw/iAiZ3tR90276NG4B1gG3A68YY9a39qQicgrwZ6ACeFpElhtjphpjVorII8AqoBG41BjjCxxzGfA84AXuMcasbK0cLadjWRqKoihthRv3VBcRGQpMBuaJyCDgY2PMuS09qTHmCeAJh33zgHkx0p8BnmnpORVFUZTW42YakSKsJV/7Yi3zWgzEWdauE9DB+jQURVHaCjfuqddtv1uNMRuTK9J+QGm/VEugKIqSFNy4p0YAiEi+MWZPovydnkuXQkG3VEuhKIqSFNy4pw4RkVXA6sD2SBG5LemSpTVx3FMVgyG3pP1EURRFaUfcBPreAkwlsI6dMeZ9rE5xRVEUpZPhdpbbDRFJviTI0nHQjnBFUTopbjrCN4jIoYAJzHb7AwKuKkVRFKVz4cbSuAi4FGviwI3AqMC2oiiK0smIa2mIiBc41xhzdjvJk954MsHfgI4IVxSlsxLX0ghM4ZF+61akinHfTbUEiqIoKcVNn8YbInIr8DDWIkwAGGPeTZpU6UpwDXDtCFcUpZPiRmkcGvj7K1uaAY5qe3HSHZ0tVlGUzo2bEeFHtocgHQJdzlVRlE5Oalbx6KiElIa6pxRF6Zyo0mgO2qehKEonR5VGs1D3lKIonRs3Exbmici1InJnYHuQiMxIvmhpiPZpKIrSyXFjadwL1AGHBLY3Ar9JmkRpjSoNRVE6N26UxgBjzI1AA4AxZh+dtfYM9mloR7iiKJ0UN0qjXkRyCdSUIjIAy/LofIQ6wlMrhqIoSqpwM7hvLvAc0FtEHgAOA76TRJnSF9G4AUVROjduBve9ICLvABOx3FKXG2O+Sbpk6Yh2hCuK0slxEz21CDgOWGyMeaotFIaInCYiK0XELyJjbelVIrJPRJYHfrfb9o0RkQ9EZK2IzBdJRQ2uSkNRlM6NG3/LH4FJwCoReVREZolITivP+yFwKvBajH3rjDGjAr+LbOl/BeYAgwK/aa2UofloR7iiKJ2chErDGPOqMeYSoD9wB/Bt4OvWnNQYs9oY87Hb/CLSAygyxiwxxhjgPmBma2RoEUHjRkeEK4rSSXHVsxuInvoW1ip+44B/JFGmfiLynoi8KiKTAmnBVQODbAykxURE5ojIMhFZVl1d3XaSaZ+GoiidnIQd4SLyMDABK4LqL1h9G34Xx70EdI+x6xpjzEKHw7YAfYwxW0VkDPCkiAwldmeCY3PfGHMHllXE2LFj29AsUKWhKErnxk3I7b3AWYFV/FxjjDmmucIYY+oIjAExxrwjIuuAA7Asi0pb1kpgc3PLbzVqaSiK0slxVBoicpQx5mUgDzg5MljJGPN4WwsjIhXANmOMT0T6Y3V4f2qM2SYiNSIyEXgLOA/4c1ufP7GA2hGuKErnJp6lcQTwMnBijH0GaLHSEJFTsCr9CuBpEVlujJkKTAZ+JSKNgA+4yBizLXDYxcDfgVzg2cCvfdGp0RVF6eQ4Kg1jzHWBf39ljPnMvk9E+rXmpMaYJ4AnYqQ/BjzmcMwyYFhrzttqdES4oiidHDe1YKxK/F9tLUjHQPs0FEXp3MTr0zgQGAoUi8iptl1FQGsH93VMtCNcUZROTrw+jcHADKCE8H6NGuDCZAqVtmhHuKIonZx4fRoLgYUicogxZkk7ypS+aEe4oiidHDfjNN4TkUuxXFUht5Qx5oKkSaUoiqKkJW46wv+JNbJ7KvAq1sC6mmQKlbZon4aiKJ0cN0pjoDHmWmCPMeYfwAnA8OSKlaZon4aiKJ0cN0qjIfB3h4gMA4qBqqRJlNaopaEoSufGTZ/GHSJSClwLLAIKgF8kVap0RTvCFUXp5LhZ7vWuwL+vYq2p0XnRPg1FUTo58Qb3/SjegcaYm9penDRHpxFRFKWTE68WLEzw6xwMObnpf+0IVxSlkxNvcN8v21OQtGXc92BVcM0odU8pitK5cbNy373EaFp3msF99k5v7QhXFKWT4yZ66inb/znAKaRi1bx0QDvCFUXp5LiJngqbGl1EHgJeSppEaYfdqlCloShK56Yl4UCDgD5tLUiHIGRpqHtKUZTOiZs+jRqsWlICf78ErkqyXGmEzbqoOtz6O35OakRRFEVJMW7cU50nvDYmNquisDvM3Zk6URRFUVKMm45wRGQE1nxTofzGmMeTJJOiKIqSprhxT90DjABWAv5AsgE6h9LQ8FpFUZQQbiyNicaYIUmXRFEURUl73ERPLRGRNlUaIvJ7EflIRFaIyBMiUmLb91MRWSsiH4vIVFv6tEDaWhG5ui3liY9aGoqiKEHcKI1/YCmOjwOV/AcisqKV530RGGaMGQF8AvwUIKCczsBaWnYacJuIeEXEC/wFOB4YApzZ1opMURRFSYwb99Q9wLnABzT1abQKY8wLts03gVmB/08GFhhj6oDPRGQtMD6wb60x5lMAEVkQyLuqLeRRFEVR3OFGaXxhjFmURBkuAB4O/N8LS4kE2RhIA9gQkT7BqUARmQPMAejTp5XjELUjXFEUJYQbpfGRiDwI/BuoCyYmCrkVkZeA7jF2XWOMWRjIcw3QCDwQPCxGfkNsN5pjbW6MuQO4A2Ds2LFa6yuKorQRbpRGLpayOM6WljDk1hhzTLz9IjIbmAEcbUyoOb8R6G3LVknT5IhO6YqiKEo74WZE+PltfVIRmYY1FckRxpi9tl2LgAdF5CagJ9Y8V0uxLJBBItIP2ITVWX5WW8ulKIqixMfN4L5+wP8RPSL8pFac91YgG3hRrEkA3zTGXGSMWSkij2B1cDcClxpjfAE5LgOeB7zAPcaYla04fzNQ75aiKEoQN+6pJ4G7sfo02ip6amCcffOAeTHSnwGeaYvzK4qiKC3DjdKoNcbMT7ok6YoaGoqiKCHcKI0/ich1wAuER0+9mzSpFEVRlLTEjdIYjjW47yjCJyw8KllCKYqiKOmJG6VxCtDfGFOfbGHSE/VPKYqiBHEz99T7QEnCXIqiKMp+jxtLoxvWqPC3Ce/TaE3IraIoitIBcaM0rku6FIqiKEqHwM2I8FfbQ5C0RScsVBRFCeFmRHgNTb3BWUAmsMcYU5RMwRRFUZT0w42lUWjfFpGZNK1x0QlQS0NRFCWIm+ipMIwxT6JjNBRFUTolbtxTp9o2PcBYOlvze+bt8MWSVEuhKIqSctxET51o+78RWI+11GrnwBgYdab1UxRF6eSkZD0NRVEUpWOSsE9DRP4hIiW27VIRuSe5YimKoijpiJuO8BHGmB3BDWPMdmB08kRSFEVR0hU3SsMjIqXBDREpw11fyH5C5+rzVxRFiYebyv+PwP9E5F9YNei3ibGy3n5L9xGplkBRFCVtcNMRfp+ILMMamyHAqcaYVUmXLNWU9YeeB0Nxr1RLoiiKkja4cjMFlMT+ryjsGAMiqZZCURQlrWj2iPDOhSoNRVEUO6o0HNEOcEVRlEhSojRE5Pci8pGIrBCRJ4LjQESkSkT2icjywO922zFjROQDEVkrIvNF2sF3pO4pRVGUMFJlabwIDDPGjAA+AX5q27fOGDMq8LvIlv5XYA4wKPCbllQJdR0NRVGUKFKiNIwxLxhjGgObbwKV8fKLSA+gyBizxBhjgPuAmUkWE+3TUBRFCScd+jQuAJ61bfcTkfdE5FURmRRI6wVstOXZGEiLiYjMEZFlIrKsurq6hWKppaEoihJJ0kZ2i8hLQPcYu64xxiwM5LkGa+bcBwL7tgB9jDFbRWQM8KSIDCV2k9+xVjfG3AHcATB27NiW1f4G7dNQFEWJIGlKwxhzTLz9IjIbmAEcHXA5YYypA+oC/78jIuuAA7AsC7sLqxLYnAy5I6RM/ikURVE6EKmKnpoGXAWcZIzZa0uvEBFv4P/+WB3enxpjtgA1IjIxEDV1HrAwuVKqe0pRFCWSVE08eCuQDbwYiJx9MxApNRn4lYg0Aj7gImPMtsAxFwN/B3Kx+kCejSy0zVH3lKIoShgpURrGmIEO6Y8BjznsWwYMS6ZcESdst1MpiqJ0FNIheipNMWifhqIoSjiqNOKhOkNRFCUMVRpOqHtKURQlClUacVFTQ1EUxY4qDUfU0lAURYmkE6313QI05FbpoDQ0NLBx40Zqa2tTLYqSxuTk5FBZWUlmZqbrY1RpOKF9GkoHZuPGjRQWFlJVVUV7rCKgdDyMMWzdupWNGzfSr18/18epe8oRDblVOi61tbWUl5erwlAcERHKy8ubbY2q0oiHfnBKB0YVhpKIlrwjqjScUPeUoihKFKo04qItNUVJR66//vpWHf/kk0+yatWqNpImNtOnT2fHjh0AFBQUJPVc7YkqDUfU0lCUdKUjKI1nnnmGkpKSpJ4jFWj0lBPGaJ+Gsl/wy3+vZNXmXW1a5pCeRVx34tC4ee677z7+8Ic/ICKMGDGCf/7zn3z++edccMEFVFdXU1FRwb333kufPn34zne+Q1FREcuWLePLL7/kxhtvZNasWWzZsoXTTz+dXbt20djYyF//+leefvpp9u3bx6hRoxg6dCgPPPAAM2fOZMOGDdTW1nL55ZczZ84cwGrhX3755Tz11FPk5uaycOFC1q1bx6JFi3j11Vf5zW9+w2OPPcaAAQNCcldXV3PRRRfxxRdfAHDLLbdw2GGHMXfuXNatW8emTZvYsGEDV155JRdeeGFMGSdNmkRVVRXLli2jS5cuobKNMVx55ZU8++yziAg///nPOf3001m8eDFz586lS5cufPjhh4wZM4b7778/LfulVGnEJf0emKJ0BFauXMm8efN444036NKlC9u2WSscXHbZZZx33nnMnj2be+65hx/84Ac8+eSTAGzZsoXXX3+djz76iJNOOolZs2bx4IMPMnXqVK655hp8Ph979+5l0qRJ3HrrrSxfvjx0vnvuuYeysjL27dvHuHHj+Na3vkV5eTl79uxh8UlwbwAAEIBJREFU4sSJzJs3jyuvvJI777yTn//855x00knMmDGDWbNmRcl++eWXc8UVV3D44YfzxRdfMHXqVFavXg3AihUrePPNN9mzZw+jR4/mhBNO4KGHHoqS0YnHH3+c5cuX8/777/PNN98wbtw4Jk+eDMB7773HypUr6dmzJ4cddhhvvPEGhx9+eJs9k7ZClYYj6p5S9g8SWQTJ4OWXX2bWrFmhVnZZWRkAS5Ys4fHHHwfg3HPP5corrwwdM3PmTDweD0OGDOGrr74CYNy4cVxwwQU0NDQwc+ZMRo0aFfN88+fP54knngBgw4YNrFmzhvLycrKyspgxYwYAY8aM4cUXX0wo+0svvRTmutq1axc1NTUAnHzyyeTm5pKbm8uRRx7J0qVLXcsI8Prrr3PmmWfi9Xrp1q0bRxxxBG+//TZFRUWMHz+eykprgdJRo0axfv36tFQa2qcRjzQ0DRWlI2CMceVasefJzs4OOx5g8uTJvPbaa/Tq1Ytzzz2X++67L6qMxYsX89JLL7FkyRLef/99Ro8eHRp7kJmZGTqH1+ulsbExoUx+v58lS5awfPlyli9fzqZNmygsLIySN7jtRsbI64qF/frdypoKVGk4oSG3itJijj76aB555BG2bt0KEHJPHXrooSxYsACABx54IGFL+vPPP6dr165ceOGFfPe73+Xdd98FLGXQ0NAAwM6dOyktLSUvL4+PPvqIN998M6F8hYWFIeshkuOOO45bb701tG13gy1cuJDa2lq2bt3K4sWLGTdunKOMsZg8eTIPP/wwPp+P6upqXnvtNcaPH59Q3nRClUZc1NJQlJYwdOhQrrnmGo444ghGjhzJj370I8ByI917772hjvE//elPcctZvHgxo0aNYvTo0Tz22GNcfvnlAMyZM4cRI0Zw9tlnM23aNBobGxkxYgTXXnstEydOTCjfGWecwe9//3tGjx7NunXrwvbNnz+fZcuWMWLECIYMGcLtt98e2jd+/HhOOOEEJk6cyLXXXkvPnj0dZYzFKaecwogRIxg5ciRHHXUUN954I927d08obzoh8cyl/YGxY8eaZcuWNf/A31XBsFlwwh/aXCZFSTarV6/moIMOSrUY+xVz586loKCAH//4x6kWpU2J9a6IyDvGmLGx8qul4YSG3CqKokSh0VNxUaWhKIrF3LlzUy1CWqCWhiP7t9tOURSlJaRMaYjIr0VkhYgsF5EXRKRnIF1EZL6IrA3sP9h2zGwRWRP4zW4HIZN+CkVRlI5EKi2N3xtjRhhjRgFPAb8IpB8PDAr85gB/BRCRMuA6YAIwHrhOREqTJp0aGoqiKFGkTGkYY+yT4eTTVE2fDNxnLN4ESkSkBzAVeNEYs80Ysx14EZiWXCnV0lAURbGT0j4NEZknIhuAs2myNHoBG2zZNgbSnNJjlTtHRJaJyLLq6uoWSqemhqKkmkWLFnHDDTcAVkf0H/7Q/iHwjz76KEOHDsXj8RAZvv/b3/6WgQMHMnjwYJ5//vlQ+nPPPcfgwYMZOHBgSP4gDz30EPPmzeOjjz7ikEMOITs7O+q6nI7/7LPPmDBhAoMGDeL000+nvr7e1TWsX7+eBx98sLmXHpOkKg0ReUlEPozxOxnAGHONMaY38ABwWfCwGEU5rb0as2Y3xtxhjBlrjBlbUVHRMuE15FZRUs5JJ53E1VdfnVIZhg0bxuOPPx6aWDDIqlWrWLBgAStXruS5557jkksuwefz4fP5uPTSS3n22WdZtWoVDz30UNhcVs899xzTpk2jrKyM+fPnR437iHf8VVddxRVXXMGaNWsoLS3l7rvvjpJ3ypQprF+/PiytLZVGUkNujTHHuMz6IPA0Vp/FRqC3bV8lsDmQPiUifXGrhYyLKg1lP+DZq+HLD9q2zO7D4fgb4ma5//77mT9/PvX19UyYMIHbbrsNr9dLQUEB3//+93nllVcoLS1lwYIFVFRUMH/+fG6//XYyMjIYMmQICxYs4O9//zvLli0Lm9YDrKk9LrroIvbu3cuAAQO45557KC0tZcqUKUyYMIFXXnmFHTt2cPfddzNp0qRWXarTIMmFCxdyxhlnkJ2dTb9+/Rg4cCBLly4FYODAgfTv3x+wRp8vXLiQIUOGYIxh+fLlHHzwwYgIXbt25emnnw4rd+nSpTGPP+igg3j55ZdDlf/s2bOZO3cuF198ccJruPrqq1m9ejWjRo1i9uzZXHHFFS2+H6mMnhpk2zwJ+Cjw/yLgvEAU1URgpzFmC/A8cJyIlAY6wI8LpCUJdU8pSktZvXo1Dz/8MG+88QbLly/H6/XywAMPALBnzx4OPvhg3n33XY444gh++ctfAnDDDTfw3nvvsWLFirCpO2Jx3nnn8bvf/Y4VK1YwfPjwUBkAjY2NLF26lFtuuSUsPUhNTQ2jRo2K+WvOwkybNm2id++m9m1lZSWbNm1yTAdr+vORI0fGnczR6fitW7dSUlJCRkZGVLmJuOGGG5g0aRLLly9vlcKA1A7uu0FEBgN+4HPgokD6M8B0YC2wFzgfwBizTUR+DbwdyPcrY8y2pEqo7illfyCBRZAM/vOf//DOO+8wbtw4APbt20fXrl0B8Hg8nH766QCcc845nHrqqQChuaRmzpzJzJkzHcveuXMnO3bs4IgjjgCsFvdpp50W2h8sb8yYMVFuGrAmK7RPQthSYk3BJCL4/f6Y6WC5po4//vgWleuUDnDvvfeG5vFau3Yt06dPJysri379+oWmjG8rUqY0jDHfckg3wKUO++4B7kmmXLaTtctpFGV/xBjD7Nmz+e1vf5swb7Die/rpp3nttddYtGgRv/71r1m5cmWLzh2cYtxpevGamhpHl9WDDz7IkCFDXJ2nsrKSDRuaYnM2btxIz549ARzTX3jhBR577LEWldulSxd2/P/27j+2qrOO4/j7s95uzUBKwWhwXfixEbVd+LEBghIjA2EOA4lZFohEKDSLiZmTmJgRidOEfyCLQ7NmzqBAiFkXO9CJiY2Z+2MmBjd0TmCbhdRJ57AVaAmLYev6+Md5bncpvXBuf116+LySk/Y857k3z/d8b/nynHPuOd3d9Pb2ksvlLnvfhoYGGhoagOScxr59+5gxY0aqOErlb4QX5aJhNlTLly+npaWFzs5OILk1+ltvvQUkz6toaWkBkn+kly5dSl9fH6dPn2bZsmXs2rWL7u5uLl68OOh7V1dXU1NTw0svvQTAgQMH+mcdaeRnGoMtaQsGJCfpm5ubuXTpEu3t7bS1tbFo0SIWLlxIW1sb7e3tvPfeezQ3N7NmzRp6enro7e1l6tSpV33fYq+XxLJly/r33f79+1m7dm3qmIvdCr5UvvfU1fjwlNmQ1NXVsWPHDlauXElfXx+VlZU0NTUxffp0JkyYwPHjx7nnnnuorq7uf77Ehg0b6OnpIYTA1q1bmTx5ctH3379/f/+J8FmzZrF3795Ri+XQoUM8/PDDdHV1sXr1aubNm0drayv19fU8+OCD1NXVkcvlaGpqoqKiAoAnn3ySVatW8cEHH7B582bq6+tpaWlhxYoPrw06c+YMCxYs4MKFC9x0003s3r2bEydOMGnSpEFfD7Bz507WrVvH9u3bmT9/Plu2bEkVw5w5c8jlcsydO5dNmzYN67yGb41ezMGH4I57Ye66kR+U2Si7nm+NPnHixKKziCxrbGyksbEx1fM+xlKpt0b3TKOYr/y03CMwswzZs2dPuYcwInxOw8zG1I04y8gSFw2zjMr6oWcbvqF8Rlw0zDKoqqqKs2fPunBYUSEEzp49S1VVVUmv8zkNswyqra2lo6ODod+w024EVVVV1NbWlvQaFw2zDKqsrGTmzJnlHoZlkA9PmZlZai4aZmaWmouGmZmllvlvhEvqIrmL7lB8FPjvCA5nPHDMNwbHnH3DiXd6CGHQJ9hlvmgMh6RXin2VPqsc843BMWffaMXrw1NmZpaai4aZmaXmonF1N+JdCx3zjcExZ9+oxOtzGmZmlppnGmZmlpqLhpmZpeaiMQhJ90l6U9JJSY+WezwjRdLtkl6U9Lqk45Ieie1TJP1eUlv8WRPbJenHcT+8Junu8kYwdJIqJP1V0uG4PlPSkRjzs5Juju23xPWTcfuMco57qCRNltQi6Y2Y7yVZz7OkrfFzfUzSM5KqspZnST+X1CnpWEFbyXmVtDH2b5O0sZQxuGgMIKkCaAK+BNQB6yWlf9r89a0X+HYI4dPAYuAbMbZHgRdCCLOBF+I6JPtgdlweAp4a+yGPmEeA1wvWdwJPxJjPA/mHLW8BzocQ7gSeiP3Gox8BvwshfAqYSxJ7ZvMs6Tbgm8CCEMJdQAWwjuzleR9w34C2kvIqaQrwGPAZYBHwWL7QpBJC8FKwAEuA1oL1bcC2co9rlGL9NfBF4E1gWmybBrwZf38aWF/Qv7/feFqA2vjHdC9wGBDJN2VzA3MOtAJL4u+52E/ljqHEeCcB7QPHneU8A7cBp4EpMW+HgVVZzDMwAzg21LwC64GnC9ov63etxTONK+U/fHkdsS1T4nR8PnAE+HgI4R2A+PNjsVtW9sVu4DtAX1yfCnSHEHrjemFc/THH7T2x/3gyC+gC9sZDcnskTSDDeQ4hvA08DvwLeIckb0fJdp7zSs3rsPLtonElDdKWqeuSJU0EngO+FUK4cLWug7SNq30h6ctAZwjhaGHzIF1Dim3jRQ64G3gqhDAfeJcPD1kMZtzHHA+vrAVmAp8AJpAcnhkoS3m+lmIxDit2F40rdQC3F6zXAv8u01hGnKRKkoLxixDCwdj8H0nT4vZpQGdsz8K++BywRtI/gWaSQ1S7gcmS8g8hK4yrP+a4vRo4N5YDHgEdQEcI4UhcbyEpIlnO8wqgPYTQFUJ4HzgIfJZs5zmv1LwOK98uGld6GZgdr7q4meRk2vNlHtOIkCTgZ8DrIYQfFmx6HshfQbGR5FxHvv1r8SqMxUBPfho8XoQQtoUQakMIM0hy+YcQwleBF4EHYreBMef3xQOx/7j6H2gI4QxwWtInY9Ny4AQZzjPJYanFkm6Nn/N8zJnNc4FS89oKrJRUE2doK2NbOuU+qXM9LsD9wD+AU8B3yz2eEYxrKck09DXg1bjcT3Is9wWgLf6cEvuL5EqyU8DfSa5MKXscw4j/C8Dh+Pss4M/ASeCXwC2xvSqun4zbZ5V73EOMdR7wSsz1r4CarOcZ+AHwBnAMOADckrU8A8+QnLN5n2TGsGUoeQU2x9hPAg2ljMG3ETEzs9R8eMrMzFJz0TAzs9RcNMzMLDUXDTMzS81Fw8zMUnPRMBtBkqZKejUuZyS9XbB+84C+rZI+co3365A0eXRHbZaeL7k1GyWSvg9cDCE8PqBdJH97fYO+8PK+HcBdIYTu0RmlWWk80zAbA5LujM95+AnwF2Ba4SxC0m8kHY3Pg2gs72jNistdu4uZjZA6km/ffh0gmXD02xhCOCfpVuAVSc+FEM6XY5BmV+OZhtnYORVCeLnItq2S/gb8ieQGcneM3bDM0vNMw2zsvDtYo6QVwOeBxSGE/0n6I8m9kcyuO55pmJVfNXAuFox6YGG5B2RWjIuGWfn9Frg1Hp76HsnTFM2uS77k1szMUvNMw8zMUnPRMDOz1Fw0zMwsNRcNMzNLzUXDzMxSc9EwM7PUXDTMzCy1/wNzQGmmL5lMzgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "env =  GridworldEnv()\n",
    "env.reset()\n",
    "#print(env.P)\n",
    "agent_DQ = Agent_DQlearning('5', env, 4,100,0,0.5)\n",
    "t = timeit.default_timer()\n",
    "agent_DQ.main_loop()\n",
    "t1_DQ = timeit.default_timer() - t\n",
    "print(\"constatnt time = \"+str(t1_DQ))\n",
    "agent_DQ2 = Agent_DQlearning('5', env, 4,100,0,0)\n",
    "t = timeit.default_timer()\n",
    "agent_DQ2.main_loop()\n",
    "t2_DQ = timeit.default_timer() - t\n",
    "print(\"dynamic time = \"+str(t2_DQ))\n",
    "plt.plot([i for i in range(1000)],agent_DQ.R_sum, label=\"constant epsilon\")\n",
    "plt.plot([i for i in range(1000)],agent_DQ2.R_sum, label=\"epsilon = 100/100+t\")\n",
    "plt.xlabel(\"Trial\")\n",
    "plt.ylabel(\"cummulative reward\")\n",
    "plt.title(\"DQ policy result\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}