MountainCar-v0-CrossEntropy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5f786d31",
   "metadata": {},
   "source": [
    "# MountainCar-v0-Cross-Entropy Method\n",
    "Implement of a corss-entropy method in policy-based methods\n",
    "\n",
    "### Import Necessary Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4c2a0c47",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pyvirtualdisplay in /home/eight/anaconda3/envs/torchunity/lib/python3.6/site-packages (3.0)\r\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import math\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "\n",
    "!python -m pip install pyvirtualdisplay\n",
    "from pyvirtualdisplay import Display\n",
    "display = Display(visible=0, size=(1400,900))\n",
    "display.start()\n",
    "\n",
    "is_ipython = 'inline' in plt.get_backend()\n",
    "if is_ipython:\n",
    "    from IPython import display\n",
    "\n",
    "plt.ion()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea41e1ee",
   "metadata": {},
   "source": [
    "### Specify the Environment, and Explore the State and Action spaces\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cecb298d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create an environment and set random seed\n",
    "env = gym.make('MountainCarContinuous-v0')\n",
    "env.seed(101)\n",
    "np.random.seed(101)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "911fb143",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "observation space:  Box([-1.2  -0.07], [0.6  0.07], (2,), float32)\n",
      " -low:  [-1.2  -0.07]\n",
      " -high:  [0.6  0.07]\n",
      "action space:  Box([-1.], [1.], (1,), float32)\n",
      " -low:  [-1.]\n",
      " -high:  [1.]\n"
     ]
    }
   ],
   "source": [
    "# Explore state (observation) space\n",
    "# state: [position, speed]\n",
    "print('observation space: ', env.observation_space)\n",
    "print(' -low: ', env.observation_space.low)\n",
    "print(' -high: ', env.observation_space.high)\n",
    "\n",
    "# Explore the action space\n",
    "# action [action]\n",
    "print('action space: ', env.action_space)\n",
    "print(' -low: ', env.action_space.low)\n",
    "print(' -high: ', env.action_space.high)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fbc360a1",
   "metadata": {},
   "source": [
    "### Watch a random agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4960b603",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score:  999\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAADnCAYAAABBu67aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAi+ElEQVR4nO3deVyNef8/8NepTtmZlDTGzthlnbFNY52UfSlFyU4RoTH4YubG2LIvZddiiYrMWB+WIQnD2McgM5g7pdFC2lTnfH5/+NXNSFLX6Tqd83o+Hv7Q6VzX+1heffZLIYQAEREVnYHcBRAR6QoGKhGRRBioREQSYaASEUmEgUpEJBGjD7zOJQBERO9S5PVFtlCJiCTCQCUikggDlYhIIgxUIiKJMFCJiCTCQCUikggDlYhIIgxUIiKJMFCJiCTCQCWiEk0Igfj4J1Cr0+QuBYoPHDDNradEpNWEEPDw6IipU6vC2LgGFAoTVKniAQOD0lAolDA0rKCJ2+a59fRDe/mJiLSeSpWN588P5P7+2bONUCgUMDauCVPTYQCAcuU6okyZ1gAAhcIYCoX0HXQGKhHpHLU6GQCQnn4TT57cBAAYGlaEgUEFKBQKmJu7o1SpJqhYsRcUijwbm4XCQCUinVe6dDMolZ8CMETVql4wNDSFgUEpye/DQCUinaFQKFG6dDMABihdunlud7906UZQKi01fn8GKhGVeKVLN0bt2tNhYFAaFSvaATAEAEm78wXBQCWiEs/IqApMTYfIXQbXoRIRSYWBSkQkEQYqEZFEGKhERBJhoBIRSYSBSkQkEQYqEZFEGKhERBJhoBIRSYSBSkQkEQYqEZFEGKhERBJhoBIRSYSBSkQkEQYqEZFEGKhERBJhoBIRSYQn9hORzklLS0NKSgoAIC4uDps2bYIQosDvHzBgAJo3bw4AKFWqFCpUqFCg9zFQiajEU6vVSEtLw/Hjx/H777/jt99+w8mTJ9967WNs374dRkav47Fx48bo06cPqlSpAmdnZwBAmTJl8nyf4gOpXfBIJyIqRkII3LlzBxcvXoS3tzdevnyJpKQkpKenA/jfA/pMTU3Rt2/fj3pgX3h4OP7888+37qVUKmFubg4AePLkSZ4XYwuViEqUly9f4uzZszh27BhCQ0Px9OnT3NesrKzw2WefoWPHjujbty8AwMTEBHXr1v2oQI2OjsaLFy8AANevX8eePXuQnJyMc+fO5fs+tlCJSOsJIXD9+nXcuXMHK1euxNWrVwEAFhYWqF27NurUqYPx48ejQYMGsLCw0EgNKSkpufe1trbOM50ZqESktbKzs3H37l0sWbIEYWFhSE1NhbGxMWrWrAkbGxuMHj0aVlZWAPBRLVAJMFCJqGTIzMzE/fv34e3tjaCgIGRmZqJy5cqoW7cupk6disGDB8PAwAAGBrKt/GSgEpF2E0Lg1q1b2LBhA3bu3In09HRUqFABY8eOhYeHB8zMzN47w17MGKhEpJ2EEMjMzMS+ffswf/58PHjwAKampnBxccG0adNQrVo1GBoayl3mmxioRKSdzp07h4kTJ+LOnTswNzeHvb09pkyZgjp16hT32GhBcdkUEWmX5ORkeHl54aeffsKzZ8/Qpk0beHt7w9raWu7SCoV7+Ymo2AkhcOjQIfTu3RtbtmxBhQoVEBwcjPDw8BIbpgC7/ERUzBITE7Fw4UL4+fkhJSUFEydOxNixY9GoUSNt7d7nhV1+IpKPWq1GfHw8RowYgWPHjqFp06bw8PDAyJEjc/fNl3RsoRKRxgkhsGvXLnh5eSEpKQkODg6YN28e6tevL3dphcUWKhHJY9euXXBzc0NWVha8vb3h7u6uM63SN+neJyIirREbGwt3d3ecOnUKrVq1wrJly9CmTRttW1MqGQYqEWlETEwMhgwZgoiICFhbWyMoKAiWlpZyl6VRXDZFRJLKzs7G5s2b0a9fP1y7dg0LFy7UizAF2EIlIgmpVCr4+PjAy8sLSqUSmzZtwtChQ+U8xKRY6cenJCKNE0LAx8cHM2bMQPv27bFt2zY4OTnpTZgCXDZFRBIQQmD9+vX47rvv0LVrVwQEBMDU1FTusjQpz2VT+vOjg4g0QqVSYd26dfjuu+/QrVs3+Pv763qYvhfHUImo0KKiojBjxgwcPXoUPXr0gJ+fHypXrix3WbJhoBJRoURFRWHAgAG4c+cO7Ozs9LplmoNdfiIqlDlz5iAmJgbff/+9PoyZFghbqERUYEIIPHr0CMePH8ezZ88QGBgIW1tbvZrJzw8DlYgK7NGjRxg0aBAePXqEgIAA2NnZlaQj9zSOP1aIqEAePHiAgQMH4vHjx/D390fv3r0Zpv/CFioRfVBUVBQGDRqE6Oho+Pv7o0+fPnKXpJUYqESUr5zZ/JiYGAQGBsLOzk7ukrQWA5WI3iunmx8TE4OdO3fC1taW3fx8MFCJKE9CCHh4eOD27dtYvnw5evbsyTD9AE5KEdE7cvbmnz17Fr169cLIkSO5NKoA2EIlorfkHMHHvfkfjz9yiChXdnY2NmzYgBkzZqBbt27w8/NjmH4EBioRAfhfmH777bfo2rUr/P399fqgk8JgoBIRVCrVWy3TwMBAtkwLgQdMExEuX76MTp06oWLFirh16xYsLCzkLknb8YBpInpXbGwsvLy8YGxsjJUrV8Lc3FzukkoszvIT6bHY2Fg4Ojri2rVr8PX1xbBhw7jWtAgYqER6KiYmBkOGDMH169cZphLhGCqRHnr58iVsbW0RGRmJ7du3w9XVlWH6cTiGSkSAWq1GWFgYLl++jPbt26NXr14MU4kwUIn0iBACu3btgpubG1q3bo19+/ZxEkpCDFQiPbJz5064ubmhRYsWCAkJQbVq1eQuSafkG6hnz55FRkZGcdVCRBqUkJCAFStWQK1WY9q0afj000/lLknn5BuoXbp0wezZs5Genl5c9RCRBiQmJsLV1RX379/HsmXL0L9/f7lL0kn5BqqDgwNWrVqFBQsWFFc9RCSxxMREuLi44NSpU1i2bBkmTpzIo/g0JN8/1cmTJ8PCwgJ79uzB7du38YElVkSkZdRqNTZs2IAjR47AxsYG7u7unNHXoHzXoQohRHh4OBwcHPDJJ5/gwIEDaNiwIf9CiEoAIQSOHDkCFxcXfPrppzhw4ADq168vd1m6Is8Q/ODCfiEEgoKCMHToULRq1QoXL16EUqnUTIlEJJlDhw5h+PDhqFatGkJDQ/H555/LXZIuKdzCfoVCATs7Ozg6OuL27dtYtmwZsrKypC+PiCSTnJyM5cuXIyUlBRMnTmSYFpMC7eWvWLEiNm7cCCEE5s+fDyEEZsyYAWNjY03XR0QfKTk5GRMnTsSFCxcwZ84cjBkzRu6S9MZH7eX/+++/0aJFC6SmpuLy5cto3ry5Zqsjoo8ihMCoUaPg5+eHmTNnYuHChTA0NJS7LF1U9L381atXh5+fH0qXLo3hw4fjr7/+kqY0IioyIQQiIyNx+PBh1KtXD66urgzTYvbRp02p1WocOnQIzs7OqF+/PsLCwlC9enXNVUhEBXLx4kUMGjQISqUSBw8eRPPmzbkiR3OkOW3KwMAAtra2sLa2xtWrVxEcHMz1qUQye/XqFXx9fRETEwNHR0dYWVkxTGVQqO0SSqUSfn5+6NmzJ+bOnYuNGzdCrVZLXRsRFUB6ejq8vLywe/duTJo0CfPmzZO7JL1VpAOmIyMjYWNjg4oVK+Ly5cuwtLSUtjoi+qA5c+Zg0aJFcHBwwI4dO1C6dGm5S9IH0h8w3b59e6xbtw5JSUlwdHREbGxsUS5HRB/p6tWr2LlzJywtLTF58mSGqcyK/AgUtVoNPz8/jBs3Dh06dMDBgwfxySefSFchEeXp3r17sLOzw/Pnz3Ho0CG0a9eO46bFRzOPQDEwMEC/fv3QunVrXLhwAUePHuUkFZGGqVQqBAQE4OHDh+jTpw+++OILhqkWkOQMr8qVKyMkJAStW7fG+PHjsXfvXikuS0R5UKlUWLhwIZYvX45hw4Zh/fr1XG+qJSR96umePXswcuRINGvWDMePH4epqWnRqiOid0RFRaFjx45QqVQ4efIkWrZsKXdJ+kjzTz0dMmQIFi9ejNu3b2P06NFISkqS8vJEei8qKgqDBw9GdnY2/P39GaZaRtJANTAwwJQpU/DFF18gLCyMi/6JJJSdnY2lS5fi5s2bGDZsGHr16iV3SfQvknb5gdf7iR88eICBAwciNjYWu3fvRo8ePThgTlQE2dnZ2LBhA2bMmIHu3bsjMDCQQ2ryKtwB04W927lz59C5c2dYWlri1q1bXEpFVATr1q2Dl5cXunbtil27djFM5af5MdQ3ffHFF5g5cybi4uIwdepUpKamaupWRDotNjYWmzZtglKpxKRJkximWqxAB0wXhomJCebNm4eHDx8iICAAVatWxZIlSzR1OyKdlHPYyd9//41NmzbB1tZW7pIoHxp9lqyJiQm8vLxQu3ZtBAYGIiIiQpO3I9IparUaO3fuzB0+Gzp0KB//rOU0NoaaewEhcOPGDXTr1g2lSpXCiRMn0Lhx46JelkinqVQq7N69G25ubmjdujWCgoJ4+JB2Kd4x1Ny7KhRo2rQpnJyc8PTpU2zevBmZmZmavi1RiRYXFwdPT09kZWVhzZo1DNMSQmNjqG/dxMgIK1asgEqlyt0m5+3tze4LUR5evHiB0aNHIzk5GStWrEDTpk3lLokKSONd/jfduHEDNjY2EELgl19+QaNGjbg+legNQgiEhYXB3t4eTZo0walTp2BmZiZ3WfQuebr8b7KyssKePXugUCgwYMAA/P7778V5eyKt9/PPP2P06NFo2rQpQkNDGaYlTLH3ubt06QJHR0fcv38fq1at4ngq0f+XnJyMH3/8EampqZg+fTrq1asnd0n0kYplDPXfFixYgLi4OAQEBKBmzZqYOXMmjI2N5SiFSCukpKRgwoQJuH79OubMmYOhQ4fKXRIVQrGOob7pn3/+QdOmTZGYmIgLFy6gbdu2mroVkVYTQmD79u0YO3Ys2rdvj/DwcJ5vqv3kH0N9k5mZGbZt24by5ctj3LhxePjwoVylEMlGCIHIyEjMnTsXderUga+vL1e/lGCytVCB1/+Y1q5dC09PT3zzzTc4duwYZ/1Jrzx//hwtWrTA48ePcfr0aXTp0kXukqhgtKuFCrxe9O/s7AxbW1tERETA19cXarVazpKIik16ejrmzJmDJ0+eYMqUKWjXrp3cJVERydpCzREfH49u3bohKioKu3btwoABA4rjtkSymjdvHhYuXAh7e3v4+fnxEdAli/a1UHOYmZlh2rRpUCgUWLVqFWJjY+UuiUijrl69ioCAAFhaWsLT05NhqiO0ooUKvD5Zx8/PD2PHjkWHDh3w888/o1KlSsV1e6Jic+/ePdja2uL58+c4fPgw2rVrx7mDkkd7W6jA6+dR9e3bF23btsWlS5dw5MgRjqeSzlGpVAgICMCjR49y/70zTHWH1rRQczx48CD3SY5XrlxBgwYNirsEIo1QqVRYsGABFi1aBEdHR/j6+qJs2bJyl0WFo90t1By1atXC/PnzkZWVhRkzZiAxMVHukogk8fDhQ2zcuBEVKlSAp6cnw1QHaV2gGhkZYfLkyfj222/x008/wcPDg4+iphIvKioKAwcORFZWFvz9/dGqVSu5SyIN0LpABQBDQ0OMHDkSzZs3x7Fjx3Ds2DGGKpVYWVlZ8Pb2xq1bt+Do6Ag7Ozu5SyIN0box1NwbC4E///wTPXv2RFJSEg4cOABra2u5yiEqNF9fX0yZMgVfffUVQkJC+Eh13VAyxlBzKBQK1KtXD87Oznj+/Dk2btyItLQ0ucsi+ihPnjzBli1bYGxsDDc3N4apjpPl+L6PMWvWLGRlZWHp0qVQKpXw9fVFmTJl5C6L6INiY2MxZMgQ3L9/H5s3b+YOQD2gtS3UHCYmJnB3d0f16tURFBSEX3/9Ve6SiD5ICIH9+/fj/PnzaN++Pezt7Xkknx7Q2jHUf7t69Srs7e2Rnp6OvXv34quvvpK7JKI8qdVq7Ny5ExMnTkSbNm2wZ88eVK1aVe6ySFp5jqGWmEAVQmDDhg3w9PSEtbU1Dh8+zP3PpJWePXuGVq1aISEhAcePH+cPf91Usial/k2hUGDcuHFwc3PDuXPnMGvWLLx69UrusojekpCQABcXFzx79gze3t7o2LGj3CVRMSoxLdQcGRkZaNu2Lf744w8EBwejf//+3AtNWiE5ORlDhgzBiRMnsHLlSkycOJHjprqrZLdQc5iYmGD9+vUwMzPDrFmz8Mcff3DRP8lOCIHTp0/j5MmTaNy4MZycnBimeqjEtVBzHDhwAAMHDkTTpk1x6dIlLqUiWR08eBAjRoxArVq1EBISgrp168pdEmmWbrRQc3Tt2hVOTk64f/8+VqxYgczMTLlLIj2VnJyMJUuWIC0tDZ6engxTPab1C/vfp2LFiti0aRMAYMGCBVCr1Zg1axaMjY1lroz0ycuXLzFhwgRcvXoVc+fOhbOzs9wlkYxKbJc/R1xcHJo1a4bExERcuHABbdu2lbsk0hNCCPj5+WHUqFFo164dIiIiOG6qP3Sry5/D3Nwc27dvR/ny5TF27Fj89ddfcpdEekAIgfPnz2P27NmoV68eNm/eDAODEv/fiYqoxLdQgdc7U9asWYNp06bBxsYGR48e5VIq0qgXL17AysoKjx8/xqlTp9C1a1e5S6LipZstVOD186iGDx8OOzs7hIeHw8fHh0upSGPS09Mxe/ZsPHnyBFOmTEH79u3lLom0hE60UHMkJCSgS5cuePDgAfbs2YN+/frJXRLpmLS0NHh5eWHr1q2YOHEiFi1axC3Q+qlk7+UvqICAALi7u6NVq1bYu3cvLC0t5S6JdMjp06fRvXt31KxZE1evXuX5pvpLPwJVCIHAwEC4ubmhZcuWCA4OZqhSkQkhcO3aNQwaNAhZWVkICQnBl19+ybF6/aW7Y6hvUigUcHZ2hrOzM86fP4/58+dDrVbLXRaVcCqVCpMmTcLjx48xf/58hinlSecCFXg9STV37ly0a9cOgYGBCAoK4iQVFZpKpcL8+fNx5coVuLi4YMiQIQxTypNOBioAfPbZZwgJCYGVlRUmTJiA3bt3M1Tpo2VlZeE///kPlixZAicnJ6xfvx5ly5aVuyzSUjo3hvpvYWFhGDp0KOrVq4czZ87A1NRU7pKoBLlz5w46dOgAIyMjhIeHo3HjxnKXRNpBP8ZQ/61fv35YsmQJoqKiMGLECCQlJcldEpUQd+/ezX0WVGBgIBo1aiR3SaTldL6FCrzeSbVq1Sp4eXmhb9++2LdvH0xMTOQui7TYkydP0KNHDzx8+BD79+9Hz549OW5Kb9LPFirwepLK3t4eTZo0wenTp3H69GmOp9J7qVQqBAUF4e7du+jRowe+/vprhikViF4EKgDUqFED+/fvR/Xq1eHs7Ixjx44xVOkdQgisW7cO//d//4fevXvD39+fh5dTgelFl/9NN2/eRLt27VCmTBn8+uuvqFOnjtwlkZZQq9VYt24dZs6ciW+++QZ+fn7cCUXvo79d/jc1bNgQXl5eSE5Oxg8//ICXL1/KXRJpiZiYGHh7e8PY2BgzZ85kmNJHK7En9heWsbEx5s2bBwMDAyxevBgA4OPjg3LlyhX52tHR0ShfvjwqVqxY5GtR8YqOjoaDgwNevHiBzZs3o127dnKXRCWQ3gUqABgZGWHu3LnIyMjA0qVLYWRkhK1btxb6gOA393nXrl0b7u7uGDhwIA8cLiGio6MxePBg/P7779i8eTMcHR05CUWFI4TI75dOu3PnjqhTp44wMzMTkZGRQq1WF+o6WVlZokOHDgKvx5xFuXLlRFBQkFCpVBJXTFJTqVRi1KhRAoCYNGkS/86ooPLMTL1uQjVq1AjBwcEoW7Ys7O3tcenSpY+e+VepVFiwYAEuX76c+7WUlBSMGTMGwcHBXEmgxYQQ2LlzJ4KCgtCpUyfMnj2bvQoqmvclrdCDFmqOK1euiJo1a4pq1aqJiIiIAr8vKytLzJs3TyiVytzW6Zu/ypcvL9avXy+ePn2qweqpMNRqtfDz8xPlypUT1tbWIiYmRu6SqGTJMzP1btnU+1y8eBGdO3dG5cqVceHCBdSoUSPf78/MzMTChQuxePFiZGdn5/u9Xbt2xe7du2FhYSFlyVQEZ86cQZ8+fWBubo7IyEhUrVpV7pKoZOGyqfy0aNECY8aMwT///IMVK1YgPT093+9fs2YNfvzxxw+GKfD6lHcnJyfExcVJVS4VQUJCApYtWwaVSgUvLy/+oCPpvK/pKvSoy58jIyNDeHh4CCMjIzF58mSRnp6e5/fdvXtXNGzYUBgbG4uyZcuKsmXLCgMDgzy7/W/+sra2FrGxscX8qehN8fHxwsbGRpQqVUps2LCh0BORpPfY5S+IzMxMTJ8+HZs2bYK7uzuWLVsGY2Pj3Nejo6PRp08fvHr1Cm3btsVnn30G4PUOrEuXLuHZs2f5Xr9Tp05wd3eHg4MDDA0NNfpZ6G0JCQkYOnQozp49ixUrVmDChAn8O6DCyrPLr5frUPNjbGwMb29vCCGwYcMGKBQKLF26NDdUzc3N0a1bN5iYmLwVtG3atEGtWrUQFBSE+Pj4914/IiICV65cwYsXLzB+/HiudywmCQkJGDZsWG6Yjh8/nmFKkuMYah5KlSqFZcuWoVOnTli3bh22bNmS+1pCQgJMTU3fCtMcZmZmGDx48Af/o2ZkZOS2gj/QQyAJxMfHw8XF5a0wNTJiW4Kkx0B9jzJlyuD777+Hubk5Vq9ejRs3bgB4/UiMrKysfN9XkFZnWlpabqjyIYKaI4TAkSNHcPToUbRo0QKjR49mmJLGMFDz0blzZ+zduxepqakYOHAgrl+/Lun109LSMHXqVDg4OOCvv/6S9Nr0OkwPHjwIT09PtG7dGkFBQShVqpTcZZEOY6B+wFdffYXg4GCkp6drJFQzMjIQGhqKgQMHMlQlpFar8fPPP2PEiBGoXbs2QkJCPri2mKioGKgfoFAo0KFDB+zbtw/p6ekYM2YM4uLi3jv2+fjx49wuvEKhgFKphFKp/OCWxhs3bmDQoEGIiYmR/DPoG7VajcOHD8PV1RW1atVCcHAwatWqxQlA0jgGagEoFAp07NgRe/fuhZGREfbu3fvO8iiFQgErKytYW1vDwMAAVapUQc+ePTFlyhRMmTIFtra2sLS0zPc+qamp+Y7P0ocJIXDo0CEMHz4cNWrUQGhoKA8Rp2LDdagfQQiB8PBwDBkyBIaGhpg/f37uOao5a1IVCgUWL16MxMTEdw4ofvHiBYKDgxEdHf3OtevXr4/9+/ejadOmxfJZdNXBgwcxYsQI1KxZE6Ghoahbt67cJZFuyrO7w0AthDNnzsDJyQnlypVDSEgIrKyscl9LTU3F1q1b8fz58zzf++LFC2zduvWtJwU0bNgQISEhaNKkiaZL12lhYWEYNWpU7php7dq15S6JdBf38kulc+fOOHr0KFq2bPnORJVKpUJKSsp73/vvE/0bNWqEAwcOMEyLIGc2f9SoUahTpw7279/PMCVZMFALqUWLFvD09Pzo2X+FQoGWLVsCeP3kgB9++AENGjTQYKW6LWc2f8KECWjQoAFCQ0M5m0+yYaAWQfv27REcHIyMjAwMHjwYN2/e/OB71Go1IiMjYWhoiKlTp2LAgAGcfS4ktVqNQ4cOwdXVFZ06dcIvv/yCmjVr8s+TZMNALYKcJVVBQUFITU3F4MGDERUVle/Zms+ePUNGRgY8PT2xYMECKJXKYqxYd+TM5ru6uqJ69epYsmQJF+2T7DgpJQEhBM6ePQtHR0dUrFgRO3bswB9//PHObL6lpSXKli2LqKgozJw5kwFQSPHx8QgJCcGsWbM4m09y4Sy/puXM/iuVStjZ2cHZ2RlRUVEAXh+4Ymtri0qVKslbZAkXHx8PV1dXHDlyBK1atUJwcDDXmZIcGKjFITw8HI6Ojnj69Ck8PDywePFilClTRu6ydEJCQgJcXFxw5swZfPnll9ixYwdq1aold1mkn7hsqjh89dVXOHjwIMaMGQMfHx/MmjULGRkZcpdV4uWcZ/rLL79g+fLlOHXqFGrWrCl3WURv4TlmElMoFGjbti2srKxgYmICHx8fxMfHw8PDA19++SVnoD+SSqXC/v374evri8jIyNzzTPm4Z9JG7PJrUEZGBry8vODj44NPP/0UISEhDNWPIIRAQEAA3NzcoFKpsHLlSh4OTdqCY6hySEtLw8KFC7Fr1y6oVCosWbIE/fv3zz0DgPIWExODvXv3Yu7cufj8888xbtw4jBkzhmFK2oKBKqfffvsN9vb2ePToEVxcXLB+/XqUL19e7rK00pMnT+Do6IiIiAh07twZe/bsyXdtL5EMOCklp9atW2P//v1o06YN9uzZA1tbWxw+fJjPlHpDdnY21q5dC1tbW9y4cQN2dnYMUypR2EItRkIIqFQqLFiwAIsXL0b58uXh7++Pr7/+Wu9bqwkJCfD398fMmTNhYmKCTZs2wdHRkZNPpK3Y5dcW2dnZ8Pf3x+rVq3Hv3j3Y2Nhg27ZtMDMz07sAycrKwoMHD+Dg4IC7d++ie/fucHd3R69evfTuz4JKFAaqtrl//z4GDBiAe/fuwdTUFKtXr4aDg4NeTLwIIZCdnY2FCxdizZo1ePnyJXr16gV/f/93DuYm0kIMVG0UFxeHXbt2YdasWTA2NkbHjh3h5+cHCwsLnV1elZGRgZ9++glLlizB7du3Ub58eWzfvh2dO3d+66xYIi3GQNVW2dnZuHbtGqZNm4aIiAh8/vnnGD9+PAYPHqxzZ3vevn0bPj4+2LRpE9RqNYYPH47vvvsOjRs3lrs0oo/BQNV2MTExCA0NxezZs5GSkoIGDRrA1dUV06dPh7GxsdzlFZoQAlFRUVizZg1CQ0MRHx+PZs2aYebMmbCzs9P7CTkqkRioJYEQAidPnsTatWtx9uxZpKamon///pgxYwasrKxK3JF/ycnJOH36NEaNGoWkpCRUrlwZrq6uWLx4MZRKpc4Oa5DOY6CWJJmZmTh79ix8fX0RFhYGpVKJQYMGoVmzZpgwYQIqVaqktWGUnZ2N//73v9iyZQsuXryIc+fOwcTEBK6urnBzc0PDhg31YuKNdBoDtSRKSUnB0qVLsW3bNjx9+hQKhQKWlpbw9PTEqFGjUKFCBa0Ip5w1to8ePcKWLVuwZcsWJCUlwcDAAE2aNEFQUBDq16/PJxSQrmCgllRCCERHR8PX1xc3btzAsWPHoFarUaNGDTRq1AjTp09H9+7dAaDYW61CCLx69Sp31j42NhZPnz5FuXLlYGNjg27dumHQoEGoUqVKsdZFpGEMVF3w6tUr/Prrr1i0aBHOnDmDjIwMlClTBo0bN0bv3r1hbW0NCwsLjc6aCyFw+fJlpKamIjAwENeuXcPNmzehVqtRuXJl9OrVC9OnT0ezZs20dliCqIgYqLpErVYjPDwct27dwtq1a/HgwYPc1ywtLWFlZYVq1arBw8MDCoUC5ubmsLS0LNS9/vzzT6SmpiI1NRVLly7Fq1evEBERgZSUFACAoaFh7qx93bp10aZNG0k+I5EWY6DqIiEEEhMTcfjwYRw9ehQRERFIS0tDYmIigP8NATRu3BitWrV66729evVChw4d3vpaYmIiVq9eDZVKlfu1EydOIC4uLvd+AGBhYQGlUgknJye0bt0aAwYM4Kw96RMGqq7LysqCSqXC3bt3cfz4cTx9+hQ7duyAEAKZmZnvPIrFyMgIhoaGb30t53vfVLZsWRgaGqJMmTJwc3NDqVKlMGzYMFSuXBlKpfKdaxDpAQaqvsnOzsaLFy8AABcuXMCJEyfeev38+fO4efPmW1+rUKECnJyc3jqYZPjw4ahVqxYUCgUqVarEQ0uIGKj0bwkJCbmBm8PIyAjVq1dn150ofwxUIiKJ8MR+IiJNYqASEUmEgUpEJBEGKhGRRBioREQSYaASEUmEgUpEJBEGKhGRRBioREQSYaASEUmEgUpEJBEGKhGRRBioREQSYaASEUmEgUpEJJEPPdCdpwwTERUQW6hERBJhoBIRSYSBSkQkEQYqEZFEGKhERBJhoBIRSeT/AYO72H3A6UF8AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "state = env.reset()\n",
    "img = plt.imshow(env.render(mode='rgb_array'))\n",
    "for t in range(1000):\n",
    "    action = env.action_space.sample()\n",
    "    img.set_data(env.render(mode='rgb_array'))\n",
    "    plt.axis('off')\n",
    "    display.display(plt.gcf())\n",
    "    display.clear_output(wait=True)\n",
    "    state, reward, done, _ = env.step(action)\n",
    "    if done:\n",
    "        print('Score: ', t+1)\n",
    "        break\n",
    "\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8be83d6",
   "metadata": {},
   "source": [
    "## Train an agent to perfrom better.\n",
    "\n",
    "Explore the state and action spaces, as well as sample them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "49f9ce64",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Observation space:  Box([-1.2  -0.07], [0.6  0.07], (2,), float32)\n",
      "state shape:  (2,)\n",
      " -low:  [-1.2  -0.07]\n",
      " -high:  [0.6  0.07]\n",
      "State space samples:\n",
      "[[-0.30670425  0.06603958]\n",
      " [-0.15175831 -0.03713643]\n",
      " [-0.5649248  -0.02363089]\n",
      " [ 0.5595818  -0.04409696]\n",
      " [-0.5084649   0.05890182]\n",
      " [ 0.30128193  0.06887893]\n",
      " [ 0.27741018  0.00126783]\n",
      " [-0.13336651 -0.0424035 ]\n",
      " [-0.51184523  0.0341886 ]\n",
      " [ 0.05956766  0.0532283 ]]\n",
      "\n",
      "\n",
      "\n",
      "Action space:  Box([-1.], [1.], (1,), float32)\n",
      "action shape:  (1,)\n",
      " -low:  [-1.]\n",
      " -high:  [1.]\n",
      "Action space samples:\n",
      "[[-0.3601283 ]\n",
      " [ 0.79807544]\n",
      " [ 0.09698661]\n",
      " [ 0.28053644]\n",
      " [-0.51310056]\n",
      " [ 0.2777974 ]\n",
      " [ 0.00765343]\n",
      " [ 0.09583376]\n",
      " [ 0.9308967 ]\n",
      " [ 0.7934322 ]]\n"
     ]
    }
   ],
   "source": [
    "# Explore state (observation) space\n",
    "# state: [position, speed]\n",
    "print('Observation space: ', env.observation_space)\n",
    "print('state shape: ', env.observation_space.shape)\n",
    "print(' -low: ', env.observation_space.low)\n",
    "print(' -high: ', env.observation_space.high)\n",
    "# Generate some samples from the state space \n",
    "print(\"State space samples:\")\n",
    "print(np.array([env.observation_space.sample() for i in range(10)]))\n",
    "\n",
    "print(\"\\n\\n\")\n",
    "\n",
    "# Explore the action space\n",
    "# action [action]\n",
    "print('Action space: ', env.action_space)\n",
    "print('action shape: ', env.action_space.shape)\n",
    "print(' -low: ', env.action_space.low)\n",
    "print(' -high: ', env.action_space.high)\n",
    "# Generate some samples from the action space\n",
    "print(\"Action space samples:\")\n",
    "print(np.array([env.action_space.sample() for i in range(10)]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cae5223e",
   "metadata": {},
   "source": [
    "# Instantiate the Agent\n",
    "\n",
    "View this tensor as the same size as other. self.view_as(other) is equivalent to self.view(other.size()).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c0214987",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent(nn.Module):\n",
    "    def __init__(self, env, h_size=16):\n",
    "        super(Agent, self).__init__()\n",
    "        self.env = env\n",
    "        # state, hidden_layer, action sizes\n",
    "        self.s_size = env.observation_space.shape[0]    # state_size\n",
    "        self.h_size = h_size\n",
    "        self.a_size = env.action_space.shape[0]         # action_size\n",
    "        \n",
    "        # Define layer\n",
    "        self.fc1 = nn.Linear(self.s_size, self.h_size)\n",
    "        self.fc2 = nn.Linear(self.h_size, self.a_size)\n",
    "        \n",
    "    def set_weights(self, weights):\n",
    "        s_size = self.s_size\n",
    "        h_size = self.h_size\n",
    "        a_size = self.a_size\n",
    "        \n",
    "        \n",
    "        # Separate the weights for each layer---\n",
    "        # Writing to a tensor created from a read-only NumPy array\n",
    "        # fc1: weight and bias resize\n",
    "        fc1_end = (s_size * h_size) + h_size\n",
    "        fc1_W = torch.from_numpy(weights[:s_size*h_size].reshape(s_size, h_size))\n",
    "        fc1_b = torch.from_numpy(weights[s_size*h_size:fc1_end])\n",
    "        \n",
    "        # fc2: weight and bias resize\n",
    "        fc2_W = torch.from_numpy(weights[fc1_end:fc1_end+(h_size*a_size)].reshape(h_size, a_size))\n",
    "        fc2_b = torch.from_numpy(weights[fc1_end+(h_size*a_size):])\n",
    "        \n",
    "        # set the weights for each layer\n",
    "        self.fc1.weight.data.copy_(fc1_W.view_as(self.fc1.weight.data))\n",
    "        self.fc1.bias.data.copy_(fc1_b.view_as(self.fc1.bias.data))\n",
    "        self.fc2.weight.data.copy_(fc2_W.view_as(self.fc2.weight.data))\n",
    "        self.fc2.bias.data.copy_(fc2_b.view_as(self.fc2.bias.data))\n",
    "        # ---\n",
    "        \n",
    "\n",
    "    def get_weights_dim(self):\n",
    "        return (self.s_size + 1) * self.h_size + (self.h_size + 1) * self.a_size\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.tanh(self.fc2(x))                   # [-1, 1] use tanh\n",
    "        return x.cpu().data\n",
    "\n",
    "    def evaluate(self, weights, gamma=1.0, max_t=5000):\n",
    "        self.set_weights(weights)\n",
    "        episode_return = 0.0\n",
    "        state = self.env.reset()\n",
    "        for t in range(max_t):\n",
    "            state = torch.from_numpy(state).float().to(device)\n",
    "            action = self.forward(state)          # action from state\n",
    "            state, reward, done, _ = self.env.step(action)\n",
    "            episode_return += reward * math.pow(gamma, t)\n",
    "            if done:\n",
    "                break\n",
    "        return episode_return\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ba6bcc56",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set torch device\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "agent = Agent(env).to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57d893f2",
   "metadata": {},
   "source": [
    "# Train the Agent with Cross-Entropy Method\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "14a5cf20",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/eight/anaconda3/envs/torchunity/lib/python3.6/site-packages/torch/nn/functional.py:1628: UserWarning: nn.functional.tanh is deprecated. Use torch.tanh instead.\n",
      "  warnings.warn(\"nn.functional.tanh is deprecated. Use torch.tanh instead.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 10\tAverage Score: -1.68\n",
      "Episode 20\tAverage Score: -1.84\n",
      "Episode 30\tAverage Score: -1.66\n",
      "Episode 40\tAverage Score: -1.57\n",
      "Episode 50\tAverage Score: -1.37\n",
      "Episode 60\tAverage Score: -1.21\n",
      "Episode 70\tAverage Score: -1.15\n",
      "Episode 80\tAverage Score: -1.03\n",
      "Episode 90\tAverage Score: -0.99\n",
      "Episode 100\tAverage Score: -0.93\n",
      "Episode 110\tAverage Score: 7.60\n",
      "Episode 120\tAverage Score: 17.15\n",
      "Episode 130\tAverage Score: 26.71\n",
      "Episode 140\tAverage Score: 36.30\n",
      "Episode 150\tAverage Score: 45.86\n",
      "Episode 160\tAverage Score: 55.28\n",
      "Episode 170\tAverage Score: 64.70\n",
      "Episode 180\tAverage Score: 74.10\n",
      "Episode 190\tAverage Score: 83.57\n",
      "\n",
      "Environment solved in 97 iterations!\tAverage Score: 90.14\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAx9ElEQVR4nO3deXhU5dn48e+dmez7RghJWBI2w04ii+KKe921rlVsabXVamtrrbZva/t7a321rVurrbhUaq3i1mqroIIKqGUJ+07CHkjIQhISQraZ5/fHORkSSCDEnMwM3J/rypWZZ87MuXNmcu551iPGGJRSSimAEH8HoJRSKnBoUlBKKeWjSUEppZSPJgWllFI+mhSUUkr5uP0dwFeRkpJiBg4c6O8wlFIqqCxbtqzCGJPa0WNBnRQGDhxIQUGBv8NQSqmgIiI7OnvMseYjEXlJRMpEZG2bsiQR+VhECu3fiXa5iMjTIlIkIqtFZLxTcSmllOqck30KLwMXHVb2ADDPGDMEmGffB7gYGGL/3A782cG4lFJKdcKxpGCMWQDsO6z4CmCmfXsmcGWb8r8ZyyIgQUTSnYpNKaVUx3p79FGaMabEvl0KpNm3M4BdbbYrtsuOICK3i0iBiBSUl5c7F6lSSp2E/DYk1ViLLh33wkvGmBnGmHxjTH5qaoed50oppbqpt5PC3tZmIft3mV2+G8hqs12mXaaUUqoX9XZSeA+YZt+eBrzbpvxWexTSJKCmTTOTUkqpXuLYPAUReQ04G0gRkWLgIeD/gDdEZDqwA7jO3vwD4BKgCKgHvulUXEqp9pZu30dkqIuRGfHdfo2GZg8eryE6PKinPikcTArGmBs7eWhqB9sa4C6nYlHqZFFT38wjszfwkwuHkRwT3qXn/PiNVfSNj+CNOybT2GKd3KPCjn1quHfWSvolRPCTC4dz76yVlNc28tb3Tut0+8YWD8u2VzE5JxkR6fLfpHqXrn2k1Ank9aU7eX3pLj5av7dL2+870MTOffVs3luLMYYH3l7DVc98SYvHy4qdVcxaurPD53m8htlrS3h35R68XsPnRRUU7KiiuKq+0309Pa+Qm15YzPMLtx7xWIvH26V4jTFsqzjQpW170qpd1awprun1/baqrm9i2ktL2LWv8+PbUzQpKHWCMMYwa6k1snvVruojHnv2syI2lu5vV7662Nquur6ZygNNLN5ayaa9tby2ZCff+/tyfvbPtdQ1thyxr20VdTQ0eymuOsgXWyqobbC2+WidlYwamj3c9PwiPlpXCkBTi5dZS3cR6hIemb2RzzaV+V5ra3kdIx76kC+KKnz365uO3CfAh+tKOef3n7GptPY4j073ebyG6TMLuOKZz3nm0yKOdbXKorI6ahuaezSGL7dUMn9zeZeT/VehSUGpIPbppjLfiX7Jtn1srThAuDuElYclhVXFNTw2ZxNPzS1sV9722+/SbfvYU9MAwC/eXUfp/gY8XsPS7YfPQYV1ew4ll+cXbgMgKTqMOXYSmLehjC+3VPKTt1ZTtr+Bj9aXUlHXxNM3jCMnNYZHPtjoO7nOXltKY4uXVxfvoHBvLec9Pp8JD8/j4ffXH1GDmLfBSiaHJ73jsb3iAGc89km7v6uusYXnF2zlWy8vZcm29n9vwfZ9VNQ1MjQtlt99uMkXg9d7ZHJo8Xi5+tkv+Onbq31lDc0efvGvtUz87VzuenV5t2Jet8d6n1qTuJM0KSgVpIrKavn2zAJuen4xOyoP8Jf5W4gNd3Pr5AFs3lvb7tv260usZqB5G8vafYtdVVxDYlQoAP9aaY0Cv+20gQBcm5dJmCuERVsqj9j3+pL9hLqEiNAQFmwuJzEqlG9M7O87gf5zRTFJ0WE0tniYPrOAxz/aTFZSJBeO6Mv0KYPYtLeW5TurAZi7wfr2O3d9GY9/vJkwdwjnDu/D8wu3ce8bq3yJwRjDwsIK3/478synRVz2x8+P+m3+o/Wl7Np3kB+/sYoDdi3okQ828PAHG5i/uZy/frGt3fZz1pUS5g5h1h2TSYkJ581luygqq2XUrz7kQzsJ+t6T8jr2N7Qwe20pRWVWbea9VXt4ZdEOosPczF5bQnltI2DVQB6ZvYELnpjPRU8uoLKusdOY1+62/t7VvdCEpUlBqSD1m/c3EBXqoqnFy/mPL+DTTeXcPXUwk3OS8ZpDJ5K6xhbeW7WH3PQ4mlq8viYesL55njOsD7Hhbj7daK0Q8IOpQ3jjjsk8fNVIxmYl8N+tlXi8hreXFTP1D5/x3PwtrN+zn6FpsYzJTABgXP9ELhmdjtfAg++s4bNN5Xw9L5PfXDmKstoGDjS1cPe5QwgJES4b04/oMBf/WLyTstoGVu6q5vzcNJo8XmavLeXavEyevnEcD148nH+v2sND760DYEt5HaX7rZrMhg6SwpbyOp6cu5k1u2sorjrY6XH7vKiSxKhQdlXV87sPNwHwRVEF552Sxg2nZjF/czkNzR7ASkQfri3lzCGpxEeGcuXYfnyysYxfvruOA00efv/hpnY1htW7rJN2iAjPfrYFsGoaiVGhPPuN8XgNzF5b4it/bv5WEqLC2FpxgHteX3FEzcgYgzGGdXtqCBHYVnGAmoM92zR1OE0KSgWZTaW1/M+/rBPvD84bwhPXj6VPXDjP3DSe28/MYbR9om5tYnlneTH1TR7+3xUjyEyM5OUvt/PTt1bzyOwNlNU2MjoznsFpMTR5vPSLjyAxOowJg5IId7uYlJPM2t01fO/vy/jxm6vYU93Anz4pYs3uGnLT48gfmAhA3oBEhveN4/6LhvHx+r20eA1Xj8/k2rxMFv/sPBb/7Dyuy7fmp8aEu7liXAb/Wb2HZz4pwhi497yh5KRGIwLTp2QDcMdZOdxxVjavLt7JmwW7WLDZqiWcMSSFjaW1vk5nr9c6cf7qvXW0VhBWdNK81NDsYcm2Sq4Ym8HX8zKZtXQX2ysOsL2ynknZSZyfm0Z9k4f/brVqR/9ZXcKemgYuGtkXgGvyMmn2GL7cUsm4/gkUltXxx0+KuO/NVazdXcOq4mpiI6za2rsr91BSc5CC7VXkDUhieN84hvSJ4T+rrKQwe20p4e4QXrrtVH5z5Ui+KKrktr8uZe1uK7F8vH4vp//fJ3xeVEFFXRPnDrdWBXK6w1sHFSsVIFpPctmpMZ1u09Ds4Zo/f0mTx8uVY/tx6+SBhLlDOD83zbdNSkw4mYmRLCgsp298BP/7n/VMGJhE3oBErhybwZ8+LWJbxQFfB/LorATWl+xnxc5qcvu1n6swOTuZp+cV8tH6vfzkwmGcNTSVS//4OQC5/eIYlhbLM59uYVJ2EgDfOyuHFo9hR2U9w/rGdvp3TJ8yiI/WlTLzvzvISIjklPRYfnFpLlvLDzAoJdq33U8uGMaa4hp++vZqYiNCGZQSzfm5aSwsrOD1pbt48J01fOv0QfSND2dhYQW/vDSXxz7cyIqdVVw+ph8AnxdWsLCwnMYWL6cOTKKh2csZQ1KIDHXxRkGxr7YwYVASw/rGEh3mYtaSXbz8xXbmby4nOyWaC0ZYx/eU9DhGZsRRUt3AzG9N4KpnvuCJuZsBqDrQxN7aBkZnxnPbaQP56xfbmbFgK1srDnD9qVZC/NrodJ6aV8ju6oPMWVvKWUNTiQl3c11+FgebPDwxdzOX/+lzfnzBMF5YuJWq+mbuf8vqn7hxQhZzN+xlVXE12anRJEWHERHqOtbH6rhpUlAqQHxeVMEtLy7h7e9NJm9AUrvHFm+tZExWAquLa6hrbGHGLXlcMKJvp681KTuZt5YVs7CwgpEZcTw/LR8R4fvnDubMoamMzUpgW8UBVu6qYlxWAsu2VwEwol9cu9cZ1z+BEf3iuHBEX+46ZzAAZw9L5bNN5eSmxzExO5lPfnyWL5GJCPdMHXLMvzUnNYbPf3ous9eW0C8+EhHh7GF9OHtY++3crhD+fHMeMxZuYfaaUq7Jy+SUdCvG336wgRCBl+w+gEtG9eWbpw9k9toSX0f73PV7+c4rBYSGhOA1hlcW7cAdIkzMTibCHUJSdBjvrykhOsxFbnocblcIZw5NZfbaUqLCXPzi0ly+Mak/4e5DJ98/35xHY4uHuIhQnrh+LAXbq9i5r56//Xc7ISJ858xsBiRHM2FQEjO/3A5A/kDr/bxybAZ/+qSIG2csonR/Aw+MGu573WmnDeSq8Rnc+/pKfvfhJiJCQ/jaqHTeX1OCCEzMTmZQSjR//WIbT87dzEOXjeAbkwYc81gfL00KSgWI1mGW/15V0i4prCmu4foZi/jx+UNxu6wW39aTTGd+e9UobpyQxa59BzlneB/iI63O5IhQFxMGWc8d1jfW921+eLr1e9Rhs5ojQl28f88Z7coeuHg40WFuxmQlABy1ZnM0EaEurhqXeczt4qNC+cmFw/nJhdYJdL/dUV7b0MJ9FwylqKyOnfvq+d21YxARxmYlMPO/O1ixs4p7Xl/BqIx4Xr99Eit3VjN9ZgFjsuKJsWdeX5CbxutLd5E3MMl3bL99RjYhItx34bB2tZZWWUlRvtujMxMYnZnArn31vPzldrzGMCbTOoZfz8tkybZ9hLtDGJlhJbKBKdE8c/N47np1OWGuEM49pU+7146LCOW5W/J49rMtDOsby/j+iczdsJeMhEhiwt2cPthK9jdPHMDZw5xZEFSTglIBYkelNTFpztpSfnlpLiEh1qzfVxZtB+DD9aX0jYskO8VqOjiaMHcIeQOSyOviF8nTc1J47pY8zh3e55jbDu8bxzM3++/iiHERoWQmRlJR18g3Jg0gISoMY4xvlvTYrESeX7iNb7ywmITIUF64NZ+oMDenDU7h4x+dSZjrUFfqxaPSeX3pLiYOOpRk8wYkkjcg8bhiykqK4owhKSwsrPD16VwyKp2H3lvHyIz4djWNC0f05ZXpEymvayQuIvSI13K7QtrVtn5z5UhC7ZgfumwEP78kl8iwnm828u3fsVdWSh2XnfZs1dL9DazYVU3egERq6pt5b9Ue4iLcrN29n63lB7hkVM9ffyokRLjwKM1Rgea7Z+VggIQoKzm2XTZjXP8EX9lL3zyVPnERvscyE6PavgxTBqfwwMXDuTbv2DWWY7n/wuGMySwlPd7aX3S4m2dvHk9y9JHLjUzOSe7y6349/9AC0qGuEBzoRmhHk4JSAWLnvnrOGJLCoq2VzFlbQt6ARN5aXkxDs5fHrxvLna8up77Jc9zfYk9ER2tLT4+P4Ltn5XDW0FSG943rdDsAV4jw3bNyeiSmUZnxjMps3/x29rBj17wCjQ5JVSoAeLyG4qp6RvSLZ1J2Mp9stGbNfrSulNz0OC4Zlc7QNKvtPl+TwlGJCA9cPPy4vo2rQzQpKBUASmoO0uwxDEiOYnJOMlvKD7Cn+iArdlVzmn1yuy4/i+yUaHK62bGrVFdo85FSAaC1P2FAUhRD06yRQC8s3EZTi5dT7U7Qb5+RzbfPyPZbjOrkoElBqQCw0x55lJUURd/4CCJDXfxjyQ4ATj3G8FOlepI2HykVAHbuq8cdIvRLiCTUFULegEQamr0MTYs55vBTpXqSJgWlAsCOffVkJkbisucmtI6bnzBIawmqd2lSUCoA7Kysp3/yodmzpw1OsX7npPgrJHWS0j4FpQLAzn31jMk6NMY9b0Ai/7zzNN/S1Er1Fk0KSvlZTX0zNQebGZDUfp2dcf11PoLqfdp8pJSftQ5H7Z8cdYwtlXKeJgWl/GzHvgMA9E/SpKD8T5OCUn7mqyloUlABQJOCUr3M6zW+awCDNfIoJSaM6HDt4lP+p0lBqV720hfbOPOxTzH2BYV37qvXWoIKGJoUlOplczfspay20XeN5B2VmhRU4NCkoJTDtpTX+WoFTS1eVuysBqC6vpmmFi8lNQfbTVxTyp80KSjloOKqeqb+YT5vFhQDsGZ3NY0tXgBqDjazu/ogXmOtjqpUIPBLUhCRe0VknYisFZHXRCRCRAaJyGIRKRKRWSKiq4CpoFfbYDURvfzldowxLNlW5Xusqr6JHZX2cFSdo6ACRK8nBRHJAO4B8o0xIwEXcAPwKPCEMWYwUAVM7+3YlOppHq/VbLS+ZD/Ld1azZFsl4W7r3666vpniqoMAZCVqUlCBwV/NR24gUkTcQBRQApwLvGU/PhO40j+hKdVz7K4EAH713jqWbNvHGUNSAag+2ExZbSMikBKjFWMVGHo9KRhjdgO/B3ZiJYMaYBlQbYxpsTcrBjI6er6I3C4iBSJSUF5e3hshK9VtHjsrjMmMp6isjj5xEdx22kAAqg80UVHXSHJ0GG6Xdu+pwNDrs2VEJBG4AhgEVANvAhd19fnGmBnADID8/HxzjM2V8iuvnRR+dMEwzhqa6iuPCnNRfbCZ8tpGUmLC/RWeUkfwx9eT84BtxphyY0wz8A5wOpBgNycBZAK7/RCbUj3Ka/cp2NfO8UmIDKW63koKqbGaFFTg8EdS2AlMEpEoERFgKrAe+BS41t5mGvCuH2JTqkfZOQGXtM8KCVFh1By0mo9StaagAog/+hQWY3UoLwfW2DHMAH4K/EhEioBk4MXejk2pntY6+kiOSAqhVNk1hRStKagA4pcVuIwxDwEPHVa8FZjgh3CUckzrTGZXyJFJoWB7FY0tXq0pqICiQx6UclDr6KMj+hSiwiirbQQgJVaHo6rAoUlBKQe1Nh+FHF5TiAz13U6NiejVmJQ6Gk0KSjmodfJaSAd9Cq109JEKJJoUlHJQa02ho9FHrXQ2swokmhSUclDr5DXpYJ4CWB3QiVGaFFTg0KSglIO8nY4+shJBcnTYEf0NSvmTJgWlHOTtpE8h0e5T0P4EFWg0KSjlIF+fwmH/afGaFFSA0qSglIMO9SkcPiTVaj7SxfBUoNGkoJSDfH0KhyWFMHcIQ9NiGJUR74+wlOqUX5a5UOpk4bUux3xEnwLAR/ee1cvRKHVsWlNQykG+ZS70P00FCf2oKuUg41v7SIedquCgSUEpB3ns5qPD5ykoFag0KSjloM5mNCsVqDQpKOWgzkYfKRWoNCko5aBD12jWpKCCgyYFpRzkaV3mQvsUVJDQpKCUg0wnV15TKlBpUlDKQYfWPtKsoIKDJgWlHNTZKqlKBSpNCko5yKuT11SQ0aSglIMOjT7ycyBKdZEmBaUc5OnkymtKBSpNCko5yGus2cyHX09BqUClSUEpB3m9RvsTVFDRpKCUg7zG6BIXKqhoUlDKQR5jdDE8FVQ0KSjlIK/XaCezCiqaFJRykNfoHAUVXPySFEQkQUTeEpGNIrJBRCaLSJKIfCwihfbvRH/EplRP8niNzlFQQcVfNYWngDnGmOHAGGAD8AAwzxgzBJhn31cqqBljdIVUFVR6PSmISDxwJvAigDGmyRhTDVwBzLQ3mwlc2duxKdXTPDr6SAUZf9QUBgHlwF9FZIWIvCAi0UCaMabE3qYUSOvoySJyu4gUiEhBeXl5L4WsVPdYk9c0Kajg4Y+k4AbGA382xowDDnBYU5GxFqE3HT3ZGDPDGJNvjMlPTU11PFilvgpr9JG/o1Cq6/zxcS0Gio0xi+37b2Elib0ikg5g/y7zQ2xK9Siv0RnNKrj0elIwxpQCu0RkmF00FVgPvAdMs8umAe/2dmxK9TSPV4ekquDi9tN+7wZeFZEwYCvwTawE9YaITAd2ANf5KTaleow1+sjfUSjVdX5JCsaYlUB+Bw9N7eVQlHKUjj5SwUa/wyjlIJ3RrIKNJgWlHOT16uQ1FVw0KSjlIGv0kb+jUKrrNCko5SCPXmRHBRlNCko5SPsUVLDRpKCUg7xGr6eggosmBaUcpH0KKthoUlDKQR4dfaSCjCYFpRxktE9BBRlNCko5yOPVGc0quHQ5KYhIZJtF7JRSXeA1Bs0JKph0KSmIyGXASmCOfX+siLznYFxKnRB09JEKNl2tKfwKmABUg29Bu0GORKTUCUTnKahg09Wk0GyMqTmsrMMroymlDtHRRyrYdHXp7HUichPgEpEhwD3Al86FpdSJweg8BRVkulpTuBsYATQC/wBqgB86FJNSJwy9noIKNsesKYiIC3jfGHMO8HPnQ1LqxOH1gmhSUEHkmDUFY4wH8IpIfC/Eo9QJxRp95O8olOq6rvYp1AFrRORj4EBroTHmHkeiUuoE4fHqkFQVXLqaFN6xf5RSx8GavKZJQQWPLiUFY8xMEQkDhtpFm4wxzc6FpdSJwWvQjmYVVLqUFETkbGAmsB0QIEtEphljFjgWmVInAF06WwWbrjYf/QG4wBizCUBEhgKvAXlOBabUiUAnr6lg09VxEaGtCQHAGLMZCHUmJKVOHLp0tgo2Xa0pFIjIC8Df7fs3AwXOhKTUiUOXzlbBpqtJ4XvAXVjLWwAsBJ51JCKlTiBeYwjReQoqiHQ1KbiBp4wxj4NvlnO4Y1EpdYKwOpq1pqCCR1e/w8wDItvcjwTm9nw4Sp1YdOlsFWy6mhQijDF1rXfs21HOhKTUiUNnNKtg09WkcEBExrfeEZF84OBX2bGIuERkhYj8x74/SEQWi0iRiMyyJ8spFdT0cpwq2HQ1KfwQeFNEForIQuB14Ptfcd8/ADa0uf8o8IQxZjBQBUz/iq+vlN95dfSRCjJHTQoicqqI9DXGLAWGA7OAZqxrNW/r7k5FJBP4GvCCfV+Ac4G37E1mAld29/WVChReg05eU0HlWDWF54Am+/Zk4GfAM1jf5Gd8hf0+CdwPeO37yUC1MabFvl8MZHT0RBG5XUQKRKSgvLz8K4SglPM8OvpIBZljJQWXMWaffft6YIYx5m1jzC+Awd3ZoYhcCpQZY5Z15/nGmBnGmHxjTH5qamp3XkKpXqOX41TB5ljzFFwi4ra/wU8Fbj+O53bmdOByEbkEiADigKeAhDb7ygR2d/P1lQoYOvpIBZtj1RReA+aLyLtYo40WAojIYKzrNB83Y8yDxphMY8xA4AbgE2PMzcCnwLX2ZtOAd7vz+koFEq/Ry3Gq4HLUb/vGmIdFZB6QDnxkjDH2QyHA3T0cy0+B10XkN8AK4MUefn2lepXXa/276OgjFUyO2QRkjFnUQdnmnti5MeYz4DP79lZgQk+8rlKBwGt/h9LWIxVMdKkupRziaU0KmhVUENGkoJRDWhtbdUiqCiaaFJRyiKe1T0H/y1QQ0Y+rUg451KegNQUVPDQpKOUQrz1fX5OCCiaaFJRyiI4+UsFIk4JSDmkdfaQzmlUw0aSglENaJ6/pjGYVTDQpKOUQOydoTUEFFU0KSjnEo30KKghpUlDKIa3NRzr6SAUTTQpKOUTnKahgpElBKYdon4IKRpoUlHJI6zIXuiCeCiaaFJRyiNGOZhWENCko5RDf5DXtU1BBRJOCUg5pXftIJ6+pYKJJQSmHeHWZCxWENCko5RBdEE8FI00KSjlERx+pYKRJQSmHePVynCoIaVJQyiFeHX2kgpAmBaUccmjtIz8HotRx0KSglEN8q6RqVlBBRJOCUg4x2qeggpAmBaUc0jr6yKX/ZSqI6MdVKYe0djTrjGYVTDQpKOUQHX2kgpEmBaUc0rr2kfYpqGCiSUEphxwafeTnQJQ6Dr3+cRWRLBH5VETWi8g6EfmBXZ4kIh+LSKH9O7G3Y1OqJxm9HKcKQv74DtMC/NgYkwtMAu4SkVzgAWCeMWYIMM++r1TQ8tjNR7pKqgomvZ4UjDElxpjl9u1aYAOQAVwBzLQ3mwlc2duxKdWTdJVUFYz82topIgOBccBiIM0YU2I/VAqkdfKc20WkQEQKysvLeydQpbrBq81HKgj5LSmISAzwNvBDY8z+to8ZqzHWdPQ8Y8wMY0y+MSY/NTW1FyJVqns0Kahg5JekICKhWAnhVWPMO3bxXhFJtx9PB8r8EZtSPUX7FFQw8sfoIwFeBDYYYx5v89B7wDT79jTg3d6OTame1LpKqlYUVDBx+2GfpwO3AGtEZKVd9jPg/4A3RGQ6sAO4zg+xKdVj9BrNKhj1elIwxnwOdPZfMrU3Y1HKSR7tU1BBSOdaKuUQvRynCkaaFJRyiF55TQUjTQpKOUT7FFQw0qSglEM8Xr2eggo+mhSUckjr5Ti1pqCCiSYFpRzi0bWPVBDSpKCUQ3SZCxWMNCko5ZDW0UfafKSCiSYFpRyi8xRUMNKkoJRDPDpPQQUhTQpKOcQYg4gOSVXBRZOCUg7xGINLE4IKMpoUlHKI12h/ggo+mhSUcojXawjR/zAVZPQjq5RDvMZoTUEFHU0KSjnE40X7FFTQ0aSglEO89ugjpYKJJgWlHOI1Rmczq6CjSUEph2ifggpGmhSUcojHCyFaU1BBRpOCUg4xxugSFyroaFJQyiEer85oVsFHk4JSDvEaXfdIBR9NCko5REcfqWCkSUEph3i82qeggo8mBaUc4jVGRx+poKNJQSmH6DwFFYw0KSjlEK+ufaSCkCYFpRzi0bWPVBAKqKQgIheJyCYRKRKRB/wdj1JfhdHRRyoIBUxSEBEX8AxwMZAL3Cgiuf6NSqnus0YfaVJQwcXt7wDamAAUGWO2AojI68AVwPreDqSpxcuW8jr2H2xmXP9EwtwheL0dX4S9sq6RMHcIsRGhx7WPTaW1bKuoIyMhilGZ8T0ZPsYYwIrVGENji5eIUJfvsXV79lNe10hiVBg5qdHtYt+7v4EVO6tobPES7g4hPNRF/oDEdttU1jWyvmQ/feMiGJIW227flXWNrCquZmhaLBkJke2OV0OzxxdHW4u3VjIqM56oMOvjeLDJw979DfRLiCTMbX1vKSqrY/PeWtLjIxiTmYAILCisoOZgM2EuIdQVQqgrhNgIN7n94gh3W/vZWVnPquJqosNd1Dd5GJoWy9A2Ma/dXUNtQwth7hAq6xrJG5BIckx4u2PZ2QS06vomPi+qsCapAfGRoUwZnIIIbNpby776Zh19pIJOICWFDGBXm/vFwMTDNxKR24HbAfr37/+Vdvji59vITonmnOF92pX/cNYKPlhTCsCYzHjOHZ7GX+Zv4X+vHMm1eZm+7f69ag/3v7WaKUNSeP7W/CNev8Xj5QezVjJ9yiDG90/0lS/YXM63ZxbQ5PEC8I/vTOS0nBTf4w3NHuasLWVyTjJpcRG+cmMMbxYUM7RvLGOzEjr9u56cW8j7a0r4+N4zeXNZMQ++s4YLctOIiwjly60V7Np3sN32A5KjyEmNobCs9ojHAG6ZNID/vXIkAJ9uKuM7Mwtoscfgf+eMbFJjwznY5CEuMpSn5xVSeaAJgKToMEZlxDMyI45lO6pYtHUfUwan8IPzhnDqwCQAdlQe4PoZi/ja6HT+7+pRfO/vy/lyS4V9fWPr9e88ZzA3Pr+I8tpGAKYMTiEm3M2cdaUd/v1h7hDuPDuHmyb05/JnPqe6vrndYy/fdiqTc5J5cm4hT80rbPfca/My+f3XxwAwZ20p9725ipsn9Sc5Ooy/zN/KQ5flcsXYDFbtqubOV5ezu7r98RrRLw5jYH3JfgDOGpra6fukVCAKpKTQJcaYGcAMgPz8fNPd12n2eHl09ka8xvDMzeO5cERfAGobmpm7voxLR6dzxpAUfvOfDawq3owrRPhsU5kvKby1rJj73lxFZKiLzzaVsb+hmbiIUPbub+DX/17HTy8aTrPHy/urS0iMCvUlhbW7a7j9lQKyU6N55OpRTJ9ZwMwvt3NaTgord1Xz3y2V/H3RDnZXHyQuws1vrx7FpaP7AfhOYhMGJvHGdyfz1rJiviiqID0+gulTBpEcE05JzUH+PH8LTS1eisrqmLdhL1FhLr7cUokIjO+fyN3nDCGnTwz7DjSxqXQ/60v2U1RWR256HNMmDyTPrhk0tXj5xbtrWbGrCoD9Dc08+PYaslOj+eWlI3h35W6eW7C13XHNTY/j918fQ3H1QdYUV7O6uIaFheWkxUUwbfIAPly3l5ufX8xLt53KlCEpzN9cDsD7q0vYULKfnZX13HFWDoOSo5lfWM5zC7by5ZZKKusamXFLHnuqD/LYh5tobPHy4MXDmXpKH5paDC1eL80eL+W1TfxrxW6enFvIP1fs5mCTh1e/PZHocDcuEe57cxXfmrmUuIhQymobuWZ8JtfkZdDY7OX5hVsp2L4PgJW7qvnhrBXERYTy3Hzrb0yICuXBd9aws7KeP35SRGpsOK9+eyJpcREYY1izu4Yn5xYS7g7h4atGMiwtlmF929eklAp0gZQUdgNZbe5n2mWO2F5xgCaPl9gIN9//x3Lev+cMhqbF8snGMpo8Xr55+kDyBiQxZUgqu/bV88p/d7CquBqAkpqD/Pq9dUwYlMS95w3lxucX8enGMi4a2Zfv/n0ZK3ZWkzcgiYyESAAKtlf59vvonI1Eh7l5ZfpEUmPDuS4/ixkLtvDI7A2+k8/ozHjuv2gYL32+jR/NWsV5p6SxYHM5T80rJCUmjOU7q9h3oIn/9+91eI1Vs/hwXSnP3ZLHc/O30mLXQBZtrWTJtn1cNKIvj14zusPmr/Nz0456nCYMSuKFhVtpbPHw2JyNlNU28NwtpzMmK4EpQ1K4+9whxES4iQgNobjqIINSogl1tXZVDQCs+EJdIbhChHvPH8oNMxbxnb8V8M6dp7FgczlZSZEkRYWxqriGx64ZzXWnWh+Dq8ZnsLemgYIdVXzr9EFcYCfuC0f2pbahpV0zUFvnndKHO15ZxryNZfzy0lxOH3yoFvbKtyfwv//ZQKhLmJydzLV5mb5jsrG0lkfnbKSyrpH731pFSkw4/7rrdLaU1XGgqYXc9HgufmoBf/h4M1MGp/D0jeNIig7zvfaQtFiuHp95RDxKBZNASgpLgSEiMggrGdwA3OTUzjbtrQXg2ZvHc+ffl/O7Dzfx/K35fLCmhLS4cMZlWd/sMxIiyUiIZHVxNe+vKaGyrpH/+edaWryG3107mqzEKFJjw3l/dQmfbCxjxc5qwlwhbC6tpbHF49tXbUMzm/fWsrCwggcvHk5qrNVuffPE/jy3YAvPzd/KxSP78vBVo3wnGleI8P1/rKCorI6FhRXEhrt5+oZx3PTCYn734Ub2N7Tw4rR8YiNC+dbLSznv8QUATJs8gDnrSnl18U6q6puZMCip223bozLiafYYVhfX8Pay3Vybl8mYNk1X/ZOjfLc7O0m37UdIiArjlekTOe/x+fz2gw0s21HFNeMzuXvqYDaU1LZrbgl1hfDszeN5fekupk8Z5CtPj48k/SjdMG5XCH+6aTyLt1Vy5pD2zTd9YiP4443jOnxe3gDrPX9zWTGb99bx/64YQUpMOClt+hhevO1UVu+q5pbJA3VkkTohBUxSMMa0iMj3gQ8BF/CSMWadU/vbXFpLiMCpA5O4/cxs/vDxZl5bspPPNpVz44T+R5xEx2QmAFaz0byNZdx3wVAGJEcDcEFuGq8u3gnA/RcNY+HmCjburaXFa+y/zWqOmLFgK0nRYdwyeYDvdbOSorhpQn/Kaxt54vqx7U6guelxgNU+vXZPDaf0i+PUQUnEhLt5bckuYiPcTBmSQrjbxbvfP50viyqICnPztdHpVNU3896qPQBMHJTc7eM0KsM6+85YsJWDzR4uHpXe7ddqlRobzh1nZfPYnE0AnDk0lT6xEfSJjThi2z5xEdwzdchx7yMyzMXZw/oce8M2RmfG4w4Rnv20CMDXpNjW+P6J7fqHlDrRBMyQVABjzAfGmKHGmBxjzMNO7mtjaS0DU6KJCHUx/YxBpMSE8+A7a2jyeLl8bL8jth+ZEU+IwFPzCgl1CTdMONTJfdW4DMJcIfzP107hzrMHM6xvLIV7aykqq2V0Zjwi8PS8QhYWVvC9s3J8o2xaPXzVKGbcmn/EyJwBydFEhrpYt7uGDSX7GdkvnlBXCKflWCf5C3L7+kbZ5KTGcMvkgVyTl0lEqIuJ2VZHbt+4CLKSIrt9nDITI0mICuXj9VbfxOTs7ieYtm47bSApMWG4Q4TJOT3zml9VRKiL3H5x7G9oYXz/hHad/EqdLAKmptDbNu+tJbef9U08KszNG3dMoqSmgSFpMR1+Y40OdzM0LZaNpbVcMqpvuyaF/IFJrPn1Bb4T9LC+sdQ3eVizu4abJw6gqcXL0u1VZKdEM+20gV2O0RUiDOsby+y1pTQ0exmZYcV71rBUPlq/l6+NPvKbbKuJg6ykMGFQ0lda019EGJURz8LCCs4YktLhkNLuiApz8+g1o9lWcYCY8MD5GI7vn8jq4houGtn5sVXqRBY4/4296GCThx376rlyXIavLDs1huzUmKM+b2xWAhtLa7nh1COHwrYmBMA34sRrICc1Gq8xbCyt5aHLR/jG3XdVbr84Vu6qBmBEP6sp59q8TOIiQjl7aOfNIzmpMdw4IYsrxmZ0uk1XtSaFqaccvVP6ePX06/WEs4elMmvpLi7pgWYypYLRSZkUisrqMAaGddIx2pnrTs3C7RKmtBnN0pG2Ha45fWKYekoa+QMTuzVm/RS7XyHcHUJOarR928VlY45s4mpLRHjk6tHHvb+OnJebxicbyzg/AE/iPe3sYX1Y9dAFx528lTpRnJRJYWOpNbHoeMeQd7WTMSbcTWZiJMVVB8lOjSEjIZKspKhjPq8juelWjKekx+F2+edENb5/InN+eKZf9u0PmhDUyeyk/PQbYEifGN/oIScMS4slMtRF+lfsrBzWNw4RfP0JSinlpJOypnBdfhbX5Wcde8Ov4DtnZnPO8D5fee2bmHA3f/lGHiMzenZ9JKWU6shJmRR6w6TsZCb10PDNjsbLK6WUE07K5iOllFId06SglFLKR5OCUkopH00KSimlfDQpKKWU8tGkoJRSykeTglJKKR9NCkoppXzEmG5f5tjvRKQc2NGNp6YAFT0cTk8J1NgCNS7Q2LpLYzt+gRoXHF9sA4wxHa7QGdRJobtEpMAYk+/vODoSqLEFalygsXWXxnb8AjUu6LnYtPlIKaWUjyYFpZRSPidrUpjh7wCOIlBjC9S4QGPrLo3t+AVqXNBDsZ2UfQpKKaU6drLWFJRSSnVAk4JSSimfkyopiMhFIrJJRIpE5AE/x5IlIp+KyHoRWSciP7DLfyUiu0Vkpf1ziZ/i2y4ia+wYCuyyJBH5WEQK7d/HvmB1z8c1rM2xWSki+0Xkh/46biLykoiUicjaNmUdHiexPG1//laLyPhejut3IrLR3vc/RSTBLh8oIgfbHLu/OBXXUWLr9P0TkQftY7ZJRC70Q2yz2sS1XURW2uW9fdw6O2f07OfNGHNS/AAuYAuQDYQBq4BcP8aTDoy3b8cCm4Fc4FfAfQFwvLYDKYeVPQY8YN9+AHg0AN7TUmCAv44bcCYwHlh7rOMEXALMBgSYBCzu5bguANz27UfbxDWw7XZ+OmYdvn/2/8QqIBwYZP8Pu3oztsMe/wPwSz8dt87OGT36eTuZagoTgCJjzFZjTBPwOnCFv4IxxpQYY5bbt2uBDUCGv+LpoiuAmfbtmcCV/gsFgKnAFmNMd2a19whjzAJg32HFnR2nK4C/GcsiIEFE0nsrLmPMR8aYFvvuIiDTiX0fSyfHrDNXAK8bYxqNMduAIqz/5V6PTUQEuA54zan9H81Rzhk9+nk7mZJCBrCrzf1iAuQkLCIDgXHAYrvo+3Z17yV/NNHYDPCRiCwTkdvtsjRjTIl9uxRI809oPjfQ/h80EI4bdH6cAukz+C2sb5GtBonIChGZLyJn+Cmmjt6/QDpmZwB7jTGFbcr8ctwOO2f06OftZEoKAUlEYoC3gR8aY/YDfwZygLFACVZ11R+mGGPGAxcDd4nImW0fNFb91G/jmUUkDLgceNMuCpTj1o6/j1NHROTnQAvwql1UAvQ3xowDfgT8Q0TiejmsgHz/DnMj7b+E+OW4dXDO8OmJz9vJlBR2A1lt7mfaZX4jIqFYb+6rxph3AIwxe40xHmOMF3geB6vKR2OM2W3/LgP+acext7X6af8u80dstouB5caYvRA4x83W2XHy+2dQRG4DLgVutk8g2E0zlfbtZVjt9kN7M66jvH9+P2YAIuIGrgZmtZb547h1dM6ghz9vJ1NSWAoMEZFB9rfMG4D3/BWM3T75IrDBGPN4m/K2bX5XAWsPf24vxBYtIrGtt7E6KNdiHa9p9mbTgHd7O7Y22n1rC4Tj1kZnx+k94FZ7VMgkoKZNtd9xInIRcD9wuTGmvk15qoi47NvZwBBga2/FZe+3s/fvPeAGEQkXkUF2bEt6MzbbecBGY0xxa0FvH7fOzhn09Oett3rOA+EHqzd+M1ZG/7mfY5mCVc1bDay0fy4BXgHW2OXvAel+iC0ba8THKmBd67ECkoF5QCEwF0jy07GLBiqB+DZlfjluWImpBGjGarOd3tlxwhoF8oz9+VsD5PdyXEVYbcytn7e/2NteY7/PK4HlwGV+OGadvn/Az+1jtgm4uLdjs8tfBr572La9fdw6O2f06OdNl7lQSinlczI1HymllDoGTQpKKaV8NCkopZTy0aSglFLKR5OCUkopH00K6qQkIh5pv9rqUVfNFZHvisitPbDf7SKS0o3nXSgiv7ZXxJx97Gco1T1ufweglJ8cNMaM7erGxhhHl0XugjOAT+3fn/s5FnUC05qCUm3Y3+QfE+taEktEZLBd/isRuc++fY+9pv1qEXndLksSkX/ZZYtEZLRdniwiH9nr37+ANaGodV/fsPexUkSea50de1g814u1fv89wJNYS0B8U0T8Nhtfndg0KaiTVeRhzUfXt3msxhgzCvgT1on4cA8A44wxo4Hv2mW/BlbYZT8D/maXPwR8bowZgbWGVH8AETkFuB443a6xeICbD9+RMWYW1mqYa+2Y1tj7vrz7f7pSndPmI3WyOlrz0Wttfj/RweOrgVdF5F/Av+yyKVjLHmCM+cSuIcRhXbTlarv8fRGpsrefCuQBS60lbYik8wUGh3JoTZ1oY62lr5QjNCkodSTTye1WX8M62V8G/FxERnVjHwLMNMY8eNSNrEuhpgBuEVkPpNvNSXcbYxZ2Y79KHZU2Hyl1pOvb/P5v2wdEJATIMsZ8CvwUiAdigIXYzT8icjZQYay17hcAN9nlFwOtF4+ZB1wrIn3sx5JEZMDhgRhj8oH3sa6i9RjW4oRjNSEop2hNQZ2sIu1v3K3mGGNah6UmishqoBFrie62XMDfRSQe69v+08aYahH5FfCS/bx6Di1l/GvgNRFZB3wJ7AQwxqwXkf/BurpdCNaqnHcBHV1adDxWR/OdwOMdPK5Uj9FVUpVqQ0S2Yy0xXOHvWJTyB20+Ukop5aM1BaWUUj5aU1BKKeWjSUEppZSPJgWllFI+mhSUUkr5aFJQSinl8/8By951peUmJKcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def crossEntropyMethod(n_iterations=500, max_t=1000, gamma=1.0, print_every=10, pop_size=50, elite_frac=0.2, sigma=0.5):\n",
    "    \"\"\"\n",
    "    Implement cross-entropy method\n",
    "    \n",
    "    Params\n",
    "    ======\n",
    "        n_iterations (int): maximum number of training iterations\n",
    "        max_t (int): maximum nuber of timesteps per episode\n",
    "        gamma (float): discount rate\n",
    "        print_every (int): how often to print average score (over last 100 episodes)\n",
    "        pop_size (int): size of population at each iteration\n",
    "        elite_frac (float): percentage of top performers to use in update\n",
    "        sigma (float): standard deviation of additive noise\n",
    "    \"\"\"\n",
    "    \n",
    "    n_elite = int(pop_size * elite_frac)\n",
    "    \n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores = []\n",
    "    best_weight = sigma * np.random.randn(agent.get_weights_dim())      # initialize parameters\n",
    "    \n",
    "    for i_iteration in range(1, n_iterations+1):\n",
    "        weights_pop = [best_weight + (sigma * np.random.randn(agent.get_weights_dim())) for i in range(pop_size)]\n",
    "        rewards = np.array([agent.evaluate(weights, gamma, max_t) for weights in weights_pop])\n",
    "        \n",
    "        elite_idxs = rewards.argsort()[-n_elite:]\n",
    "        elite_weights = [weights_pop[i] for i in elite_idxs]\n",
    "        best_weight = np.array(elite_weights).mean(axis=0)\n",
    "        \n",
    "        reward = agent.evaluate(best_weight, gamma=1.0)\n",
    "        scores_deque.append(reward)\n",
    "        scores.append(reward)\n",
    "        \n",
    "        \n",
    "        # Checkpint\n",
    "        torch.save(agent.state_dict(), 'checkpoint.pth')\n",
    "        \n",
    "        # Monitor\n",
    "        if i_iteration % print_every == 0:\n",
    "            print('Episode {}\\tAverage Score: {:.2f}'.format(i_iteration, np.mean(scores_deque)))\n",
    "\n",
    "        if np.mean(scores_deque)>=90.0:\n",
    "            print('\\nEnvironment solved in {:d} iterations!\\tAverage Score: {:.2f}'.format(i_iteration-100, np.mean(scores_deque)))\n",
    "            break\n",
    "    return scores\n",
    "\n",
    "scores = crossEntropyMethod()\n",
    "\n",
    "# plot the scores\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b19a210",
   "metadata": {},
   "source": [
    "# Simulate Trained Agent\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "de99254f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAADnCAYAAABBu67aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAjGElEQVR4nO3deVyN+eIH8M+pTosRpjIVY8syI0skM8VorENlC6VI2UmMhq5hLuZeXHuMrcQwLShUsvOyV8oyk92gZjBX26jQTuec5/eHX137knN6Tud83n/dKZ3nY+74+D7f7/f5PhJBEEBERB9OR+wARESagoVKRKQkLFQiIiVhoRIRKQkLlYhISfTe8n1uASAiepnkVV/kCJWISElYqERESsJCJSJSEhYqEZGSsFCJiJSEhUpEpCQsVCIiJWGhEhEpCQuViEhJWKhEVK0JgoCcnHQoFMViR4HkLQdM89FTIlJrgiBgypTO+O47C+jrN4REYoBPPpkCHR0jSCRS6OrWUsVlX/no6due5SciUntyuQwPH+6q+Of799dDIpFAX78RTEyGAwBq1uyMGjU6AAAkEn1IJMq/QWehEpHGUSjyAQAlJZeRnn4ZAKCrWxs6OrUgkUhQt+4kGBq2Qu3aLpBIXjnYrBQWKhFpPCOjNpBK6wHQhYVFAHR1TaCjY6j067BQiUhjSCRSGBm1AaADI6O2Fbf7RkYtIZVaqvz6LFQiqvaMjKzRpMl06OgYoXZtZwC6AKDU2/l3wUIlompPT+8TmJgMFTsG96ESESkLC5WISElYqERESsI5VCKqtrKzs3H37l1kZWXh4cOHqFOnjqh5+OgpEVUrmZmZyMzMxPLly3Hjxg1cuHABALB//344OztXVQw+ekpE1U9RURGKioqwbt065OXl4eDBg7hz5w7kcvlzv27Pnj1wcnKq8q1Sz2KhEpHaefz4MXJycrBlyxacPHkSSUlJKCwshEKheO3PJCYmQiaTQSqVVmHS5/GWn4hEJwgCBEHAzZs3ce7cOYSEhODPP/9Ednb2O3+Gnp4etm3bBjc3NxUmrcBbfiJSL3l5eUhOTkZBQQGWLl2KrKwsZGZmVuqzZDIZiovFPROVI1QiqlJyuRwXL17E4sWLkZaWhosXLyrts7t06YKjR49CX19faZ/5Gq8cobJQiahKFRUVoWPHjvj9998/+LPq1KkDHR0dPHr0CHK5HBYWFrhw4QIsLCyUkPSNXlmo3NhPRFWqRo0amDx5cqV/XiKRoH379ujbty98fX3h5+cHa2tr1KtXD1lZWQgPD1di2vfDQiWiKiWRSGBrawsTE5NK/bydnR369u0LOzs7GBgYQEdHB8bGxnB3d4elpSUSExNFm0tloRJRlbOzs0PdunXf++dq1qyJDh06QFdX96Xv1alTB66urigoKHhpj2pVYaESUZXT1dWFv7//c1+TSCSwsrJC586dMWnSJHTq1AlNmjR5bqO+qanpG+dH69evj+DgYBgbG6sq+htx2xQRVTmJRIKWLVuiZs2aKCwshI6ODrp06QJHR8eK0ec333wDmUxWsbH/TZv6y5WVlSEnJ0fV8V+LI1QiEoWjoyPatGkDADAzM8NXX3310q28np4eunXrBjs7OzRu3BiffvrpGz/TyMgILVq0UFnmt+EIlYhEM3DgQCQnJ+PLL7+Ent6r60hXVxcDBgzAuHHjIJVKER0djfT09Ff+2latWsHMzAzFxcUoLCwE8PREqpCQELxli+hzXF1d0bZtWwCAoaEhatWq9U4/x0IlIlFIJBI4ODhU/O83HWoikUigp6eHOnXqwM3NDTt27EBGRkbF9wVBgFQqxd27d7Fw4UL89ttvOHr0KABAoVC896r/5s2bKwre2toa/fr1wyeffAIvLy8AT7d+vTInN/YTkVhu3bqFXr16oX379mjfvv1rf52NjQ369+8PHZ2ns5SPHj1CfHw8rl69ijNnzuD06dOQyWR48uQJSkpKAPzvBX0mJibo37//e51CFR8fjz/++KPin8sLu3xnQnp6Op/lJyL10qJFCyQkJODKlSv47bffXnlbLpFIYGFhUVGmBQUFSEhIwOHDhxETE4OsrKyKX2tjY4NPP/0UnTt3Rv/+/QEABgYGaNq06XsV6r179/Do0SMAwMWLFxEZGYn8/HwkJCS88ec4QiUi0clkMsTHx+P06dMvreabmppi9OjRuHnzJq5fv44VK1YgJSUFAGBubo4mTZrAysoKEyZMwGeffQZzc3OVZCwsLKy4rqOjI5/lJyL1pVAocOrUKVy7dq3ia82bN4eBgQF+/vlnxMXFoaioCPr6+mjUqBF69+6NMWPGwMbGBgCq+mBpFioRqbfyc1GfPHmCW7duITAwEFFRUXjy5AlMTU3RtGlTfPfddxgyZAh0dHQqpgFEwEIlIvUmCAKuXLmCdevWYcuWLSgpKUGtWrUwbtw4TJkyBWZmZq9dYa9iLFQiUk/lo9IdO3Zg3rx5SEtLg4mJCUaMGIFp06ahfv36r3x+X0QsVCJSTwkJCfDz88P169dRt25duLm5YerUqbCyshL1pXtvwG1TRKRe8vPzERAQgD179uD+/fuws7PDsmXL4OjoKHa0SuGz/ERU5QRBwL59+9C3b19s3LgRtWrVws6dOxEfH19tyxTgLT8RVbG8vDwsWLAAoaGhKCwshJ+fH8aNG4eWLVuq6+39q/CWn4jEo1AokJOTg5EjR+LQoUNo3bo1pkyZglGjRr32YJTqhiNUIlI5QRCwdetWBAQE4MGDB3B3d8fcuXPRvHlzsaNVFkeoRCSOrVu3wtfXF2VlZVi2bBkmTZqkMaPSZ2ne74iI1EZmZiYmTZqEY8eOwdbWFkuXLoWdnZ267SlVGhYqEalERkYGhg4disTERDg6OiIqKgqWlpZix1IpbpsiIqWSyWTYsGEDBgwYgAsXLmDBggVaUaYAR6hEpERyuRxBQUEICAiAVCpFSEgIhg0bJuYhJlVKO36XRKRygiAgKCgIM2bMgIODAzZt2gRPT0+tKVOA26aISAkEQcDatWvx/fffo3v37ggPD4eJiYnYsVTpldumtOevDiJSCblcjjVr1uD7779Hjx49EBYWpull+lqcQyWiSktNTcWMGTNw8OBB9OrVC6GhoTA1NRU7lmhYqERUKampqXB1dcX169fh7Oys1SPTcrzlJ6JKmT17NjIyMvDjjz9qw5zpO+EIlYjemSAIuHPnDg4fPoz79+8jIiICTk5OWrWS/yYsVCJ6Z3fu3MHgwYNx584dhIeHw9nZuToduady/GuFiN5JWloaBg0ahLt37yIsLAx9+/Zlmb6AI1QieqvU1FQMHjwY9+7dQ1hYGPr16yd2JLXEQiWiNypfzc/IyEBERAScnZ3FjqS2WKhE9Frlt/kZGRnYsmULnJyceJv/BixUInolQRAwZcoUXL16FcuXL0efPn1Ypm/BRSkiekn5s/mnTp2Ci4sLRo0axa1R74AjVCJ6TvkRfHw2//3xrxwiqiCTybBu3TrMmDEDPXr0QGhoKMv0PbBQiQjA/8r0H//4B7p3746wsDCtPuikMlioRAS5XP7cyDQiIoIj00rgAdNEhPPnz+Orr75C7dq1ceXKFZibm4sdSd3xgGkiellmZiYCAgKgr6+PFStWoG7dumJHqra4yk+kxTIzM+Hh4YELFy4gODgYw4cP517TD8BCJdJSGRkZGDp0KC5evMgyVRLOoRJpoYKCAjg5OSEpKQmbN2+Gj48Py/T9cA6ViACFQoG4uDicP38eDg4OcHFxYZkqCQuVSIsIgoCtW7fC19cXHTp0wI4dO7gIpUQsVCItsmXLFvj6+qJdu3aIjo5G/fr1xY6kUd5YqKdOnUJpaWlVZSEiFcrNzUVgYCAUCgWmTZuGevXqiR1J47yxULt164YffvgBJSUlVZWHiFQgLy8PPj4+uHXrFpYuXYqBAweKHUkjvbFQ3d3dsXLlSsyfP7+q8hCRkuXl5WHEiBE4duwYli5dCj8/Px7FpyJv/Lf67bffwtzcHJGRkbh69SressWKiNSMQqHAunXrcODAAfTu3RuTJk3iir4KvXEfqiAIQnx8PNzd3fHxxx9j165d+Pzzz/l/CFE1IAgCDhw4gBEjRqBevXrYtWsXmjdvLnYsTfHKEnzrxn5BEBAVFYVhw4bB1tYWZ86cgVQqVU1EIlKaffv2wdvbG/Xr10dMTAxatGghdiRNUrmN/RKJBM7OzvDw8MDVq1exdOlSlJWVKT8eESlNfn4+li9fjsLCQvj5+bFMq8g7Pctfu3ZtrF+/HoIgYN68eRAEATNmzIC+vr6q8xHRe8rPz4efnx+Sk5Mxe/ZsjB07VuxIWuO9nuX/66+/0K5dOxQVFeH8+fNo27atatMR0XsRBAGjR49GaGgoZs6ciQULFkBXV1fsWJrow5/lb9CgAUJDQ2FkZARvb2/8+eefyolGRB9MEAQkJSVh//79aNasGXx8fFimVey9T5tSKBTYt28fvLy80Lx5c8TFxaFBgwaqS0hE7+TMmTMYPHgwpFIpdu/ejbZt23JHjuoo57QpHR0dODk5wdHRESkpKdi5cyf3pxKJ7PHjxwgODkZGRgY8PDxgY2PDMhVBpR6XkEqlCA0NRZ8+fTBnzhysX78eCoVC2dmI6B2UlJQgICAA27Ztw+TJkzF37lyxI2mtDzpgOikpCb1790bt2rVx/vx5WFpaKjcdEb3V7NmzsXDhQri7u+OXX36BkZGR2JG0gfIPmHZwcMCaNWvw4MEDeHh4IDMz80M+jojeU0pKCrZs2QJLS0t8++23LFORffArUBQKBUJDQzF+/Hh06tQJu3fvxscff6y8hET0Sjdv3oSzszMePnyIffv2wd7envOmVUc1r0DR0dHBgAED0KFDByQnJ+PgwYNcpCJSMblcjvDwcNy+fRv9+vXDF198wTJVA0o5w8vU1BTR0dHo0KEDJkyYgO3btyvjY4noFeRyORYsWIDly5dj+PDhWLt2LfebqgmlvvU0MjISo0aNQps2bXD48GGYmJh8WDoieklqaio6d+4MuVyOo0ePon379mJH0kaqf+vp0KFDsWjRIly9ehVjxozBgwcPlPnxRFovNTUVQ4YMgUwmQ1hYGMtUzSi1UHV0dDB16lR88cUXiIuL46Z/IiWSyWRYsmQJLl++jOHDh8PFxUXsSPQCpd7yA0+fJ05LS8OgQYOQmZmJbdu2oVevXpwwJ/oAMpkM69atw4wZM9CzZ09ERERwSk1clTtgurJXS0hIQNeuXWFpaYkrV65wKxXRB1izZg0CAgLQvXt3bN26lWUqPtXPoT7riy++wMyZM5GdnY3vvvsORUVFqroUkUbLzMxESEgIpFIpJk+ezDJVY+90wHRlGBgYYO7cubh9+zbCw8NhYWGBxYsXq+pyRBqp/LCTv/76CyEhIXBychI7Er2BSt8la2BggICAADRp0gQRERFITExU5eWINIpCocCWLVsqps+GDRvG1z+rOZXNoVZ8gCDg0qVL6NGjBwwNDXHkyBFYW1t/6McSaTS5XI5t27bB19cXHTp0QFRUFA8fUi9VO4dacVWJBK1bt4anpyeysrKwYcMGPHnyRNWXJarWsrOz4e/vj7KyMqxatYplWk2obA71uYvo6SEwMBByubziMblly5bx9oXoFR49eoQxY8YgPz8fgYGBaN26tdiR6B2p/Jb/WZcuXULv3r0hCAJOnDiBli1bcn8q0TMEQUBcXBzc3NzQqlUrHDt2DGZmZmLHopeJc8v/LBsbG0RGRkIikcDV1RXXrl2ryssTqb29e/dizJgxaN26NWJiYlim1UyV33N369YNHh4euHXrFlauXMn5VKL/l5+fj//85z8oKirC9OnT0axZM7Ej0XuqkjnUF82fPx/Z2dkIDw9Ho0aNMHPmTOjr64sRhUgtFBYWYuLEibh48SJmz56NYcOGiR2JKqFK51Cf9ffff6N169bIy8tDcnIyOnbsqKpLEak1QRCwefNmjBs3Dg4ODoiPj+f5pupP/DnUZ5mZmWHTpk0wNjbG+PHjcfv2bbGiEIlGEAQkJSVhzpw5sLKyQnBwMHe/VGOijVCBp/8xrV69Gv7+/vjmm29w6NAhrvqTVnn48CHatWuHu3fv4vjx4+jWrZvYkejdqNcIFXi66d/LywtOTk5ITExEcHAwFAqFmJGIqkxJSQlmz56N9PR0TJ06Ffb29mJHog8k6gi1XE5ODnr06IHU1FRs3boVrq6uVXFZIlHNnTsXCxYsgJubG0JDQ/kK6OpF/Uao5czMzDBt2jRIJBKsXLkSmZmZYkciUqmUlBSEh4fD0tIS/v7+LFMNoRYjVODpyTqhoaEYN24cOnXqhL1796JOnTpVdXmiKnPz5k04OTnh4cOH2L9/P+zt7bl2UP2o7wgVePo+qv79+6Njx444e/YsDhw4wPlU0jhyuRzh4eG4c+dOxX/vLFPNoTYj1HJpaWkVb3L89ddf8dlnn1V1BCKVkMvlmD9/PhYuXAgPDw8EBwfjo48+EjsWVY56j1DLNW7cGPPmzUNZWRlmzJiBvLw8sSMRKcXt27exfv161KpVC/7+/ixTDaR2haqnp4dvv/0W//jHP7Bnzx5MmTKFr6Kmai81NRWDBg1CWVkZwsLCYGtrK3YkUgG1K1QA0NXVxahRo9C2bVscOnQIhw4dYqlStVVWVoZly5bhypUr8PDwgLOzs9iRSEXUbg614sKCgD/++AN9+vTBgwcPsGvXLjg6OooVh6jSgoODMXXqVHTp0gXR0dF8pbpmqB5zqOUkEgmaNWsGLy8vPHz4EOvXr0dxcbHYsYjeS3p6OjZu3Ah9fX34+vqyTDWcKMf3vY9Zs2ahrKwMS5YsgVQqRXBwMGrUqCF2LKK3yszMxNChQ3Hr1i1s2LCBTwBqAbUdoZYzMDDApEmT0KBBA0RFReHcuXNiRyJ6K0EQEBsbi9OnT8PBwQFubm48kk8LqO0c6otSUlLg5uaGkpISbN++HV26dBE7EtErKRQKbNmyBX5+frCzs0NkZCQsLCzEjkXK9co51GpTqIIgYN26dfD394ejoyP279/P559JLd2/fx+2trbIzc3F4cOH+Ze/Zqpei1IvkkgkGD9+PHx9fZGQkIBZs2bh8ePHYsciek5ubi5GjBiB+/fvY9myZejcubPYkagKVZsRarnS0lJ07NgRv//+O3bu3ImBAwfyWWhSC/n5+Rg6dCiOHDmCFStWwM/Pj/Ommqt6j1DLGRgYYO3atTAzM8OsWbPw+++/c9M/iU4QBBw/fhxHjx6FtbU1PD09WaZaqNqNUMvt2rULgwYNQuvWrXH27FlupSJR7d69GyNHjkTjxo0RHR2Npk2bih2JVEszRqjlunfvDk9PT9y6dQuBgYF48uSJ2JFIS+Xn52Px4sUoLi6Gv78/y1SLqf3G/tepXbs2QkJCAADz58+HQqHArFmzoK+vL3Iy0iYFBQWYOHEiUlJSMGfOHHh5eYkdiURUbW/5y2VnZ6NNmzbIy8tDcnIyOnbsKHYk0hKCICA0NBSjR4+Gvb09EhMTOW+qPTTrlr9c3bp1sXnzZhgbG2PcuHH4888/xY5EWkAQBJw+fRo//PADmjVrhg0bNkBHp9r/caIPVO1HqMDTJ1NWrVqFadOmoXfv3jh48CC3UpFKPXr0CDY2Nrh79y6OHTuG7t27ix2JqpZmjlCBp++j8vb2hrOzM+Lj4xEUFMStVKQyJSUl+OGHH5Ceno6pU6fCwcFB7EikJjRihFouNzcX3bp1Q1paGiIjIzFgwACxI5GGKS4uRkBAAH7++Wf4+flh4cKFfARaO1XvZ/nfVXh4OCZNmgRbW1ts374dlpaWYkciDXL8+HH07NkTjRo1QkpKCs831V7aUaiCICAiIgK+vr5o3749du7cyVKlDyYIAi5cuIDBgwejrKwM0dHR+PLLLzlXr700dw71WRKJBF5eXvDy8sLp06cxb948KBQKsWNRNSeXyzF58mTcvXsX8+bNY5nSK2lcoQJPF6nmzJkDe3t7REREICoqiotUVGlyuRzz5s3Dr7/+ihEjRmDo0KEsU3oljSxUAPj0008RHR0NGxsbTJw4Edu2bWOp0nsrKyvDv//9byxevBienp5Yu3YtPvroI7FjkZrSuDnUF8XFxWHYsGFo1qwZTp48CRMTE7EjUTVy/fp1dOrUCXp6eoiPj4e1tbXYkUg9aMcc6osGDBiAxYsXIzU1FSNHjsSDBw/EjkTVxI0bNyreBRUREYGWLVuKHYnUnMaPUIGnT1KtXLkSAQEB6N+/P3bs2AEDAwOxY5EaS09PR69evXD79m3ExsaiT58+nDelZ2nnCBV4ukjl5uaGVq1a4fjx4zh+/DjnU+m15HI5oqKicOPGDfTq1Qtff/01y5TeiVYUKgA0bNgQsbGxaNCgAby8vHDo0CGWKr1EEASsWbMG//znP9G3b1+EhYXx8HJ6Z1pxy/+sy5cvw97eHjVq1MC5c+dgZWUldiRSEwqFAmvWrMHMmTPxzTffIDQ0lE9C0eto7y3/sz7//HMEBAQgPz8f//rXv1BQUCB2JFITGRkZWLZsGfT19TFz5kyWKb23antif2Xp6+tj7ty50NHRwaJFiwAAQUFBqFmzpsjJSEz37t2Du7s7Hj16hA0bNsDe3l7sSFQNaV2hAoCenh7mzJmD0tJSLFmyBHp6evj55595QLCWunfvHoYMGYJr165hw4YN8PDw4CIUVYrWNoiuri58fHxgZWWFvXv34uzZs1yk0kIKhQI//vgjzp49i5EjR/KxUvogWrco9aKUlBQMGjQIMpmMJwhpmWdPJrO1tcWOHTt4Mhm9Ky5KvYqtrS1iYmKgp6eHIUOGICkpSexIVAUEQUB4eDj8/PxgZ2fHMiWl0PoRarkzZ86ga9euMDU1RXJyMho2bCh2JFKhkydPol+/fqhbty6SkpJgYWEhdiSqXjhCfZN27dph7Nix+PvvvxEYGIiSkhKxI5GK5ObmYunSpZDL5QgICIC5ubnYkUhDaOUq/6sYGhoiMDAQOjo6CAoKAgAsWbIEhoaGIicjZcrNzcXw4cNx6tQpBAYGwtfXl3PmpDQs1GcYGBhg+fLlEAQBwcHBkEgkWLp0KfT19cWORkqQm5uLYcOGVZTphAkTWKakVCzUF+jr62PZsmUQBAHr1q2DRCLBkiVLWKrV3Isj0wkTJkBXV1fsWKRhuCj1GsXFxXBxcUFCQgJWrVoFPz8/sSNRJeXk5MDb2xsnTpzA8uXLMWHCBOjpcSxBH0Q73nqqTCdPnoSnpydq1qxZ8ToVql7K95r6+PjA3t4eJ06c4Lw4KQMLtTLi4+Ph4eEBIyMjxMTEoF27dmJHonckCAJ2796N0aNHw8rKCjExMWjUqJHYsUgzsFArQxAEJCUlwc3NDYaGhoiNjWWpVgMKhQL79u2Dt7c3mjZtWlGmXIQiJWGhVpYgCDh9+jTc3NxQs2ZNxMbGonXr1vzDqaYUCgX2798Pb29vNGrUCLGxsTz3lpSNG/srSyKRoHPnzti+fTsKCwsxePBgXL9+nYepqCFBECpGpg0bNkRMTAzLlKoMC/UdSSQSdOnSBVFRUcjPz4erqyuuXbsmdix6wZ49e+Dj41MxMm3atKnYkUiLsFDfg0Qiwddff42oqCgUFBTA1dUVly5dEjsW/b+4uDiMGjUKVlZW2LVrF8uUqhwLtRK6du2KgwcPon379hg0aBAuXrwodiSt9uJqfmxsLJo0aSJ2LNJCXJT6AElJSRgyZAhX/0VUvpo/YcIENG7cGFFRUWjYsCEXDEnVuCilbA4ODti5cydKS0sxZMgQXL58mQtVVai8TH18fPDVV1/hxIkT3BpFouII9QMJgoCEhAQMHToUxsbG2LVrF6ytrfmHWsUEQcDevXvh4+ODBg0acM6UqhpHqKpQvvofGRmJ/Px8DBo0CNeuXeNIVYVycnIQEhJSsZrPMiV1wRGqEpU/+y+VSuHu7o758+fDyMhI7FgaJScnBz4+Pjhw4ABsbW2xc+dO7jMlMXCEqmpdu3bF9u3bIZPJsGLFCsycORPFxcVix9IYubm5FadGde3alZv2Se2wUJWsS5cu2L17N8aOHYugoCDMmjULpaWlYseq9srPMy0/gu/YsWM86ITUDg+FVDKJRIKOHTvCxsYGBgYGCAoKQk5ODqZMmcJXVFeCXC5HbGwsgoODkZSUVHE4tI4OxwKkfjiHqkKlpaUICAhAUFAQ6tWrh+joaJbqeyh/1bOvry/kcjlWrFjBw6FJXfC0KTEUFxdjwYIF2Lp1K+RyORYvXoyBAweiZs2aYkdTaxkZGdi+fTvmzJmDFi1aYPz48Rg7dizLlNQFC1VMv/32G9zc3HDnzh2MGDECa9euhbGxsdix1FJ6ejo8PDyQmJiIrl27IjIyEhYWFmLHInoWV/nF1KFDB8TGxsLOzg6RkZFwcnLC/v37uV/1GTKZDKtXr4aTkxMuXboEZ2dnlilVKxyhViFBECCXyzF//nwsWrQIxsbGCAsLw9dff631o9Xc3FyEhYVh5syZMDAwQEhICDw8PLj4ROqKt/zqQiaTISwsDD/99BNu3ryJ3r17Y9OmTTAzM9O6AikrK0NaWhrc3d1x48YN9OzZE5MmTYKLi4vW/bugaoWFqm5u3boFV1dX3Lx5EyYmJvjpp5/g7u6uFQsvgiBAJpNhwYIFWLVqFQoKCuDi4oKwsDB8/PHHYscjehsWqjrKzs7G1q1bMWvWLOjr66Nz584IDQ2Fubm5xm6vKi0txZ49e7B48WJcvXoVxsbG2Lx5M7p27YratWuLHY/oXbBQ1ZVMJsOFCxcwbdo0JCYmokWLFpgwYQKGDBmChg0bih1Pqa5evYqgoCCEhIRAoVDA29sb33//PaytrcWORvQ+WKjqLiMjAzExMfjhhx9QWFiIzz77DD4+Ppg+fTr09fXFjldpgiAgNTUVq1atQkxMDHJyctCmTRvMnDkTzs7OWr8gR9USC7U6EAQBR48exerVq3Hq1CkUFRVh4MCBmDFjBmxsbGBoaCh2xPeSn5+P48ePY/To0Xjw4AFMTU3h4+ODRYsWQSqVauy0Bmk8Fmp18uTJE5w6dQrBwcGIi4uDVCrF4MGD0aZNG0ycOBF16tRR2zKSyWT473//i40bN+LMmTNISEiAgYEBfHx84Ovri88//1wrFt5Io7FQq6PCwkIsWbIEmzZtQlZWFiQSCSwtLeHv74/Ro0ejVq1aalFO5Xts79y5g40bN2Ljxo148OABdHR00KpVK0RFRaF58+aQSqViRyVSBhZqdSUIAu7du4fg4GBcunQJhw4dgkKhQMOGDdGyZUtMnz4dPXv2BIAqH7UKgoDHjx9XrNpnZmYiKysLNWvWRO/evdGjRw8MHjwYn3zySZXmIlIxFqomePz4Mc6dO4eFCxfi5MmTKC0tRY0aNWBtbY2+ffvC0dER5ubmKl01FwQB58+fR1FRESIiInDhwgVcvnwZCoUCpqamcHFxwfTp09GmTRu1nZYg+kAsVE2iUCgQHx+PK1euYPXq1UhLS6v4nqWlJWxsbFC/fn1MmTIFEokEdevWhaWlZaWu9ccff6CoqAhFRUVYsmQJHj9+jMTERBQWFgIAdHV1K1btmzZtCjs7O6X8HonUGAtVEwmCgLy8POzfvx8HDx5EYmIiiouLkZeXB+B/UwDW1tawtbV97mddXFzQqVOn576Wl5eHn376CXK5vOJrR44cQXZ2dsX1AMDc3BxSqRSenp7o0KEDXF1duWpP2oSFqunKysogl8tx48YNHD58GFlZWfjll18gCAKePHny0qtY9PT0oKur+9zXyn/tsz766CPo6uqiRo0a8PX1haGhIYYPHw5TU1NIpdKXPoNIC7BQtY1MJsOjR48AAMnJyThy5Mhz3z99+jQuX7783Ndq1aoFT0/P5w4m8fb2RuPGjSGRSFCnTh0eWkLEQqUX5ebmVhRuOT09PTRo0IC37kRvxkIlIlISnthPRKRKLFQiIiVhoRIRKQkLlYhISVioRERKwkIlIlISFioRkZKwUImIlISFSkSkJCxUIiIlYaESESkJC5WISElYqERESsJCJSJSEhYqEZGSvO2F7jxlmIjoHXGESkSkJCxUIiIlYaESESkJC5WISElYqERESsJCJSJSkv8Dq9EG/LSNVlkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# load the weights from file\n",
    "agent.load_state_dict(torch.load('checkpoint.pth'))\n",
    "\n",
    "state = env.reset()\n",
    "img = plt.imshow(env.render(mode='rgb_array'))\n",
    "while True:\n",
    "    state = torch.from_numpy(state).float().to(device)\n",
    "    with torch.no_grad():\n",
    "        action = agent(state)\n",
    "#     action = env.action_space.sample()\n",
    "    img.set_data(env.render(mode='rgb_array'))\n",
    "    plt.axis('off')\n",
    "    display.display(plt.gcf())\n",
    "    display.clear_output(wait=True)\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    state = next_state\n",
    "    if done:\n",
    "#         print('Score: ', t+1)\n",
    "        break\n",
    "\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78e9b6e4",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "473c43b2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}