policy-gradient-pytorch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyOXRycCDDGL5aPVI/76t7a6",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "816eefbd0cd34a1c97ea1c80f8938853": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_909f8840553c44d0ba508e831f6e4dd0",
              "IPY_MODEL_be16a55054b0499eacfbf9d9eb85ecd2",
              "IPY_MODEL_b7b1fc85c5244519b747070c23bd7987",
              "IPY_MODEL_755a5d83edfd470189d50fa5aa89c987",
              "IPY_MODEL_760bcd38b24643d1990e873bb725db35"
            ],
            "layout": "IPY_MODEL_7c268ae59e234f158da90c2089d32145"
          }
        },
        "909f8840553c44d0ba508e831f6e4dd0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6d09c583d930484ba02b084025bd3a0d",
            "placeholder": "​",
            "style": "IPY_MODEL_89e44b0f399547d3af7ae44970593315",
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
          }
        },
        "be16a55054b0499eacfbf9d9eb85ecd2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "PasswordModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "PasswordModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "PasswordView",
            "continuous_update": true,
            "description": "Token:",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_9fdf785a66b44743ad4590306c070fca",
            "placeholder": "​",
            "style": "IPY_MODEL_ceabf729106f48ad9ebe76b4d7f1473f",
            "value": ""
          }
        },
        "b7b1fc85c5244519b747070c23bd7987": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "CheckboxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Add token as git credential?",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_ecc5fd9e734946b483e7b5eae43b7c2d",
            "style": "IPY_MODEL_58eeff673a634aa28913f0d3c3b9abe7",
            "value": true
          }
        },
        "755a5d83edfd470189d50fa5aa89c987": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ButtonView",
            "button_style": "",
            "description": "Login",
            "disabled": false,
            "icon": "",
            "layout": "IPY_MODEL_d80f71f4cfb2420ab36456ed25dff266",
            "style": "IPY_MODEL_4e2416645aa8453c9f1dd326691ed9f2",
            "tooltip": ""
          }
        },
        "760bcd38b24643d1990e873bb725db35": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_56cf2622ad384980bd8165617c59e4d6",
            "placeholder": "​",
            "style": "IPY_MODEL_a6be8db66996434b86417cc92c051b21",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
          }
        },
        "7c268ae59e234f158da90c2089d32145": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": "center",
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "flex",
            "flex": null,
            "flex_flow": "column",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "6d09c583d930484ba02b084025bd3a0d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "89e44b0f399547d3af7ae44970593315": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9fdf785a66b44743ad4590306c070fca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ceabf729106f48ad9ebe76b4d7f1473f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ecc5fd9e734946b483e7b5eae43b7c2d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "58eeff673a634aa28913f0d3c3b9abe7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d80f71f4cfb2420ab36456ed25dff266": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4e2416645aa8453c9f1dd326691ed9f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "button_color": null,
            "font_weight": ""
          }
        },
        "56cf2622ad384980bd8165617c59e4d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a6be8db66996434b86417cc92c051b21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/michunie99/RL-course-huggin-face/blob/main/policy-gradient-pytorch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Setting up the virtual display"
      ],
      "metadata": {
        "id": "xlxijbZfySP-"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "CJF-GR2Oqh62"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "!apt install python-opengl\n",
        "!apt install ffmpeg\n",
        "!apt install xvfb\n",
        "!pip install pyvirtualdisplay\n",
        "!pip install pyglet==1.5.1"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Virtual display\n",
        "from pyvirtualdisplay import Display\n",
        "\n",
        "virtual_display = Display(visible=0, size=(1400, 900))\n",
        "virtual_display.start()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IqSHyDYByRsG",
        "outputId": "6ba7db06-9216-46a2-8e2b-e7c94609d177"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<pyvirtualdisplay.display.Display at 0x7f181d647ac0>"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "!pip install -r https://raw.githubusercontent.com/huggingface/deep-rl-class/main/notebooks/unit4/requirements-unit4.txt"
      ],
      "metadata": {
        "id": "6mbTYRqayaX6"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "All nessesary imports"
      ],
      "metadata": {
        "id": "DNvMjEhtynVw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "from collections import deque\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "# PyTorch\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torch.distributions import Categorical\n",
        "\n",
        "# Gym\n",
        "import gym\n",
        "import gym_pygame\n",
        "\n",
        "# Hugging Face Hub\n",
        "from huggingface_hub import notebook_login # To log to our Hugging Face account to be able to upload models to the Hub.\n",
        "import imageio"
      ],
      "metadata": {
        "id": "ytb5KX2pybLE"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "metadata": {
        "id": "cYkixYkwyqzX"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8QbApxo-yxp9",
        "outputId": "1d616576-3b06-49a8-96d3-ccca787237f9"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cuda:0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Playing CartPole-v1"
      ],
      "metadata": {
        "id": "YlA6kHVay15t"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "env_id = \"CartPole-v1\"\n",
        "# Create the env\n",
        "env = gym.make(env_id)\n",
        "\n",
        "# Create the evaluation env\n",
        "eval_env = gym.make(env_id)\n",
        "\n",
        "# Get the state space and action space\n",
        "s_size = env.observation_space.shape[0]\n",
        "a_size = env.action_space.n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-rXWaaHHy0Fm",
        "outputId": "54f2d155-bd9b-4cd8-c119-7a26ac892d6a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/gym/core.py:317: DeprecationWarning: \u001b[33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n",
            "/usr/local/lib/python3.8/dist-packages/gym/wrappers/step_api_compatibility.py:39: DeprecationWarning: \u001b[33mWARN: Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"_____OBSERVATION SPACE_____ \\n\")\n",
        "print(\"The State Space is: \", s_size)\n",
        "print(\"Sample observation\", env.observation_space.sample())  # Get a random observation"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rODpWDQQzMV8",
        "outputId": "bd5e38a3-fe9c-4571-dc45-9a8997cba6b9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "_____OBSERVATION SPACE_____ \n",
            "\n",
            "The State Space is:  4\n",
            "Sample observation [ 4.2000484e+00 -1.3047681e+38  1.1323833e-02  2.3354901e+38]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"\\n _____ACTION SPACE_____ \\n\")\n",
        "print(\"The Action Space is: \", a_size)\n",
        "print(\"Action Space Sample\", env.action_space.sample())  # Take a random action"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sye0S2HIzNv4",
        "outputId": "c7283245-c1dd-41fb-d9ae-37323706c329"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " _____ACTION SPACE_____ \n",
            "\n",
            "The Action Space is:  2\n",
            "Action Space Sample 1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class Policy(nn.Module):\n",
        "    def __init__(self, s_size, a_size, h_size):\n",
        "        super(Policy, self).__init__()\n",
        "        # Create two fully connected layers\n",
        "        self.fc1 = nn.Linear(s_size, h_size)\n",
        "        self.fc2 = nn.Linear(h_size, a_size)\n",
        "\n",
        "\n",
        "    def forward(self, x):\n",
        "        # Define the forward pass\n",
        "        # state goes to fc1 then we apply ReLU activation function\n",
        "        x = F.relu(self.fc1(x))\n",
        "        # fc1 outputs goes to fc2\n",
        "        x = self.fc2(x)\n",
        "        # We output the softmax (dim=1 to avoid calculating softmax on batch size)\n",
        "        return F.softmax(x, dim=1)\n",
        "\n",
        "    def act(self, state):\n",
        "        \"\"\"\n",
        "        Given a state, take action\n",
        "        \"\"\"\n",
        "        state = torch.from_numpy(state).float().unsqueeze(0).to(device)\n",
        "        probs = self.forward(state).cpu()\n",
        "        m = Categorical(probs)\n",
        "        action = m.sample()\n",
        "        return action.item(), m.log_prob(action)"
      ],
      "metadata": {
        "id": "VW24AeoxzOMb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def reinforce(policy, optimizer, n_training_episodes, max_t, gamma, print_every):\n",
        "    # Help us to calculate the score during the training\n",
        "    scores_deque = deque(maxlen=100)\n",
        "    scores = []\n",
        "    # Line 3 of pseudocode\n",
        "    for i_episode in range(1, n_training_episodes+1):\n",
        "        saved_log_probs = []\n",
        "        rewards = []\n",
        "        state = env.reset()\n",
        "        # Line 4 of pseudocode\n",
        "        for t in range(max_t):\n",
        "            action, log_prob = policy.act(state)\n",
        "            saved_log_probs.append(log_prob)\n",
        "            state, reward, done, _ = env.step(action)\n",
        "            rewards.append(reward)\n",
        "            if done:\n",
        "                break\n",
        "        scores_deque.append(sum(rewards))\n",
        "        scores.append(sum(rewards))\n",
        "\n",
        "        # Line 6 of pseudocode: calculate the return\n",
        "        returns = deque(maxlen=max_t)\n",
        "        n_steps = len(rewards)\n",
        "        # Compute the discounted returns at each timestep,\n",
        "        # as the sum of the gamma-discounted return at time t (G_t) + the reward at time t\n",
        "\n",
        "        # In O(N) time, where N is the number of time steps\n",
        "        # (this definition of the discounted return G_t follows the definition of this quantity\n",
        "        # shown at page 44 of Sutton&Barto 2017 2nd draft)\n",
        "        # G_t = r_(t+1) + r_(t+2) + ...\n",
        "\n",
        "        # Given this formulation, the returns at each timestep t can be computed\n",
        "        # by re-using the computed future returns G_(t+1) to compute the current return G_t\n",
        "        # G_t = r_(t+1) + gamma*G_(t+1)\n",
        "        # G_(t-1) = r_t + gamma* G_t\n",
        "        # (this follows a dynamic programming approach, with which we memorize solutions in order\n",
        "        # to avoid computing them multiple times)\n",
        "\n",
        "        # This is correct since the above is equivalent to (see also page 46 of Sutton&Barto 2017 2nd draft)\n",
        "        # G_(t-1) = r_t + gamma*r_(t+1) + gamma*gamma*r_(t+2) + ...\n",
        "\n",
        "\n",
        "        ## Given the above, we calculate the returns at timestep t as:\n",
        "        #               gamma[t] * return[t] + reward[t]\n",
        "        #\n",
        "        ## We compute this starting from the last timestep to the first, in order\n",
        "        ## to employ the formula presented above and avoid redundant computations that would be needed\n",
        "        ## if we were to do it from first to last.\n",
        "\n",
        "        ## Hence, the queue \"returns\" will hold the returns in chronological order, from t=0 to t=n_steps\n",
        "        ## thanks to the appendleft() function which allows to append to the position 0 in constant time O(1)\n",
        "        ## a normal python list would instead require O(N) to do this.\n",
        "        for t in range(n_steps)[::-1]:\n",
        "            disc_return_t = (returns[0] if len(returns)>0 else 0)\n",
        "            returns.appendleft(rewards[t] + gamma * disc_return_t) # TODO: complete here\n",
        "\n",
        "        ## standardization of the returns is employed to make training more stable\n",
        "        eps = np.finfo(np.float32).eps.item()\n",
        "\n",
        "        ## eps is the smallest representable float, which is\n",
        "        # added to the standard deviation of the returns to avoid numerical instabilities\n",
        "        returns = torch.tensor(returns)\n",
        "        returns = (returns - returns.mean()) / (returns.std() + eps)\n",
        "\n",
        "        # Line 7:\n",
        "        policy_loss = []\n",
        "        for log_prob, disc_return in zip(saved_log_probs, returns):\n",
        "            policy_loss.append(-log_prob * disc_return)\n",
        "        policy_loss = torch.cat(policy_loss).sum()\n",
        "\n",
        "        # Line 8: PyTorch prefers gradient descent\n",
        "        optimizer.zero_grad()\n",
        "        policy_loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        if i_episode % print_every == 0:\n",
        "            print('Episode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_deque)))\n",
        "\n",
        "    return scores"
      ],
      "metadata": {
        "id": "j6ERjwFg0V6C"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cartpole_hyperparameters = {\n",
        "    \"h_size\": 128,\n",
        "    \"n_training_episodes\": 2500,\n",
        "    \"n_evaluation_episodes\": 10,\n",
        "    \"max_t\": 1000,\n",
        "    \"gamma\": 0.98,\n",
        "    \"lr\": 1e-3,\n",
        "    \"env_id\": env_id,\n",
        "    \"state_space\": s_size,\n",
        "    \"action_space\": a_size,\n",
        "}"
      ],
      "metadata": {
        "id": "WH8mFd3B694n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create policy and place it to the device\n",
        "cartpole_policy = Policy(\n",
        "    cartpole_hyperparameters[\"state_space\"],\n",
        "    cartpole_hyperparameters[\"action_space\"],\n",
        "    cartpole_hyperparameters[\"h_size\"],\n",
        ").to(device)\n",
        "cartpole_optimizer = optim.Adam(cartpole_policy.parameters(), lr=cartpole_hyperparameters[\"lr\"])"
      ],
      "metadata": {
        "id": "dzhmFq0G9-qS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "scores = reinforce(\n",
        "    cartpole_policy,\n",
        "    cartpole_optimizer,\n",
        "    cartpole_hyperparameters[\"n_training_episodes\"],\n",
        "    cartpole_hyperparameters[\"max_t\"],\n",
        "    cartpole_hyperparameters[\"gamma\"],\n",
        "    100,\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U8WLTQoU7Axg",
        "outputId": "85d86837-5981-4024-c7d7-ead1a3502d38"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Episode 100\tAverage Score: 26.95\n",
            "Episode 200\tAverage Score: 68.61\n",
            "Episode 300\tAverage Score: 125.98\n",
            "Episode 400\tAverage Score: 240.71\n",
            "Episode 500\tAverage Score: 374.16\n",
            "Episode 600\tAverage Score: 419.28\n",
            "Episode 700\tAverage Score: 455.35\n",
            "Episode 800\tAverage Score: 465.63\n",
            "Episode 900\tAverage Score: 477.39\n",
            "Episode 1000\tAverage Score: 454.65\n",
            "Episode 1100\tAverage Score: 478.74\n",
            "Episode 1200\tAverage Score: 487.44\n",
            "Episode 1300\tAverage Score: 480.58\n",
            "Episode 1400\tAverage Score: 483.23\n",
            "Episode 1500\tAverage Score: 482.95\n",
            "Episode 1600\tAverage Score: 495.62\n",
            "Episode 1700\tAverage Score: 496.80\n",
            "Episode 1800\tAverage Score: 467.88\n",
            "Episode 1900\tAverage Score: 498.53\n",
            "Episode 2000\tAverage Score: 478.02\n",
            "Episode 2100\tAverage Score: 478.69\n",
            "Episode 2200\tAverage Score: 494.40\n",
            "Episode 2300\tAverage Score: 500.00\n",
            "Episode 2400\tAverage Score: 485.35\n",
            "Episode 2500\tAverage Score: 486.74\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plt.plot(scores)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "id": "jY1q3BXaDI2J",
        "outputId": "1d511695-9cf3-405f-b4f9-8cb420a0807c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f006c947220>]"
            ]
          },
          "metadata": {},
          "execution_count": 104
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2deZwU1bn3f8/s+74wzgDDwLDJIuPIIogIOiJco3FJjImiIWIStwRNJHqvmnh9Re/rTfSN14REo8ZE4zUmEuMSghg1ERVcQBRkQEBGYAZkGXZm5rx/dPXQ013VXdVVp+pU9fP9fOYz3aeqz3lOLb/z1FNnISEEGIZhmGCR5rUBDMMwjPOwuDMMwwQQFneGYZgAwuLOMAwTQFjcGYZhAkiG1wYAQEVFhaivr/faDIZhGF+xcuXKnUKISr1tSoh7fX09VqxY4bUZDMMwvoKINhtt47AMwzBMAGFxZxiGCSAs7gzDMAGExZ1hGCaAsLgzDMMEEFPiTkSbiGg1Eb1PRCu0tDIiWkJE67X/pVo6EdEDRNRKRKuIqElmBRiGYZhYrHjuZwghThJCNGvfFwBYKoRoBLBU+w4A5wBo1P7mAXjIKWMZhmEYc9jp534egGna58cAvArgZi39cRGaS3g5EZUQUY0QYpsdQ/3C0o934IPP9uD7Zw0FEcVsF0Lg2Xfb8JdVn6O+PB9fHDiK+op8tIysxqjaYmzbewj/s2wD/rVhJ4b1K8QLq7cDAL4zbTC6ewQWvbYxJs+rThuE19fvxNrtnbjxrKG4bkYj3li/E796fSPGDyrDY//ahI79R5BOhKlDK/HK2nZkZ6ThSFcPLj65Dv+7cmufvH71+qe4bOJAHDrWjU92dCIvKx3vbNqN7h6BjDRCWX4W2juPAAC+PmEAlm/chVmja7B84y4MqSpERhrht8s3Y2h1AfYcPIb2ziOoKMhCY1Uhygqy8NdV23BKfSnWbutE55EujB9Uhn5FOTh0rBsj+hUCAPYd7sKj/9rUp56zx9Sg83AX2vcdRo8Q+GTHfowbUAIC8O6WPZg9ugZbvjiIM4aFxnS8tn4n9h/pQk+PQFl+FtKIUF6QhZriXKzY/AUOHOnCrNE1CJ+lDR0HcLS7BwXZGXhvy25s2nUQI2uKUJqfiZMHlAIAXlnXjs07D2JwVQEqCrKx7/AxEIDa0lwU5WSi83AXaktycLRb4NV17WgZWQ0A2LHvCDZ/cQBHu3pw8Gg39h06hotOrsM/N+xCv+IcbN97GCs370YaAYMq8jFpcDnWbuvEqYPLda+zNzfuwrB+hSjLy4rZ9tKa7fhkx35MH16FUScU6V+oBry0ZjvOHFGNZ1ZuRXvnEQzvV4i12ztx5ohq1JfnIS8r3VQ+KzbvRm1JLmqKc3rTXl6zA9NHVCEzLXTEDx3rxuq2vfj+mUNx++I1WLu9EyV5mRhbV4KxdcUAgNaO/Wht34+bWoZh3m9X4oKmWtSV5PbmebRbYNnadpx9YnVvWrcQeHDZBnx32mBkpMXeg06ybF0HVrftxYVNddi6+yA6D3chI53QebgL5flZGFCWh7rSXGzceQCHjnbjxBOK8MArrbiwqQ5Hurpx+Fg3rpveiLH9Sxy3jczM505EnwLYDUAA+KUQYhER7RFClGjbCcBuIUQJET0PYKEQ4g1t21IANwshVkTlOQ8hzx4DBgw4efNmw774vqJ+wV8BAL/4RhNmjqqJ2b584y5csmi57m83LZyN7zyxEi9+uN2WDZsWzu61w2+E20M3lxmQXSaRvbyjfYTIvOJt09seDzM2mslPz75EaUbl6O0TaYPZsmRg9pwmOv//ef4ofGPiwKRsIKKVEdGUPpj13KcIIdqIqArAEiJaG7lRCCGIyNLlK4RYBGARADQ3NwduxZB9h7t00/cbpIdZ375fhjnSuXnmcNzz0lq0jKzG3z7akXQ+n949GwAw5o6XDY9hIl64/jSs3LIb//HnD3W3VxdlY8e+0JPHvKkNuGXWCACI2yBu+D+z0Nq+H2f/7LWE5S++djLu+9sn+McnHfjNlafgjGFVunm/eMNpOOf+1+Pm9fatM1BVmNMnrb3zMMbftRTA8eMVZmPHfky/7x+936O3x+MvH3yO6558D7UluWjbc0h3HzP5HevuQeOtL/bZf8lHO3DV4ytw5ogq/HrOKQCAsT/+G/YeOha3HL3jFmnD3EffwdK17XjkimZMHx7y3k+87SUcONpt2t5kOXCkCyfe/nLC/T69+7iz9cgVzfjmo31H4ycr7IkwFXMXQrRp/9sB/AnAeAA7iKgGALT/7drubQD6R/y8TktjINeT8Jr3bzsLP7/U+/fniY5xpBdlZSWyjHRzJy/N5Ek2sx/B2gWjFwpUFSdWgQvn0Ndb988xkElCcSeifCIqDH8G0ALgQwCLAczRdpsD4Dnt82IAl2u9ZiYC2Jsq8XYzBPW6ExAoyctCVob3vWtlHGMrWZptBMzYabUuTlTd7vEz+3NZj+u8dGgIM2GZagB/0lrDDAC/F0K8RETvAHiaiOYC2AzgK9r+LwCYBaAVwEEAVzpuNcPEgSz4uzK8vHTTnnvifaxaZ6c6rkuizzU4mWNt9UnMDgnFXQixEcBYnfRdAGbopAsA1zhiHeMb3LxoE0Fk/sYz6+URmRftdNM9NEyEZSwqiArnwazNsrRdxbDMHeeOxBcHjjrzaGUSJab8ZY6j3mXJhBlYnofKwmx0aN1AjTAr7sl67vEEPFLXrp8+xJQd0WW5pY09DoRP3La5b9nmC71i8iAAwLJ17Qn2dA7vA6QpxvynP/DahMBj5T436+UREYgIV09tSLiv2ReqZsq2HHOP2L/lxH7WfhzOw6aLYTrm7qDr7pcwu5ttEIu7y+w5qN/1i3EOr5/KDx/rPh5yiCM65jx393rLJNJHZ8YDHc9E+Dzo7vV1lggW9wBi1Hc4dbDyStV5BpTnmdrPlI0e9JYxwmy4Kb7oHRd0J7xtfzcPcmFxDyD7PBB3lbwwGS9UrZCdkX5cZOPYIaUrJOl/NvXbBL9z+kVlKnaFdPNlL4u7JBR/Ygs0Khx7M2EZU+Ju8XdOPLEY5WC2t1B8AYucO8C0SUwSsLgzjqBCF7wwVrwjL7vNmRqharUrpMR+7ua7eJorRdbTnopdIb2AxV0SyV62fF3ax8ohlPUI71hYJtly4XyD6/QEi04cej2T3ArLqH6vsrhLJlGfaMZ5iGJv+hvPGtpnu2zM9ZaR0RUy+codj7nr5+GM5x7ZW4aRCYu7JAjA39Zsxyl3/R3/bN1p6jdbdh3EJzv8OSukShAoRhSvMuif7uUjvJmS9bzvuBFtB6pjGHN3Oiwj66nJpXOazJORmy97WdwlsnLLbgDAqq17Te1/z8trE+/E+AJzYRkJnruN3ybMW8HeMuz9G8PirhI+vlJV6gqpAk71lrGKTK/VbG8ZsyjcY9EUHHNPZTy6eP/0Hk+fH43RjejWY/I5o2KnApASc7e2u6VMnAnLOIt6FqkDi3sA+dnfP3G9TJW6QgLe2xMdlvmfrzdh+Y/6TqIqZ/oB/c+W8jBIT3NELdS6ToIMi7tMPLqOvX7anTKkwmMLYPrYywpjRIdliChGHM0It5558WyW2aiZnRAtPvKvTte6QrpSSvKwuMvEa5X1iLu+PMprE2IgEJ797qn48zWT+6R7OVSdTNx9lgXEid4yBiKeoWBYhjGG53N3AdVfvDiNMx6e8zQNKHWtLFO9ZczkY2OEarJevNGvfnV5c1L5mcvdOVzrCqnodR6GPXcXMOscco8TeUTeh14tppyM2KoiHzOGV6GhssCBnJy9xvVy44nDQrC4M46gWsOkiijGQ0Y/d8Y9VD81LO4u4PYNqrDj4hqqPzKbRa8eZmtm9RD48TWk/qyZwTj3dmFxV4g3N+zy2oSk8brrYSKMrHPzET5ac1Q9Yvo9dJzKnT0Pt2BxV4jdvAQf4yG9E4dJaHZUjoMni+oPCCzuTEpg9Kju6cRhEoouzD7eAU5N7VHTqiDC4i4Jjvt5i9mj72pYxql8EkxG1lCZn1S+Mo9EEO8H1evE4i4JIVTrP+IeKlzzfohvq/qeQv/82bNVrxGdPbrGVp6A/7pC8pS/jO9QqSmz0ri4Nf2AVYZWO9Gn3BpuNzX3X3KSyyWmFjxCVSJq+mXBpWVkNTLT03BCSa7XpugS3ZDEa1eevnoSNu86aLM8Wz93FL1GNCM91re87d9G4swR1ebzNVlWKsLiLgkiUsiXlY8KIYYxdcW4dnqj7jYnZku0gpnpB+JRkpeFkrwsp8wxhczr1Ww4YlBFPgaU50m0JHXgsIwkIi9m72XPXVTwnLw2wW5YJh6yG1LdgVNu9XP3/tIJDOy5S+IHz6zq/ZxKHryqKNHgKF4iRf1n/A177kwg8TpMZG4NVdlWqOhWSHqBrWJVdVBy4jAiSiei94joee37ICJ6i4haiegPRJSlpWdr31u17fVyTPcPbpxOBRxTXyKra5rMsIws4pnq3OWV2rNCuokVz/0GAB9HfL8HwE+FEEMA7AYwV0ufC2C3lv5TbT9GMl5re2RXSK9tUZXYvveyj1Ry+TuznB7jNaZOIxHVAZgN4NfadwIwHcAz2i6PAThf+3ye9h3a9hmkQsCTSSnMXnGyLk2pYRnTv7PmwcqcWya2lOSIHn3LXSGNMdtG/wzADwH0aN/LAewRQnRp37cCqNU+1wL4DAC07Xu1/RmJeH1Bex3jVg0/hmXicf0M/S6mbhM5d04kfRdjCchBt0lCcSeifwPQLoRY6WTBRDSPiFYQ0YqOjg4ns2YYJYluAP3UHA6pcn/ErB4s2+Yx47lPBvAlItoE4CmEwjH3AyghonAzWgegTfvcBqA/AGjbiwHETFQuhFgkhGgWQjRXVlbaqoTquDpoRgFS5an499+aYLjN7iAmO9gt0svzV5tgdLGMdWeDSkJxF0L8SAhRJ4SoB3AJgFeEEF8HsAzARdpucwA8p31erH2Htv0VkeLPSalde7mcfaL5oepOM/KEIlu/T1aEEv3M7uXmlTRWFGRjaHVhUr/leywWO+/FbwYwn4haEYqpP6ylPwygXEufD2CBPRMZM6Sqs2IU61fSe1PQJJUYUGZiTqCo88qaboylEapCiFcBvKp93ghgvM4+hwFc7IBtgcENnfHac1FpVkgr2HmotPsSWdZlwW2IuvCUvwxjk3gC53VDKBvb1fPoqScZu7khM4bFPSB4HYWI9GLd7BZpt95uhm9iBjEpqkyKmgXA2LbIY5mblQ7AuNtkqsDizjDJ4pEKJipWhll2G6I0LQOdKdx7sfNEFfnbETWhF913Xzg6+QwDQGo3bUxgUdUrjkT6KlAKMW1YJa6cXI/vTBssvazwUc1X0HN380lRvdozjAWSuVe8Ev7IYp+/boo3RsRBZqOQkZ6G28890XY+fmi0VYHDMozjJHsDvnXLDOtlGXWFdCFmYkdoRtUWO2dIFH4dxJRM46LiU4oqsLgzylBdlOO1CYEgWcGLt1iHrMaSPXF5sLgHgAIFY4t6FGRn4MmrJjqbaYQ4FOVmWv65vX7uFvd3SMlkx227XXSH0/vO+GX596q3DY9eeYpnZbO4uwDPmBhi3tQGTBosb4LQS8cP6P2cyh5hslUPS2t3T0/c/ZwkLS25pQD9QGNVAaYNq/KsfBZ3F5A9elO1C162PWeO0J9PJiNePzsDlJymwCZ2rzYXtV25a9dJvH4fwOLOKEm8G+OuL4/q/Ww4qMVRawzKsNgwqC5kYft6dMIjrkyhIb+IlILF3QU4LJN6yFysQ/Ygpu4edWXWTIOqivUVBVmels/iHhCC1oDEq03ktiCGVeySrLj1xtxdnHzH5vtUpXnw0iZPy2dxZ5zHAb01e58b9XbxeoEUvcU6/NIOZei85EzG9PGDyhLuY1XQTS3WYS1LKdQU56C8INtTG1jcGV/jhuf+0U/Otvwbs2GZn331JMt5J8LuutvJLpgRyUc/ORu/i7NSVaoSdkYmNchfVprFnVESswJlvF/yot9Qmd/ne16WvHEE54+rTbyTRexGN5yIjuRlZSDTRO+lPmEZZaLl8slIl++UsLgzrmHlETzurpLvixdvOA2jTUwPEO+hQTcs45DhfgnvmCFocfZEuPmOiMWdcRy/v9zNzkhHZaG38VI7yJnyV/45NSP0qjZsmxbOxpg6efMFJQOLu03qF/wV859+32szfIGVG9N0WMZgR7si8MOZwwAA8QZQqtqI+XWBbMZZWNwd4Nl327w2IXB3pMyn9WITc9Dka3H2NAddRVW9zmi8ipSkWohGNizuLuDKTZ1CN0bfJf2M9jHmN1eOR01xcGeg9EkbkhSqPi2pCIu7C/jVI7l55vCkfudEYybzFq4tycUVp9bbzsepRrtlpP5cOYblJjg6Mi43llRn6F+aCwCYMqRCelks7owhbiyJZkRYoO67eGzc/epK8+QbI5mfX9qEl753mtdm9OKVkJtqlHzeyjRUFuCtW2Zg3tQG6WWxuLuAK2EZn1/0Vog8ntfPaDTYxz8HJCsjDWX5zs1DYrfmPn3Q9A3VRTmuXJ8s7oyShC/9REKjN1Tej6geS3ZlVki/xi8VhcWdUZLo23zGcP1FDwy7QjprjiM4JZA+eihxHBWrPlax/u1h/LE+GxMX1fwdJ29Ar2/muKNQvTZOEipXKytDLX/07VtnoDDb+vKObsDizriGlafueGEZN8XHrUiBSg2Fas5CJNEjh29sGYYtXxzEhIbEM1DKoKpQ3S61LO4BQCFdcAyzAmP0YsquWMoQW9Xj6vFQ5QV19DEceUIRlsw/vU+ayo2Tm6j1jONzvFrBxsuLubrI/BwsyeiDGpKij5/FWg+vRZHfpzoLi7uD/OafnxpuE0Kgo/OIi9a4g96UtU54eXZz8PPLyzvPH5V4J8aQYDW5ycPi7iBtew4Zbnti+WacctffpZQbxItZRP2PxEzj0VxfhtqSXPt2OOhNxjM7XE55fhYumzjQuUIt4PV1dNu5Iw23hecDUiQ65AsSijsR5RDR20T0ARGtIaIfa+mDiOgtImoloj8QUZaWnq19b9W218utgjrEm2Tq9fU7XbREDicU52D26BpcOmGA16YkpCgnE/9cMD3p35tpQFhonGVynCH5t84e4aIlwcDMC9UjAKYLIfYTUSaAN4joRQDzAfxUCPEUEf0CwFwAD2n/dwshhhDRJQDuAfBVSfYrRdw1NSULgWydefDSJkxprEBxbiY27zqA37+1RWp5FPWfkY/SIW+ljVOThJ67CLFf+5qp/QkA0wE8o6U/BuB87fN52ndo22eQKq/aJZMekNGSesweU2NqqlzAWUH2+p5O9srVW0M1XlbhJeZS405JHj485jEVcyeidCJ6H0A7gCUANgDYI4To0nbZCiD8Zq0WwGcAoG3fCyBmNVgimkdEK4hoRUdHh71aKEK8Nkx2zwqvRdBNoo9kQ0U+7r1ojJSyVOzBIasBYOEMFqb6uQshugGcREQlAP4EILm5YPvmuQjAIgBobm5W8BayTtxVe3xy59QU52Db3sNem9GLmcP2yk3TPCnX1O99ct5VJ5UWz3YKS71lhBB7ACwDMAlACRGFG4c6AOHliNoA9AcAbXsxgF2OWKs4XoZlnCo5W7Hh3bKYNboGmemEi0+uc63MuNHJXu3i1iCS8fVluH76kN7vfnGSVMBMb5lKzWMHEeUCOAvAxwiJ/EXabnMAPKd9Xqx9h7b9FZEC071194j4YRmPLspyB6eSNYv0l8cO5N+/LA/r75qFxupC6WVZgcWrL09/exLmtwyz9JvAi41JzLhpNQCWEdEqAO8AWCKEeB7AzQDmE1ErQjH1h7X9HwZQrqXPB7DAebPV43t/UHOR7MbqAq9N8DXx3BICMLDc/cVCgjQy1qwQB989dJ6EMXchxCoA43TSNwIYr5N+GMDFjljnI/7yweeBWDiCCZHolE0ZUoGM9DQsvmYKdh44ghn3/SNxnnG2sXaZw0zDxndbCJ44zAVSIColDVWP3LB+oXBOcV4mivOcm/I11YQp1errJqnx9kwB/PIo/eVx9l8wJqrr/ZecJDV/JzHy4Lm9dgbTYRmpVgQTFncH2XfomG66l2EZqyKUkyn/kjjvpNjJxozwR5MYy/CakGdfWXB81kwzc8sw+pygzRM0uCrfY0v8A4dlHOTRf20y3uiDHiRWGVDm/stEt0lWdG9qGYazRlRjVK21JdjMnMdUfIVz+tBKPH31JDQPLPXaFN/A4h4Q2PNzFruhn8z0NDTXe7M6UFAZP4iPpxU4LOMSXjhbFzbVoaLQ/GIajuFAZcu1cEaa3hXq4sF00kvmXlOx8BGRB4u7S3hxY9/3lbG+vXme+NYE3H3BaGUXH3aaVB1en5q1dgcW9yQRQuDP77Ul3tEl7LYd/YpyMHVoJb56Sn9nDLJJbUkuvjZ+gOc3v4xw13knnWC4zS+9qhj14Zh7kry5YZelUale3bJmtamiMAuPfzNmTFrKIutB64PbWpCXnW4rj0SmDe9XhA0dB5CXpf7tzU2ZPNQ/+4qy73BX4p085OqpDV6b4Ah6N78s4f3HD6ahK2qRc6OykrXBaMCTk08I/3XxGFw2aWBv90GV8frJLMhwWMYF7nz+I9e7r11+ar27BUbg1/eGA8vzMbjS3Fw8TodrcjJD3ny4f7wd8rIyMLEhZgkFJsVgcU8SlQTsl5edHJOmkHm+xq0upmX5WXjyqon4f1+LmcaJYZKCxd0l9hqMXjVLRYH+1L3Xz2jEqToLC/c2PgF87nWj4fKicZw0uByFOYl7B3GXSsYMLO5JYvX2enWdvaUEjeKnow1GQIZ7XZjpYjd+UBn+8/zRyRvnEW5onNUynr9uihxDXMALP4CbKXmwuOvw8bZ9+O2bmwCEpvKtX/BXtO/zduk5vZvg+eum4KyR1fr7W7hrnr56Ek7qX5KcYR6iojBYnW4g1ckI8KLyXsO9ZXQ45/7XAQCXTarHk29vAQCsb9+PqqIcL82KoTZObwizt8zt5450xhgP4PCEs3hxNMNLU05KwRfAaQT0SHxcYnFPEteFxWp5JnefrBOvV53wsTd7RHIz7fUrZ+SRmR4KHjyQgi+S37+9Bd3d8tSdxT0BfpmQK3pkY2/M3Sf2J4OZ9u7pqyehf1kS/b0VfihQ2DTLZKSHatPV0+OxJe5TZOLluR045m6SyBuqo/MIrnp8havlf338gJi0eOKmesSiocL+vNxmhuqPH1SGmmL1B/OkKhnazHBdDnqwF50cWnBmWILFz4MOi3sSfLKj0/UyL26Ov0JSdK8YxbUdf7h6kv3pDlSvJJOQ04dVAgDyspwLnZ079gRsWjjb9RG6qg0cY3FPAi+8YqsxftVfNlYWZmPq0MqkfktR/xn/8uMvnYh//GBa7xTPfuaHZw/D76+a4LUZvbC4JyDsEa/cvBv7DocGIqk4c19szD1Eoph7ca7aU+qGva9xA2K7aspsv1Q8x2EUb7ctkZmehoHlwVg6LyM9DfUK1YXF3ST3LfkE81yOs9shkQBcPbUBr9x4Oqptdu88Y1gVgNDoShmMPKEIf58/Fd+dNiRmm8oCzDBew71lLLCmbR8Af3hOiUaoZqQTGkxOkhWPiQ3l2LRwtu184jGkSv/FmB/OA8N4BXvuCdALa/hCUzQjg9gVMlwlX5wHhvEIFvckUP1lJRBsr7ZHa7FknocgHz8mNWBxj4MwcHv9cONHm/ilscZLu6lAk84LUyPCp8UHp0EKfnAuGO9hcY+DUUjDD7dWtADMGl3jkSUhMtMpbgPzx++caj1TP5wIhvEIfqEaB4GoaVA1MVHRcYq2SbXp3NffNSvudkveqCqVYhiFYc89DkZhGRWJNjVG7BVskJIl3AMoTWbMXVrOTJBRSTFY3ONgfKLUv/WD3Ac87OX3U2wKZoZRiYTiTkT9iWgZEX1ERGuI6AYtvYyIlhDReu1/qZZORPQAEbUS0SoiapJdCVkYxtwV0c14Aq6KjTIozs3Ez756Eh6fa3NuGsYUa++c6bUJvkGl286M594F4EYhxEgAEwFcQ0QjASwAsFQI0QhgqfYdAM4B0Kj9zQPwkONWu8iG9v0xaaqcwMgBSkZi7qPIkiXOH1dre3QtY44cng/flyQUdyHENiHEu9rnTgAfA6gFcB6Ax7TdHgNwvvb5PACPixDLAZQQkbddNZJEQGDXgaO933snrPKBW2z0gpVhmNTAUm8ZIqoHMA7AWwCqhRDbtE3bAYQX86wF8FnEz7Zqadsi0kBE8xDy7DFgQOxc5Srg666QyvWX0eeJuRPwequ9xcNl4IcGnGHiYVrciagAwB8BfE8IsS/y4hdCCCKypCJCiEUAFgFAc3Oz2goUhR/u+9jeMuaNLszJwNVTGxy2SJ8pjRWY0ui/pf6i+fXlzehXzGGiVEclITMl7kSUiZCw/04I8ayWvIOIaoQQ27SwS7uW3gagf8TP67Q032Hsuauh7nFfqNrId/UdZ/f5rvq0wCpw5sjqxDsxjIskFHcKuXwPA/hYCPHfEZsWA5gDYKH2/7mI9GuJ6CkAEwDsjQjf+AqjGRVV8dyN7AOOe+p2X6h+/JOZMfXN1VbNGVlTZC9zHyCzLz0TPFS6Wsx47pMBXAZgNRG9r6XdgpCoP01EcwFsBvAVbdsLAGYBaAVwEMCVjlrsIrEDg1Q6dfGJtjRZy3N1lj+rKszBM9+ehJEnBFfcw8crPc0/55xhIkko7kKIN2CsDTN09hcArrFplxIYOb1+0PiwjbJigM31ZZJyVot0P5xshtGBR6jGwXBWSEUevuIPYlLDRr8SPvNpKeq5//vsEV6bwNiExZ1hdAg37KkalvnWae70lgoaKvWWYXGPQ/SJCjvDaYoctT4jVBPs66Yjn5WuyAGyQbeQPzkZw8jE/3ehRIx6mryytl1/g4J4MbNlTmY63rn1TJwzqp/rZTtFd0/Yc/fYEBcZpy2YcvHJdR5b4l9UcgV4Pvd4GOjivS+tc9cOAyJj7okk3G0HtLIwGxk+VsZecU8hz72uNE/6YueMe/j37nOBg8e6vDbBNirEAO++YLTXJlimpyf0P1VfqDL+hz33OFz00Jtem2CaSAkaPyi2m6IXPXxumNGI1v9ArpYAAA2uSURBVPb9ni/xlww1JTkYX1+G+S1DvTaFYZKCxT0ObXsOeW1CXIxGqH7/TDUEaUhVAV684TSvzUiKzPQ0PP3tSV6bwTBJw2GZgBPU+dwZhokPi7sFVIu+Wgq1qGY8wzBS4bBMBEIILFvnn26OZmDHnWFSE/bcI3j23TZ889EVXpsRQ0me/pS7qs9ayTCMd7C4R7Btr5ovUFOprzXDMM6Q0uIuhMAXEWukJmL3wWOoX/BXiRbpYzQJWGTMPXKfQRX5OvsCw/sVOm4bE1xqeGUpX5PS4v7L1zai6c4l2Lr7oNemxMXKOJrVd7Sguuj4TRk5/cBL35uKG89So5ukX5k9pgaleZn4+kQ11/11kj9fMxm/ueIUr81gkiSlX6gu/XgHAKBt9yHUleZ5bI0xZmYmTDSHDE8B7Aw1xbl477YWr81wheqinD6OAuMvUtpzV2Ve9kQYzUwYb5k9I1jjGSY1SGlxD6NSd0E9L/1r4/vr7NkX9swZhokktcVdMT185IpmLLtxWkz6NWcM0d3fypOHYlVlbFJfnoc7zx/ltRmMwqR0zD2MKkP0pw+v1k234pVHV0WVujHO8uoPzvDaBEZxUlrcU8mbDULU5rlrJuNod4/XZjCML0hpcQ+TzItJFcjOjI2qRWu4X+umx9j+JV6bwDC+IaVj7n71ZisKsrD2zpnIyUz32hSGYRQlpcXdr/x27gRDYQ+On84wjB04LAMoq4iv3Hg61m3vjEkfUVMUk5boIcQvffoZhnGGlPLcj3X34Mm3t6BHW/x4+cYvpJdZW5Kb9G8bKgtwjskl6ozaJ+4twzCpSUqJ+6LXNuJHz67GMyu39kmXqX/FufrT9bqNX98vMAyTHCkl7uEZIPcdPuZamW45zqzdDMNEklLiHiY6VHG0qwcbOvZ7Y4xkOCzDMKlJSr1QDU/bEt33+z+e+xBbdx/ClZPr3TdKY1Rt7EvSaK46bRDGDShNKn/27BkmtUgpcQ8P44/2ZrfuDq3AtO9Ql+NlJpqKFwjNwZ6Vkfgh6tbZIw233XT2MNy+eA1yo7pIBmkQE8Mw5kmoKET0CBG1E9GHEWllRLSEiNZr/0u1dCKiB4iolYhWEVGTTOOTxbhniTdCWJiTiewMewOS5pxaj00LZyMz3eCUsuvOMCmFmZj7owBmRqUtALBUCNEIYKn2HQDOAdCo/c0D8JAzZjpDIn3r4QA1wzABIaG4CyFeAxDdIfw8AI9pnx8DcH5E+uMixHIAJURkrqO2G4Rj7gYa3uOAti+7aZr9TBiGYWySbG+ZaiHENu3zdgDhuWprAXwWsd9WLS0GIppHRCuIaEVHR0eSZlgjPErTyENf/MHntvK/+vQG3cWpvYQfRhgmNbHdFVKEAtWWJUQIsUgI0SyEaK6srLRrhikiB/L0OOGmR3H6UOfq0b8s+ZGteoQbtnBsPyudJx1jmCCTbG+ZHURUI4TYpoVd2rX0NgCRa8LVaWnK0SVB3PWauGQ85/f+4yzd6XyTIbr4y08diH2Hj+Hq0xscyZ9hGDVJVkEWA5ijfZ4D4LmI9Mu1XjMTAeyNCN94Tthx37X/KIb++4uO5+9Uc1Gan4W8LGd7qYafWrIz0nFjyzCeLphhAk5CBSGiJwFMA1BBRFsB3A5gIYCniWgugM0AvqLt/gKAWQBaARwEcKUEm5MmLHB/WWUvts4wDKM6CcVdCPE1g00zdPYVAK6xa1QyvLdlN+59aR0e++Z4wwFB4bizrJeMSr68VNEmhmGkE5i5ZRb8cTXe3LhLd46Yp97egnc2fYHDx7q1lNRTPB7DxDCpRWCmH6A4fdgXPLvaFRv0hvp7Pfzf6/IZhvGGwHjuYcyImYyOMoD5sMzdF4zG9TMa5RhhAPGE7gyTUgRG3I0mBdObL8bNOWT0iqorzcX8s4a6Un5zfRkAoLIw25XyGIZRg+CEZQzS9bz03QflLNahYgDkppZhuLCpTrmRswzDyCUwnrsR3bJiMHGYN/X4AKEMo1kaXSI9jTCkqsBTGxiGcZ/AiHv0C9Up97yC25770BNxv2XWiN7Piy47mUeDMgzjOsETdy04snX3ITz+5mZ0urleqk6AvX9ZHn50zog+acQdExmGkUyAYu76L1Rf/HC7azZEFn1Ty1Cc1qg/kVhuFg/9ZxhGLoHz3KNZ394pvezfXzUBeVnpGNe/pDft2umNGBvxPcx/XTQGTQNi0xmGYZwkMJ57mOjAyBPLt0gv89TBFfjoJ9GLVelzcfPxSTPfvnUGurpV7GPDMIzfCYy4hx13r9ZBTYaqwhyvTWAYJqAEJiwTjssIADc89Z63thiQmc4vUhmGcYcAeu7Ac++rN6XvL77RhOH9irw2g2GYFCE44q64UzxzlDrrhDMME3yCE5bRMFr82i6ja4ul5MswDCOD4Hju2v+PPt/naL7ZGWmoKspGcW6mo/kyDMPIJDDivueQnJGoH9zeAgBYtrYdb7TulFIGwzCM0wRC3Ht6BDZ2HADgfOw9vJD0STzwiGEYH+H7mLsQAg23vND7XfH3qgzDMK4QAHH32gKGYRj18L24f773UJ/vnUe6bOdZmBMbrTJqRP7325Nsl8cwDOM0vhb3w8e6MeWeZX3S7n1pnSP5RlNdpD9VQE4Gz/DIMIx6+FrcH/3XJin53tgyLCYtPY2j+QzDxKcsPwsAcM0ZQzy2xOe9ZTIdWMLuL9dOwbk/f6NP2rdPH4xPOw5g6lD9+dgZhmH0yMlMx6aFs702A4Dvxd2eN31Ty1CMrtMfeXrPRWNs5c0wDOMlvg7L2PXcL2iqc8gShmEYtfC1uG/s2G/5Nw2V+b2f07QRT1dOrgcAXHRyHTJMxta/Nn4AAKB/Wa5lGxiGYWTj67DMe1v2WP5Nv6Kc3tGsYR2//dwTcfu5JwIA/u/FYxPmEY6p3X3BaMvlMwzDuIGvPfewICdLlUH3RiMevLQJT8ydYKtMhmEYN/C15z6sXyGqi7KxY9+R3rSSvEzcf8k4nK71dFm2rh2Hjnbjzuc/wra9hzFhUDlumNGIgeX5RtkaMnsMz8nOMIw/IBlrjhLRTAD3A0gH8GshxMJ4+zc3N4sVK1YkXd6WXQfx5/fbsLptL+69cAxKtb6mevvVleYijfusMwwTAIhopRCiWW+b4547EaUDeBDAWQC2AniHiBYLIT5yuqwwA8rzcP2MRlP7MQzDpAIyYu7jAbQKITYKIY4CeArAeRLKYRiGYQyQIe61AD6L+L5VS2MYhmFcwrPeMkQ0j4hWENGKjo4Or8xgGIYJJDLEvQ1A/4jvdVpaH4QQi4QQzUKI5spKnsOFYRjGSWSI+zsAGoloEBFlAbgEwGIJ5TAMwzAGON5bRgjRRUTXAngZoa6Qjwgh1jhdDsMwDGOMlEFMQogXALyQcEeGYRhGCr6efoBhGIbRR8oIVctGEHUA2JzkzysA7HTQHD/AdU4NuM6pgZ06DxRC6PZIUULc7UBEK4yG3wYVrnNqwHVODWTVmcMyDMMwAYTFnWEYJoAEQdwXeW2AB3CdUwOuc2ogpc6+j7kzDMMwsQTBc2cYhmGiYHFnGIYJIL4WdyKaSUTriKiViBZ4bY+TENEmIlpNRO8T0QotrYyIlhDReu1/qZZORPSAdhxWEVGTt9abg4geIaJ2IvowIs1yHYlojrb/eiKa40VdzGBQ3zuIqE07z+8T0ayIbT/S6ruOiM6OSPfNdU9E/YloGRF9RERriOgGLT3I59mozu6eayGEL/8QmrdmA4AGAFkAPgAw0mu7HKzfJgAVUWn3AligfV4A4B7t8ywALwIgABMBvOW1/SbrOBVAE4APk60jgDIAG7X/pdrnUq/rZqG+dwC4SWffkdo1nQ1gkHatp/vtugdQA6BJ+1wI4BOtbkE+z0Z1dvVc+9lzT8UVn84D8Jj2+TEA50ekPy5CLAdQQkTKr+YthHgNwBdRyVbreDaAJUKIL4QQuwEsATBTvvXWMaivEecBeEoIcUQI8SmAVoSueV9d90KIbUKId7XPnQA+RmjxniCfZ6M6GyHlXPtZ3IO+4pMA8DciWklE87S0aiHENu3zdgDV2ucgHQurdQxC3a/VQhCPhMMTCGB9iagewDgAbyFFznNUnQEXz7WfxT3oTBFCNAE4B8A1RDQ1cqMIPc8Fuh9rKtQRwEMABgM4CcA2APd5a44ciKgAwB8BfE8IsS9yW1DPs06dXT3XfhZ3Uys++RUhRJv2vx3AnxB6RNsRDrdo/9u13YN0LKzW0dd1F0LsEEJ0CyF6APwKofMMBKi+RJSJkMj9TgjxrJYc6POsV2e3z7WfxT2wKz4RUT4RFYY/A2gB8CFC9Qv3EpgD4Dnt82IAl2s9DSYC2BvxyOs3rNbxZQAtRFSqPea2aGm+IOrdyJcROs9AqL6XEFE2EQ0C0AjgbfjsuiciAvAwgI+FEP8dsSmw59mozq6fa6/fLNt8Kz0LoTfRGwDc6rU9DtarAaE34x8AWBOuG4ByAEsBrAfwdwBlWjoBeFA7DqsBNHtdB5P1fBKhx9NjCMUT5yZTRwDfROglVCuAK72ul8X6/larzyrtxq2J2P9Wrb7rAJwTke6b6x7AFIRCLqsAvK/9zQr4eTaqs6vnmqcfYBiGCSB+DsswDMMwBrC4MwzDBBAWd4ZhmADC4s4wDBNAWNwZhmECCIs7wzBMAGFxZxiGCSD/H/OPHoe4tc/UAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate_agent(env, max_steps, n_eval_episodes, policy):\n",
        "    \"\"\"\n",
        "    Evaluate the agent for ``n_eval_episodes`` episodes and returns average reward and std of reward.\n",
        "    :param env: The evaluation environment\n",
        "    :param n_eval_episodes: Number of episode to evaluate the agent\n",
        "    :param policy: The Reinforce agent\n",
        "    \"\"\"\n",
        "    episode_rewards = []\n",
        "    for episode in range(n_eval_episodes):\n",
        "        state = env.reset()\n",
        "        step = 0\n",
        "        done = False\n",
        "        total_rewards_ep = 0\n",
        "\n",
        "        for step in range(max_steps):\n",
        "            action, _ = policy.act(state)\n",
        "            new_state, reward, done, info = env.step(action)\n",
        "            total_rewards_ep += reward\n",
        "\n",
        "            if done:\n",
        "                break\n",
        "            state = new_state\n",
        "        episode_rewards.append(total_rewards_ep)\n",
        "    mean_reward = np.mean(episode_rewards)\n",
        "    std_reward = np.std(episode_rewards)\n",
        "\n",
        "    return mean_reward, std_reward"
      ],
      "metadata": {
        "id": "KmtP7cXv-JNe"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "evaluate_agent(\n",
        "    eval_env, cartpole_hyperparameters[\"max_t\"], cartpole_hyperparameters[\"n_evaluation_episodes\"], cartpole_policy\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cykZfw8c-L6E",
        "outputId": "b00b418b-ddcb-495c-934b-603689cc7cc5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(500.0, 0.0)"
            ]
          },
          "metadata": {},
          "execution_count": 106
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Push to hub"
      ],
      "metadata": {
        "id": "Y9Rh9RVuES2K"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from huggingface_hub import HfApi, snapshot_download\n",
        "from huggingface_hub.repocard import metadata_eval_result, metadata_save\n",
        "\n",
        "from pathlib import Path\n",
        "import datetime\n",
        "import json\n",
        "import imageio\n",
        "\n",
        "import tempfile\n",
        "\n",
        "import os"
      ],
      "metadata": {
        "id": "SyQA8jP-C7MN"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def record_video(env, policy, out_directory, fps=30):\n",
        "    \"\"\"\n",
        "    Generate a replay video of the agent\n",
        "    :param env\n",
        "    :param Qtable: Qtable of our agent\n",
        "    :param out_directory\n",
        "    :param fps: how many frame per seconds (with taxi-v3 and frozenlake-v1 we use 1)\n",
        "    \"\"\"\n",
        "    images = []\n",
        "    done = False\n",
        "    state = env.reset()\n",
        "    img = env.render(mode=\"rgb_array\")\n",
        "    images.append(img)\n",
        "    while not done:\n",
        "        # Take the action (index) that have the maximum expected future reward given that state\n",
        "        action, _ = policy.act(state)\n",
        "        state, reward, done, info = env.step(action)  # We directly put next_state = state for recording logic\n",
        "        img = env.render(mode=\"rgb_array\")\n",
        "        images.append(img)\n",
        "    imageio.mimsave(out_directory, [np.array(img) for i, img in enumerate(images)], fps=fps)"
      ],
      "metadata": {
        "id": "Kgk3x01JEVXS"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def push_to_hub(repo_id,\n",
        "                model,\n",
        "                hyperparameters,\n",
        "                eval_env,\n",
        "                video_fps=30\n",
        "                ):\n",
        "  \"\"\"\n",
        "  Evaluate, Generate a video and Upload a model to Hugging Face Hub.\n",
        "  This method does the complete pipeline:\n",
        "  - It evaluates the model\n",
        "  - It generates the model card\n",
        "  - It generates a replay video of the agent\n",
        "  - It pushes everything to the Hub\n",
        "\n",
        "  :param repo_id: repo_id: id of the model repository from the Hugging Face Hub\n",
        "  :param model: the pytorch model we want to save\n",
        "  :param hyperparameters: training hyperparameters\n",
        "  :param eval_env: evaluation environment\n",
        "  :param video_fps: how many frame per seconds to record our video replay\n",
        "  \"\"\"\n",
        "\n",
        "  _, repo_name = repo_id.split(\"/\")\n",
        "  api = HfApi()\n",
        "\n",
        "  # Step 1: Create the repo\n",
        "  repo_url = api.create_repo(\n",
        "        repo_id=repo_id,\n",
        "        exist_ok=True,\n",
        "  )\n",
        "\n",
        "  with tempfile.TemporaryDirectory() as tmpdirname:\n",
        "    local_directory = Path(tmpdirname)\n",
        "\n",
        "    # Step 2: Save the model\n",
        "    torch.save(model, local_directory / \"model.pt\")\n",
        "\n",
        "    # Step 3: Save the hyperparameters to JSON\n",
        "    with open(local_directory / \"hyperparameters.json\", \"w\") as outfile:\n",
        "      json.dump(hyperparameters, outfile)\n",
        "\n",
        "    # Step 4: Evaluate the model and build JSON\n",
        "    mean_reward, std_reward = evaluate_agent(eval_env,\n",
        "                                            hyperparameters[\"max_t\"],\n",
        "                                            hyperparameters[\"n_evaluation_episodes\"],\n",
        "                                            model)\n",
        "    # Get datetime\n",
        "    eval_datetime = datetime.datetime.now()\n",
        "    eval_form_datetime = eval_datetime.isoformat()\n",
        "\n",
        "    evaluate_data = {\n",
        "          \"env_id\": hyperparameters[\"env_id\"],\n",
        "          \"mean_reward\": mean_reward,\n",
        "          \"n_evaluation_episodes\": hyperparameters[\"n_evaluation_episodes\"],\n",
        "          \"eval_datetime\": eval_form_datetime,\n",
        "    }\n",
        "\n",
        "    # Write a JSON file\n",
        "    with open(local_directory / \"results.json\", \"w\") as outfile:\n",
        "        json.dump(evaluate_data, outfile)\n",
        "\n",
        "    # Step 5: Create the model card\n",
        "    env_name = hyperparameters[\"env_id\"]\n",
        "\n",
        "    metadata = {}\n",
        "    metadata[\"tags\"] = [\n",
        "          env_name,\n",
        "          \"reinforce\",\n",
        "          \"reinforcement-learning\",\n",
        "          \"custom-implementation\",\n",
        "          \"deep-rl-class\"\n",
        "      ]\n",
        "\n",
        "    # Add metrics\n",
        "    eval = metadata_eval_result(\n",
        "        model_pretty_name=repo_name,\n",
        "        task_pretty_name=\"reinforcement-learning\",\n",
        "        task_id=\"reinforcement-learning\",\n",
        "        metrics_pretty_name=\"mean_reward\",\n",
        "        metrics_id=\"mean_reward\",\n",
        "        metrics_value=f\"{mean_reward:.2f} +/- {std_reward:.2f}\",\n",
        "        dataset_pretty_name=env_name,\n",
        "        dataset_id=env_name,\n",
        "      )\n",
        "\n",
        "    # Merges both dictionaries\n",
        "    metadata = {**metadata, **eval}\n",
        "\n",
        "    model_card = f\"\"\"\n",
        "  # **Reinforce** Agent playing **{env_id}**\n",
        "  This is a trained model of a **Reinforce** agent playing **{env_id}** .\n",
        "  To learn to use this model and train yours check Unit 4 of the Deep Reinforcement Learning Course: https://huggingface.co/deep-rl-course/unit4/introduction\n",
        "  \"\"\"\n",
        "\n",
        "    readme_path = local_directory / \"README.md\"\n",
        "    readme = \"\"\n",
        "    if readme_path.exists():\n",
        "        with readme_path.open(\"r\", encoding=\"utf8\") as f:\n",
        "          readme = f.read()\n",
        "    else:\n",
        "      readme = model_card\n",
        "\n",
        "    with readme_path.open(\"w\", encoding=\"utf-8\") as f:\n",
        "      f.write(readme)\n",
        "\n",
        "    # Save our metrics to Readme metadata\n",
        "    metadata_save(readme_path, metadata)\n",
        "\n",
        "    # Step 6: Record a video\n",
        "    video_path =  local_directory / \"replay.mp4\"\n",
        "    record_video(env, model, video_path, video_fps)\n",
        "\n",
        "    # Step 7. Push everything to the Hub\n",
        "    api.upload_folder(\n",
        "          repo_id=repo_id,\n",
        "          folder_path=local_directory,\n",
        "          path_in_repo=\".\",\n",
        "    )\n",
        "\n",
        "    print(f\"Your model is pushed to the Hub. You can view your model here: {repo_url}\")"
      ],
      "metadata": {
        "id": "oNslXPhTEXFL"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "notebook_login()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 443,
          "referenced_widgets": [
            "816eefbd0cd34a1c97ea1c80f8938853",
            "909f8840553c44d0ba508e831f6e4dd0",
            "be16a55054b0499eacfbf9d9eb85ecd2",
            "b7b1fc85c5244519b747070c23bd7987",
            "755a5d83edfd470189d50fa5aa89c987",
            "760bcd38b24643d1990e873bb725db35",
            "7c268ae59e234f158da90c2089d32145",
            "6d09c583d930484ba02b084025bd3a0d",
            "89e44b0f399547d3af7ae44970593315",
            "9fdf785a66b44743ad4590306c070fca",
            "ceabf729106f48ad9ebe76b4d7f1473f",
            "ecc5fd9e734946b483e7b5eae43b7c2d",
            "58eeff673a634aa28913f0d3c3b9abe7",
            "d80f71f4cfb2420ab36456ed25dff266",
            "4e2416645aa8453c9f1dd326691ed9f2",
            "56cf2622ad384980bd8165617c59e4d6",
            "a6be8db66996434b86417cc92c051b21"
          ]
        },
        "id": "vOzH0ZV0Eds5",
        "outputId": "e0a743d6-0203-4332-a693-51845e400772"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Token is valid.\n",
            "Your token has been saved in your configured git credential helpers (store).\n",
            "Your token has been saved to /root/.huggingface/token\n",
            "Login successful\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "repo_id = \"Michunie/CartPole-v1-policy-grad-torch\"  # TODO Define your repo id {username/Reinforce-{model-id}}\n",
        "push_to_hub(\n",
        "    repo_id,\n",
        "    cartpole_policy,  # The model we want to save\n",
        "    cartpole_hyperparameters,  # Hyperparameters\n",
        "    eval_env,  # Evaluation environment\n",
        "    video_fps=30\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "heHcakJ5Eeu3",
        "outputId": "52d69828-11a3-4b2c-8496-2ef6763cb611"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/gym/core.py:43: DeprecationWarning: \u001b[33mWARN: The argument mode in render method is deprecated; use render_mode during environment initialization instead.\n",
            "See here for more information: https://www.gymlibrary.ml/content/api/\u001b[0m\n",
            "  deprecation(\n",
            "WARNING:imageio_ffmpeg:IMAGEIO FFMPEG_WRITER WARNING: input image is not divisible by macro_block_size=16, resizing from (600, 400) to (608, 400) to ensure video compatibility with most codecs and players. To prevent resizing, make your input image divisible by the macro_block_size or set the macro_block_size to 1 (risking incompatibility).\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Your model is pushed to the Hub. You can view your model here: https://huggingface.co/Michunie/CartPole-v1-policy-grad-torch\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# PixelCopter"
      ],
      "metadata": {
        "id": "dUqgsZXREf2p"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "env_id = \"Pixelcopter-PLE-v0\"\n",
        "env = gym.make(env_id)\n",
        "eval_env = gym.make(env_id)\n",
        "s_size = env.observation_space.shape[0]\n",
        "a_size = env.action_space.n"
      ],
      "metadata": {
        "id": "SaAClTXJRktj",
        "outputId": "e17881e1-d22f-4db5-9add-dfb2b983b7c0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "couldn't import doomish\n",
            "Couldn't import doom\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/gym/core.py:317: DeprecationWarning: \u001b[33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n",
            "/usr/local/lib/python3.8/dist-packages/gym/wrappers/step_api_compatibility.py:39: DeprecationWarning: \u001b[33mWARN: Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"_____OBSERVATION SPACE_____ \\n\")\n",
        "print(\"The State Space is: \", s_size)\n",
        "print(\"Sample observation\", env.observation_space.sample())  # Get a random observation"
      ],
      "metadata": {
        "id": "owXphUpWRl9e",
        "outputId": "7092447e-c61c-48b5-eabd-ca68b0512963",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "_____OBSERVATION SPACE_____ \n",
            "\n",
            "The State Space is:  7\n",
            "Sample observation [-0.16741583 -0.14717987  0.94071686  1.0691606  -3.3566318  -0.2657506\n",
            "  1.0085341 ]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"\\n _____ACTION SPACE_____ \\n\")\n",
        "print(\"The Action Space is: \", a_size)\n",
        "print(\"Action Space Sample\", env.action_space.sample())  # Take a random action"
      ],
      "metadata": {
        "id": "7hFbVMXYRm76",
        "outputId": "130ceb7c-f445-404f-c03d-a7efc7b66846",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " _____ACTION SPACE_____ \n",
            "\n",
            "The Action Space is:  2\n",
            "Action Space Sample 1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class Policy(nn.Module):\n",
        "    def __init__(self, s_size, a_size, h_size):\n",
        "        super(Policy, self).__init__()\n",
        "        # Define the three layers here\n",
        "        self.fc1 = nn.Linear(s_size, h_size)\n",
        "        self.fc2 = nn.Linear(h_size, h_size * 2)\n",
        "        self.fc3 = nn.Linear(h_size * 2, a_size)\n",
        "\n",
        "    def forward(self, x):\n",
        "        # Define the forward process here\n",
        "        x = F.relu(self.fc1(x))\n",
        "        x = F.relu(self.fc2(x))\n",
        "        x = self.fc3(x)\n",
        "        return F.softmax(x, dim=1)\n",
        "\n",
        "    def act(self, state):\n",
        "        state = torch.from_numpy(state).float().unsqueeze(0).to(device)\n",
        "        probs = self.forward(state).cpu()\n",
        "        m = Categorical(probs)\n",
        "        action = m.sample()\n",
        "        return action.item(), m.log_prob(action)"
      ],
      "metadata": {
        "id": "C9eaVonbEhCF"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pixelcopter_hyperparameters = {\n",
        "    \"h_size\": 64,\n",
        "    \"n_training_episodes\": 50000,#50000,\n",
        "    \"n_evaluation_episodes\": 10,\n",
        "    \"max_t\": 10000,\n",
        "    \"gamma\": 0.99,\n",
        "    \"lr\": 5e-5,\n",
        "    \"env_id\": env_id,\n",
        "    \"state_space\": s_size,\n",
        "    \"action_space\": a_size,\n",
        "}"
      ],
      "metadata": {
        "id": "mut8imU_jcLz"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create policy and place it to the device\n",
        "#torch.manual_seed(1)\n",
        "pixelcopter_policy = Policy(\n",
        "    pixelcopter_hyperparameters[\"state_space\"],\n",
        "    pixelcopter_hyperparameters[\"action_space\"],\n",
        "    pixelcopter_hyperparameters[\"h_size\"],\n",
        ").to(device)\n",
        "pixelcopter_optimizer = optim.Adam(pixelcopter_policy.parameters(), lr=pixelcopter_hyperparameters[\"lr\"])"
      ],
      "metadata": {
        "id": "lu4kf6JSjiNT"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "scores = reinforce(\n",
        "    pixelcopter_policy,\n",
        "    pixelcopter_optimizer,\n",
        "    pixelcopter_hyperparameters[\"n_training_episodes\"],\n",
        "    pixelcopter_hyperparameters[\"max_t\"],\n",
        "    pixelcopter_hyperparameters[\"gamma\"],\n",
        "    1000,\n",
        ")"
      ],
      "metadata": {
        "id": "sFG0b3E4jkE_",
        "outputId": "5af7c67e-94aa-4878-d585-e2d61c9b45d7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Episode 1000\tAverage Score: 39.86\n",
            "Episode 2000\tAverage Score: 54.29\n",
            "Episode 3000\tAverage Score: 48.98\n",
            "Episode 4000\tAverage Score: 31.86\n",
            "Episode 5000\tAverage Score: 52.69\n",
            "Episode 6000\tAverage Score: 47.29\n",
            "Episode 7000\tAverage Score: 73.40\n",
            "Episode 8000\tAverage Score: 57.90\n",
            "Episode 9000\tAverage Score: 49.50\n",
            "Episode 10000\tAverage Score: 69.35\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plt.plot(scores)"
      ],
      "metadata": {
        "id": "33aXK7Ddjq9-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "torch.save(pixelcopter_policy.state_dict(), \"pixelcopter-policy-v2.pt\")"
      ],
      "metadata": {
        "id": "6fFbbr3yiGDL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "evaluate_agent(\n",
        "    eval_env, pixelcopter_hyperparameters[\"max_t\"], pixelcopter_hyperparameters[\"n_evaluation_episodes\"], pixelcopter_policy\n",
        ")"
      ],
      "metadata": {
        "id": "ecGBP1sARsVw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "repo_id = \"Michunie/Reinforce-PixelCopter\"  # TODO Define your repo id {username/Reinforce-{model-id}}\n",
        "push_to_hub(\n",
        "    repo_id,\n",
        "    pixelcopter_policy,  # The model we want to save\n",
        "    pixelcopter_hyperparameters,  # Hyperparameters\n",
        "    eval_env,  # Evaluation environment\n",
        "    video_fps=30\n",
        ")"
      ],
      "metadata": {
        "id": "6HoXqRhtzRB3",
        "outputId": "621a2639-7af6-4b5b-dfcd-60ab3376d7d6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/gym/core.py:43: DeprecationWarning: \u001b[33mWARN: The argument mode in render method is deprecated; use render_mode during environment initialization instead.\n",
            "See here for more information: https://www.gymlibrary.ml/content/api/\u001b[0m\n",
            "  deprecation(\n",
            "/usr/local/lib/python3.8/dist-packages/gym/utils/passive_env_checker.py:280: UserWarning: \u001b[33mWARN: No render modes was declared in the environment (env.metadata['render_modes'] is None or not defined), you may have trouble when calling `.render()`.\u001b[0m\n",
            "  logger.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Your model is pushed to the Hub. You can view your model here: https://huggingface.co/Michunie/Reinforce-PixelCopter\n"
          ]
        }
      ]
    }
  ]
}