RL.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RL - Reinforcement Learning"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pyglet==1.5.27\n",
      "  Downloading pyglet-1.5.27-py3-none-any.whl (1.1 MB)\n",
      "Installing collected packages: pyglet\n",
      "  Attempting uninstall: pyglet\n",
      "    Found existing installation: pyglet 2.0.7\n",
      "    Uninstalling pyglet-2.0.7:\n",
      "      Successfully uninstalled pyglet-2.0.7\n",
      "Successfully installed pyglet-1.5.27\n"
     ]
    }
   ],
   "source": [
    "# !pip install stable-baselines3[extra]\n",
    "# !pip install pyglet==1.5.27"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import gym\n",
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "from stable_baselines3.common.evaluation import evaluate_policy"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "environment_name = 'CartPole-v0'\n",
    "env = gym.make(environment_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'CartPole-v0'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "environment_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode:1 Score:14.0\n",
      "Episode:2 Score:35.0\n",
      "Episode:3 Score:23.0\n",
      "Episode:4 Score:14.0\n",
      "Episode:5 Score:25.0\n"
     ]
    }
   ],
   "source": [
    "episodes = 5\n",
    "for episode in range(1, episodes+1):\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    score = 0\n",
    "\n",
    "    while not done:\n",
    "        env.render()\n",
    "        action = env.action_space.sample()\n",
    "        n_state, reward, done, info = env.step(action)\n",
    "        score+=reward\n",
    "    print('Episode:{} Score:{}'.format(episode, score))\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\EI12753\\Anaconda3\\lib\\site-packages\\gym\\envs\\classic_control\\cartpole.py:150: UserWarning: \u001b[33mWARN: You are calling 'step()' even though this environment has already returned done = True. You should always call 'reset()' once you receive 'done = True' -- any further steps are undefined behavior.\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([ 0.05982491,  1.139573  , -0.25251535, -2.149712  ], dtype=float32),\n",
       " 0.0,\n",
       " True,\n",
       " {})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.step(1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Understand the Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Discrete(2)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.action_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Box([-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38], [4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38], (4,), float32)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.observation_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# actiona sapce 0 means push cart to left and 1 means push cart to right\n",
    "env.action_space.sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 3.8196471e+00, -8.4892563e+37, -7.6101586e-02,  1.9936354e+37],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.observation_space.sample()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make your directroies first\n",
    "log_path = os.path.join('Training', 'Logs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Training\\\\Logs'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cpu device\n"
     ]
    }
   ],
   "source": [
    "env = gym.make(environment_name)\n",
    "env = DummyVecEnv([lambda: env])\n",
    "model = PPO('MlpPolicy', env, verbose=1, tensorboard_log=log_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PPO??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logging to Training\\Logs\\PPO_1\n",
      "-----------------------------\n",
      "| time/              |      |\n",
      "|    fps             | 1221 |\n",
      "|    iterations      | 1    |\n",
      "|    time_elapsed    | 1    |\n",
      "|    total_timesteps | 2048 |\n",
      "-----------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 835         |\n",
      "|    iterations           | 2           |\n",
      "|    time_elapsed         | 4           |\n",
      "|    total_timesteps      | 4096        |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.008214259 |\n",
      "|    clip_fraction        | 0.0873      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.687      |\n",
      "|    explained_variance   | 0.00242     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 10.3        |\n",
      "|    n_updates            | 10          |\n",
      "|    policy_gradient_loss | -0.013      |\n",
      "|    value_loss           | 51.5        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 788         |\n",
      "|    iterations           | 3           |\n",
      "|    time_elapsed         | 7           |\n",
      "|    total_timesteps      | 6144        |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.009680622 |\n",
      "|    clip_fraction        | 0.0573      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.672      |\n",
      "|    explained_variance   | 0.0776      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 12.9        |\n",
      "|    n_updates            | 20          |\n",
      "|    policy_gradient_loss | -0.0161     |\n",
      "|    value_loss           | 35.6        |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 690          |\n",
      "|    iterations           | 4            |\n",
      "|    time_elapsed         | 11           |\n",
      "|    total_timesteps      | 8192         |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0077995164 |\n",
      "|    clip_fraction        | 0.076        |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.64        |\n",
      "|    explained_variance   | 0.249        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 18.2         |\n",
      "|    n_updates            | 30           |\n",
      "|    policy_gradient_loss | -0.0189      |\n",
      "|    value_loss           | 49.5         |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 687         |\n",
      "|    iterations           | 5           |\n",
      "|    time_elapsed         | 14          |\n",
      "|    total_timesteps      | 10240       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.007940204 |\n",
      "|    clip_fraction        | 0.0731      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.61       |\n",
      "|    explained_variance   | 0.256       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 20.9        |\n",
      "|    n_updates            | 40          |\n",
      "|    policy_gradient_loss | -0.0185     |\n",
      "|    value_loss           | 64.8        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 675         |\n",
      "|    iterations           | 6           |\n",
      "|    time_elapsed         | 18          |\n",
      "|    total_timesteps      | 12288       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.009608548 |\n",
      "|    clip_fraction        | 0.0935      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.604      |\n",
      "|    explained_variance   | 0.533       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 21          |\n",
      "|    n_updates            | 50          |\n",
      "|    policy_gradient_loss | -0.0176     |\n",
      "|    value_loss           | 53          |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 646         |\n",
      "|    iterations           | 7           |\n",
      "|    time_elapsed         | 22          |\n",
      "|    total_timesteps      | 14336       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.011538024 |\n",
      "|    clip_fraction        | 0.0943      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.571      |\n",
      "|    explained_variance   | 0.613       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 21.2        |\n",
      "|    n_updates            | 60          |\n",
      "|    policy_gradient_loss | -0.0115     |\n",
      "|    value_loss           | 54.6        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| time/                   |             |\n",
      "|    fps                  | 638         |\n",
      "|    iterations           | 8           |\n",
      "|    time_elapsed         | 25          |\n",
      "|    total_timesteps      | 16384       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.005244945 |\n",
      "|    clip_fraction        | 0.0723      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.583      |\n",
      "|    explained_variance   | 0.845       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 4.3         |\n",
      "|    n_updates            | 70          |\n",
      "|    policy_gradient_loss | -0.0122     |\n",
      "|    value_loss           | 28.6        |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 601          |\n",
      "|    iterations           | 9            |\n",
      "|    time_elapsed         | 30           |\n",
      "|    total_timesteps      | 18432        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0037054992 |\n",
      "|    clip_fraction        | 0.0298       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.574       |\n",
      "|    explained_variance   | 0.821        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 1.23         |\n",
      "|    n_updates            | 80           |\n",
      "|    policy_gradient_loss | -0.00514     |\n",
      "|    value_loss           | 21           |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| time/                   |              |\n",
      "|    fps                  | 581          |\n",
      "|    iterations           | 10           |\n",
      "|    time_elapsed         | 35           |\n",
      "|    total_timesteps      | 20480        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0065331194 |\n",
      "|    clip_fraction        | 0.0254       |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -0.57        |\n",
      "|    explained_variance   | 0.651        |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | 7.36         |\n",
      "|    n_updates            | 90           |\n",
      "|    policy_gradient_loss | -0.0054      |\n",
      "|    value_loss           | 23.8         |\n",
      "------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x1f30d117550>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.learn(total_timesteps=20000)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "PPO_Path=os.path.join('Training', 'Saved Models', 'PPO_Model_Cartpole')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(PPO_Path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "del model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Training\\\\Saved Models\\\\PPO_Model_Cartpole'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PPO_Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\EI12753\\Documents\\VS-Code\\GitHub\\TietoEvry\\DI\\Reinforcement_Learning\\RL.ipynb Cell 26\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/EI12753/Documents/VS-Code/GitHub/TietoEvry/DI/Reinforcement_Learning/RL.ipynb#X61sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m model\u001b[39m.\u001b[39mlearn(total_timesteps\u001b[39m=\u001b[39m\u001b[39m20000\u001b[39m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "model.learn(total_timesteps=20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = PPO.load(PPO_Path, env=env)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\EI12753\\Anaconda3\\lib\\site-packages\\stable_baselines3\\common\\evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(200.0, 0.0)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_policy(model, env, n_eval_episodes=10, render=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode:1 Score:[200.]\n",
      "Episode:2 Score:[200.]\n",
      "Episode:3 Score:[200.]\n",
      "Episode:4 Score:[200.]\n",
      "Episode:5 Score:[200.]\n"
     ]
    }
   ],
   "source": [
    "episodes = 5\n",
    "for episode in range(1, episodes+1):\n",
    "    obs = env.reset()\n",
    "    done = False\n",
    "    score = 0\n",
    "\n",
    "    while not done:\n",
    "        env.render()\n",
    "        action, _ = model.predict(obs)\n",
    "        obs, reward, done, info = env.step(action)\n",
    "        score+=reward\n",
    "    print('Episode:{} Score:{}'.format(episode, score))\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;31mSignature:\u001b[0m\n",
      "\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m\n",
      "\u001b[0m    \u001b[0mobservation\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mDict\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\n",
      "\u001b[0m    \u001b[0mstate\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mTuple\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m...\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\n",
      "\u001b[0m    \u001b[0mepisode_start\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\n",
      "\u001b[0m    \u001b[0mdeterministic\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\n",
      "\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[0mTuple\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mOptional\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mTuple\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m...\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mSource:\u001b[0m   \n",
      "    \u001b[1;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m\n",
      "\u001b[0m        \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\n",
      "\u001b[0m        \u001b[0mobservation\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mDict\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\n",
      "\u001b[0m        \u001b[0mstate\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mTuple\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m...\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\n",
      "\u001b[0m        \u001b[0mepisode_start\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\n",
      "\u001b[0m        \u001b[0mdeterministic\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\n",
      "\u001b[0m    \u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[0mTuple\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mOptional\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mTuple\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m...\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\n",
      "\u001b[0m        \u001b[1;34m\"\"\"\n",
      "        Get the policy action from an observation (and optional hidden state).\n",
      "        Includes sugar-coating to handle different observations (e.g. normalizing images).\n",
      "\n",
      "        :param observation: the input observation\n",
      "        :param state: The last hidden states (can be None, used in recurrent policies)\n",
      "        :param episode_start: The last masks (can be None, used in recurrent policies)\n",
      "            this correspond to beginning of episodes,\n",
      "            where the hidden states of the RNN must be reset.\n",
      "        :param deterministic: Whether or not to return deterministic actions.\n",
      "        :return: the model's action and the next hidden state\n",
      "            (used in recurrent policies)\n",
      "        \"\"\"\u001b[0m\u001b[1;33m\n",
      "\u001b[0m        \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpolicy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobservation\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepisode_start\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mFile:\u001b[0m      c:\\users\\ei12753\\anaconda3\\lib\\site-packages\\stable_baselines3\\common\\base_class.py\n",
      "\u001b[1;31mType:\u001b[0m      method\n"
     ]
    }
   ],
   "source": [
    "model.predict??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Discrete(2)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.action_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Box([-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38], [4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38], (4,), float32)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.observation_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[ 0.01647124, -0.17852287,  0.04943347,  0.3274649 ]],\n",
       "       dtype=float32),\n",
       " array([1.], dtype=float32),\n",
       " array([False]),\n",
       " [{}])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.step(action)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Viewing logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_log_path = os.path.join(log_path, 'PPO_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Training\\\\Logs\\\\PPO_1'"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_log_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "^C\n"
     ]
    }
   ],
   "source": [
    "!tensorboard --logdir=(training_log_path)\n",
    "\n",
    "## got to localhost:6006"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adding a callback to the training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}