RL PPO with stable-baselines.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pygame 2.4.0 (SDL 2.26.4, Python 3.10.9)\n",
      "Hello from the pygame community. https://www.pygame.org/contribute.html\n",
      "life: \t1\tscore: \t0\tfps: \t0\n",
      "-----------------\n",
      "-               -\n",
      "-               -\n",
      "-               -\n",
      "-               -\n",
      "-               -\n",
      "-               -\n",
      "-               -\n",
      "-       >       -\n",
      "-               -\n",
      "-               -\n",
      "-               -\n",
      "-               -\n",
      "-               -\n",
      "-               -\n",
      "-       F       -\n",
      "-----------------\n"
     ]
    }
   ],
   "source": [
    "from utils import make_env\n",
    "\n",
    "\n",
    "env = make_env()\n",
    "env.reset()\n",
    "\n",
    "_ = env.render(\"rgb_array\")\n",
    "_ = env.render(\"print\")\n",
    "\n",
    "env.close()\n",
    "del env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "\n",
    "vec_env = make_vec_env(lambda: make_env(1000), n_envs=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cuda device\n"
     ]
    }
   ],
   "source": [
    "from stable_baselines3 import PPO\n",
    "\n",
    "\n",
    "n_steps = 5_000_000\n",
    "\n",
    "model = PPO(\"MlpPolicy\", vec_env, verbose=1, tensorboard_log=f\"logs/{n_steps: _}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logging to logs/ 5_000_000\\PPO_1\n",
      "---------------------------------\n",
      "| rollout/           |          |\n",
      "|    ep_len_mean     | 739      |\n",
      "|    ep_rew_mean     | 2.63     |\n",
      "| time/              |          |\n",
      "|    fps             | 752      |\n",
      "|    iterations      | 1        |\n",
      "|    time_elapsed    | 21       |\n",
      "|    total_timesteps | 16384    |\n",
      "---------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 694        |\n",
      "|    ep_rew_mean          | 2.76       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 559        |\n",
      "|    iterations           | 2          |\n",
      "|    time_elapsed         | 58         |\n",
      "|    total_timesteps      | 32768      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.01364282 |\n",
      "|    clip_fraction        | 0.139      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.6       |\n",
      "|    explained_variance   | 0.126      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.0158     |\n",
      "|    n_updates            | 10         |\n",
      "|    policy_gradient_loss | -0.0181    |\n",
      "|    value_loss           | 0.0335     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 686         |\n",
      "|    ep_rew_mean          | 2.87        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 514         |\n",
      "|    iterations           | 3           |\n",
      "|    time_elapsed         | 95          |\n",
      "|    total_timesteps      | 49152       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.013403153 |\n",
      "|    clip_fraction        | 0.148       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.58       |\n",
      "|    explained_variance   | 0.115       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0191     |\n",
      "|    n_updates            | 20          |\n",
      "|    policy_gradient_loss | -0.0183     |\n",
      "|    value_loss           | 0.0275      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 681         |\n",
      "|    ep_rew_mean          | 2.9         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 489         |\n",
      "|    iterations           | 4           |\n",
      "|    time_elapsed         | 134         |\n",
      "|    total_timesteps      | 65536       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014158369 |\n",
      "|    clip_fraction        | 0.158       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.55       |\n",
      "|    explained_variance   | 0.0784      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00552    |\n",
      "|    n_updates            | 30          |\n",
      "|    policy_gradient_loss | -0.0203     |\n",
      "|    value_loss           | 0.0295      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 678        |\n",
      "|    ep_rew_mean          | 3.06       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 471        |\n",
      "|    iterations           | 5          |\n",
      "|    time_elapsed         | 173        |\n",
      "|    total_timesteps      | 81920      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.01191399 |\n",
      "|    clip_fraction        | 0.138      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.52      |\n",
      "|    explained_variance   | 0.00277    |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0108    |\n",
      "|    n_updates            | 40         |\n",
      "|    policy_gradient_loss | -0.0182    |\n",
      "|    value_loss           | 0.0271     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 669         |\n",
      "|    ep_rew_mean          | 3.17        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 460         |\n",
      "|    iterations           | 6           |\n",
      "|    time_elapsed         | 213         |\n",
      "|    total_timesteps      | 98304       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.013949834 |\n",
      "|    clip_fraction        | 0.16        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.5        |\n",
      "|    explained_variance   | 0.0133      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0179     |\n",
      "|    n_updates            | 50          |\n",
      "|    policy_gradient_loss | -0.0207     |\n",
      "|    value_loss           | 0.0331      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 684         |\n",
      "|    ep_rew_mean          | 3.14        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 452         |\n",
      "|    iterations           | 7           |\n",
      "|    time_elapsed         | 253         |\n",
      "|    total_timesteps      | 114688      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014206588 |\n",
      "|    clip_fraction        | 0.166       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.46       |\n",
      "|    explained_variance   | -0.00907    |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0376     |\n",
      "|    n_updates            | 60          |\n",
      "|    policy_gradient_loss | -0.0217     |\n",
      "|    value_loss           | 0.0294      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 712         |\n",
      "|    ep_rew_mean          | 3.18        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 449         |\n",
      "|    iterations           | 8           |\n",
      "|    time_elapsed         | 291         |\n",
      "|    total_timesteps      | 131072      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014100311 |\n",
      "|    clip_fraction        | 0.165       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.44       |\n",
      "|    explained_variance   | -0.0464     |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0267     |\n",
      "|    n_updates            | 70          |\n",
      "|    policy_gradient_loss | -0.0213     |\n",
      "|    value_loss           | 0.0265      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 711         |\n",
      "|    ep_rew_mean          | 3.18        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 451         |\n",
      "|    iterations           | 9           |\n",
      "|    time_elapsed         | 326         |\n",
      "|    total_timesteps      | 147456      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.013112858 |\n",
      "|    clip_fraction        | 0.156       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.38       |\n",
      "|    explained_variance   | 0.0161      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0345     |\n",
      "|    n_updates            | 80          |\n",
      "|    policy_gradient_loss | -0.0226     |\n",
      "|    value_loss           | 0.0238      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 744         |\n",
      "|    ep_rew_mean          | 3.17        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 452         |\n",
      "|    iterations           | 10          |\n",
      "|    time_elapsed         | 361         |\n",
      "|    total_timesteps      | 163840      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014113702 |\n",
      "|    clip_fraction        | 0.166       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.34       |\n",
      "|    explained_variance   | 0.062       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.026      |\n",
      "|    n_updates            | 90          |\n",
      "|    policy_gradient_loss | -0.0226     |\n",
      "|    value_loss           | 0.0252      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 746         |\n",
      "|    ep_rew_mean          | 3.23        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 454         |\n",
      "|    iterations           | 11          |\n",
      "|    time_elapsed         | 396         |\n",
      "|    total_timesteps      | 180224      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.012984214 |\n",
      "|    clip_fraction        | 0.155       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.31       |\n",
      "|    explained_variance   | 0.135       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0284     |\n",
      "|    n_updates            | 100         |\n",
      "|    policy_gradient_loss | -0.0214     |\n",
      "|    value_loss           | 0.0234      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 732         |\n",
      "|    ep_rew_mean          | 3.22        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 455         |\n",
      "|    iterations           | 12          |\n",
      "|    time_elapsed         | 431         |\n",
      "|    total_timesteps      | 196608      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014760892 |\n",
      "|    clip_fraction        | 0.169       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.25       |\n",
      "|    explained_variance   | -0.045      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0434     |\n",
      "|    n_updates            | 110         |\n",
      "|    policy_gradient_loss | -0.0244     |\n",
      "|    value_loss           | 0.027       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 737         |\n",
      "|    ep_rew_mean          | 3.33        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 456         |\n",
      "|    iterations           | 13          |\n",
      "|    time_elapsed         | 466         |\n",
      "|    total_timesteps      | 212992      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.012875687 |\n",
      "|    clip_fraction        | 0.15        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.22       |\n",
      "|    explained_variance   | 0.0673      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0397     |\n",
      "|    n_updates            | 120         |\n",
      "|    policy_gradient_loss | -0.0222     |\n",
      "|    value_loss           | 0.0245      |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| rollout/                |              |\n",
      "|    ep_len_mean          | 713          |\n",
      "|    ep_rew_mean          | 3.34         |\n",
      "| time/                   |              |\n",
      "|    fps                  | 457          |\n",
      "|    iterations           | 14           |\n",
      "|    time_elapsed         | 501          |\n",
      "|    total_timesteps      | 229376       |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0139559135 |\n",
      "|    clip_fraction        | 0.15         |\n",
      "|    clip_range           | 0.2          |\n",
      "|    entropy_loss         | -1.21        |\n",
      "|    explained_variance   | 0.0489       |\n",
      "|    learning_rate        | 0.0003       |\n",
      "|    loss                 | -0.0151      |\n",
      "|    n_updates            | 130          |\n",
      "|    policy_gradient_loss | -0.0235      |\n",
      "|    value_loss           | 0.0257       |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 718         |\n",
      "|    ep_rew_mean          | 3.24        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 458         |\n",
      "|    iterations           | 15          |\n",
      "|    time_elapsed         | 536         |\n",
      "|    total_timesteps      | 245760      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.015356608 |\n",
      "|    clip_fraction        | 0.175       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.13       |\n",
      "|    explained_variance   | 0.07        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.007      |\n",
      "|    n_updates            | 140         |\n",
      "|    policy_gradient_loss | -0.0251     |\n",
      "|    value_loss           | 0.0246      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 731         |\n",
      "|    ep_rew_mean          | 3.23        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 458         |\n",
      "|    iterations           | 16          |\n",
      "|    time_elapsed         | 571         |\n",
      "|    total_timesteps      | 262144      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.013144247 |\n",
      "|    clip_fraction        | 0.161       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.11       |\n",
      "|    explained_variance   | 0.251       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00333    |\n",
      "|    n_updates            | 150         |\n",
      "|    policy_gradient_loss | -0.0243     |\n",
      "|    value_loss           | 0.0235      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 749         |\n",
      "|    ep_rew_mean          | 3.18        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 459         |\n",
      "|    iterations           | 17          |\n",
      "|    time_elapsed         | 606         |\n",
      "|    total_timesteps      | 278528      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019768363 |\n",
      "|    clip_fraction        | 0.178       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.01       |\n",
      "|    explained_variance   | 0.228       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0034      |\n",
      "|    n_updates            | 160         |\n",
      "|    policy_gradient_loss | -0.0248     |\n",
      "|    value_loss           | 0.0255      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 779         |\n",
      "|    ep_rew_mean          | 3.14        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 459         |\n",
      "|    iterations           | 18          |\n",
      "|    time_elapsed         | 641         |\n",
      "|    total_timesteps      | 294912      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.014781395 |\n",
      "|    clip_fraction        | 0.162       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.04       |\n",
      "|    explained_variance   | 0.174       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0382     |\n",
      "|    n_updates            | 170         |\n",
      "|    policy_gradient_loss | -0.0241     |\n",
      "|    value_loss           | 0.0202      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 810        |\n",
      "|    ep_rew_mean          | 3.21       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 460        |\n",
      "|    iterations           | 19         |\n",
      "|    time_elapsed         | 676        |\n",
      "|    total_timesteps      | 311296     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.01955967 |\n",
      "|    clip_fraction        | 0.195      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1         |\n",
      "|    explained_variance   | 0.247      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0491    |\n",
      "|    n_updates            | 180        |\n",
      "|    policy_gradient_loss | -0.0281    |\n",
      "|    value_loss           | 0.0203     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 837         |\n",
      "|    ep_rew_mean          | 3.3         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 460         |\n",
      "|    iterations           | 20          |\n",
      "|    time_elapsed         | 711         |\n",
      "|    total_timesteps      | 327680      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019052116 |\n",
      "|    clip_fraction        | 0.163       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.943      |\n",
      "|    explained_variance   | 0.381       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0374     |\n",
      "|    n_updates            | 190         |\n",
      "|    policy_gradient_loss | -0.0248     |\n",
      "|    value_loss           | 0.0176      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 845         |\n",
      "|    ep_rew_mean          | 3.12        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 460         |\n",
      "|    iterations           | 21          |\n",
      "|    time_elapsed         | 746         |\n",
      "|    total_timesteps      | 344064      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016053695 |\n",
      "|    clip_fraction        | 0.143       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.826      |\n",
      "|    explained_variance   | 0.398       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0623     |\n",
      "|    n_updates            | 200         |\n",
      "|    policy_gradient_loss | -0.0231     |\n",
      "|    value_loss           | 0.0192      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 846        |\n",
      "|    ep_rew_mean          | 3.16       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 461        |\n",
      "|    iterations           | 22         |\n",
      "|    time_elapsed         | 781        |\n",
      "|    total_timesteps      | 360448     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.01766817 |\n",
      "|    clip_fraction        | 0.147      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.813     |\n",
      "|    explained_variance   | 0.299      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0537    |\n",
      "|    n_updates            | 210        |\n",
      "|    policy_gradient_loss | -0.0244    |\n",
      "|    value_loss           | 0.0158     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 843         |\n",
      "|    ep_rew_mean          | 3.29        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 461         |\n",
      "|    iterations           | 23          |\n",
      "|    time_elapsed         | 816         |\n",
      "|    total_timesteps      | 376832      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016244132 |\n",
      "|    clip_fraction        | 0.164       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.834      |\n",
      "|    explained_variance   | 0.257       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0139     |\n",
      "|    n_updates            | 220         |\n",
      "|    policy_gradient_loss | -0.0247     |\n",
      "|    value_loss           | 0.0212      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 844         |\n",
      "|    ep_rew_mean          | 3.17        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 461         |\n",
      "|    iterations           | 24          |\n",
      "|    time_elapsed         | 851         |\n",
      "|    total_timesteps      | 393216      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019366637 |\n",
      "|    clip_fraction        | 0.155       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.805      |\n",
      "|    explained_variance   | 0.342       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0281     |\n",
      "|    n_updates            | 230         |\n",
      "|    policy_gradient_loss | -0.0247     |\n",
      "|    value_loss           | 0.0202      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 862         |\n",
      "|    ep_rew_mean          | 3.11        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 462         |\n",
      "|    iterations           | 25          |\n",
      "|    time_elapsed         | 886         |\n",
      "|    total_timesteps      | 409600      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016339371 |\n",
      "|    clip_fraction        | 0.157       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.856      |\n",
      "|    explained_variance   | 0.219       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0369     |\n",
      "|    n_updates            | 240         |\n",
      "|    policy_gradient_loss | -0.0257     |\n",
      "|    value_loss           | 0.0166      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 872         |\n",
      "|    ep_rew_mean          | 3.1         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 462         |\n",
      "|    iterations           | 26          |\n",
      "|    time_elapsed         | 921         |\n",
      "|    total_timesteps      | 425984      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.022114165 |\n",
      "|    clip_fraction        | 0.169       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.801      |\n",
      "|    explained_variance   | 0.268       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0231     |\n",
      "|    n_updates            | 250         |\n",
      "|    policy_gradient_loss | -0.0247     |\n",
      "|    value_loss           | 0.0197      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 891         |\n",
      "|    ep_rew_mean          | 3.18        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 462         |\n",
      "|    iterations           | 27          |\n",
      "|    time_elapsed         | 956         |\n",
      "|    total_timesteps      | 442368      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.025513448 |\n",
      "|    clip_fraction        | 0.166       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.679      |\n",
      "|    explained_variance   | 0.307       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.041      |\n",
      "|    n_updates            | 260         |\n",
      "|    policy_gradient_loss | -0.0245     |\n",
      "|    value_loss           | 0.0163      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 918         |\n",
      "|    ep_rew_mean          | 3.16        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 462         |\n",
      "|    iterations           | 28          |\n",
      "|    time_elapsed         | 991         |\n",
      "|    total_timesteps      | 458752      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.022400228 |\n",
      "|    clip_fraction        | 0.16        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.695      |\n",
      "|    explained_variance   | 0.155       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.0462      |\n",
      "|    n_updates            | 270         |\n",
      "|    policy_gradient_loss | -0.0247     |\n",
      "|    value_loss           | 0.0198      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 897         |\n",
      "|    ep_rew_mean          | 3.28        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 462         |\n",
      "|    iterations           | 29          |\n",
      "|    time_elapsed         | 1027        |\n",
      "|    total_timesteps      | 475136      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016780581 |\n",
      "|    clip_fraction        | 0.16        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.733      |\n",
      "|    explained_variance   | 0.236       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0195     |\n",
      "|    n_updates            | 280         |\n",
      "|    policy_gradient_loss | -0.0252     |\n",
      "|    value_loss           | 0.0191      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 914         |\n",
      "|    ep_rew_mean          | 3.06        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 459         |\n",
      "|    iterations           | 30          |\n",
      "|    time_elapsed         | 1070        |\n",
      "|    total_timesteps      | 491520      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020054389 |\n",
      "|    clip_fraction        | 0.154       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.634      |\n",
      "|    explained_variance   | 0.48        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0305     |\n",
      "|    n_updates            | 290         |\n",
      "|    policy_gradient_loss | -0.0244     |\n",
      "|    value_loss           | 0.0187      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 918         |\n",
      "|    ep_rew_mean          | 3.02        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 454         |\n",
      "|    iterations           | 31          |\n",
      "|    time_elapsed         | 1116        |\n",
      "|    total_timesteps      | 507904      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.027810723 |\n",
      "|    clip_fraction        | 0.172       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.582      |\n",
      "|    explained_variance   | 0.355       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0269     |\n",
      "|    n_updates            | 300         |\n",
      "|    policy_gradient_loss | -0.0238     |\n",
      "|    value_loss           | 0.0158      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 912        |\n",
      "|    ep_rew_mean          | 3.06       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 450        |\n",
      "|    iterations           | 32         |\n",
      "|    time_elapsed         | 1162       |\n",
      "|    total_timesteps      | 524288     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02561608 |\n",
      "|    clip_fraction        | 0.165      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.593     |\n",
      "|    explained_variance   | 0.199      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0139    |\n",
      "|    n_updates            | 310        |\n",
      "|    policy_gradient_loss | -0.024     |\n",
      "|    value_loss           | 0.0144     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 915         |\n",
      "|    ep_rew_mean          | 3.18        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 447         |\n",
      "|    iterations           | 33          |\n",
      "|    time_elapsed         | 1208        |\n",
      "|    total_timesteps      | 540672      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.021494083 |\n",
      "|    clip_fraction        | 0.156       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.586      |\n",
      "|    explained_variance   | 0.279       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.065      |\n",
      "|    n_updates            | 320         |\n",
      "|    policy_gradient_loss | -0.0233     |\n",
      "|    value_loss           | 0.019       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 918         |\n",
      "|    ep_rew_mean          | 3.26        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 443         |\n",
      "|    iterations           | 34          |\n",
      "|    time_elapsed         | 1255        |\n",
      "|    total_timesteps      | 557056      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.032110058 |\n",
      "|    clip_fraction        | 0.166       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.554      |\n",
      "|    explained_variance   | 0.276       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0173     |\n",
      "|    n_updates            | 330         |\n",
      "|    policy_gradient_loss | -0.024      |\n",
      "|    value_loss           | 0.0195      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 943         |\n",
      "|    ep_rew_mean          | 3.15        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 440         |\n",
      "|    iterations           | 35          |\n",
      "|    time_elapsed         | 1301        |\n",
      "|    total_timesteps      | 573440      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.017856233 |\n",
      "|    clip_fraction        | 0.146       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.537      |\n",
      "|    explained_variance   | 0.385       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0361     |\n",
      "|    n_updates            | 340         |\n",
      "|    policy_gradient_loss | -0.0234     |\n",
      "|    value_loss           | 0.0179      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 941         |\n",
      "|    ep_rew_mean          | 3.09        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 437         |\n",
      "|    iterations           | 36          |\n",
      "|    time_elapsed         | 1347        |\n",
      "|    total_timesteps      | 589824      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.050217792 |\n",
      "|    clip_fraction        | 0.203       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.506      |\n",
      "|    explained_variance   | 0.402       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0401     |\n",
      "|    n_updates            | 350         |\n",
      "|    policy_gradient_loss | -0.0272     |\n",
      "|    value_loss           | 0.0163      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 932        |\n",
      "|    ep_rew_mean          | 3.2        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 434        |\n",
      "|    iterations           | 37         |\n",
      "|    time_elapsed         | 1393       |\n",
      "|    total_timesteps      | 606208     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03007529 |\n",
      "|    clip_fraction        | 0.146      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.521     |\n",
      "|    explained_variance   | 0.433      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0388    |\n",
      "|    n_updates            | 360        |\n",
      "|    policy_gradient_loss | -0.0205    |\n",
      "|    value_loss           | 0.0164     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 945         |\n",
      "|    ep_rew_mean          | 3.16        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 432         |\n",
      "|    iterations           | 38          |\n",
      "|    time_elapsed         | 1439        |\n",
      "|    total_timesteps      | 622592      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.027847715 |\n",
      "|    clip_fraction        | 0.158       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.449      |\n",
      "|    explained_variance   | 0.37        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0539     |\n",
      "|    n_updates            | 370         |\n",
      "|    policy_gradient_loss | -0.0228     |\n",
      "|    value_loss           | 0.0161      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 941         |\n",
      "|    ep_rew_mean          | 3.01        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 430         |\n",
      "|    iterations           | 39          |\n",
      "|    time_elapsed         | 1485        |\n",
      "|    total_timesteps      | 638976      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.031804338 |\n",
      "|    clip_fraction        | 0.147       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.444      |\n",
      "|    explained_variance   | 0.331       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0243     |\n",
      "|    n_updates            | 380         |\n",
      "|    policy_gradient_loss | -0.0209     |\n",
      "|    value_loss           | 0.0189      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 936         |\n",
      "|    ep_rew_mean          | 2.78        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 427         |\n",
      "|    iterations           | 40          |\n",
      "|    time_elapsed         | 1532        |\n",
      "|    total_timesteps      | 655360      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.028730033 |\n",
      "|    clip_fraction        | 0.152       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.457      |\n",
      "|    explained_variance   | 0.389       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0512     |\n",
      "|    n_updates            | 390         |\n",
      "|    policy_gradient_loss | -0.0204     |\n",
      "|    value_loss           | 0.013       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 925         |\n",
      "|    ep_rew_mean          | 2.78        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 425         |\n",
      "|    iterations           | 41          |\n",
      "|    time_elapsed         | 1578        |\n",
      "|    total_timesteps      | 671744      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.059255145 |\n",
      "|    clip_fraction        | 0.141       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.418      |\n",
      "|    explained_variance   | 0.246       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0051     |\n",
      "|    n_updates            | 400         |\n",
      "|    policy_gradient_loss | -0.0192     |\n",
      "|    value_loss           | 0.0171      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 925         |\n",
      "|    ep_rew_mean          | 2.69        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 423         |\n",
      "|    iterations           | 42          |\n",
      "|    time_elapsed         | 1624        |\n",
      "|    total_timesteps      | 688128      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.044300802 |\n",
      "|    clip_fraction        | 0.189       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.44       |\n",
      "|    explained_variance   | 0.376       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00882    |\n",
      "|    n_updates            | 410         |\n",
      "|    policy_gradient_loss | -0.0214     |\n",
      "|    value_loss           | 0.0143      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 918         |\n",
      "|    ep_rew_mean          | 2.5         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 421         |\n",
      "|    iterations           | 43          |\n",
      "|    time_elapsed         | 1670        |\n",
      "|    total_timesteps      | 704512      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.054189935 |\n",
      "|    clip_fraction        | 0.157       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.356      |\n",
      "|    explained_variance   | 0.319       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00914    |\n",
      "|    n_updates            | 420         |\n",
      "|    policy_gradient_loss | -0.0171     |\n",
      "|    value_loss           | 0.015       |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 922        |\n",
      "|    ep_rew_mean          | 2.39       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 419        |\n",
      "|    iterations           | 44         |\n",
      "|    time_elapsed         | 1716       |\n",
      "|    total_timesteps      | 720896     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.06383287 |\n",
      "|    clip_fraction        | 0.151      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.357     |\n",
      "|    explained_variance   | 0.234      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.0244     |\n",
      "|    n_updates            | 430        |\n",
      "|    policy_gradient_loss | -0.019     |\n",
      "|    value_loss           | 0.0143     |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 924        |\n",
      "|    ep_rew_mean          | 2.29       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 418        |\n",
      "|    iterations           | 45         |\n",
      "|    time_elapsed         | 1762       |\n",
      "|    total_timesteps      | 737280     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.01900949 |\n",
      "|    clip_fraction        | 0.118      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.333     |\n",
      "|    explained_variance   | 0.383      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0279    |\n",
      "|    n_updates            | 440        |\n",
      "|    policy_gradient_loss | -0.0167    |\n",
      "|    value_loss           | 0.0097     |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 934        |\n",
      "|    ep_rew_mean          | 2.21       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 416        |\n",
      "|    iterations           | 46         |\n",
      "|    time_elapsed         | 1809       |\n",
      "|    total_timesteps      | 753664     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.09138796 |\n",
      "|    clip_fraction        | 0.186      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.375     |\n",
      "|    explained_variance   | 0.42       |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0185    |\n",
      "|    n_updates            | 450        |\n",
      "|    policy_gradient_loss | -0.0133    |\n",
      "|    value_loss           | 0.012      |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 957         |\n",
      "|    ep_rew_mean          | 2.25        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 415         |\n",
      "|    iterations           | 47          |\n",
      "|    time_elapsed         | 1855        |\n",
      "|    total_timesteps      | 770048      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020467378 |\n",
      "|    clip_fraction        | 0.13        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.355      |\n",
      "|    explained_variance   | 0.446       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0314     |\n",
      "|    n_updates            | 460         |\n",
      "|    policy_gradient_loss | -0.0174     |\n",
      "|    value_loss           | 0.0114      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 957        |\n",
      "|    ep_rew_mean          | 2.1        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 413        |\n",
      "|    iterations           | 48         |\n",
      "|    time_elapsed         | 1901       |\n",
      "|    total_timesteps      | 786432     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.07240994 |\n",
      "|    clip_fraction        | 0.136      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.346     |\n",
      "|    explained_variance   | 0.274      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0465    |\n",
      "|    n_updates            | 470        |\n",
      "|    policy_gradient_loss | -0.0147    |\n",
      "|    value_loss           | 0.0152     |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 969        |\n",
      "|    ep_rew_mean          | 1.91       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 412        |\n",
      "|    iterations           | 49         |\n",
      "|    time_elapsed         | 1945       |\n",
      "|    total_timesteps      | 802816     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.10067861 |\n",
      "|    clip_fraction        | 0.136      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.343     |\n",
      "|    explained_variance   | 0.281      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0106    |\n",
      "|    n_updates            | 480        |\n",
      "|    policy_gradient_loss | -0.0173    |\n",
      "|    value_loss           | 0.0102     |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 958        |\n",
      "|    ep_rew_mean          | 1.97       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 413        |\n",
      "|    iterations           | 50         |\n",
      "|    time_elapsed         | 1979       |\n",
      "|    total_timesteps      | 819200     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.04122699 |\n",
      "|    clip_fraction        | 0.164      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.372     |\n",
      "|    explained_variance   | 0.328      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0327    |\n",
      "|    n_updates            | 490        |\n",
      "|    policy_gradient_loss | -0.0171    |\n",
      "|    value_loss           | 0.00849    |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 961         |\n",
      "|    ep_rew_mean          | 2.01        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 414         |\n",
      "|    iterations           | 51          |\n",
      "|    time_elapsed         | 2014        |\n",
      "|    total_timesteps      | 835584      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.021276455 |\n",
      "|    clip_fraction        | 0.128       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.322      |\n",
      "|    explained_variance   | 0.0819      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.026      |\n",
      "|    n_updates            | 500         |\n",
      "|    policy_gradient_loss | -0.0173     |\n",
      "|    value_loss           | 0.0123      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 962         |\n",
      "|    ep_rew_mean          | 1.85        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 415         |\n",
      "|    iterations           | 52          |\n",
      "|    time_elapsed         | 2049        |\n",
      "|    total_timesteps      | 851968      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.031653337 |\n",
      "|    clip_fraction        | 0.16        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.383      |\n",
      "|    explained_variance   | 0.216       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0326     |\n",
      "|    n_updates            | 510         |\n",
      "|    policy_gradient_loss | -0.021      |\n",
      "|    value_loss           | 0.0157      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 963         |\n",
      "|    ep_rew_mean          | 1.74        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 416         |\n",
      "|    iterations           | 53          |\n",
      "|    time_elapsed         | 2084        |\n",
      "|    total_timesteps      | 868352      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.037357256 |\n",
      "|    clip_fraction        | 0.178       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.32       |\n",
      "|    explained_variance   | 0.513       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0425     |\n",
      "|    n_updates            | 520         |\n",
      "|    policy_gradient_loss | -0.00857    |\n",
      "|    value_loss           | 0.00707     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 964         |\n",
      "|    ep_rew_mean          | 1.71        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 417         |\n",
      "|    iterations           | 54          |\n",
      "|    time_elapsed         | 2119        |\n",
      "|    total_timesteps      | 884736      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.030455992 |\n",
      "|    clip_fraction        | 0.157       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.319      |\n",
      "|    explained_variance   | 0.255       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0313     |\n",
      "|    n_updates            | 530         |\n",
      "|    policy_gradient_loss | -0.0167     |\n",
      "|    value_loss           | 0.0081      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 968         |\n",
      "|    ep_rew_mean          | 1.65        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 418         |\n",
      "|    iterations           | 55          |\n",
      "|    time_elapsed         | 2154        |\n",
      "|    total_timesteps      | 901120      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.024982493 |\n",
      "|    clip_fraction        | 0.135       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.342      |\n",
      "|    explained_variance   | 0.289       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.000666    |\n",
      "|    n_updates            | 540         |\n",
      "|    policy_gradient_loss | -0.0167     |\n",
      "|    value_loss           | 0.01        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 973         |\n",
      "|    ep_rew_mean          | 1.57        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 418         |\n",
      "|    iterations           | 56          |\n",
      "|    time_elapsed         | 2190        |\n",
      "|    total_timesteps      | 917504      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.031761795 |\n",
      "|    clip_fraction        | 0.137       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.344      |\n",
      "|    explained_variance   | 0.246       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0137     |\n",
      "|    n_updates            | 550         |\n",
      "|    policy_gradient_loss | -0.0187     |\n",
      "|    value_loss           | 0.0096      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 979         |\n",
      "|    ep_rew_mean          | 1.32        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 419         |\n",
      "|    iterations           | 57          |\n",
      "|    time_elapsed         | 2224        |\n",
      "|    total_timesteps      | 933888      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020370573 |\n",
      "|    clip_fraction        | 0.107       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.247      |\n",
      "|    explained_variance   | 0.432       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.046      |\n",
      "|    n_updates            | 560         |\n",
      "|    policy_gradient_loss | -0.0116     |\n",
      "|    value_loss           | 0.00562     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 971         |\n",
      "|    ep_rew_mean          | 1.66        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 420         |\n",
      "|    iterations           | 58          |\n",
      "|    time_elapsed         | 2259        |\n",
      "|    total_timesteps      | 950272      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.043118376 |\n",
      "|    clip_fraction        | 0.134       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.295      |\n",
      "|    explained_variance   | 0.35        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0404     |\n",
      "|    n_updates            | 570         |\n",
      "|    policy_gradient_loss | -0.0174     |\n",
      "|    value_loss           | 0.00862     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 966         |\n",
      "|    ep_rew_mean          | 1.75        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 421         |\n",
      "|    iterations           | 59          |\n",
      "|    time_elapsed         | 2294        |\n",
      "|    total_timesteps      | 966656      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.023132863 |\n",
      "|    clip_fraction        | 0.136       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.318      |\n",
      "|    explained_variance   | 0.141       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0316     |\n",
      "|    n_updates            | 580         |\n",
      "|    policy_gradient_loss | -0.0166     |\n",
      "|    value_loss           | 0.0145      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 965         |\n",
      "|    ep_rew_mean          | 1.73        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 421         |\n",
      "|    iterations           | 60          |\n",
      "|    time_elapsed         | 2329        |\n",
      "|    total_timesteps      | 983040      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020570142 |\n",
      "|    clip_fraction        | 0.12        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.288      |\n",
      "|    explained_variance   | 0.208       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0135     |\n",
      "|    n_updates            | 590         |\n",
      "|    policy_gradient_loss | -0.0155     |\n",
      "|    value_loss           | 0.0143      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 966         |\n",
      "|    ep_rew_mean          | 1.76        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 422         |\n",
      "|    iterations           | 61          |\n",
      "|    time_elapsed         | 2364        |\n",
      "|    total_timesteps      | 999424      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.052118063 |\n",
      "|    clip_fraction        | 0.144       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.297      |\n",
      "|    explained_variance   | 0.475       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00595    |\n",
      "|    n_updates            | 600         |\n",
      "|    policy_gradient_loss | -0.0144     |\n",
      "|    value_loss           | 0.00863     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 966         |\n",
      "|    ep_rew_mean          | 1.77        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 423         |\n",
      "|    iterations           | 62          |\n",
      "|    time_elapsed         | 2399        |\n",
      "|    total_timesteps      | 1015808     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.024127707 |\n",
      "|    clip_fraction        | 0.126       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.31       |\n",
      "|    explained_variance   | 0.325       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0109     |\n",
      "|    n_updates            | 610         |\n",
      "|    policy_gradient_loss | -0.0156     |\n",
      "|    value_loss           | 0.00801     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 969         |\n",
      "|    ep_rew_mean          | 1.75        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 423         |\n",
      "|    iterations           | 63          |\n",
      "|    time_elapsed         | 2434        |\n",
      "|    total_timesteps      | 1032192     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.027677435 |\n",
      "|    clip_fraction        | 0.113       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.254      |\n",
      "|    explained_variance   | 0.49        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0327     |\n",
      "|    n_updates            | 620         |\n",
      "|    policy_gradient_loss | -0.0157     |\n",
      "|    value_loss           | 0.00747     |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 972        |\n",
      "|    ep_rew_mean          | 1.44       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 424        |\n",
      "|    iterations           | 64         |\n",
      "|    time_elapsed         | 2469       |\n",
      "|    total_timesteps      | 1048576    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02455661 |\n",
      "|    clip_fraction        | 0.144      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.372     |\n",
      "|    explained_variance   | 0.444      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0148    |\n",
      "|    n_updates            | 630        |\n",
      "|    policy_gradient_loss | -0.0147    |\n",
      "|    value_loss           | 0.00737    |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 974         |\n",
      "|    ep_rew_mean          | 1.47        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 425         |\n",
      "|    iterations           | 65          |\n",
      "|    time_elapsed         | 2504        |\n",
      "|    total_timesteps      | 1064960     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.024588855 |\n",
      "|    clip_fraction        | 0.134       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.281      |\n",
      "|    explained_variance   | 0.278       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0344     |\n",
      "|    n_updates            | 640         |\n",
      "|    policy_gradient_loss | -0.0181     |\n",
      "|    value_loss           | 0.00946     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 979         |\n",
      "|    ep_rew_mean          | 1.5         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 425         |\n",
      "|    iterations           | 66          |\n",
      "|    time_elapsed         | 2539        |\n",
      "|    total_timesteps      | 1081344     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.036231846 |\n",
      "|    clip_fraction        | 0.142       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.259      |\n",
      "|    explained_variance   | 0.285       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0176     |\n",
      "|    n_updates            | 650         |\n",
      "|    policy_gradient_loss | -0.0168     |\n",
      "|    value_loss           | 0.0122      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 968        |\n",
      "|    ep_rew_mean          | 1.63       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 426        |\n",
      "|    iterations           | 67         |\n",
      "|    time_elapsed         | 2573       |\n",
      "|    total_timesteps      | 1097728    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03188927 |\n",
      "|    clip_fraction        | 0.144      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.285     |\n",
      "|    explained_variance   | 0.0903     |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0184    |\n",
      "|    n_updates            | 660        |\n",
      "|    policy_gradient_loss | -0.0167    |\n",
      "|    value_loss           | 0.0101     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 964         |\n",
      "|    ep_rew_mean          | 1.82        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 427         |\n",
      "|    iterations           | 68          |\n",
      "|    time_elapsed         | 2608        |\n",
      "|    total_timesteps      | 1114112     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.037227497 |\n",
      "|    clip_fraction        | 0.165       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.301      |\n",
      "|    explained_variance   | 0.193       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0272     |\n",
      "|    n_updates            | 670         |\n",
      "|    policy_gradient_loss | -0.0186     |\n",
      "|    value_loss           | 0.0133      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 962         |\n",
      "|    ep_rew_mean          | 2.01        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 427         |\n",
      "|    iterations           | 69          |\n",
      "|    time_elapsed         | 2643        |\n",
      "|    total_timesteps      | 1130496     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.031820126 |\n",
      "|    clip_fraction        | 0.111       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.213      |\n",
      "|    explained_variance   | 0.317       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0178     |\n",
      "|    n_updates            | 680         |\n",
      "|    policy_gradient_loss | -0.017      |\n",
      "|    value_loss           | 0.0119      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 962         |\n",
      "|    ep_rew_mean          | 2.13        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 428         |\n",
      "|    iterations           | 70          |\n",
      "|    time_elapsed         | 2678        |\n",
      "|    total_timesteps      | 1146880     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.037686333 |\n",
      "|    clip_fraction        | 0.176       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.28       |\n",
      "|    explained_variance   | 0.206       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0359     |\n",
      "|    n_updates            | 690         |\n",
      "|    policy_gradient_loss | -0.0193     |\n",
      "|    value_loss           | 0.0155      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 954         |\n",
      "|    ep_rew_mean          | 2.12        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 428         |\n",
      "|    iterations           | 71          |\n",
      "|    time_elapsed         | 2713        |\n",
      "|    total_timesteps      | 1163264     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.031515643 |\n",
      "|    clip_fraction        | 0.161       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.272      |\n",
      "|    explained_variance   | 0.17        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0408     |\n",
      "|    n_updates            | 700         |\n",
      "|    policy_gradient_loss | -0.0199     |\n",
      "|    value_loss           | 0.011       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 962         |\n",
      "|    ep_rew_mean          | 1.91        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 429         |\n",
      "|    iterations           | 72          |\n",
      "|    time_elapsed         | 2748        |\n",
      "|    total_timesteps      | 1179648     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019646866 |\n",
      "|    clip_fraction        | 0.109       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.257      |\n",
      "|    explained_variance   | 0.311       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0274     |\n",
      "|    n_updates            | 710         |\n",
      "|    policy_gradient_loss | -0.0154     |\n",
      "|    value_loss           | 0.0136      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 968         |\n",
      "|    ep_rew_mean          | 1.87        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 429         |\n",
      "|    iterations           | 73          |\n",
      "|    time_elapsed         | 2783        |\n",
      "|    total_timesteps      | 1196032     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019904554 |\n",
      "|    clip_fraction        | 0.125       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.338      |\n",
      "|    explained_variance   | 0.564       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0512     |\n",
      "|    n_updates            | 720         |\n",
      "|    policy_gradient_loss | -0.0164     |\n",
      "|    value_loss           | 0.00812     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 978         |\n",
      "|    ep_rew_mean          | 1.76        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 430         |\n",
      "|    iterations           | 74          |\n",
      "|    time_elapsed         | 2818        |\n",
      "|    total_timesteps      | 1212416     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.022715833 |\n",
      "|    clip_fraction        | 0.132       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.331      |\n",
      "|    explained_variance   | 0.278       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0199     |\n",
      "|    n_updates            | 730         |\n",
      "|    policy_gradient_loss | -0.0167     |\n",
      "|    value_loss           | 0.01        |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 969        |\n",
      "|    ep_rew_mean          | 1.6        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 430        |\n",
      "|    iterations           | 75         |\n",
      "|    time_elapsed         | 2853       |\n",
      "|    total_timesteps      | 1228800    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.05586247 |\n",
      "|    clip_fraction        | 0.187      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.375     |\n",
      "|    explained_variance   | 0.662      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0296    |\n",
      "|    n_updates            | 740        |\n",
      "|    policy_gradient_loss | -0.015     |\n",
      "|    value_loss           | 0.00601    |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 955        |\n",
      "|    ep_rew_mean          | 1.61       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 431        |\n",
      "|    iterations           | 76         |\n",
      "|    time_elapsed         | 2888       |\n",
      "|    total_timesteps      | 1245184    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03775985 |\n",
      "|    clip_fraction        | 0.141      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.302     |\n",
      "|    explained_variance   | 0.409      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0148    |\n",
      "|    n_updates            | 750        |\n",
      "|    policy_gradient_loss | -0.0165    |\n",
      "|    value_loss           | 0.00983    |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 965        |\n",
      "|    ep_rew_mean          | 1.63       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 431        |\n",
      "|    iterations           | 77         |\n",
      "|    time_elapsed         | 2923       |\n",
      "|    total_timesteps      | 1261568    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.04561594 |\n",
      "|    clip_fraction        | 0.144      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.285     |\n",
      "|    explained_variance   | 0.12       |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0199    |\n",
      "|    n_updates            | 760        |\n",
      "|    policy_gradient_loss | -0.0156    |\n",
      "|    value_loss           | 0.0146     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 964         |\n",
      "|    ep_rew_mean          | 1.78        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 431         |\n",
      "|    iterations           | 78          |\n",
      "|    time_elapsed         | 2958        |\n",
      "|    total_timesteps      | 1277952     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019205175 |\n",
      "|    clip_fraction        | 0.119       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.279      |\n",
      "|    explained_variance   | 0.259       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0196     |\n",
      "|    n_updates            | 770         |\n",
      "|    policy_gradient_loss | -0.0148     |\n",
      "|    value_loss           | 0.012       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 960         |\n",
      "|    ep_rew_mean          | 1.78        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 432         |\n",
      "|    iterations           | 79          |\n",
      "|    time_elapsed         | 2993        |\n",
      "|    total_timesteps      | 1294336     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.021542411 |\n",
      "|    clip_fraction        | 0.124       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.268      |\n",
      "|    explained_variance   | 0.412       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.023      |\n",
      "|    n_updates            | 780         |\n",
      "|    policy_gradient_loss | -0.0173     |\n",
      "|    value_loss           | 0.01        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 945         |\n",
      "|    ep_rew_mean          | 1.98        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 432         |\n",
      "|    iterations           | 80          |\n",
      "|    time_elapsed         | 3028        |\n",
      "|    total_timesteps      | 1310720     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.033724543 |\n",
      "|    clip_fraction        | 0.124       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.234      |\n",
      "|    explained_variance   | 0.26        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0293     |\n",
      "|    n_updates            | 790         |\n",
      "|    policy_gradient_loss | -0.0176     |\n",
      "|    value_loss           | 0.0128      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 955        |\n",
      "|    ep_rew_mean          | 2.2        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 433        |\n",
      "|    iterations           | 81         |\n",
      "|    time_elapsed         | 3063       |\n",
      "|    total_timesteps      | 1327104    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.19615549 |\n",
      "|    clip_fraction        | 0.177      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.266     |\n",
      "|    explained_variance   | 0.372      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.00553   |\n",
      "|    n_updates            | 800        |\n",
      "|    policy_gradient_loss | -0.0181    |\n",
      "|    value_loss           | 0.013      |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 969         |\n",
      "|    ep_rew_mean          | 2.17        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 433         |\n",
      "|    iterations           | 82          |\n",
      "|    time_elapsed         | 3098        |\n",
      "|    total_timesteps      | 1343488     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.042909957 |\n",
      "|    clip_fraction        | 0.189       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.306      |\n",
      "|    explained_variance   | 0.302       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0259     |\n",
      "|    n_updates            | 810         |\n",
      "|    policy_gradient_loss | -0.0212     |\n",
      "|    value_loss           | 0.0148      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 965        |\n",
      "|    ep_rew_mean          | 2.16       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 433        |\n",
      "|    iterations           | 83         |\n",
      "|    time_elapsed         | 3133       |\n",
      "|    total_timesteps      | 1359872    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03876203 |\n",
      "|    clip_fraction        | 0.151      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.255     |\n",
      "|    explained_variance   | 0.422      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0281    |\n",
      "|    n_updates            | 820        |\n",
      "|    policy_gradient_loss | -0.0219    |\n",
      "|    value_loss           | 0.00997    |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 961        |\n",
      "|    ep_rew_mean          | 2.04       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 434        |\n",
      "|    iterations           | 84         |\n",
      "|    time_elapsed         | 3168       |\n",
      "|    total_timesteps      | 1376256    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.19881135 |\n",
      "|    clip_fraction        | 0.145      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.26      |\n",
      "|    explained_variance   | 0.381      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0391    |\n",
      "|    n_updates            | 830        |\n",
      "|    policy_gradient_loss | -0.0193    |\n",
      "|    value_loss           | 0.00888    |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 960         |\n",
      "|    ep_rew_mean          | 2.1         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 434         |\n",
      "|    iterations           | 85          |\n",
      "|    time_elapsed         | 3203        |\n",
      "|    total_timesteps      | 1392640     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.046040922 |\n",
      "|    clip_fraction        | 0.147       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.321      |\n",
      "|    explained_variance   | 0.516       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0298     |\n",
      "|    n_updates            | 840         |\n",
      "|    policy_gradient_loss | -0.0139     |\n",
      "|    value_loss           | 0.0105      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 965        |\n",
      "|    ep_rew_mean          | 2.01       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 435        |\n",
      "|    iterations           | 86         |\n",
      "|    time_elapsed         | 3238       |\n",
      "|    total_timesteps      | 1409024    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02250947 |\n",
      "|    clip_fraction        | 0.138      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.294     |\n",
      "|    explained_variance   | 0.454      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0279    |\n",
      "|    n_updates            | 850        |\n",
      "|    policy_gradient_loss | -0.0188    |\n",
      "|    value_loss           | 0.0122     |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 966        |\n",
      "|    ep_rew_mean          | 1.87       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 435        |\n",
      "|    iterations           | 87         |\n",
      "|    time_elapsed         | 3273       |\n",
      "|    total_timesteps      | 1425408    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.06765215 |\n",
      "|    clip_fraction        | 0.145      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.262     |\n",
      "|    explained_variance   | 0.452      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0527    |\n",
      "|    n_updates            | 860        |\n",
      "|    policy_gradient_loss | -0.0148    |\n",
      "|    value_loss           | 0.0122     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 966         |\n",
      "|    ep_rew_mean          | 1.92        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 435         |\n",
      "|    iterations           | 88          |\n",
      "|    time_elapsed         | 3308        |\n",
      "|    total_timesteps      | 1441792     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.033650395 |\n",
      "|    clip_fraction        | 0.134       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.276      |\n",
      "|    explained_variance   | 0.566       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00951    |\n",
      "|    n_updates            | 870         |\n",
      "|    policy_gradient_loss | -0.017      |\n",
      "|    value_loss           | 0.00829     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 957         |\n",
      "|    ep_rew_mean          | 1.92        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 436         |\n",
      "|    iterations           | 89          |\n",
      "|    time_elapsed         | 3343        |\n",
      "|    total_timesteps      | 1458176     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.023721471 |\n",
      "|    clip_fraction        | 0.154       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.382      |\n",
      "|    explained_variance   | 0.39        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0334     |\n",
      "|    n_updates            | 880         |\n",
      "|    policy_gradient_loss | -0.018      |\n",
      "|    value_loss           | 0.0115      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 932        |\n",
      "|    ep_rew_mean          | 2.17       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 436        |\n",
      "|    iterations           | 90         |\n",
      "|    time_elapsed         | 3378       |\n",
      "|    total_timesteps      | 1474560    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.09576962 |\n",
      "|    clip_fraction        | 0.181      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.286     |\n",
      "|    explained_variance   | 0.551      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0331    |\n",
      "|    n_updates            | 890        |\n",
      "|    policy_gradient_loss | -0.0187    |\n",
      "|    value_loss           | 0.00968    |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 934        |\n",
      "|    ep_rew_mean          | 2.1        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 436        |\n",
      "|    iterations           | 91         |\n",
      "|    time_elapsed         | 3413       |\n",
      "|    total_timesteps      | 1490944    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.07071915 |\n",
      "|    clip_fraction        | 0.169      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.316     |\n",
      "|    explained_variance   | 0.494      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0441    |\n",
      "|    n_updates            | 900        |\n",
      "|    policy_gradient_loss | -0.000291  |\n",
      "|    value_loss           | 0.0136     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 932         |\n",
      "|    ep_rew_mean          | 2.05        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 437         |\n",
      "|    iterations           | 92          |\n",
      "|    time_elapsed         | 3448        |\n",
      "|    total_timesteps      | 1507328     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.028873488 |\n",
      "|    clip_fraction        | 0.151       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.313      |\n",
      "|    explained_variance   | 0.428       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0255     |\n",
      "|    n_updates            | 910         |\n",
      "|    policy_gradient_loss | -0.0172     |\n",
      "|    value_loss           | 0.0129      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 925         |\n",
      "|    ep_rew_mean          | 2.09        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 437         |\n",
      "|    iterations           | 93          |\n",
      "|    time_elapsed         | 3483        |\n",
      "|    total_timesteps      | 1523712     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.025522355 |\n",
      "|    clip_fraction        | 0.128       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.264      |\n",
      "|    explained_variance   | 0.448       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0174     |\n",
      "|    n_updates            | 920         |\n",
      "|    policy_gradient_loss | -0.0175     |\n",
      "|    value_loss           | 0.0132      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 917         |\n",
      "|    ep_rew_mean          | 2.12        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 437         |\n",
      "|    iterations           | 94          |\n",
      "|    time_elapsed         | 3518        |\n",
      "|    total_timesteps      | 1540096     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.025099888 |\n",
      "|    clip_fraction        | 0.124       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.244      |\n",
      "|    explained_variance   | 0.287       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0266     |\n",
      "|    n_updates            | 930         |\n",
      "|    policy_gradient_loss | -0.0157     |\n",
      "|    value_loss           | 0.00989     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 926         |\n",
      "|    ep_rew_mean          | 2.04        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 438         |\n",
      "|    iterations           | 95          |\n",
      "|    time_elapsed         | 3553        |\n",
      "|    total_timesteps      | 1556480     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.035419714 |\n",
      "|    clip_fraction        | 0.131       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.247      |\n",
      "|    explained_variance   | 0.188       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00409    |\n",
      "|    n_updates            | 940         |\n",
      "|    policy_gradient_loss | -0.0172     |\n",
      "|    value_loss           | 0.0107      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 953         |\n",
      "|    ep_rew_mean          | 1.98        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 438         |\n",
      "|    iterations           | 96          |\n",
      "|    time_elapsed         | 3588        |\n",
      "|    total_timesteps      | 1572864     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.022925958 |\n",
      "|    clip_fraction        | 0.125       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.225      |\n",
      "|    explained_variance   | 0.206       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0225     |\n",
      "|    n_updates            | 950         |\n",
      "|    policy_gradient_loss | -0.0181     |\n",
      "|    value_loss           | 0.0132      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 944        |\n",
      "|    ep_rew_mean          | 2          |\n",
      "| time/                   |            |\n",
      "|    fps                  | 438        |\n",
      "|    iterations           | 97         |\n",
      "|    time_elapsed         | 3623       |\n",
      "|    total_timesteps      | 1589248    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02317325 |\n",
      "|    clip_fraction        | 0.12       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.242     |\n",
      "|    explained_variance   | 0.295      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.00485   |\n",
      "|    n_updates            | 960        |\n",
      "|    policy_gradient_loss | -0.0181    |\n",
      "|    value_loss           | 0.0117     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 953         |\n",
      "|    ep_rew_mean          | 1.98        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 438         |\n",
      "|    iterations           | 98          |\n",
      "|    time_elapsed         | 3658        |\n",
      "|    total_timesteps      | 1605632     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.026994232 |\n",
      "|    clip_fraction        | 0.143       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.291      |\n",
      "|    explained_variance   | 0.347       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0299     |\n",
      "|    n_updates            | 970         |\n",
      "|    policy_gradient_loss | -0.0176     |\n",
      "|    value_loss           | 0.0129      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 958         |\n",
      "|    ep_rew_mean          | 1.77        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 439         |\n",
      "|    iterations           | 99          |\n",
      "|    time_elapsed         | 3693        |\n",
      "|    total_timesteps      | 1622016     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.047183875 |\n",
      "|    clip_fraction        | 0.157       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.276      |\n",
      "|    explained_variance   | 0.415       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0339     |\n",
      "|    n_updates            | 980         |\n",
      "|    policy_gradient_loss | -0.0183     |\n",
      "|    value_loss           | 0.0127      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 958         |\n",
      "|    ep_rew_mean          | 1.86        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 439         |\n",
      "|    iterations           | 100         |\n",
      "|    time_elapsed         | 3728        |\n",
      "|    total_timesteps      | 1638400     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019882575 |\n",
      "|    clip_fraction        | 0.0987      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.23       |\n",
      "|    explained_variance   | 0.4         |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0256     |\n",
      "|    n_updates            | 990         |\n",
      "|    policy_gradient_loss | -0.0133     |\n",
      "|    value_loss           | 0.00845     |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 969        |\n",
      "|    ep_rew_mean          | 2.01       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 439        |\n",
      "|    iterations           | 101        |\n",
      "|    time_elapsed         | 3763       |\n",
      "|    total_timesteps      | 1654784    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.05718589 |\n",
      "|    clip_fraction        | 0.124      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.25      |\n",
      "|    explained_variance   | 0.393      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0392    |\n",
      "|    n_updates            | 1000       |\n",
      "|    policy_gradient_loss | -0.0174    |\n",
      "|    value_loss           | 0.0122     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 963         |\n",
      "|    ep_rew_mean          | 2.05        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 439         |\n",
      "|    iterations           | 102         |\n",
      "|    time_elapsed         | 3798        |\n",
      "|    total_timesteps      | 1671168     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.021558356 |\n",
      "|    clip_fraction        | 0.121       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.224      |\n",
      "|    explained_variance   | 0.33        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.00551     |\n",
      "|    n_updates            | 1010        |\n",
      "|    policy_gradient_loss | -0.0168     |\n",
      "|    value_loss           | 0.0142      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 975        |\n",
      "|    ep_rew_mean          | 1.96       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 440        |\n",
      "|    iterations           | 103        |\n",
      "|    time_elapsed         | 3833       |\n",
      "|    total_timesteps      | 1687552    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03985922 |\n",
      "|    clip_fraction        | 0.0964     |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.175     |\n",
      "|    explained_variance   | 0.477      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0296    |\n",
      "|    n_updates            | 1020       |\n",
      "|    policy_gradient_loss | -0.0149    |\n",
      "|    value_loss           | 0.0126     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 977         |\n",
      "|    ep_rew_mean          | 1.8         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 440         |\n",
      "|    iterations           | 104         |\n",
      "|    time_elapsed         | 3868        |\n",
      "|    total_timesteps      | 1703936     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.030274967 |\n",
      "|    clip_fraction        | 0.144       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.296      |\n",
      "|    explained_variance   | 0.126       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00948    |\n",
      "|    n_updates            | 1030        |\n",
      "|    policy_gradient_loss | -0.016      |\n",
      "|    value_loss           | 0.011       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 974         |\n",
      "|    ep_rew_mean          | 1.98        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 440         |\n",
      "|    iterations           | 105         |\n",
      "|    time_elapsed         | 3903        |\n",
      "|    total_timesteps      | 1720320     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.034277823 |\n",
      "|    clip_fraction        | 0.146       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.25       |\n",
      "|    explained_variance   | 0.306       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0255     |\n",
      "|    n_updates            | 1040        |\n",
      "|    policy_gradient_loss | -0.014      |\n",
      "|    value_loss           | 0.00847     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 980         |\n",
      "|    ep_rew_mean          | 1.88        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 440         |\n",
      "|    iterations           | 106         |\n",
      "|    time_elapsed         | 3938        |\n",
      "|    total_timesteps      | 1736704     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020800805 |\n",
      "|    clip_fraction        | 0.138       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.294      |\n",
      "|    explained_variance   | 0.334       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0173     |\n",
      "|    n_updates            | 1050        |\n",
      "|    policy_gradient_loss | -0.0169     |\n",
      "|    value_loss           | 0.00894     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 976         |\n",
      "|    ep_rew_mean          | 1.69        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 441         |\n",
      "|    iterations           | 107         |\n",
      "|    time_elapsed         | 3973        |\n",
      "|    total_timesteps      | 1753088     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.017615974 |\n",
      "|    clip_fraction        | 0.101       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.193      |\n",
      "|    explained_variance   | 0.328       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.038      |\n",
      "|    n_updates            | 1060        |\n",
      "|    policy_gradient_loss | -0.0139     |\n",
      "|    value_loss           | 0.00943     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 978         |\n",
      "|    ep_rew_mean          | 1.65        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 441         |\n",
      "|    iterations           | 108         |\n",
      "|    time_elapsed         | 4008        |\n",
      "|    total_timesteps      | 1769472     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.040418632 |\n",
      "|    clip_fraction        | 0.147       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.274      |\n",
      "|    explained_variance   | 0.186       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | 0.165       |\n",
      "|    n_updates            | 1070        |\n",
      "|    policy_gradient_loss | -0.015      |\n",
      "|    value_loss           | 0.00994     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 983         |\n",
      "|    ep_rew_mean          | 1.7         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 441         |\n",
      "|    iterations           | 109         |\n",
      "|    time_elapsed         | 4043        |\n",
      "|    total_timesteps      | 1785856     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019183774 |\n",
      "|    clip_fraction        | 0.116       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.243      |\n",
      "|    explained_variance   | 0.256       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00999    |\n",
      "|    n_updates            | 1080        |\n",
      "|    policy_gradient_loss | -0.0158     |\n",
      "|    value_loss           | 0.013       |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 980        |\n",
      "|    ep_rew_mean          | 1.84       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 441        |\n",
      "|    iterations           | 110        |\n",
      "|    time_elapsed         | 4078       |\n",
      "|    total_timesteps      | 1802240    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.12349033 |\n",
      "|    clip_fraction        | 0.165      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.269     |\n",
      "|    explained_variance   | 0.316      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | 0.134      |\n",
      "|    n_updates            | 1090       |\n",
      "|    policy_gradient_loss | -0.0164    |\n",
      "|    value_loss           | 0.0102     |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 982        |\n",
      "|    ep_rew_mean          | 1.86       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 442        |\n",
      "|    iterations           | 111        |\n",
      "|    time_elapsed         | 4113       |\n",
      "|    total_timesteps      | 1818624    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03522118 |\n",
      "|    clip_fraction        | 0.132      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.253     |\n",
      "|    explained_variance   | 0.351      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0259    |\n",
      "|    n_updates            | 1100       |\n",
      "|    policy_gradient_loss | -0.0171    |\n",
      "|    value_loss           | 0.0123     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 981         |\n",
      "|    ep_rew_mean          | 1.83        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 442         |\n",
      "|    iterations           | 112         |\n",
      "|    time_elapsed         | 4148        |\n",
      "|    total_timesteps      | 1835008     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.025962587 |\n",
      "|    clip_fraction        | 0.148       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.303      |\n",
      "|    explained_variance   | 0.357       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0222     |\n",
      "|    n_updates            | 1110        |\n",
      "|    policy_gradient_loss | -0.019      |\n",
      "|    value_loss           | 0.0118      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 970         |\n",
      "|    ep_rew_mean          | 1.8         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 442         |\n",
      "|    iterations           | 113         |\n",
      "|    time_elapsed         | 4182        |\n",
      "|    total_timesteps      | 1851392     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.016840426 |\n",
      "|    clip_fraction        | 0.118       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.3        |\n",
      "|    explained_variance   | 0.0436      |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0186     |\n",
      "|    n_updates            | 1120        |\n",
      "|    policy_gradient_loss | -0.0158     |\n",
      "|    value_loss           | 0.0103      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 956         |\n",
      "|    ep_rew_mean          | 1.83        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 442         |\n",
      "|    iterations           | 114         |\n",
      "|    time_elapsed         | 4218        |\n",
      "|    total_timesteps      | 1867776     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.028879194 |\n",
      "|    clip_fraction        | 0.118       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.22       |\n",
      "|    explained_variance   | 0.229       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0124     |\n",
      "|    n_updates            | 1130        |\n",
      "|    policy_gradient_loss | -0.015      |\n",
      "|    value_loss           | 0.0115      |\n",
      "-----------------------------------------\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 950       |\n",
      "|    ep_rew_mean          | 1.8       |\n",
      "| time/                   |           |\n",
      "|    fps                  | 443       |\n",
      "|    iterations           | 115       |\n",
      "|    time_elapsed         | 4253      |\n",
      "|    total_timesteps      | 1884160   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.0385094 |\n",
      "|    clip_fraction        | 0.146     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.249    |\n",
      "|    explained_variance   | 0.233     |\n",
      "|    learning_rate        | 0.0003    |\n",
      "|    loss                 | -0.0468   |\n",
      "|    n_updates            | 1140      |\n",
      "|    policy_gradient_loss | -0.0203   |\n",
      "|    value_loss           | 0.0131    |\n",
      "---------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 940        |\n",
      "|    ep_rew_mean          | 1.75       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 443        |\n",
      "|    iterations           | 116        |\n",
      "|    time_elapsed         | 4288       |\n",
      "|    total_timesteps      | 1900544    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02349641 |\n",
      "|    clip_fraction        | 0.12       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.216     |\n",
      "|    explained_variance   | 0.445      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.00912   |\n",
      "|    n_updates            | 1150       |\n",
      "|    policy_gradient_loss | -0.0172    |\n",
      "|    value_loss           | 0.00808    |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 927         |\n",
      "|    ep_rew_mean          | 1.81        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 443         |\n",
      "|    iterations           | 117         |\n",
      "|    time_elapsed         | 4323        |\n",
      "|    total_timesteps      | 1916928     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.021840088 |\n",
      "|    clip_fraction        | 0.131       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.276      |\n",
      "|    explained_variance   | 0.321       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0307     |\n",
      "|    n_updates            | 1160        |\n",
      "|    policy_gradient_loss | -0.0192     |\n",
      "|    value_loss           | 0.01        |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 927         |\n",
      "|    ep_rew_mean          | 1.9         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 443         |\n",
      "|    iterations           | 118         |\n",
      "|    time_elapsed         | 4358        |\n",
      "|    total_timesteps      | 1933312     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.039806128 |\n",
      "|    clip_fraction        | 0.149       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.315      |\n",
      "|    explained_variance   | 0.276       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0331     |\n",
      "|    n_updates            | 1170        |\n",
      "|    policy_gradient_loss | -0.0195     |\n",
      "|    value_loss           | 0.0146      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 935         |\n",
      "|    ep_rew_mean          | 1.95        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 443         |\n",
      "|    iterations           | 119         |\n",
      "|    time_elapsed         | 4393        |\n",
      "|    total_timesteps      | 1949696     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020795621 |\n",
      "|    clip_fraction        | 0.141       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.276      |\n",
      "|    explained_variance   | 0.322       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0168     |\n",
      "|    n_updates            | 1180        |\n",
      "|    policy_gradient_loss | -0.0172     |\n",
      "|    value_loss           | 0.0123      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 954         |\n",
      "|    ep_rew_mean          | 1.94        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 443         |\n",
      "|    iterations           | 120         |\n",
      "|    time_elapsed         | 4428        |\n",
      "|    total_timesteps      | 1966080     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.021730319 |\n",
      "|    clip_fraction        | 0.122       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.25       |\n",
      "|    explained_variance   | 0.315       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00945    |\n",
      "|    n_updates            | 1190        |\n",
      "|    policy_gradient_loss | -0.0152     |\n",
      "|    value_loss           | 0.013       |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 952         |\n",
      "|    ep_rew_mean          | 2.03        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 444         |\n",
      "|    iterations           | 121         |\n",
      "|    time_elapsed         | 4463        |\n",
      "|    total_timesteps      | 1982464     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.022492152 |\n",
      "|    clip_fraction        | 0.118       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.287      |\n",
      "|    explained_variance   | 0.356       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0385     |\n",
      "|    n_updates            | 1200        |\n",
      "|    policy_gradient_loss | -0.0157     |\n",
      "|    value_loss           | 0.0112      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 953         |\n",
      "|    ep_rew_mean          | 2.05        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 444         |\n",
      "|    iterations           | 122         |\n",
      "|    time_elapsed         | 4498        |\n",
      "|    total_timesteps      | 1998848     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020143604 |\n",
      "|    clip_fraction        | 0.123       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.221      |\n",
      "|    explained_variance   | 0.461       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0182     |\n",
      "|    n_updates            | 1210        |\n",
      "|    policy_gradient_loss | -0.0171     |\n",
      "|    value_loss           | 0.0102      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 950        |\n",
      "|    ep_rew_mean          | 1.95       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 444        |\n",
      "|    iterations           | 123        |\n",
      "|    time_elapsed         | 4533       |\n",
      "|    total_timesteps      | 2015232    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02390182 |\n",
      "|    clip_fraction        | 0.121      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.216     |\n",
      "|    explained_variance   | 0.555      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0265    |\n",
      "|    n_updates            | 1220       |\n",
      "|    policy_gradient_loss | -0.0159    |\n",
      "|    value_loss           | 0.00829    |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 955        |\n",
      "|    ep_rew_mean          | 1.99       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 444        |\n",
      "|    iterations           | 124        |\n",
      "|    time_elapsed         | 4568       |\n",
      "|    total_timesteps      | 2031616    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02842904 |\n",
      "|    clip_fraction        | 0.109      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.21      |\n",
      "|    explained_variance   | 0.479      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.00574   |\n",
      "|    n_updates            | 1230       |\n",
      "|    policy_gradient_loss | -0.0162    |\n",
      "|    value_loss           | 0.013      |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 948         |\n",
      "|    ep_rew_mean          | 1.88        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 444         |\n",
      "|    iterations           | 125         |\n",
      "|    time_elapsed         | 4603        |\n",
      "|    total_timesteps      | 2048000     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.017118149 |\n",
      "|    clip_fraction        | 0.116       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.24       |\n",
      "|    explained_variance   | 0.273       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0385     |\n",
      "|    n_updates            | 1240        |\n",
      "|    policy_gradient_loss | -0.0152     |\n",
      "|    value_loss           | 0.00765     |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 943        |\n",
      "|    ep_rew_mean          | 1.8        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 445        |\n",
      "|    iterations           | 126        |\n",
      "|    time_elapsed         | 4638       |\n",
      "|    total_timesteps      | 2064384    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02302307 |\n",
      "|    clip_fraction        | 0.147      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.352     |\n",
      "|    explained_variance   | 0.121      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0234    |\n",
      "|    n_updates            | 1250       |\n",
      "|    policy_gradient_loss | -0.0174    |\n",
      "|    value_loss           | 0.0148     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 942         |\n",
      "|    ep_rew_mean          | 1.72        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 445         |\n",
      "|    iterations           | 127         |\n",
      "|    time_elapsed         | 4673        |\n",
      "|    total_timesteps      | 2080768     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.027428817 |\n",
      "|    clip_fraction        | 0.109       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.223      |\n",
      "|    explained_variance   | 0.44        |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00947    |\n",
      "|    n_updates            | 1260        |\n",
      "|    policy_gradient_loss | -0.0141     |\n",
      "|    value_loss           | 0.00723     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 952         |\n",
      "|    ep_rew_mean          | 1.73        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 445         |\n",
      "|    iterations           | 128         |\n",
      "|    time_elapsed         | 4708        |\n",
      "|    total_timesteps      | 2097152     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.032403644 |\n",
      "|    clip_fraction        | 0.147       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.31       |\n",
      "|    explained_variance   | 0.314       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0156     |\n",
      "|    n_updates            | 1270        |\n",
      "|    policy_gradient_loss | -0.0177     |\n",
      "|    value_loss           | 0.0108      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 952         |\n",
      "|    ep_rew_mean          | 1.64        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 445         |\n",
      "|    iterations           | 129         |\n",
      "|    time_elapsed         | 4743        |\n",
      "|    total_timesteps      | 2113536     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020720232 |\n",
      "|    clip_fraction        | 0.119       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.269      |\n",
      "|    explained_variance   | 0.329       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0099     |\n",
      "|    n_updates            | 1280        |\n",
      "|    policy_gradient_loss | -0.0178     |\n",
      "|    value_loss           | 0.00869     |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 953        |\n",
      "|    ep_rew_mean          | 1.7        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 445        |\n",
      "|    iterations           | 130        |\n",
      "|    time_elapsed         | 4778       |\n",
      "|    total_timesteps      | 2129920    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02671453 |\n",
      "|    clip_fraction        | 0.154      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.256     |\n",
      "|    explained_variance   | 0.392      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0117    |\n",
      "|    n_updates            | 1290       |\n",
      "|    policy_gradient_loss | -0.0196    |\n",
      "|    value_loss           | 0.00951    |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 957         |\n",
      "|    ep_rew_mean          | 1.7         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 445         |\n",
      "|    iterations           | 131         |\n",
      "|    time_elapsed         | 4813        |\n",
      "|    total_timesteps      | 2146304     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.019944048 |\n",
      "|    clip_fraction        | 0.118       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.273      |\n",
      "|    explained_variance   | 0.258       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0145     |\n",
      "|    n_updates            | 1300        |\n",
      "|    policy_gradient_loss | -0.0172     |\n",
      "|    value_loss           | 0.0126      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 951        |\n",
      "|    ep_rew_mean          | 2.08       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 446        |\n",
      "|    iterations           | 132        |\n",
      "|    time_elapsed         | 4848       |\n",
      "|    total_timesteps      | 2162688    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.05593154 |\n",
      "|    clip_fraction        | 0.163      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.274     |\n",
      "|    explained_variance   | 0.276      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0235    |\n",
      "|    n_updates            | 1310       |\n",
      "|    policy_gradient_loss | -0.0212    |\n",
      "|    value_loss           | 0.0151     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 953         |\n",
      "|    ep_rew_mean          | 2.12        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 446         |\n",
      "|    iterations           | 133         |\n",
      "|    time_elapsed         | 4883        |\n",
      "|    total_timesteps      | 2179072     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.022772985 |\n",
      "|    clip_fraction        | 0.127       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.25       |\n",
      "|    explained_variance   | 0.339       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00449    |\n",
      "|    n_updates            | 1320        |\n",
      "|    policy_gradient_loss | -0.0176     |\n",
      "|    value_loss           | 0.016       |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 943        |\n",
      "|    ep_rew_mean          | 2.18       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 446        |\n",
      "|    iterations           | 134        |\n",
      "|    time_elapsed         | 4918       |\n",
      "|    total_timesteps      | 2195456    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.02460384 |\n",
      "|    clip_fraction        | 0.141      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.271     |\n",
      "|    explained_variance   | 0.243      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0224    |\n",
      "|    n_updates            | 1330       |\n",
      "|    policy_gradient_loss | -0.0191    |\n",
      "|    value_loss           | 0.0136     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 938         |\n",
      "|    ep_rew_mean          | 2.28        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 446         |\n",
      "|    iterations           | 135         |\n",
      "|    time_elapsed         | 4954        |\n",
      "|    total_timesteps      | 2211840     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.024609936 |\n",
      "|    clip_fraction        | 0.12        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.263      |\n",
      "|    explained_variance   | 0.324       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0422     |\n",
      "|    n_updates            | 1340        |\n",
      "|    policy_gradient_loss | -0.0164     |\n",
      "|    value_loss           | 0.0117      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 947        |\n",
      "|    ep_rew_mean          | 2.23       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 446        |\n",
      "|    iterations           | 136        |\n",
      "|    time_elapsed         | 4991       |\n",
      "|    total_timesteps      | 2228224    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.01992891 |\n",
      "|    clip_fraction        | 0.136      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.282     |\n",
      "|    explained_variance   | 0.448      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0393    |\n",
      "|    n_updates            | 1350       |\n",
      "|    policy_gradient_loss | -0.0161    |\n",
      "|    value_loss           | 0.00986    |\n",
      "----------------------------------------\n",
      "---------------------------------------\n",
      "| rollout/                |           |\n",
      "|    ep_len_mean          | 946       |\n",
      "|    ep_rew_mean          | 2.29      |\n",
      "| time/                   |           |\n",
      "|    fps                  | 446       |\n",
      "|    iterations           | 137       |\n",
      "|    time_elapsed         | 5026      |\n",
      "|    total_timesteps      | 2244608   |\n",
      "| train/                  |           |\n",
      "|    approx_kl            | 0.1402319 |\n",
      "|    clip_fraction        | 0.191     |\n",
      "|    clip_range           | 0.2       |\n",
      "|    entropy_loss         | -0.318    |\n",
      "|    explained_variance   | 0.473     |\n",
      "|    learning_rate        | 0.0003    |\n",
      "|    loss                 | -0.0393   |\n",
      "|    n_updates            | 1360      |\n",
      "|    policy_gradient_loss | -0.03     |\n",
      "|    value_loss           | 0.0158    |\n",
      "---------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 928         |\n",
      "|    ep_rew_mean          | 2.05        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 446         |\n",
      "|    iterations           | 138         |\n",
      "|    time_elapsed         | 5061        |\n",
      "|    total_timesteps      | 2260992     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.058511328 |\n",
      "|    clip_fraction        | 0.14        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.249      |\n",
      "|    explained_variance   | 0.301       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00707    |\n",
      "|    n_updates            | 1370        |\n",
      "|    policy_gradient_loss | -0.0172     |\n",
      "|    value_loss           | 0.0124      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 932        |\n",
      "|    ep_rew_mean          | 2.09       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 446        |\n",
      "|    iterations           | 139        |\n",
      "|    time_elapsed         | 5096       |\n",
      "|    total_timesteps      | 2277376    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.05748818 |\n",
      "|    clip_fraction        | 0.177      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.298     |\n",
      "|    explained_variance   | 0.323      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0228    |\n",
      "|    n_updates            | 1380       |\n",
      "|    policy_gradient_loss | -0.0213    |\n",
      "|    value_loss           | 0.0123     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 929         |\n",
      "|    ep_rew_mean          | 2.21        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 447         |\n",
      "|    iterations           | 140         |\n",
      "|    time_elapsed         | 5131        |\n",
      "|    total_timesteps      | 2293760     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.030683152 |\n",
      "|    clip_fraction        | 0.122       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.213      |\n",
      "|    explained_variance   | 0.373       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0205     |\n",
      "|    n_updates            | 1390        |\n",
      "|    policy_gradient_loss | -0.0153     |\n",
      "|    value_loss           | 0.0113      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 916         |\n",
      "|    ep_rew_mean          | 2.24        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 447         |\n",
      "|    iterations           | 141         |\n",
      "|    time_elapsed         | 5166        |\n",
      "|    total_timesteps      | 2310144     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.029830465 |\n",
      "|    clip_fraction        | 0.142       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.275      |\n",
      "|    explained_variance   | 0.356       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0384     |\n",
      "|    n_updates            | 1400        |\n",
      "|    policy_gradient_loss | -0.0201     |\n",
      "|    value_loss           | 0.0167      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 913        |\n",
      "|    ep_rew_mean          | 2.2        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 446        |\n",
      "|    iterations           | 142        |\n",
      "|    time_elapsed         | 5205       |\n",
      "|    total_timesteps      | 2326528    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.14179869 |\n",
      "|    clip_fraction        | 0.147      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.234     |\n",
      "|    explained_variance   | 0.464      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0344    |\n",
      "|    n_updates            | 1410       |\n",
      "|    policy_gradient_loss | -0.0192    |\n",
      "|    value_loss           | 0.0116     |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 917        |\n",
      "|    ep_rew_mean          | 2.19       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 446        |\n",
      "|    iterations           | 143        |\n",
      "|    time_elapsed         | 5242       |\n",
      "|    total_timesteps      | 2342912    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03899983 |\n",
      "|    clip_fraction        | 0.155      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.24      |\n",
      "|    explained_variance   | 0.421      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.045     |\n",
      "|    n_updates            | 1420       |\n",
      "|    policy_gradient_loss | -0.0214    |\n",
      "|    value_loss           | 0.0119     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 921         |\n",
      "|    ep_rew_mean          | 2.17        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 446         |\n",
      "|    iterations           | 144         |\n",
      "|    time_elapsed         | 5280        |\n",
      "|    total_timesteps      | 2359296     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.025015559 |\n",
      "|    clip_fraction        | 0.142       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.3        |\n",
      "|    explained_variance   | 0.441       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0336     |\n",
      "|    n_updates            | 1430        |\n",
      "|    policy_gradient_loss | -0.0165     |\n",
      "|    value_loss           | 0.00988     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 927         |\n",
      "|    ep_rew_mean          | 2.25        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 446         |\n",
      "|    iterations           | 145         |\n",
      "|    time_elapsed         | 5315        |\n",
      "|    total_timesteps      | 2375680     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.030028103 |\n",
      "|    clip_fraction        | 0.132       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.23       |\n",
      "|    explained_variance   | 0.419       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0202     |\n",
      "|    n_updates            | 1440        |\n",
      "|    policy_gradient_loss | -0.0151     |\n",
      "|    value_loss           | 0.0118      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 951        |\n",
      "|    ep_rew_mean          | 1.96       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 447        |\n",
      "|    iterations           | 146        |\n",
      "|    time_elapsed         | 5350       |\n",
      "|    total_timesteps      | 2392064    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.12199864 |\n",
      "|    clip_fraction        | 0.163      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.274     |\n",
      "|    explained_variance   | 0.419      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0444    |\n",
      "|    n_updates            | 1450       |\n",
      "|    policy_gradient_loss | 0.153      |\n",
      "|    value_loss           | 0.0125     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 954         |\n",
      "|    ep_rew_mean          | 2.15        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 447         |\n",
      "|    iterations           | 147         |\n",
      "|    time_elapsed         | 5385        |\n",
      "|    total_timesteps      | 2408448     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.025869627 |\n",
      "|    clip_fraction        | 0.148       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.284      |\n",
      "|    explained_variance   | 0.338       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0406     |\n",
      "|    n_updates            | 1460        |\n",
      "|    policy_gradient_loss | -0.018      |\n",
      "|    value_loss           | 0.0102      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 956        |\n",
      "|    ep_rew_mean          | 2.13       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 447        |\n",
      "|    iterations           | 148        |\n",
      "|    time_elapsed         | 5420       |\n",
      "|    total_timesteps      | 2424832    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.04188674 |\n",
      "|    clip_fraction        | 0.146      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.251     |\n",
      "|    explained_variance   | 0.382      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0238    |\n",
      "|    n_updates            | 1470       |\n",
      "|    policy_gradient_loss | -0.0175    |\n",
      "|    value_loss           | 0.0168     |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 963         |\n",
      "|    ep_rew_mean          | 2.06        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 447         |\n",
      "|    iterations           | 149         |\n",
      "|    time_elapsed         | 5455        |\n",
      "|    total_timesteps      | 2441216     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.027832523 |\n",
      "|    clip_fraction        | 0.125       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.238      |\n",
      "|    explained_variance   | 0.444       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0268     |\n",
      "|    n_updates            | 1480        |\n",
      "|    policy_gradient_loss | -0.0178     |\n",
      "|    value_loss           | 0.0132      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 969         |\n",
      "|    ep_rew_mean          | 2.19        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 447         |\n",
      "|    iterations           | 150         |\n",
      "|    time_elapsed         | 5489        |\n",
      "|    total_timesteps      | 2457600     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.092133544 |\n",
      "|    clip_fraction        | 0.137       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.204      |\n",
      "|    explained_variance   | 0.572       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.032      |\n",
      "|    n_updates            | 1490        |\n",
      "|    policy_gradient_loss | -0.0183     |\n",
      "|    value_loss           | 0.0108      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 952        |\n",
      "|    ep_rew_mean          | 2.24       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 447        |\n",
      "|    iterations           | 151        |\n",
      "|    time_elapsed         | 5524       |\n",
      "|    total_timesteps      | 2473984    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.11275491 |\n",
      "|    clip_fraction        | 0.186      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.255     |\n",
      "|    explained_variance   | 0.503      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0169    |\n",
      "|    n_updates            | 1500       |\n",
      "|    policy_gradient_loss | -0.0212    |\n",
      "|    value_loss           | 0.0148     |\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 958        |\n",
      "|    ep_rew_mean          | 2.24       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 447        |\n",
      "|    iterations           | 152        |\n",
      "|    time_elapsed         | 5560       |\n",
      "|    total_timesteps      | 2490368    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.06285341 |\n",
      "|    clip_fraction        | 0.159      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.275     |\n",
      "|    explained_variance   | 0.503      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0202    |\n",
      "|    n_updates            | 1510       |\n",
      "|    policy_gradient_loss | -0.0238    |\n",
      "|    value_loss           | 0.014      |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 951         |\n",
      "|    ep_rew_mean          | 2           |\n",
      "| time/                   |             |\n",
      "|    fps                  | 448         |\n",
      "|    iterations           | 153         |\n",
      "|    time_elapsed         | 5595        |\n",
      "|    total_timesteps      | 2506752     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.023681791 |\n",
      "|    clip_fraction        | 0.131       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.278      |\n",
      "|    explained_variance   | 0.342       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.00789    |\n",
      "|    n_updates            | 1520        |\n",
      "|    policy_gradient_loss | -0.0154     |\n",
      "|    value_loss           | 0.0103      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 938         |\n",
      "|    ep_rew_mean          | 1.93        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 448         |\n",
      "|    iterations           | 154         |\n",
      "|    time_elapsed         | 5630        |\n",
      "|    total_timesteps      | 2523136     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.022661842 |\n",
      "|    clip_fraction        | 0.117       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.217      |\n",
      "|    explained_variance   | 0.471       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0177     |\n",
      "|    n_updates            | 1530        |\n",
      "|    policy_gradient_loss | -0.0141     |\n",
      "|    value_loss           | 0.0112      |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 946         |\n",
      "|    ep_rew_mean          | 1.76        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 448         |\n",
      "|    iterations           | 155         |\n",
      "|    time_elapsed         | 5665        |\n",
      "|    total_timesteps      | 2539520     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.030201806 |\n",
      "|    clip_fraction        | 0.145       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.242      |\n",
      "|    explained_variance   | 0.399       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0166     |\n",
      "|    n_updates            | 1540        |\n",
      "|    policy_gradient_loss | -0.0184     |\n",
      "|    value_loss           | 0.00904     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 928         |\n",
      "|    ep_rew_mean          | 1.83        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 448         |\n",
      "|    iterations           | 156         |\n",
      "|    time_elapsed         | 5700        |\n",
      "|    total_timesteps      | 2555904     |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.080859914 |\n",
      "|    clip_fraction        | 0.113       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.201      |\n",
      "|    explained_variance   | 0.537       |\n",
      "|    learning_rate        | 0.0003      |\n",
      "|    loss                 | -0.0382     |\n",
      "|    n_updates            | 1550        |\n",
      "|    policy_gradient_loss | 0.0126      |\n",
      "|    value_loss           | 0.0078      |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 907        |\n",
      "|    ep_rew_mean          | 2.11       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 448        |\n",
      "|    iterations           | 157        |\n",
      "|    time_elapsed         | 5739       |\n",
      "|    total_timesteps      | 2572288    |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.08797717 |\n",
      "|    clip_fraction        | 0.171      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.28      |\n",
      "|    explained_variance   | 0.453      |\n",
      "|    learning_rate        | 0.0003     |\n",
      "|    loss                 | -0.0326    |\n",
      "|    n_updates            | 1560       |\n",
      "|    policy_gradient_loss | -0.0211    |\n",
      "|    value_loss           | 0.0124     |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[4], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlearn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtotal_timesteps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn_steps\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlog_interval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m      3\u001b[0m model\u001b[38;5;241m.\u001b[39msave(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodels/stablebaselines/Snake DQN\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\pt_env\\lib\\site-packages\\stable_baselines3\\ppo\\ppo.py:308\u001b[0m, in \u001b[0;36mPPO.learn\u001b[1;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[0;32m    299\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mlearn\u001b[39m(\n\u001b[0;32m    300\u001b[0m     \u001b[38;5;28mself\u001b[39m: SelfPPO,\n\u001b[0;32m    301\u001b[0m     total_timesteps: \u001b[38;5;28mint\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    306\u001b[0m     progress_bar: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[0;32m    307\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m SelfPPO:\n\u001b[1;32m--> 308\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlearn\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    309\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtotal_timesteps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtotal_timesteps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    310\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcallback\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallback\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    311\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlog_interval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlog_interval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    312\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtb_log_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtb_log_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    313\u001b[0m \u001b[43m        \u001b[49m\u001b[43mreset_num_timesteps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreset_num_timesteps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    314\u001b[0m \u001b[43m        \u001b[49m\u001b[43mprogress_bar\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprogress_bar\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    315\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\pt_env\\lib\\site-packages\\stable_baselines3\\common\\on_policy_algorithm.py:271\u001b[0m, in \u001b[0;36mOnPolicyAlgorithm.learn\u001b[1;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[0;32m    268\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlogger\u001b[38;5;241m.\u001b[39mrecord(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtime/total_timesteps\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_timesteps, exclude\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtensorboard\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    269\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlogger\u001b[38;5;241m.\u001b[39mdump(step\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_timesteps)\n\u001b[1;32m--> 271\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    273\u001b[0m callback\u001b[38;5;241m.\u001b[39mon_training_end()\n\u001b[0;32m    275\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\pt_env\\lib\\site-packages\\stable_baselines3\\ppo\\ppo.py:210\u001b[0m, in \u001b[0;36mPPO.train\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    207\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muse_sde:\n\u001b[0;32m    208\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpolicy\u001b[38;5;241m.\u001b[39mreset_noise(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch_size)\n\u001b[1;32m--> 210\u001b[0m values, log_prob, entropy \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpolicy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mevaluate_actions\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrollout_data\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mobservations\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mactions\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    211\u001b[0m values \u001b[38;5;241m=\u001b[39m values\u001b[38;5;241m.\u001b[39mflatten()\n\u001b[0;32m    212\u001b[0m \u001b[38;5;66;03m# Normalize advantage\u001b[39;00m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\pt_env\\lib\\site-packages\\stable_baselines3\\common\\policies.py:691\u001b[0m, in \u001b[0;36mActorCriticPolicy.evaluate_actions\u001b[1;34m(self, obs, actions)\u001b[0m\n\u001b[0;32m    689\u001b[0m features \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mextract_features(obs)\n\u001b[0;32m    690\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshare_features_extractor:\n\u001b[1;32m--> 691\u001b[0m     latent_pi, latent_vf \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmlp_extractor\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfeatures\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    692\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    693\u001b[0m     pi_features, vf_features \u001b[38;5;241m=\u001b[39m features\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\pt_env\\lib\\site-packages\\torch\\nn\\modules\\module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\pt_env\\lib\\site-packages\\stable_baselines3\\common\\torch_layers.py:218\u001b[0m, in \u001b[0;36mMlpExtractor.forward\u001b[1;34m(self, features)\u001b[0m\n\u001b[0;32m    213\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, features: th\u001b[38;5;241m.\u001b[39mTensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[th\u001b[38;5;241m.\u001b[39mTensor, th\u001b[38;5;241m.\u001b[39mTensor]:\n\u001b[0;32m    214\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    215\u001b[0m \u001b[38;5;124;03m    :return: latent_policy, latent_value of the specified network.\u001b[39;00m\n\u001b[0;32m    216\u001b[0m \u001b[38;5;124;03m        If all layers are shared, then ``latent_policy == latent_value``\u001b[39;00m\n\u001b[0;32m    217\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 218\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mforward_actor(features), \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward_critic\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfeatures\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\pt_env\\lib\\site-packages\\stable_baselines3\\common\\torch_layers.py:224\u001b[0m, in \u001b[0;36mMlpExtractor.forward_critic\u001b[1;34m(self, features)\u001b[0m\n\u001b[0;32m    223\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward_critic\u001b[39m(\u001b[38;5;28mself\u001b[39m, features: th\u001b[38;5;241m.\u001b[39mTensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m th\u001b[38;5;241m.\u001b[39mTensor:\n\u001b[1;32m--> 224\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalue_net\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfeatures\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\pt_env\\lib\\site-packages\\torch\\nn\\modules\\module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\pt_env\\lib\\site-packages\\torch\\nn\\modules\\container.py:217\u001b[0m, in \u001b[0;36mSequential.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    215\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m):\n\u001b[0;32m    216\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m module \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m:\n\u001b[1;32m--> 217\u001b[0m         \u001b[38;5;28minput\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[43mmodule\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m    218\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28minput\u001b[39m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\pt_env\\lib\\site-packages\\torch\\nn\\modules\\module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\pt_env\\lib\\site-packages\\torch\\nn\\modules\\linear.py:114\u001b[0m, in \u001b[0;36mLinear.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    113\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[1;32m--> 114\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlinear\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.learn(total_timesteps=n_steps, log_interval=1)\n",
    "\n",
    "model.save(\"models/stablebaselines/Snake PPO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0, 0\n",
      "FPS: 172.4442429168419\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "\n",
    "vec_env.close()\n",
    "del vec_env\n",
    "\n",
    "env = make_env()\n",
    "\n",
    "model = PPO.load(\"models/stablebaselines/Snake PPO\")\n",
    "\n",
    "state = env.reset()\n",
    "rewards = 0\n",
    "done = False\n",
    "\n",
    "frames = []\n",
    "start = time.time()\n",
    "for _ in range(5000):\n",
    "    action, _states = model.predict(state)\n",
    "    state, reward, done, info = env.step(action)\n",
    "\n",
    "    rewards += reward\n",
    "    print(\"\\r{}, {}\".format(rewards, reward), end=\"\")\n",
    "\n",
    "    obs = env.render(\"rgb_array\")\n",
    "    frames.append(obs)\n",
    "\n",
    "    if done:\n",
    "        break\n",
    "\n",
    "fps = len(frames) / (time.time() - start)\n",
    "print(f\"\\nFPS: {fps}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import PIL\n",
    "\n",
    "import os\n",
    "\n",
    "\n",
    "image_path = os.path.join(\"rl videos\", f\"snake_ppo_stablebaselines_{n_steps}_steps.gif\")\n",
    "\n",
    "frame_images = []\n",
    "for frame in frames:\n",
    "    frame_images.append(PIL.Image.fromarray(frame))\n",
    "\n",
    "frame_images[0].save(image_path, format='GIF',\n",
    "                     append_images=frame_images[1:],\n",
    "                     save_all=True,\n",
    "                     duration=30,\n",
    "                     loop=0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.9 ('pt_env')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "vscode": {
   "interpreter": {
    "hash": "33ffef3325e3a534c0147e772ed31e50103af8c12a21f8b2f2c8601a52e63a0b"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}