Hyperparameter-tuning RL A2C.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pygame 2.4.0 (SDL 2.26.4, Python 3.10.10)\n",
      "Hello from the pygame community. https://www.pygame.org/contribute.html\n"
     ]
    }
   ],
   "source": [
    "from hyperopt import hp\n",
    "from hyperopt import fmin, tpe\n",
    "from hyperopt import space_eval\n",
    "\n",
    "from stable_baselines3 import A2C\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "# from stable_baselines3.common.vec_env import SubprocVecEnv\n",
    "# from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "import PIL\n",
    "\n",
    "import time\n",
    "\n",
    "from utils import make_env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dict_to_str(args):\n",
    "    string = \"\"\n",
    "\n",
    "    for key, value in args.items():\n",
    "        if isinstance(value, int):\n",
    "            string += f\", {key}_{value}\"\n",
    "        else:\n",
    "            string += f\", {key}_{value: .5f}\"\n",
    "\n",
    "    return string[2:]\n",
    "\n",
    "def save_frames_as_gif(frames, file_path):\n",
    "    frame_images = []\n",
    "    for frame in frames:\n",
    "        frame_images.append(PIL.Image.fromarray(frame))\n",
    "\n",
    "    frame_images[0].save(file_path, format='GIF',\n",
    "                        append_images=frame_images[1:],\n",
    "                        save_all=True,\n",
    "                        duration=30,\n",
    "                        loop=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rwd_func_1(info, prev_info):\n",
    "    reward = 0\n",
    "\n",
    "    reward += 1 * (info[\"score\"] - prev_info[\"score\"])\n",
    "\n",
    "    return reward\n",
    "\n",
    "def rwd_func_2(info, prev_info):\n",
    "    reward = 0\n",
    "\n",
    "    reward -= 0.1\n",
    "\n",
    "    reward += 1.1 * (info[\"score\"] - prev_info[\"score\"])\n",
    "    \n",
    "    if info[\"life\"] < prev_info[\"life\"]:\n",
    "        reward -= 0.9\n",
    "\n",
    "    return reward\n",
    "\n",
    "def rwd_func_3(info, prev_info):\n",
    "    reward = 0\n",
    "\n",
    "    if (prev_info[\"head pos\"][0] == info[\"size\"]-1 and info[\"head pos\"][0] == 0) or (\n",
    "        prev_info[\"head pos\"][1] == info[\"size\"]-1 and info[\"head pos\"][1] == 0) or (\n",
    "        prev_info[\"head pos\"][0] == 0 and info[\"head pos\"][0] == info[\"size\"]-1) or (\n",
    "        prev_info[\"head pos\"][1] == 0 and info[\"head pos\"][1] == info[\"size\"]-1):\n",
    "        reward -= 0.5\n",
    "\n",
    "    reward -= 0.1\n",
    "\n",
    "    reward += 1.1 * (info[\"score\"] - prev_info[\"score\"])\n",
    "    \n",
    "    if info[\"life\"] < prev_info[\"life\"]:\n",
    "        reward -= 0.9\n",
    "\n",
    "    return reward\n",
    "\n",
    "def rwd_func_4(info, prev_info):\n",
    "    reward = 0\n",
    "\n",
    "    if info[\"head food dist\"] < prev_info[\"head food dist\"]:\n",
    "        reward += 0.2\n",
    "    else:\n",
    "        reward -= 0.1\n",
    "\n",
    "    if (prev_info[\"head pos\"][0] == info[\"size\"]-1 and info[\"head pos\"][0] == 0) or (\n",
    "        prev_info[\"head pos\"][1] == info[\"size\"]-1 and info[\"head pos\"][1] == 0) or (\n",
    "        prev_info[\"head pos\"][0] == 0 and info[\"head pos\"][0] == info[\"size\"]-1) or (\n",
    "        prev_info[\"head pos\"][1] == 0 and info[\"head pos\"][1] == info[\"size\"]-1):\n",
    "        reward -= 0.5\n",
    "\n",
    "    reward -= 0.1\n",
    "\n",
    "    reward += 1.1 * (info[\"score\"] - prev_info[\"score\"])\n",
    "    \n",
    "    if info[\"life\"] < prev_info[\"life\"]:\n",
    "        reward -= 0.9\n",
    "\n",
    "    return reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def objective(args):\n",
    "    global best_avg_rwd, expr_no, total_timesteps, fixed_args\n",
    "    args.update(fixed_args)\n",
    "\n",
    "    start = time.time()\n",
    "\n",
    "\n",
    "    # if args[\"n_steps\"] >= 128 and args[\"n_envs\"] >= 16:\n",
    "    #     args[\"n_envs\"] = 16\n",
    "    # if args[\"n_steps\"] >= 256 and args[\"n_envs\"] >= 8:\n",
    "    #     args[\"n_envs\"] = 8\n",
    "    # if args[\"n_steps\"] >= 512 and args[\"n_envs\"] >= 4:\n",
    "    #     args[\"n_envs\"] = 4\n",
    "\n",
    "    print(f\"---Chosen Hyperparameters of opt-epoch:\")\n",
    "    print(args)\n",
    "\n",
    "\n",
    "    match args[\"env_rwd_func\"]:\n",
    "        case 0:\n",
    "            env_rwd_func = None\n",
    "        case 1:\n",
    "            env_rwd_func = rwd_func_1\n",
    "        case 2:\n",
    "            env_rwd_func = rwd_func_2\n",
    "        case 3:\n",
    "            env_rwd_func = rwd_func_3\n",
    "        case 4:\n",
    "            env_rwd_func = rwd_func_4\n",
    "    \n",
    "    vec_env = make_vec_env(\n",
    "        lambda: make_env(env_rwd_func=env_rwd_func, max_step=args[\"env_max_step\"], num_stack=args[\"env_num_stack\"]), \n",
    "        n_envs=args[\"n_envs\"], \n",
    "        # vec_env_cls=SubprocVecEnv\n",
    "    )\n",
    "\n",
    "\n",
    "    args_str = dict_to_str(args)\n",
    "    env_num_stack = args[\"env_num_stack\"]\n",
    "    del args[\"env_rwd_func\"], args[\"env_max_step\"], args[\"env_num_stack\"], args[\"n_envs\"]\n",
    "    \n",
    "\n",
    "    model = A2C(\n",
    "        \"MlpPolicy\", vec_env, \n",
    "        **args,\n",
    "        verbose=0, tensorboard_log=f\"logs/A2C hp-tuning/no.{expr_no}\"\n",
    "    )\n",
    "    model.learn(total_timesteps=total_timesteps, log_interval=1, tb_log_name=args_str) # total_timesteps=15_000_000, \n",
    "\n",
    "\n",
    "    vec_env.close()\n",
    "    del vec_env\n",
    "    env = make_env(max_step=1000, num_stack=env_num_stack)\n",
    "\n",
    "\n",
    "    n_episode = 10\n",
    "    # avg_rwd, _ = evaluate_policy(model, env, n_eval_episodes=trial_num, deterministic=False)\n",
    "    avg_rwd = 0\n",
    "    bestscore = 0\n",
    "    bestscore_frames = None\n",
    "    for episode in range(n_episode):\n",
    "        state = env.reset()\n",
    "        done = False\n",
    "        score = 0\n",
    "        frames = []\n",
    "        while not done:\n",
    "            action, _states = model.predict(state, deterministic=False)\n",
    "            state, reward, done, info = env.step(action)\n",
    "            obs = env.render(\"rgb_array\")\n",
    "            frames.append(obs)\n",
    "        score = info[\"score\"]\n",
    "        avg_rwd += score\n",
    "        print(f\"an agent on environment: {episode+1}/{n_episode} Episodes, score: {score}\")\n",
    "        if score > bestscore:\n",
    "            bestscore = score\n",
    "            bestscore_frames = frames.copy()\n",
    "    avg_rwd /= n_episode\n",
    "\n",
    "    if avg_rwd > best_avg_rwd:\n",
    "        best_avg_rwd = avg_rwd\n",
    "\n",
    "        print(\"Saving new best model.\")\n",
    "        model.save(f\"models/hp/A2C/no.{expr_no}/{avg_rwd}.zip\")\n",
    "\n",
    "        print(\"Saving new best model's gif.\")            \n",
    "        save_frames_as_gif(bestscore_frames, f\"rl videos/hp/A2C/no.{expr_no}/{avg_rwd}.gif\")\n",
    "\n",
    "\n",
    "    env.close()\n",
    "    del env\n",
    "    del model\n",
    "\n",
    "\n",
    "    print(f\"---Time taken for opt-epoch: {(time.time() - start)/60} Minutes\")\n",
    "    print(f'---Average Episode Reward of opt-epoch: {avg_rwd}')\n",
    "    print(\"\\n\")\n",
    "    return -avg_rwd/10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_space():\n",
    "    return {\n",
    "        'env_rwd_func': hp.choice('env_rwd_func', [0, 1, 2, 3, 4]),\n",
    "        'env_max_step': hp.choice('env_max_step', [500, 1000, 1500, 2000, 2500, 3000, 5000]),\n",
    "        'env_num_stack': hp.choice('env_num_stack', [2, 3, 4, 5, 6, 7, 8]),\n",
    "        'n_envs': hp.choice('n_envs', [1, 2, 4, 6, 8, 10, 16, 20, 32, 64, 128, 250, 500, 1000]),\n",
    "        'learning_rate': hp.choice('learning_rate', [1e-6, 3e-6, 6e-6, 1e-5, 3e-5, 6e-5, 1e-4, 3e-4, 6e-4, 1e-3, 3e-3, 6e-3, 1e-2]),\n",
    "        'n_steps': hp.choice('n_steps', [4, 5, 8, 10, 16, 20, 32, 50, 64, 128, 256, 400, 512, 800, 1024, 1500, 2048]),\n",
    "        'gamma': hp.choice('gamma', [0.70, 0.80, 0.90, 0.95, 0.99, 0.995]),\n",
    "        # 'gae_lambda': hp.uniform('gae_lambda', 0.01, 1.0),\n",
    "        'normalize_advantage': hp.choice('normalize_advantage', [True, False]),\n",
    "        # 'ent_coef': hp.uniform('ent_coef', 0.0, 0.5),\n",
    "        # 'vf_coef': hp.uniform('vf_coef', 0.1, 1),\n",
    "        # 'max_grad_norm': hp.uniform('max_grad_norm', 0.3, 30),\n",
    "        'use_rms_prop': hp.choice('use_rms_prop', [True, False])\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---Chosen Hyperparameters of opt-epoch:                \n",
      "{'env_max_step': 5000, 'env_num_stack': 8, 'env_rwd_func': 4, 'gamma': 0.95, 'learning_rate': 0.0001, 'n_envs': 1, 'n_steps': 20, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "  0%|          | 0/100 [00:00<?, ?trial/s, best loss=?]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_24360\\352386517.py:41: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if info[\"head food dist\"] < prev_info[\"head food dist\"]:\n",
      "\n",
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_24360\\352386517.py:48: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  prev_info[\"head pos\"][0] == 0 and info[\"head pos\"][0] == info[\"size\"]-1) or (\n",
      "\n",
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_24360\\352386517.py:49: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  prev_info[\"head pos\"][1] == 0 and info[\"head pos\"][1] == info[\"size\"]-1):\n",
      "\n",
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_24360\\352386517.py:46: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if (prev_info[\"head pos\"][0] == info[\"size\"]-1 and info[\"head pos\"][0] == 0) or (\n",
      "\n",
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_24360\\352386517.py:47: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  prev_info[\"head pos\"][1] == info[\"size\"]-1 and info[\"head pos\"][1] == 0) or (\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\Ran agent on environment: 1/10 Episodes, score: 5     \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 8     \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 0     \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4     \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3     \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 6     \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5     \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 2     \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 6     \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 7    \n",
      "Saving new best model.                                 \n",
      "Saving new best model's gif.                           \n",
      "---Time taken for opt-epoch: 51.15685174067815 Minutes \n",
      "---Average Episode Reward of opt-epoch: 4.6            \n",
      "---Chosen Hyperparameters of opt-epoch:                                 \n",
      "{'env_max_step': 5000, 'env_num_stack': 8, 'env_rwd_func': 1, 'gamma': 0.99, 'learning_rate': 0.0003, 'n_envs': 4, 'n_steps': 800, 'normalize_advantage': False, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 5                      \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                       \n",
      "---Time taken for opt-epoch: 18.360319264729817 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 3.7                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'env_max_step': 1000, 'env_num_stack': 8, 'env_rwd_func': 4, 'gamma': 0.995, 'learning_rate': 0.001, 'n_envs': 8, 'n_steps': 32, 'normalize_advantage': True, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 2                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 9                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 2                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 5                       \n",
      "---Time taken for opt-epoch: 13.974841245015462 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 4.0                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'env_max_step': 2500, 'env_num_stack': 5, 'env_rwd_func': 4, 'gamma': 0.8, 'learning_rate': 6e-05, 'n_envs': 32, 'n_steps': 50, 'normalize_advantage': True, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                       \n",
      "---Time taken for opt-epoch: 8.996386524041494 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 3.7                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'env_max_step': 3000, 'env_num_stack': 7, 'env_rwd_func': 0, 'gamma': 0.7, 'learning_rate': 3e-05, 'n_envs': 16, 'n_steps': 64, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                       \n",
      "---Time taken for opt-epoch: 10.719624626636506 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 3.6                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'env_max_step': 1000, 'env_num_stack': 4, 'env_rwd_func': 2, 'gamma': 0.995, 'learning_rate': 6e-06, 'n_envs': 4, 'n_steps': 256, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 2                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                      \n",
      "---Time taken for opt-epoch: 18.479649500052133 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 3.2                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'env_max_step': 2500, 'env_num_stack': 6, 'env_rwd_func': 0, 'gamma': 0.95, 'learning_rate': 3e-05, 'n_envs': 4, 'n_steps': 2048, 'normalize_advantage': False, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 6                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                      \n",
      "---Time taken for opt-epoch: 18.43468484481176 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 3.4                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'env_max_step': 1500, 'env_num_stack': 6, 'env_rwd_func': 2, 'gamma': 0.95, 'learning_rate': 3e-05, 'n_envs': 4, 'n_steps': 1024, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 5                       \n",
      "---Time taken for opt-epoch: 18.45039716164271 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 3.9                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'env_max_step': 3000, 'env_num_stack': 5, 'env_rwd_func': 2, 'gamma': 0.8, 'learning_rate': 1e-06, 'n_envs': 16, 'n_steps': 128, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 5                       \n",
      "---Time taken for opt-epoch: 10.547717229525249 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 4.0                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'env_max_step': 2500, 'env_num_stack': 7, 'env_rwd_func': 2, 'gamma': 0.995, 'learning_rate': 0.01, 'n_envs': 1, 'n_steps': 16, 'normalize_advantage': False, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 1                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0                      \n",
      "---Time taken for opt-epoch: 51.24016671180725 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 0.1                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'env_max_step': 500, 'env_num_stack': 5, 'env_rwd_func': 0, 'gamma': 0.7, 'learning_rate': 6e-05, 'n_envs': 16, 'n_steps': 128, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 6                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 6                        \n",
      "---Time taken for opt-epoch: 10.549483772118887 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 4.3                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'env_max_step': 2500, 'env_num_stack': 7, 'env_rwd_func': 0, 'gamma': 0.7, 'learning_rate': 0.01, 'n_envs': 8, 'n_steps': 256, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 1                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 1                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0                        \n",
      "---Time taken for opt-epoch: 13.446199516455332 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 0.2                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'env_max_step': 1000, 'env_num_stack': 2, 'env_rwd_func': 1, 'gamma': 0.7, 'learning_rate': 0.0003, 'n_envs': 1, 'n_steps': 2048, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                        \n",
      "---Time taken for opt-epoch: 42.05436238050461 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 3.8                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'env_max_step': 3000, 'env_num_stack': 4, 'env_rwd_func': 1, 'gamma': 0.995, 'learning_rate': 0.01, 'n_envs': 20, 'n_steps': 10, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0                        \n",
      "---Time taken for opt-epoch: 10.820776323477427 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 0.0                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'env_max_step': 1500, 'env_num_stack': 4, 'env_rwd_func': 0, 'gamma': 0.995, 'learning_rate': 3e-06, 'n_envs': 6, 'n_steps': 256, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 6                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                        \n",
      "---Time taken for opt-epoch: 14.69631596406301 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 4.0                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'env_max_step': 5000, 'env_num_stack': 6, 'env_rwd_func': 4, 'gamma': 0.9, 'learning_rate': 0.006, 'n_envs': 4, 'n_steps': 512, 'normalize_advantage': True, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 6                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 9                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 7                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 1                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                        \n",
      "---Time taken for opt-epoch: 18.601537080605826 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 4.4                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'env_max_step': 1000, 'env_num_stack': 7, 'env_rwd_func': 4, 'gamma': 0.95, 'learning_rate': 1e-05, 'n_envs': 8, 'n_steps': 400, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 6                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 6                        \n",
      "---Time taken for opt-epoch: 13.092989806334177 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 4.2                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'env_max_step': 500, 'env_num_stack': 4, 'env_rwd_func': 3, 'gamma': 0.99, 'learning_rate': 0.006, 'n_envs': 2, 'n_steps': 50, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      " 17%|█▋        | 17/100 [5:43:38<23:59:24, 1040.54s/trial, best loss: -4.6]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_24360\\352386517.py:25: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  prev_info[\"head pos\"][0] == 0 and info[\"head pos\"][0] == info[\"size\"]-1) or (\n",
      "\n",
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_24360\\352386517.py:23: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if (prev_info[\"head pos\"][0] == info[\"size\"]-1 and info[\"head pos\"][0] == 0) or (\n",
      "\n",
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_24360\\352386517.py:24: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  prev_info[\"head pos\"][1] == info[\"size\"]-1 and info[\"head pos\"][1] == 0) or (\n",
      "\n",
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_24360\\352386517.py:26: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  prev_info[\"head pos\"][1] == 0 and info[\"head pos\"][1] == info[\"size\"]-1):\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\Ran agent on environment: 1/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 1                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 1                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0                        \n",
      "---Time taken for opt-epoch: 26.524602631727856 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 0.2                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'env_max_step': 1000, 'env_num_stack': 7, 'env_rwd_func': 4, 'gamma': 0.95, 'learning_rate': 0.0001, 'n_envs': 64, 'n_steps': 64, 'normalize_advantage': True, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 6                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 8                        \n",
      "---Time taken for opt-epoch: 8.15829553604126 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.3                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'env_max_step': 2500, 'env_num_stack': 3, 'env_rwd_func': 4, 'gamma': 0.95, 'learning_rate': 0.001, 'n_envs': 64, 'n_steps': 32, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 6                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 9                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                       \n",
      "---Time taken for opt-epoch: 8.452770086129506 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 4.4                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'env_max_step': 5000, 'env_num_stack': 8, 'env_rwd_func': 4, 'gamma': 0.9, 'learning_rate': 0.0006, 'n_envs': 250, 'n_steps': 20, 'normalize_advantage': True, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 6                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 9                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 1                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                       \n",
      "---Time taken for opt-epoch: 7.506938401858012 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 4.3                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'env_max_step': 2000, 'env_num_stack': 6, 'env_rwd_func': 3, 'gamma': 0.9, 'learning_rate': 0.0001, 'n_envs': 4, 'n_steps': 512, 'normalize_advantage': False, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                       \n",
      "---Time taken for opt-epoch: 18.826319324970246 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 3.4                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'env_max_step': 5000, 'env_num_stack': 3, 'env_rwd_func': 4, 'gamma': 0.95, 'learning_rate': 0.003, 'n_envs': 64, 'n_steps': 20, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 1                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 8                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 1                       \n",
      "---Time taken for opt-epoch: 8.704436798890432 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 2.6                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'env_max_step': 5000, 'env_num_stack': 8, 'env_rwd_func': 4, 'gamma': 0.9, 'learning_rate': 0.006, 'n_envs': 4, 'n_steps': 512, 'normalize_advantage': True, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 2                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 1                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 6                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0                       \n",
      "---Time taken for opt-epoch: 19.232978304227192 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 2.2                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'env_max_step': 5000, 'env_num_stack': 6, 'env_rwd_func': 4, 'gamma': 0.9, 'learning_rate': 0.0001, 'n_envs': 1, 'n_steps': 1500, 'normalize_advantage': False, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 5                       \n",
      "---Time taken for opt-epoch: 42.98598795334498 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 3.7                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'env_max_step': 2000, 'env_num_stack': 3, 'env_rwd_func': 3, 'gamma': 0.95, 'learning_rate': 0.001, 'n_envs': 64, 'n_steps': 4, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 1                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 2                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 1                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 1                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 1                        \n",
      "---Time taken for opt-epoch: 9.163355632623036 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 1.6                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'env_max_step': 2500, 'env_num_stack': 3, 'env_rwd_func': 4, 'gamma': 0.95, 'learning_rate': 0.001, 'n_envs': 250, 'n_steps': 8, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 2                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 6                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 7                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 2                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 6                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 9                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 10                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                        \n",
      "Saving new best model.                                                     \n",
      "Saving new best model's gif.                                               \n",
      "---Time taken for opt-epoch: 7.8838241736094155 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 5.0                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'env_max_step': 500, 'env_num_stack': 2, 'env_rwd_func': 4, 'gamma': 0.95, 'learning_rate': 0.0006, 'n_envs': 250, 'n_steps': 8, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 7                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 7                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 6                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 6                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                       \n",
      "---Time taken for opt-epoch: 7.6781865874926245 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 4.3                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'env_max_step': 2500, 'env_num_stack': 3, 'env_rwd_func': 1, 'gamma': 0.95, 'learning_rate': 0.0001, 'n_envs': 250, 'n_steps': 8, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 5                       \n",
      "---Time taken for opt-epoch: 7.3704881310462955 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 3.3                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'env_max_step': 2000, 'env_num_stack': 8, 'env_rwd_func': 4, 'gamma': 0.8, 'learning_rate': 0.001, 'n_envs': 2, 'n_steps': 5, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 1                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0                       \n",
      "---Time taken for opt-epoch: 44.173352626959485 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 0.1                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'env_max_step': 1500, 'env_num_stack': 8, 'env_rwd_func': 3, 'gamma': 0.99, 'learning_rate': 1e-06, 'n_envs': 1, 'n_steps': 800, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 5                         \n",
      "---Time taken for opt-epoch: 45.4591108640035 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 3.7                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 5000, 'env_num_stack': 3, 'env_rwd_func': 4, 'gamma': 0.99, 'learning_rate': 3e-06, 'n_envs': 128, 'n_steps': 8, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 6                         \n",
      "---Time taken for opt-epoch: 8.199621975421906 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 3.9                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 2500, 'env_num_stack': 8, 'env_rwd_func': 1, 'gamma': 0.95, 'learning_rate': 6e-06, 'n_envs': 10, 'n_steps': 20, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                         \n",
      "---Time taken for opt-epoch: 13.63044842084249 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.1                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 5000, 'env_num_stack': 3, 'env_rwd_func': 4, 'gamma': 0.8, 'learning_rate': 1e-05, 'n_envs': 4, 'n_steps': 1024, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                         \n",
      "---Time taken for opt-epoch: 19.45306913057963 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 3.5                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 3000, 'env_num_stack': 2, 'env_rwd_func': 4, 'gamma': 0.95, 'learning_rate': 0.003, 'n_envs': 4, 'n_steps': 1500, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 5                         \n",
      "Saving new best model.                                                      \n",
      "Saving new best model's gif.                                                \n",
      "---Time taken for opt-epoch: 18.673084700107573 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 5.1                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 3000, 'env_num_stack': 2, 'env_rwd_func': 4, 'gamma': 0.95, 'learning_rate': 0.003, 'n_envs': 4, 'n_steps': 1500, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 0                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 6                         \n",
      "---Time taken for opt-epoch: 19.601985482374825 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 3.4                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 3000, 'env_num_stack': 2, 'env_rwd_func': 2, 'gamma': 0.95, 'learning_rate': 0.003, 'n_envs': 4, 'n_steps': 1500, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 5                         \n",
      "---Time taken for opt-epoch: 19.848909576733906 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 4.2                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 3000, 'env_num_stack': 2, 'env_rwd_func': 4, 'gamma': 0.99, 'learning_rate': 0.003, 'n_envs': 4, 'n_steps': 1500, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 0                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 8                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 2                         \n",
      "---Time taken for opt-epoch: 18.887023969491324 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 4.1                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 3000, 'env_num_stack': 2, 'env_rwd_func': 2, 'gamma': 0.95, 'learning_rate': 0.001, 'n_envs': 1000, 'n_steps': 16, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                         \n",
      "---Time taken for opt-epoch: 7.150490907828013 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 3.2                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 2500, 'env_num_stack': 5, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.0003, 'n_envs': 20, 'n_steps': 4, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 14                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 10                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 12                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 7                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 12                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 12                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 12                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 12                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 10                       \n",
      "Saving new best model.                                                     \n",
      "Saving new best model's gif.                                               \n",
      "---Time taken for opt-epoch: 12.213562889893849 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 10.6                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 3000, 'env_num_stack': 5, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.0003, 'n_envs': 20, 'n_steps': 4, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 15                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 13                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 15                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 11                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 13                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 11                        \n",
      "Saving new best model.                                                      \n",
      "Saving new best model's gif.                                                \n",
      "---Time taken for opt-epoch: 12.039117995897929 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 11.0                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1500, 'env_num_stack': 5, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.0003, 'n_envs': 20, 'n_steps': 4, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 17                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 12                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 11                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 16                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 9                         \n",
      "---Time taken for opt-epoch: 12.016139753659566 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 10.7                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1500, 'env_num_stack': 5, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.0003, 'n_envs': 20, 'n_steps': 4, 'normalize_advantage': False, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 11                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 11                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 12                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 11                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 8                         \n",
      "---Time taken for opt-epoch: 11.70241813659668 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 9.9                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1500, 'env_num_stack': 5, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.0003, 'n_envs': 20, 'n_steps': 4, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 8                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 12                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 12                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 10                        \n",
      "---Time taken for opt-epoch: 12.054593992233276 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 9.6                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1500, 'env_num_stack': 5, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.0003, 'n_envs': 20, 'n_steps': 4, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 11                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 16                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 11                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 12                        \n",
      "---Time taken for opt-epoch: 11.82610253492991 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 9.9                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1500, 'env_num_stack': 5, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 6e-05, 'n_envs': 32, 'n_steps': 10, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                         \n",
      "---Time taken for opt-epoch: 9.33412830432256 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 3.7                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 500, 'env_num_stack': 5, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.0003, 'n_envs': 20, 'n_steps': 4, 'normalize_advantage': False, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 12                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 11                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 9                         \n",
      "---Time taken for opt-epoch: 12.230601974328358 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 8.0                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1500, 'env_num_stack': 5, 'env_rwd_func': 0, 'gamma': 0.7, 'learning_rate': 3e-05, 'n_envs': 6, 'n_steps': 400, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                         \n",
      "---Time taken for opt-epoch: 15.098573112487793 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 4.2                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 3000, 'env_num_stack': 5, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.0003, 'n_envs': 16, 'n_steps': 128, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 8                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 8                         \n",
      "---Time taken for opt-epoch: 10.945403138796488 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 6.3                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1000, 'env_num_stack': 5, 'env_rwd_func': 0, 'gamma': 0.995, 'learning_rate': 6e-06, 'n_envs': 20, 'n_steps': 50, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                         \n",
      "---Time taken for opt-epoch: 10.078860652446746 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 3.3                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 2000, 'env_num_stack': 4, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 1e-06, 'n_envs': 4, 'n_steps': 2048, 'normalize_advantage': False, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                        \n",
      "---Time taken for opt-epoch: 18.81161828438441 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 3.7                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1500, 'env_num_stack': 7, 'env_rwd_func': 1, 'gamma': 0.7, 'learning_rate': 0.0003, 'n_envs': 4, 'n_steps': 32, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                         \n",
      "---Time taken for opt-epoch: 21.3700666030248 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 4.3                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 3000, 'env_num_stack': 5, 'env_rwd_func': 2, 'gamma': 0.8, 'learning_rate': 0.01, 'n_envs': 128, 'n_steps': 64, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 8                         \n",
      "---Time taken for opt-epoch: 7.873458520571391 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 8.3                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 500, 'env_num_stack': 5, 'env_rwd_func': 0, 'gamma': 0.995, 'learning_rate': 3e-06, 'n_envs': 10, 'n_steps': 5, 'normalize_advantage': False, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                         \n",
      "---Time taken for opt-epoch: 15.97022674481074 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.5                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1000, 'env_num_stack': 6, 'env_rwd_func': 3, 'gamma': 0.8, 'learning_rate': 6e-05, 'n_envs': 20, 'n_steps': 4, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 0                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                         \n",
      "---Time taken for opt-epoch: 12.498015983899434 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 1.9                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1500, 'env_num_stack': 4, 'env_rwd_func': 0, 'gamma': 0.7, 'learning_rate': 1e-05, 'n_envs': 4, 'n_steps': 800, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                         \n",
      "---Time taken for opt-epoch: 18.74307049512863 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 3.9                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 3000, 'env_num_stack': 7, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 3e-05, 'n_envs': 6, 'n_steps': 256, 'normalize_advantage': False, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                         \n",
      "---Time taken for opt-epoch: 15.583211763699849 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 3.4                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 2000, 'env_num_stack': 5, 'env_rwd_func': 1, 'gamma': 0.995, 'learning_rate': 0.0006, 'n_envs': 8, 'n_steps': 10, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 11                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 11                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 10                        \n",
      "---Time taken for opt-epoch: 16.28373276392619 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 9.0                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 500, 'env_num_stack': 6, 'env_rwd_func': 2, 'gamma': 0.9, 'learning_rate': 0.0003, 'n_envs': 2, 'n_steps': 16, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 6                         \n",
      "---Time taken for opt-epoch: 29.455179317792258 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 6.4                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'env_max_step': 1000, 'env_num_stack': 5, 'env_rwd_func': 3, 'gamma': 0.99, 'learning_rate': 0.01, 'n_envs': 8, 'n_steps': 400, 'normalize_advantage': False, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                           \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                           \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 5                           \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 2                           \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                           \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                           \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 6                           \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 5                           \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 6                           \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 7                          \n",
      "---Time taken for opt-epoch: 14.279938395818075 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.4                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'env_max_step': 1500, 'env_num_stack': 7, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.006, 'n_envs': 4, 'n_steps': 1024, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 11                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 13                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 11                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 11                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 11                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 10                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 12                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 10                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 15                         \n",
      "Saving new best model.                                                       \n",
      "Saving new best model's gif.                                                 \n",
      "---Time taken for opt-epoch: 19.302212611834207 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 11.3                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'env_max_step': 3000, 'env_num_stack': 7, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.006, 'n_envs': 4, 'n_steps': 1024, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 15                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 19                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 12                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 10                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 11                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 8                           \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 13                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 18                         \n",
      "Saving new best model.                                                       \n",
      "Saving new best model's gif.                                                 \n",
      "---Time taken for opt-epoch: 19.218917846679688 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 12.4                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'env_max_step': 2000, 'env_num_stack': 7, 'env_rwd_func': 1, 'gamma': 0.9, 'learning_rate': 0.006, 'n_envs': 4, 'n_steps': 1024, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 11                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                           \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 10                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 10                         \n",
      "---Time taken for opt-epoch: 19.103668896357217 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 8.9                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'env_max_step': 1500, 'env_num_stack': 7, 'env_rwd_func': 3, 'gamma': 0.7, 'learning_rate': 0.006, 'n_envs': 4, 'n_steps': 1024, 'normalize_advantage': True, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                           \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                           \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 5                           \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                           \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 1                           \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 1                           \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 1                           \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 1                           \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 6                          \n",
      "---Time taken for opt-epoch: 19.354078694184622 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 3.6                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'env_max_step': 3000, 'env_num_stack': 7, 'env_rwd_func': 0, 'gamma': 0.99, 'learning_rate': 0.006, 'n_envs': 4, 'n_steps': 1024, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                           \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 10                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 8                           \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 10                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 5                           \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 11                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 8                          \n",
      "---Time taken for opt-epoch: 19.074048745632172 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 8.4                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'env_max_step': 3000, 'env_num_stack': 7, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.006, 'n_envs': 4, 'n_steps': 1024, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                           \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 10                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 8                           \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 9                          \n",
      "---Time taken for opt-epoch: 19.02261425256729 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 8.6                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'env_max_step': 3000, 'env_num_stack': 7, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.006, 'n_envs': 4, 'n_steps': 1024, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 8                           \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 13                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 12                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 19                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 12                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 9                          \n",
      "---Time taken for opt-epoch: 19.127573692798613 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 10.9                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'env_max_step': 3000, 'env_num_stack': 7, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.006, 'n_envs': 4, 'n_steps': 1024, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 10                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 16                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 11                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 11                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 11                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 11                         \n",
      "---Time taken for opt-epoch: 19.470729700724284 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 10.6                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'env_max_step': 3000, 'env_num_stack': 7, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.006, 'n_envs': 16, 'n_steps': 128, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 12                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 8                           \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 13                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 8                           \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                           \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 9                           \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 10                         \n",
      "---Time taken for opt-epoch: 10.989810836315154 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 9.2                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'env_max_step': 3000, 'env_num_stack': 7, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 6e-06, 'n_envs': 4, 'n_steps': 2048, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                         \n",
      "---Time taken for opt-epoch: 18.691299295425416 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 3.7                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 3000, 'env_num_stack': 7, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 1e-06, 'n_envs': 4, 'n_steps': 512, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 5                         \n",
      "---Time taken for opt-epoch: 20.539728307723998 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 4.0                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 5000, 'env_num_stack': 4, 'env_rwd_func': 2, 'gamma': 0.995, 'learning_rate': 0.006, 'n_envs': 64, 'n_steps': 50, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 5                         \n",
      "---Time taken for opt-epoch: 8.09140241543452 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 3.7                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 2500, 'env_num_stack': 8, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 3e-06, 'n_envs': 1, 'n_steps': 32, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 5                        \n",
      "---Time taken for opt-epoch: 46.81685829559962 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 3.4                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'env_max_step': 3000, 'env_num_stack': 7, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 1e-05, 'n_envs': 500, 'n_steps': 64, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 5                           \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                           \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                           \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                           \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                           \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                           \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                           \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                           \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                           \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                          \n",
      "---Time taken for opt-epoch: 7.364441990852356 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 3.5                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'env_max_step': 500, 'env_num_stack': 6, 'env_rwd_func': 1, 'gamma': 0.9, 'learning_rate': 0.0006, 'n_envs': 4, 'n_steps': 1024, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 5                         \n",
      "---Time taken for opt-epoch: 18.80127768913905 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 3.7                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1000, 'env_num_stack': 7, 'env_rwd_func': 3, 'gamma': 0.8, 'learning_rate': 0.0001, 'n_envs': 2, 'n_steps': 20, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 0                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 1                         \n",
      "---Time taken for opt-epoch: 29.031371609369913 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 1.6                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1500, 'env_num_stack': 3, 'env_rwd_func': 0, 'gamma': 0.99, 'learning_rate': 6e-05, 'n_envs': 8, 'n_steps': 5, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 8                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 8                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 5                         \n",
      "---Time taken for opt-epoch: 18.29380451043447 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 6.5                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 3000, 'env_num_stack': 8, 'env_rwd_func': 2, 'gamma': 0.8, 'learning_rate': 3e-05, 'n_envs': 4, 'n_steps': 1024, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                         \n",
      "---Time taken for opt-epoch: 19.084919246037803 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 3.9                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 5000, 'env_num_stack': 4, 'env_rwd_func': 0, 'gamma': 0.7, 'learning_rate': 0.006, 'n_envs': 4, 'n_steps': 512, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 8                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 9                         \n",
      "---Time taken for opt-epoch: 20.812806864579517 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 8.3                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 2000, 'env_num_stack': 7, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.01, 'n_envs': 4, 'n_steps': 800, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 8                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 8                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 8                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 9                         \n",
      "---Time taken for opt-epoch: 19.332928184668223 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 8.9                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 2500, 'env_num_stack': 2, 'env_rwd_func': 0, 'gamma': 0.995, 'learning_rate': 6e-06, 'n_envs': 6, 'n_steps': 16, 'normalize_advantage': False, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                         \n",
      "---Time taken for opt-epoch: 17.57550896803538 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 3.1                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1500, 'env_num_stack': 7, 'env_rwd_func': 1, 'gamma': 0.8, 'learning_rate': 1e-06, 'n_envs': 8, 'n_steps': 256, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                         \n",
      "---Time taken for opt-epoch: 13.966749246915182 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 3.3                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 3000, 'env_num_stack': 6, 'env_rwd_func': 0, 'gamma': 0.9, 'learning_rate': 0.006, 'n_envs': 1, 'n_steps': 10, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 0                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 0                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 0                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 0                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 0                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 0                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 0                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 0                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0                         \n",
      "---Time taken for opt-epoch: 61.52016716400782 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 0.1                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 500, 'env_num_stack': 3, 'env_rwd_func': 3, 'gamma': 0.8, 'learning_rate': 0.001, 'n_envs': 16, 'n_steps': 128, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 1                         \n",
      "---Time taken for opt-epoch: 11.18420535326004 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 2.6                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1500, 'env_num_stack': 8, 'env_rwd_func': 2, 'gamma': 0.99, 'learning_rate': 0.0001, 'n_envs': 128, 'n_steps': 8, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                         \n",
      "---Time taken for opt-epoch: 7.655570356051127 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 3.3                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1000, 'env_num_stack': 7, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.0006, 'n_envs': 4, 'n_steps': 400, 'normalize_advantage': False, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 12                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 12                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 7                         \n",
      "---Time taken for opt-epoch: 18.660806274414064 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 9.7                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 3000, 'env_num_stack': 4, 'env_rwd_func': 0, 'gamma': 0.7, 'learning_rate': 1e-05, 'n_envs': 4, 'n_steps': 2048, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                         \n",
      "---Time taken for opt-epoch: 18.354274996121724 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 4.1                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 5000, 'env_num_stack': 2, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 3e-06, 'n_envs': 20, 'n_steps': 50, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 5                         \n",
      "---Time taken for opt-epoch: 9.975714218616485 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.0                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 2500, 'env_num_stack': 7, 'env_rwd_func': 1, 'gamma': 0.995, 'learning_rate': 6e-05, 'n_envs': 10, 'n_steps': 32, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                        \n",
      "---Time taken for opt-epoch: 12.277546751499177 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 3.4                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'env_max_step': 1500, 'env_num_stack': 6, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.003, 'n_envs': 4, 'n_steps': 1024, 'normalize_advantage': True, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 9                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 11                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 18                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 14                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 7                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 7                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 10                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 9                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 6                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 12                       \n",
      "---Time taken for opt-epoch: 19.06808598836263 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 10.3                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'env_max_step': 2000, 'env_num_stack': 5, 'env_rwd_func': 3, 'gamma': 0.9, 'learning_rate': 0.0003, 'n_envs': 2, 'n_steps': 4, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 1                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 1                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 1                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0                        \n",
      "---Time taken for opt-epoch: 44.36963443358739 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 0.8                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 3000, 'env_num_stack': 3, 'env_rwd_func': 2, 'gamma': 0.8, 'learning_rate': 3e-05, 'n_envs': 8, 'n_steps': 64, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                         \n",
      "---Time taken for opt-epoch: 13.245016475518545 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 3.1                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 3000, 'env_num_stack': 7, 'env_rwd_func': 0, 'gamma': 0.99, 'learning_rate': 0.006, 'n_envs': 32, 'n_steps': 20, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 8                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 16                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 13                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 8                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 12                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 10                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 12                        \n",
      "---Time taken for opt-epoch: 9.311142845948536 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 10.1                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1500, 'env_num_stack': 8, 'env_rwd_func': 4, 'gamma': 0.95, 'learning_rate': 0.006, 'n_envs': 4, 'n_steps': 1024, 'normalize_advantage': False, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 2                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                         \n",
      "---Time taken for opt-epoch: 18.89231744209925 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 2.6                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 500, 'env_num_stack': 5, 'env_rwd_func': 0, 'gamma': 0.7, 'learning_rate': 0.001, 'n_envs': 6, 'n_steps': 5, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 1                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 0                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0                         \n",
      "---Time taken for opt-epoch: 20.40144592523575 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 3.3                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1000, 'env_num_stack': 4, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.0003, 'n_envs': 4, 'n_steps': 800, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 6                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 9                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 8                         \n",
      "---Time taken for opt-epoch: 18.524278644720713 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 7.5                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 3000, 'env_num_stack': 2, 'env_rwd_func': 1, 'gamma': 0.995, 'learning_rate': 6e-06, 'n_envs': 8, 'n_steps': 256, 'normalize_advantage': False, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 7                          \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 5                          \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                          \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                          \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                         \n",
      "---Time taken for opt-epoch: 13.714527889092762 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 3.9                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'env_max_step': 1500, 'env_num_stack': 7, 'env_rwd_func': 3, 'gamma': 0.8, 'learning_rate': 0.0001, 'n_envs': 250, 'n_steps': 4, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 1                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 6                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                       \n",
      "---Time taken for opt-epoch: 7.9807753443717955 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 3.3                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'env_max_step': 2000, 'env_num_stack': 5, 'env_rwd_func': 0, 'gamma': 0.8, 'learning_rate': 0.01, 'n_envs': 4, 'n_steps': 1500, 'normalize_advantage': False, 'use_rms_prop': False}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 11                      \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 13                      \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 15                      \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 10                      \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 9                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 7                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 17                      \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 9                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 12                     \n",
      "---Time taken for opt-epoch: 20.592031129201253 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 10.6                             \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'env_max_step': 5000, 'env_num_stack': 7, 'env_rwd_func': 2, 'gamma': 0.9, 'learning_rate': 1e-06, 'n_envs': 1, 'n_steps': 16, 'normalize_advantage': True, 'use_rms_prop': True}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                      \n",
      "---Time taken for opt-epoch: 58.241806809107466 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 3.6                              \n",
      "100%|██████████| 100/100 [30:38:43<00:00, 1103.23s/trial, best loss: -12.4]\n"
     ]
    }
   ],
   "source": [
    "best_avg_rwd = float(\"-inf\")\n",
    "expr_no = 1\n",
    "total_timesteps = 1_000_000\n",
    "\n",
    "best = fmin(objective, get_space(), algo=tpe.suggest, max_evals=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'env_max_step': 5,\n",
       " 'env_num_stack': 5,\n",
       " 'env_rwd_func': 0,\n",
       " 'gamma': 1,\n",
       " 'learning_rate': 11,\n",
       " 'n_envs': 13,\n",
       " 'n_steps': 14,\n",
       " 'normalize_advantage': 0,\n",
       " 'use_rms_prop': 0}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'env_max_step': 3000,\n",
       " 'env_num_stack': 7,\n",
       " 'env_rwd_func': 0,\n",
       " 'gamma': 0.8,\n",
       " 'learning_rate': 0.006,\n",
       " 'n_envs': 1000,\n",
       " 'n_steps': 1024,\n",
       " 'normalize_advantage': True,\n",
       " 'use_rms_prop': True}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "space_eval(get_space(), best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_space():\n",
    "    return {\n",
    "        'env_rwd_func': hp.choice('env_rwd_func', [0, 1, 2, 3]),\n",
    "        'env_max_step': hp.choice('env_max_step', [1500, 2000, 2500, 3000, 3500]),\n",
    "        'env_num_stack': hp.choice('env_num_stack', [4, 5, 6, 7]),\n",
    "        'n_envs': hp.choice('n_envs', [8, 16, 64, 128, 256, 512, 1024]),\n",
    "        'learning_rate': hp.choice('learning_rate', [1e-4, 3e-4, 4e-4, 5e-4, 6e-4, 7e-4, 9e-4, 1e-3, 3e-3, 6e-3, 7e-3, 8e-3, 1e-2, 3e-2, 6e-2, 1e-1]),\n",
    "        'n_steps': hp.choice('n_steps', [1, 4, 5, 8, 10, 16, 20, 32, 50, 64, 128, 256, 400, 512, 800, 1024, 1500, 2048]),\n",
    "        'gamma': hp.uniform('gamma', 0.75, 0.85),\n",
    "        'gae_lambda': hp.uniform('gae_lambda', 0.01, 1.0),\n",
    "        'normalize_advantage': hp.choice('normalize_advantage', [True, False]),\n",
    "        'ent_coef': hp.uniform('ent_coef', 0.0, 0.5),\n",
    "        'vf_coef': hp.uniform('vf_coef', 0.1, 1),\n",
    "        'max_grad_norm': hp.uniform('max_grad_norm', 0.3, 30),\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---Chosen Hyperparameters of opt-epoch:               \n",
      "{'ent_coef': 0.19218002626830366, 'env_max_step': 2000, 'env_num_stack': 7, 'env_rwd_func': 0, 'gae_lambda': 0.678666281055582, 'gamma': 0.845807024412953, 'learning_rate': 0.1, 'max_grad_norm': 15.692779125887494, 'n_envs': 128, 'n_steps': 800, 'normalize_advantage': True, 'vf_coef': 0.40962507638362533}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 0    \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 0    \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 2    \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 1    \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 0    \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 0    \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 0    \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 0    \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0    \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0   \n",
      "Saving new best model.                                \n",
      "Saving new best model's gif.                          \n",
      "  0%|          | 0/50 [15:49<?, ?trial/s, best loss=?]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Hosein\\anaconda3\\envs\\pt_env\\lib\\site-packages\\stable_baselines3\\common\\save_util.py:278: UserWarning: Path 'models\\hp\\A2C\\no.2' does not exist. Will create it.\n",
      "  warnings.warn(f\"Path '{path.parent}' does not exist. Will create it.\")\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---Time taken for opt-epoch: 15.94200219710668 Minutes\n",
      "---Average Episode Reward of opt-epoch: 0.3           \n",
      "---Chosen Hyperparameters of opt-epoch:                               \n",
      "{'ent_coef': 0.46834323915969894, 'env_max_step': 2000, 'env_num_stack': 6, 'env_rwd_func': 2, 'gae_lambda': 0.8357702577941016, 'gamma': 0.8445595450270638, 'learning_rate': 0.0007, 'max_grad_norm': 6.655502137794559, 'n_envs': 16, 'n_steps': 2048, 'normalize_advantage': True, 'vf_coef': 0.15079755043808077}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                    \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                    \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                    \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                    \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                    \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                    \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                    \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                    \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                    \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                   \n",
      "Saving new best model.                                                \n",
      "Saving new best model's gif.                                          \n",
      "---Time taken for opt-epoch: 21.369914813836417 Minutes               \n",
      "---Average Episode Reward of opt-epoch: 3.5                           \n",
      "---Chosen Hyperparameters of opt-epoch:                                \n",
      "{'ent_coef': 0.04445207269922691, 'env_max_step': 3000, 'env_num_stack': 6, 'env_rwd_func': 1, 'gae_lambda': 0.41658943543187177, 'gamma': 0.7713142284054719, 'learning_rate': 0.008, 'max_grad_norm': 25.786015026264472, 'n_envs': 8, 'n_steps': 64, 'normalize_advantage': False, 'vf_coef': 0.4797757357403165}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 5                     \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                      \n",
      "Saving new best model.                                                   \n",
      "Saving new best model's gif.                                             \n",
      "---Time taken for opt-epoch: 28.388479697704316 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 3.9                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.37772817888686455, 'env_max_step': 2500, 'env_num_stack': 5, 'env_rwd_func': 3, 'gae_lambda': 0.14153677601443007, 'gamma': 0.8049593362631462, 'learning_rate': 0.03, 'max_grad_norm': 27.61829559491207, 'n_envs': 128, 'n_steps': 800, 'normalize_advantage': True, 'vf_coef': 0.5220842603902834}\n",
      "  6%|▌         | 3/50 [1:05:42<18:17:58, 1401.67s/trial, best loss: -3.9]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_28180\\352386517.py:26: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  prev_info[\"head pos\"][1] == 0 and info[\"head pos\"][1] == info[\"size\"]-1):\n",
      "\n",
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_28180\\352386517.py:23: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if (prev_info[\"head pos\"][0] == info[\"size\"]-1 and info[\"head pos\"][0] == 0) or (\n",
      "\n",
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_28180\\352386517.py:24: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  prev_info[\"head pos\"][1] == info[\"size\"]-1 and info[\"head pos\"][1] == 0) or (\n",
      "\n",
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_28180\\352386517.py:25: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  prev_info[\"head pos\"][0] == 0 and info[\"head pos\"][0] == info[\"size\"]-1) or (\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\Ran agent on environment: 1/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 2                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 1                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 1                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0                      \n",
      "---Time taken for opt-epoch: 15.582719135284425 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 0.4                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.26335964765958647, 'env_max_step': 1500, 'env_num_stack': 4, 'env_rwd_func': 1, 'gae_lambda': 0.3285690066317015, 'gamma': 0.8454179919010765, 'learning_rate': 0.007, 'max_grad_norm': 28.64785788037572, 'n_envs': 256, 'n_steps': 256, 'normalize_advantage': True, 'vf_coef': 0.31852949140201436}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 6                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 6                      \n",
      "Saving new best model.                                                   \n",
      "Saving new best model's gif.                                             \n",
      "---Time taken for opt-epoch: 15.432211720943451 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 4.1                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.25746938980725176, 'env_max_step': 3000, 'env_num_stack': 4, 'env_rwd_func': 2, 'gae_lambda': 0.9986381793248246, 'gamma': 0.8255646622566397, 'learning_rate': 0.008, 'max_grad_norm': 8.967970749691572, 'n_envs': 512, 'n_steps': 1500, 'normalize_advantage': True, 'vf_coef': 0.8902628719725834}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 6                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 6                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                      \n",
      "---Time taken for opt-epoch: 17.189907801151275 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 4.0                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.4993044338289911, 'env_max_step': 3000, 'env_num_stack': 7, 'env_rwd_func': 2, 'gae_lambda': 0.026740969455769488, 'gamma': 0.8076697420721095, 'learning_rate': 0.06, 'max_grad_norm': 15.984320239985973, 'n_envs': 256, 'n_steps': 50, 'normalize_advantage': False, 'vf_coef': 0.6429410104358494}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 7                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 6                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 6                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                      \n",
      "Saving new best model.                                                   \n",
      "Saving new best model's gif.                                             \n",
      "---Time taken for opt-epoch: 15.069838571548463 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 4.6                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.35769352839370955, 'env_max_step': 1500, 'env_num_stack': 4, 'env_rwd_func': 1, 'gae_lambda': 0.9557259230538205, 'gamma': 0.8015125970920615, 'learning_rate': 0.0005, 'max_grad_norm': 15.67694673556705, 'n_envs': 256, 'n_steps': 400, 'normalize_advantage': False, 'vf_coef': 0.6589026942073283}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 6                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 6                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 6                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                      \n",
      "---Time taken for opt-epoch: 14.607407490412394 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 4.3                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.20374696669479742, 'env_max_step': 2000, 'env_num_stack': 5, 'env_rwd_func': 2, 'gae_lambda': 0.7453261482768608, 'gamma': 0.8038168663260643, 'learning_rate': 0.0005, 'max_grad_norm': 6.712806634766867, 'n_envs': 256, 'n_steps': 1500, 'normalize_advantage': False, 'vf_coef': 0.5439690320947882}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 6                      \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                     \n",
      "---Time taken for opt-epoch: 17.14523229598999 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 3.3                             \n",
      "---Chosen Hyperparameters of opt-epoch:                                 \n",
      "{'ent_coef': 0.1014931694115177, 'env_max_step': 2000, 'env_num_stack': 6, 'env_rwd_func': 2, 'gae_lambda': 0.3748705447361421, 'gamma': 0.7603644855479926, 'learning_rate': 0.0006, 'max_grad_norm': 15.22174042088706, 'n_envs': 128, 'n_steps': 8, 'normalize_advantage': True, 'vf_coef': 0.46590946795302246}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                      \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 6                      \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 5                      \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 5                     \n",
      "---Time taken for opt-epoch: 16.24770839214325 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 3.8                             \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.27271533934220576, 'env_max_step': 2500, 'env_num_stack': 4, 'env_rwd_func': 3, 'gae_lambda': 0.647215938711168, 'gamma': 0.8043045969840061, 'learning_rate': 0.03, 'max_grad_norm': 15.964044205501756, 'n_envs': 1024, 'n_steps': 400, 'normalize_advantage': False, 'vf_coef': 0.7441466528182514}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0                      \n",
      "---Time taken for opt-epoch: 15.972014613946278 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 0.0                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.29771797243982895, 'env_max_step': 2000, 'env_num_stack': 5, 'env_rwd_func': 3, 'gae_lambda': 0.20911061830163313, 'gamma': 0.7981704762476467, 'learning_rate': 0.0004, 'max_grad_norm': 20.50777161220117, 'n_envs': 16, 'n_steps': 1500, 'normalize_advantage': True, 'vf_coef': 0.47141649537750774}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 6                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 6                      \n",
      "---Time taken for opt-epoch: 22.794637898604076 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 4.3                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'ent_coef': 0.23788540843393197, 'env_max_step': 3500, 'env_num_stack': 7, 'env_rwd_func': 3, 'gae_lambda': 0.029395469839854624, 'gamma': 0.8421892280632027, 'learning_rate': 0.0004, 'max_grad_norm': 2.7728906317905877, 'n_envs': 256, 'n_steps': 2048, 'normalize_advantage': True, 'vf_coef': 0.6498478805901913}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 6                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                       \n",
      "---Time taken for opt-epoch: 16.291430767377218 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 4.1                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'ent_coef': 0.33574879330916685, 'env_max_step': 1500, 'env_num_stack': 4, 'env_rwd_func': 3, 'gae_lambda': 0.4910935058350641, 'gamma': 0.799333129144503, 'learning_rate': 0.0007, 'max_grad_norm': 13.79442179017807, 'n_envs': 128, 'n_steps': 128, 'normalize_advantage': False, 'vf_coef': 0.8297972726952709}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                       \n",
      "---Time taken for opt-epoch: 16.25416716337204 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 3.2                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'ent_coef': 0.16121870384082415, 'env_max_step': 1500, 'env_num_stack': 6, 'env_rwd_func': 2, 'gae_lambda': 0.9102391215864578, 'gamma': 0.764805975891365, 'learning_rate': 0.0005, 'max_grad_norm': 27.541434441195335, 'n_envs': 8, 'n_steps': 128, 'normalize_advantage': True, 'vf_coef': 0.8183860551246003}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 6                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 1                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                       \n",
      "---Time taken for opt-epoch: 30.76889015038808 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 3.7                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'ent_coef': 0.3759695502903435, 'env_max_step': 3500, 'env_num_stack': 6, 'env_rwd_func': 2, 'gae_lambda': 0.9003921032536819, 'gamma': 0.7528946061636869, 'learning_rate': 0.007, 'max_grad_norm': 5.334720635996464, 'n_envs': 256, 'n_steps': 128, 'normalize_advantage': False, 'vf_coef': 0.9635767271738983}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                       \n",
      "---Time taken for opt-epoch: 15.685320834318796 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 3.8                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'ent_coef': 0.38119468123286127, 'env_max_step': 3000, 'env_num_stack': 6, 'env_rwd_func': 0, 'gae_lambda': 0.11906788662577442, 'gamma': 0.804027839971512, 'learning_rate': 0.0001, 'max_grad_norm': 19.593226771623335, 'n_envs': 256, 'n_steps': 1500, 'normalize_advantage': False, 'vf_coef': 0.19290432194807194}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                       \n",
      "---Time taken for opt-epoch: 17.244610810279845 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 3.4                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'ent_coef': 0.3308512743734311, 'env_max_step': 3000, 'env_num_stack': 7, 'env_rwd_func': 1, 'gae_lambda': 0.8656436187580384, 'gamma': 0.8483929909913892, 'learning_rate': 0.008, 'max_grad_norm': 4.922505322083451, 'n_envs': 256, 'n_steps': 32, 'normalize_advantage': False, 'vf_coef': 0.31660617243722255}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 2                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                       \n",
      "---Time taken for opt-epoch: 15.09386924902598 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 3.1                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'ent_coef': 0.29945282307409765, 'env_max_step': 2000, 'env_num_stack': 5, 'env_rwd_func': 0, 'gae_lambda': 0.8346975188301697, 'gamma': 0.7740311424455495, 'learning_rate': 0.0007, 'max_grad_norm': 29.40905605987295, 'n_envs': 16, 'n_steps': 512, 'normalize_advantage': False, 'vf_coef': 0.148534866771008}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                      \n",
      "---Time taken for opt-epoch: 21.076501949628195 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 3.4                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.4887069980047146, 'env_max_step': 2000, 'env_num_stack': 4, 'env_rwd_func': 1, 'gae_lambda': 0.6668316252622513, 'gamma': 0.8375334061911863, 'learning_rate': 0.0007, 'max_grad_norm': 29.526501611883923, 'n_envs': 8, 'n_steps': 2048, 'normalize_advantage': False, 'vf_coef': 0.6415180108658953}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 6                      \n",
      "---Time taken for opt-epoch: 25.963454365730286 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 4.2                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'ent_coef': 0.4431180348636147, 'env_max_step': 1500, 'env_num_stack': 7, 'env_rwd_func': 1, 'gae_lambda': 0.5479708693429023, 'gamma': 0.7859960572405523, 'learning_rate': 0.0003, 'max_grad_norm': 11.744566900818366, 'n_envs': 512, 'n_steps': 50, 'normalize_advantage': False, 'vf_coef': 0.6458596161477347}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 7                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                       \n",
      "---Time taken for opt-epoch: 14.21350973447164 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 3.9                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'ent_coef': 0.4343919876511443, 'env_max_step': 1500, 'env_num_stack': 7, 'env_rwd_func': 1, 'gae_lambda': 0.2596814204932565, 'gamma': 0.817503493974666, 'learning_rate': 0.06, 'max_grad_norm': 20.05873127542483, 'n_envs': 64, 'n_steps': 50, 'normalize_advantage': False, 'vf_coef': 0.7176335962028343}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 6                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                      \n",
      "---Time taken for opt-epoch: 15.755683243274689 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 4.1                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.005479863864322393, 'env_max_step': 3000, 'env_num_stack': 5, 'env_rwd_func': 3, 'gae_lambda': 0.01904490708742036, 'gamma': 0.790021351779972, 'learning_rate': 0.06, 'max_grad_norm': 23.480683486003226, 'n_envs': 16, 'n_steps': 1024, 'normalize_advantage': True, 'vf_coef': 0.27821618449123875}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 1                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0                      \n",
      "---Time taken for opt-epoch: 21.651923580964407 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 0.1                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.1406580671588061, 'env_max_step': 3000, 'env_num_stack': 5, 'env_rwd_func': 3, 'gae_lambda': 0.19090699280451975, 'gamma': 0.8141320936430139, 'learning_rate': 0.0009, 'max_grad_norm': 19.648045843594776, 'n_envs': 1024, 'n_steps': 20, 'normalize_advantage': True, 'vf_coef': 0.3993901696286828}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 6                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 8                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 7                      \n",
      "Saving new best model.                                                   \n",
      "Saving new best model's gif.                                             \n",
      "---Time taken for opt-epoch: 14.49059725999832 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 5.0                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.11973960993642516, 'env_max_step': 3000, 'env_num_stack': 7, 'env_rwd_func': 3, 'gae_lambda': 0.10955948680666155, 'gamma': 0.8203331724062867, 'learning_rate': 0.0009, 'max_grad_norm': 23.401627949073376, 'n_envs': 1024, 'n_steps': 20, 'normalize_advantage': True, 'vf_coef': 0.40076270485339227}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 6                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 7                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 7                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 7                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 9                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 2                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 8                      \n",
      "Saving new best model.                                                   \n",
      "Saving new best model's gif.                                             \n",
      "---Time taken for opt-epoch: 14.666505563259125 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 5.7                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.11511017587068491, 'env_max_step': 3000, 'env_num_stack': 5, 'env_rwd_func': 3, 'gae_lambda': 0.12928347857382183, 'gamma': 0.827261777212633, 'learning_rate': 0.0009, 'max_grad_norm': 22.970948197057687, 'n_envs': 1024, 'n_steps': 20, 'normalize_advantage': True, 'vf_coef': 0.374410665983754}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 7                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 8                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 8                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 6                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                      \n",
      "---Time taken for opt-epoch: 14.477004607518515 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 4.8                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.07672875436674031, 'env_max_step': 3000, 'env_num_stack': 7, 'env_rwd_func': 3, 'gae_lambda': 0.28257119458606395, 'gamma': 0.8188949506523011, 'learning_rate': 0.0009, 'max_grad_norm': 18.411767225241455, 'n_envs': 1024, 'n_steps': 20, 'normalize_advantage': True, 'vf_coef': 0.21186182025632527}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 2                      \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                      \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                      \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 2                      \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 6                      \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                      \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 6                      \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                     \n",
      "---Time taken for opt-epoch: 14.618076574802398 Minutes                 \n",
      "---Average Episode Reward of opt-epoch: 3.9                             \n",
      "---Chosen Hyperparameters of opt-epoch:                                 \n",
      "{'ent_coef': 0.1410428779894618, 'env_max_step': 3000, 'env_num_stack': 5, 'env_rwd_func': 3, 'gae_lambda': 0.2118920701707951, 'gamma': 0.8334947488246677, 'learning_rate': 0.0009, 'max_grad_norm': 23.87417251799134, 'n_envs': 1024, 'n_steps': 20, 'normalize_advantage': True, 'vf_coef': 0.38390694526170765}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 6                      \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 7                      \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 8                      \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 6                      \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                      \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 5                      \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                      \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                     \n",
      "---Time taken for opt-epoch: 14.4479731520017 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 4.8                             \n",
      "---Chosen Hyperparameters of opt-epoch:                                 \n",
      "{'ent_coef': 0.011874390262348211, 'env_max_step': 3500, 'env_num_stack': 7, 'env_rwd_func': 0, 'gae_lambda': 0.08615770805410888, 'gamma': 0.815493672875537, 'learning_rate': 0.006, 'max_grad_norm': 21.84386244450146, 'n_envs': 1024, 'n_steps': 16, 'normalize_advantage': True, 'vf_coef': 0.10264691687178268}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 11                     \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 11                     \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                      \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 11                     \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 12                     \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 8                      \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 11                     \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 10                     \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 9                      \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 12                    \n",
      "Saving new best model.                                                  \n",
      "Saving new best model's gif.                                            \n",
      "---Time taken for opt-epoch: 14.525335709253946 Minutes                 \n",
      "---Average Episode Reward of opt-epoch: 10.4                            \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.013658141461427187, 'env_max_step': 3500, 'env_num_stack': 7, 'env_rwd_func': 0, 'gae_lambda': 0.0704803079075291, 'gamma': 0.8269921672693458, 'learning_rate': 0.006, 'max_grad_norm': 25.18496911721966, 'n_envs': 64, 'n_steps': 16, 'normalize_advantage': True, 'vf_coef': 0.27243036277970656}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 8                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 7                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 7                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 8                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 9                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 7                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 9                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 7                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 9                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 9                      \n",
      "---Time taken for opt-epoch: 16.52520762284597 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 8.0                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.011672016535517523, 'env_max_step': 3500, 'env_num_stack': 7, 'env_rwd_func': 0, 'gae_lambda': 0.053580129960976663, 'gamma': 0.8317980915217109, 'learning_rate': 0.006, 'max_grad_norm': 26.227864639256122, 'n_envs': 64, 'n_steps': 16, 'normalize_advantage': True, 'vf_coef': 0.1087334272040681}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 8                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 8                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 10                      \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 6                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 10                      \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 10                     \n",
      "---Time taken for opt-epoch: 16.3895822485288 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 6.4                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.04119811822775363, 'env_max_step': 3500, 'env_num_stack': 7, 'env_rwd_func': 0, 'gae_lambda': 0.4510812576604396, 'gamma': 0.7917146061119349, 'learning_rate': 0.006, 'max_grad_norm': 21.6457662358397, 'n_envs': 64, 'n_steps': 16, 'normalize_advantage': True, 'vf_coef': 0.10252573787866043}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 9                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 9                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 10                      \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 9                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 11                      \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 10                      \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 10                      \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 6                      \n",
      "---Time taken for opt-epoch: 16.37879951794942 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 8.6                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.05183009351446517, 'env_max_step': 3500, 'env_num_stack': 7, 'env_rwd_func': 0, 'gae_lambda': 0.4920150215883913, 'gamma': 0.7856422048511698, 'learning_rate': 0.006, 'max_grad_norm': 13.045253975460401, 'n_envs': 64, 'n_steps': 1, 'normalize_advantage': True, 'vf_coef': 0.10618655948265314}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 9                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 2                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 6                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 1                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 9                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 8                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 9                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 7                      \n",
      "---Time taken for opt-epoch: 20.92301085392634 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 5.8                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'ent_coef': 0.04550826124447565, 'env_max_step': 3500, 'env_num_stack': 7, 'env_rwd_func': 0, 'gae_lambda': 0.5682882089894901, 'gamma': 0.7770048380224973, 'learning_rate': 0.001, 'max_grad_norm': 17.485784053614267, 'n_envs': 64, 'n_steps': 16, 'normalize_advantage': True, 'vf_coef': 0.19738281011459535}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 13                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 9                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 9                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 9                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 7                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 10                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 12                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 9                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 10                      \n",
      "---Time taken for opt-epoch: 16.393809839089712 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 9.7                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'ent_coef': 0.19443946317346278, 'env_max_step': 3500, 'env_num_stack': 7, 'env_rwd_func': 0, 'gae_lambda': 0.569869498127813, 'gamma': 0.7776501452358039, 'learning_rate': 0.001, 'max_grad_norm': 18.116893338558196, 'n_envs': 512, 'n_steps': 10, 'normalize_advantage': True, 'vf_coef': 0.2128249352566035}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                        \n",
      "---Time taken for opt-epoch: 14.168329354127248 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 3.5                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.075929475941479, 'env_max_step': 2500, 'env_num_stack': 7, 'env_rwd_func': 0, 'gae_lambda': 0.7309670151641122, 'gamma': 0.7522427737405164, 'learning_rate': 0.001, 'max_grad_norm': 11.1001236189539, 'n_envs': 64, 'n_steps': 4, 'normalize_advantage': True, 'vf_coef': 0.1518107247874066}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 9                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 6                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 10                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 14                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 9                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 9                       \n",
      "---Time taken for opt-epoch: 17.283776497840883 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 7.7                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'ent_coef': 0.0350298023675491, 'env_max_step': 3500, 'env_num_stack': 7, 'env_rwd_func': 0, 'gae_lambda': 0.6057360655770619, 'gamma': 0.8120064480916159, 'learning_rate': 0.01, 'max_grad_norm': 17.597651697306162, 'n_envs': 8, 'n_steps': 16, 'normalize_advantage': True, 'vf_coef': 0.2834408822038732}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0                       \n",
      "---Time taken for opt-epoch: 30.349611552556357 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 0.0                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.22447010224058322, 'env_max_step': 3500, 'env_num_stack': 7, 'env_rwd_func': 0, 'gae_lambda': 0.3844266265391492, 'gamma': 0.7795586093131031, 'learning_rate': 0.003, 'max_grad_norm': 21.537356262774292, 'n_envs': 128, 'n_steps': 5, 'normalize_advantage': True, 'vf_coef': 0.5903878982516402}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                        \n",
      "---Time taken for opt-epoch: 15.291897638638813 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 3.7                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.07737643700836672, 'env_max_step': 2500, 'env_num_stack': 7, 'env_rwd_func': 0, 'gae_lambda': 0.746696842377551, 'gamma': 0.7655353026896486, 'learning_rate': 0.001, 'max_grad_norm': 9.065333710978662, 'n_envs': 1024, 'n_steps': 64, 'normalize_advantage': True, 'vf_coef': 0.33884466859704304}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 6                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                        \n",
      "---Time taken for opt-epoch: 14.28738892475764 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 4.4                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.1783905750856612, 'env_max_step': 3500, 'env_num_stack': 6, 'env_rwd_func': 0, 'gae_lambda': 0.3260633449218534, 'gamma': 0.7936932380464259, 'learning_rate': 0.1, 'max_grad_norm': 9.642564470036296, 'n_envs': 512, 'n_steps': 256, 'normalize_advantage': True, 'vf_coef': 0.24014277354849378}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 2                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 1                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 2                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 1                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0                        \n",
      "---Time taken for opt-epoch: 14.987402832508087 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 0.6                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.0027531182775551166, 'env_max_step': 3500, 'env_num_stack': 4, 'env_rwd_func': 0, 'gae_lambda': 0.4401782821228353, 'gamma': 0.8085538321880519, 'learning_rate': 0.0006, 'max_grad_norm': 16.8932558543066, 'n_envs': 64, 'n_steps': 800, 'normalize_advantage': True, 'vf_coef': 0.16592755390540037}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 6                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 9                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 7                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 10                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 7                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 8                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                        \n",
      "---Time taken for opt-epoch: 16.37266807158788 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 6.2                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.09435234020260719, 'env_max_step': 3500, 'env_num_stack': 7, 'env_rwd_func': 0, 'gae_lambda': 0.80202742577083, 'gamma': 0.7576646357840917, 'learning_rate': 0.0003, 'max_grad_norm': 14.4983010000163, 'n_envs': 1024, 'n_steps': 16, 'normalize_advantage': True, 'vf_coef': 0.5164155014604686}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 7                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 6                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 4                        \n",
      "---Time taken for opt-epoch: 14.238043816884359 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 4.2                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.05918047039410639, 'env_max_step': 2500, 'env_num_stack': 7, 'env_rwd_func': 0, 'gae_lambda': 0.5416004188859509, 'gamma': 0.7693964774515576, 'learning_rate': 0.03, 'max_grad_norm': 0.35545905839915903, 'n_envs': 128, 'n_steps': 4, 'normalize_advantage': True, 'vf_coef': 0.9956890914536201}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                        \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0                       \n",
      "---Time taken for opt-epoch: 15.73686840136846 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 0.0                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                   \n",
      "{'ent_coef': 0.022054379651980035, 'env_max_step': 3500, 'env_num_stack': 4, 'env_rwd_func': 2, 'gae_lambda': 0.9975640594218754, 'gamma': 0.7827585770714667, 'learning_rate': 0.0001, 'max_grad_norm': 27.0215929677388, 'n_envs': 16, 'n_steps': 8, 'normalize_advantage': True, 'vf_coef': 0.451526073979952}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 6                        \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 9                        \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 7                        \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 5                        \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 10                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 9                        \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 7                        \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 11                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 9                       \n",
      "---Time taken for opt-epoch: 23.715911030769348 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 7.8                               \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.1511955969017444, 'env_max_step': 3500, 'env_num_stack': 6, 'env_rwd_func': 0, 'gae_lambda': 0.629027445144115, 'gamma': 0.7741797787980378, 'learning_rate': 0.001, 'max_grad_norm': 12.798122845118845, 'n_envs': 64, 'n_steps': 10, 'normalize_advantage': True, 'vf_coef': 0.33584899503856525}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 5                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 6                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                        \n",
      "---Time taken for opt-epoch: 16.34403924147288 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 4.0                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.22100711892438274, 'env_max_step': 2000, 'env_num_stack': 7, 'env_rwd_func': 2, 'gae_lambda': 0.37419584329737504, 'gamma': 0.7955752091765378, 'learning_rate': 0.003, 'max_grad_norm': 25.23098565251277, 'n_envs': 8, 'n_steps': 512, 'normalize_advantage': True, 'vf_coef': 0.4387211985840106}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 5                        \n",
      "---Time taken for opt-epoch: 27.669026362895966 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 3.4                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.10334035357424026, 'env_max_step': 2500, 'env_num_stack': 4, 'env_rwd_func': 0, 'gae_lambda': 0.7234578648235006, 'gamma': 0.7583153102563289, 'learning_rate': 0.01, 'max_grad_norm': 22.127526526875027, 'n_envs': 1024, 'n_steps': 400, 'normalize_advantage': True, 'vf_coef': 0.5094224905621467}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 2                         \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 4                         \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                         \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 1                         \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 2                         \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 2                         \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 2                         \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                         \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 1                        \n",
      "---Time taken for opt-epoch: 14.660668726762136 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 1.7                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.17272207165388198, 'env_max_step': 3500, 'env_num_stack': 6, 'env_rwd_func': 1, 'gae_lambda': 0.6936984635691728, 'gamma': 0.8083487097846285, 'learning_rate': 0.1, 'max_grad_norm': 16.568594148199598, 'n_envs': 512, 'n_steps': 5, 'normalize_advantage': True, 'vf_coef': 0.576759395454489}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 1                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 0                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 0                      \n",
      "---Time taken for opt-epoch: 14.515406231085459 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 0.1                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.13120959194331155, 'env_max_step': 1500, 'env_num_stack': 7, 'env_rwd_func': 0, 'gae_lambda': 0.798006358461497, 'gamma': 0.8234611739618884, 'learning_rate': 0.007, 'max_grad_norm': 28.050494967671405, 'n_envs': 128, 'n_steps': 1024, 'normalize_advantage': True, 'vf_coef': 0.13018974771638614}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                      \n",
      "---Time taken for opt-epoch: 15.437271960576375 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 3.3                              \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.28361858309881977, 'env_max_step': 3500, 'env_num_stack': 7, 'env_rwd_func': 2, 'gae_lambda': 0.610262574042218, 'gamma': 0.8427568798332821, 'learning_rate': 0.008, 'max_grad_norm': 18.773444973570932, 'n_envs': 16, 'n_steps': 32, 'normalize_advantage': False, 'vf_coef': 0.17955122599199017}\n",
      "\\Ran agent on environment: 1/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 2/10 Episodes, score: 5                       \n",
      "\\Ran agent on environment: 3/10 Episodes, score: 8                       \n",
      "\\Ran agent on environment: 4/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 5/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 6/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 7/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 8/10 Episodes, score: 4                       \n",
      "\\Ran agent on environment: 9/10 Episodes, score: 3                       \n",
      "\\Ran agent on environment: 10/10 Episodes, score: 3                      \n",
      "---Time taken for opt-epoch: 21.649936640262602 Minutes                  \n",
      "---Average Episode Reward of opt-epoch: 3.9                              \n",
      "100%|██████████| 50/50 [14:46:20<00:00, 1063.60s/trial, best loss: -10.4]\n"
     ]
    }
   ],
   "source": [
    "best_avg_rwd = float(\"-inf\")\n",
    "expr_no = 2\n",
    "total_timesteps = 2_000_000\n",
    "\n",
    "best = fmin(objective, get_space(), algo=tpe.suggest, max_evals=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ent_coef': 0.011874390262348211,\n",
       " 'env_max_step': 4,\n",
       " 'env_num_stack': 3,\n",
       " 'env_rwd_func': 0,\n",
       " 'gae_lambda': 0.08615770805410888,\n",
       " 'gamma': 0.815493672875537,\n",
       " 'learning_rate': 9,\n",
       " 'max_grad_norm': 21.84386244450146,\n",
       " 'n_envs': 6,\n",
       " 'n_steps': 5,\n",
       " 'normalize_advantage': 0,\n",
       " 'vf_coef': 0.10264691687178268}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ent_coef': 0.011874390262348211,\n",
       " 'env_max_step': 3500,\n",
       " 'env_num_stack': 7,\n",
       " 'env_rwd_func': 0,\n",
       " 'gae_lambda': 0.08615770805410888,\n",
       " 'gamma': 0.815493672875537,\n",
       " 'learning_rate': 0.006,\n",
       " 'max_grad_norm': 21.84386244450146,\n",
       " 'n_envs': 1024,\n",
       " 'n_steps': 16,\n",
       " 'normalize_advantage': True,\n",
       " 'vf_coef': 0.10264691687178268}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "space_eval(get_space(), best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_space():\n",
    "    return {\n",
    "        'env_rwd_func': hp.choice('env_rwd_func', [0, 1, 2, 3]),\n",
    "        'env_max_step': hp.choice('env_max_step', [0, 1500, 3500]),\n",
    "        'learning_rate': hp.choice('learning_rate', [1e-4, 3e-4, 4e-4, 6e-4, 7e-4, 1e-3, 3e-3, 6e-3, 7e-3, 8e-3, 9e-3, 1e-2]),\n",
    "        'n_steps': hp.choice('n_steps', [3, 4, 5, 6, 8, 10, 16, 32, 64, 128, 256, 512, 1024]),\n",
    "        'gae_lambda': hp.uniform('gae_lambda', 0.001, 1.0),\n",
    "        'ent_coef': hp.uniform('ent_coef', 0.0, 0.2),\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---Chosen Hyperparameters of opt-epoch:                \n",
      "{'ent_coef': 0.06715835167057442, 'env_max_step': 1500, 'env_rwd_func': 1, 'gae_lambda': 0.15806189871721002, 'learning_rate': 0.003, 'n_steps': 16, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3       \n",
      "an agent on environment: 2/10 Episodes, score: 4       \n",
      "an agent on environment: 3/10 Episodes, score: 3       \n",
      "an agent on environment: 4/10 Episodes, score: 4       \n",
      "an agent on environment: 5/10 Episodes, score: 3       \n",
      "an agent on environment: 6/10 Episodes, score: 3       \n",
      "an agent on environment: 7/10 Episodes, score: 4       \n",
      "an agent on environment: 8/10 Episodes, score: 3       \n",
      "an agent on environment: 9/10 Episodes, score: 3       \n",
      "an agent on environment: 10/10 Episodes, score: 7      \n",
      "Saving new best model.                                 \n",
      "Saving new best model's gif.                           \n",
      "  0%|          | 0/100 [37:33<?, ?trial/s, best loss=?]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Hosein\\anaconda3\\envs\\pt_env\\lib\\site-packages\\stable_baselines3\\common\\save_util.py:278: UserWarning: Path 'models\\hp\\A2C\\no.3' does not exist. Will create it.\n",
      "  warnings.warn(f\"Path '{path.parent}' does not exist. Will create it.\")\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---Time taken for opt-epoch: 37.6095109462738 Minutes  \n",
      "---Average Episode Reward of opt-epoch: 3.7            \n",
      "---Chosen Hyperparameters of opt-epoch:                                  \n",
      "{'ent_coef': 0.033317625650326255, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.959359039151814, 'learning_rate': 0.0007, 'n_steps': 64, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 11                        \n",
      "an agent on environment: 2/10 Episodes, score: 9                           \n",
      "an agent on environment: 3/10 Episodes, score: 3                           \n",
      "an agent on environment: 4/10 Episodes, score: 11                          \n",
      "an agent on environment: 5/10 Episodes, score: 12                          \n",
      "an agent on environment: 6/10 Episodes, score: 5                           \n",
      "an agent on environment: 7/10 Episodes, score: 9                           \n",
      "an agent on environment: 8/10 Episodes, score: 4                           \n",
      "an agent on environment: 9/10 Episodes, score: 3                           \n",
      "an agent on environment: 10/10 Episodes, score: 5                          \n",
      "Saving new best model.                                                     \n",
      "Saving new best model's gif.                                               \n",
      "---Time taken for opt-epoch: 37.63052788178126 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 7.2                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.02547327731107898, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.14385972877138234, 'learning_rate': 0.003, 'n_steps': 16, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 10                          \n",
      "an agent on environment: 2/10 Episodes, score: 11                          \n",
      "an agent on environment: 3/10 Episodes, score: 9                           \n",
      "an agent on environment: 4/10 Episodes, score: 9                           \n",
      "an agent on environment: 5/10 Episodes, score: 9                           \n",
      "an agent on environment: 6/10 Episodes, score: 9                           \n",
      "an agent on environment: 7/10 Episodes, score: 9                           \n",
      "an agent on environment: 8/10 Episodes, score: 9                           \n",
      "an agent on environment: 9/10 Episodes, score: 11                          \n",
      "an agent on environment: 10/10 Episodes, score: 11                         \n",
      "Saving new best model.                                                     \n",
      "Saving new best model's gif.                                               \n",
      "---Time taken for opt-epoch: 37.39024978876114 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 9.7                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.14632043248837154, 'env_max_step': 1500, 'env_rwd_func': 2, 'gae_lambda': 0.6636404715867795, 'learning_rate': 0.01, 'n_steps': 256, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                           \n",
      "an agent on environment: 2/10 Episodes, score: 3                           \n",
      "an agent on environment: 3/10 Episodes, score: 5                           \n",
      "an agent on environment: 4/10 Episodes, score: 4                           \n",
      "an agent on environment: 5/10 Episodes, score: 3                           \n",
      "an agent on environment: 6/10 Episodes, score: 7                           \n",
      "an agent on environment: 7/10 Episodes, score: 4                           \n",
      "an agent on environment: 8/10 Episodes, score: 3                           \n",
      "an agent on environment: 9/10 Episodes, score: 9                           \n",
      "an agent on environment: 10/10 Episodes, score: 4                          \n",
      "---Time taken for opt-epoch: 37.52320793469747 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 4.5                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.14277726438749555, 'env_max_step': 0, 'env_rwd_func': 0, 'gae_lambda': 0.8551762673365707, 'learning_rate': 0.0006, 'n_steps': 1024, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                           \n",
      "an agent on environment: 2/10 Episodes, score: 3                           \n",
      "an agent on environment: 3/10 Episodes, score: 3                           \n",
      "an agent on environment: 4/10 Episodes, score: 4                           \n",
      "an agent on environment: 5/10 Episodes, score: 3                           \n",
      "an agent on environment: 6/10 Episodes, score: 3                           \n",
      "an agent on environment: 7/10 Episodes, score: 4                           \n",
      "an agent on environment: 8/10 Episodes, score: 3                           \n",
      "an agent on environment: 9/10 Episodes, score: 4                           \n",
      "an agent on environment: 10/10 Episodes, score: 4                          \n",
      "---Time taken for opt-epoch: 37.61735448439916 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 3.4                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.1260644186540602, 'env_max_step': 0, 'env_rwd_func': 1, 'gae_lambda': 0.5209029650165705, 'learning_rate': 0.0001, 'n_steps': 128, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 5                           \n",
      "an agent on environment: 2/10 Episodes, score: 3                           \n",
      "an agent on environment: 3/10 Episodes, score: 5                           \n",
      "an agent on environment: 4/10 Episodes, score: 4                           \n",
      "an agent on environment: 5/10 Episodes, score: 5                           \n",
      "an agent on environment: 6/10 Episodes, score: 3                           \n",
      "an agent on environment: 7/10 Episodes, score: 4                           \n",
      "an agent on environment: 8/10 Episodes, score: 4                           \n",
      "an agent on environment: 9/10 Episodes, score: 3                           \n",
      "an agent on environment: 10/10 Episodes, score: 3                          \n",
      "---Time taken for opt-epoch: 37.603095392386116 Minutes                    \n",
      "---Average Episode Reward of opt-epoch: 3.9                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.07198960893777673, 'env_max_step': 0, 'env_rwd_func': 2, 'gae_lambda': 0.029228777093511276, 'learning_rate': 0.009, 'n_steps': 16, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                           \n",
      "an agent on environment: 2/10 Episodes, score: 3                           \n",
      "an agent on environment: 3/10 Episodes, score: 7                           \n",
      "an agent on environment: 4/10 Episodes, score: 4                           \n",
      "an agent on environment: 5/10 Episodes, score: 3                           \n",
      "an agent on environment: 6/10 Episodes, score: 6                           \n",
      "an agent on environment: 7/10 Episodes, score: 7                           \n",
      "an agent on environment: 8/10 Episodes, score: 6                           \n",
      "an agent on environment: 9/10 Episodes, score: 3                           \n",
      "an agent on environment: 10/10 Episodes, score: 7                          \n",
      "---Time taken for opt-epoch: 37.22530393997828 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 4.9                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.08262575956875767, 'env_max_step': 1500, 'env_rwd_func': 2, 'gae_lambda': 0.5222086724677301, 'learning_rate': 0.006, 'n_steps': 128, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 4                           \n",
      "an agent on environment: 2/10 Episodes, score: 4                           \n",
      "an agent on environment: 3/10 Episodes, score: 4                           \n",
      "an agent on environment: 4/10 Episodes, score: 5                           \n",
      "an agent on environment: 5/10 Episodes, score: 3                           \n",
      "an agent on environment: 6/10 Episodes, score: 5                           \n",
      "an agent on environment: 7/10 Episodes, score: 4                           \n",
      "an agent on environment: 8/10 Episodes, score: 6                           \n",
      "an agent on environment: 9/10 Episodes, score: 3                           \n",
      "an agent on environment: 10/10 Episodes, score: 4                          \n",
      "---Time taken for opt-epoch: 37.95236725012462 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 4.2                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.19198827189501522, 'env_max_step': 1500, 'env_rwd_func': 3, 'gae_lambda': 0.7488365879253016, 'learning_rate': 0.0003, 'n_steps': 5, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "  8%|▊         | 8/100 [5:00:33<57:40:39, 2256.95s/trial, best loss: -0.97]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_6904\\352386517.py:25: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  prev_info[\"head pos\"][0] == 0 and info[\"head pos\"][0] == info[\"size\"]-1) or (\n",
      "\n",
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_6904\\352386517.py:26: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  prev_info[\"head pos\"][1] == 0 and info[\"head pos\"][1] == info[\"size\"]-1):\n",
      "\n",
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_6904\\352386517.py:24: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  prev_info[\"head pos\"][1] == info[\"size\"]-1 and info[\"head pos\"][1] == 0) or (\n",
      "\n",
      "C:\\Users\\Hosein\\AppData\\Local\\Temp\\ipykernel_6904\\352386517.py:23: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if (prev_info[\"head pos\"][0] == info[\"size\"]-1 and info[\"head pos\"][0] == 0) or (\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "an agent on environment: 1/10 Episodes, score: 3                           \n",
      "an agent on environment: 2/10 Episodes, score: 6                           \n",
      "an agent on environment: 3/10 Episodes, score: 3                           \n",
      "an agent on environment: 4/10 Episodes, score: 4                           \n",
      "an agent on environment: 5/10 Episodes, score: 1                           \n",
      "an agent on environment: 6/10 Episodes, score: 3                           \n",
      "an agent on environment: 7/10 Episodes, score: 6                           \n",
      "an agent on environment: 8/10 Episodes, score: 3                           \n",
      "an agent on environment: 9/10 Episodes, score: 4                           \n",
      "an agent on environment: 10/10 Episodes, score: 4                          \n",
      "---Time taken for opt-epoch: 37.87392356793086 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 3.7                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                    \n",
      "{'ent_coef': 0.12236043308595015, 'env_max_step': 0, 'env_rwd_func': 3, 'gae_lambda': 0.5126985843139311, 'learning_rate': 0.006, 'n_steps': 10, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 9                           \n",
      "an agent on environment: 2/10 Episodes, score: 9                           \n",
      "an agent on environment: 3/10 Episodes, score: 7                           \n",
      "an agent on environment: 4/10 Episodes, score: 3                           \n",
      "an agent on environment: 5/10 Episodes, score: 5                           \n",
      "an agent on environment: 6/10 Episodes, score: 3                           \n",
      "an agent on environment: 7/10 Episodes, score: 10                          \n",
      "an agent on environment: 8/10 Episodes, score: 3                           \n",
      "an agent on environment: 9/10 Episodes, score: 3                           \n",
      "an agent on environment: 10/10 Episodes, score: 5                          \n",
      "---Time taken for opt-epoch: 37.83785657485326 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 5.7                                \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.1950284182576104, 'env_max_step': 3500, 'env_rwd_func': 3, 'gae_lambda': 0.15075429262264453, 'learning_rate': 0.01, 'n_steps': 8, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                            \n",
      "an agent on environment: 2/10 Episodes, score: 5                            \n",
      "an agent on environment: 3/10 Episodes, score: 5                            \n",
      "an agent on environment: 4/10 Episodes, score: 3                            \n",
      "an agent on environment: 5/10 Episodes, score: 3                            \n",
      "an agent on environment: 6/10 Episodes, score: 3                            \n",
      "an agent on environment: 7/10 Episodes, score: 4                            \n",
      "an agent on environment: 8/10 Episodes, score: 5                            \n",
      "an agent on environment: 9/10 Episodes, score: 3                            \n",
      "an agent on environment: 10/10 Episodes, score: 5                           \n",
      "---Time taken for opt-epoch: 37.68459082444509 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 3.9                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.08913691349407951, 'env_max_step': 3500, 'env_rwd_func': 1, 'gae_lambda': 0.45990241475204735, 'learning_rate': 0.0007, 'n_steps': 32, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                            \n",
      "an agent on environment: 2/10 Episodes, score: 3                            \n",
      "an agent on environment: 3/10 Episodes, score: 5                            \n",
      "an agent on environment: 4/10 Episodes, score: 6                            \n",
      "an agent on environment: 5/10 Episodes, score: 3                            \n",
      "an agent on environment: 6/10 Episodes, score: 3                            \n",
      "an agent on environment: 7/10 Episodes, score: 3                            \n",
      "an agent on environment: 8/10 Episodes, score: 3                            \n",
      "an agent on environment: 9/10 Episodes, score: 4                            \n",
      "an agent on environment: 10/10 Episodes, score: 4                           \n",
      "---Time taken for opt-epoch: 37.115939390659335 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 3.7                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.020896120618077132, 'env_max_step': 1500, 'env_rwd_func': 0, 'gae_lambda': 0.6984559600764565, 'learning_rate': 0.006, 'n_steps': 256, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 9                            \n",
      "an agent on environment: 2/10 Episodes, score: 9                            \n",
      "an agent on environment: 3/10 Episodes, score: 10                           \n",
      "an agent on environment: 4/10 Episodes, score: 11                           \n",
      "an agent on environment: 5/10 Episodes, score: 10                           \n",
      "an agent on environment: 6/10 Episodes, score: 9                            \n",
      "an agent on environment: 7/10 Episodes, score: 6                            \n",
      "an agent on environment: 8/10 Episodes, score: 9                            \n",
      "an agent on environment: 9/10 Episodes, score: 9                            \n",
      "an agent on environment: 10/10 Episodes, score: 9                           \n",
      "---Time taken for opt-epoch: 37.96341777642568 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 9.1                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.04579657944982327, 'env_max_step': 3500, 'env_rwd_func': 3, 'gae_lambda': 0.8264794317889221, 'learning_rate': 0.0006, 'n_steps': 512, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 5                            \n",
      "an agent on environment: 2/10 Episodes, score: 3                            \n",
      "an agent on environment: 3/10 Episodes, score: 3                            \n",
      "an agent on environment: 4/10 Episodes, score: 3                            \n",
      "an agent on environment: 5/10 Episodes, score: 4                            \n",
      "an agent on environment: 6/10 Episodes, score: 3                            \n",
      "an agent on environment: 7/10 Episodes, score: 5                            \n",
      "an agent on environment: 8/10 Episodes, score: 3                            \n",
      "an agent on environment: 9/10 Episodes, score: 3                            \n",
      "an agent on environment: 10/10 Episodes, score: 3                           \n",
      "---Time taken for opt-epoch: 38.30993368228277 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 3.5                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.020330214855697238, 'env_max_step': 1500, 'env_rwd_func': 1, 'gae_lambda': 0.8689050227877306, 'learning_rate': 0.0003, 'n_steps': 8, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                            \n",
      "an agent on environment: 2/10 Episodes, score: 3                            \n",
      "an agent on environment: 3/10 Episodes, score: 3                            \n",
      "an agent on environment: 4/10 Episodes, score: 3                            \n",
      "an agent on environment: 5/10 Episodes, score: 3                            \n",
      "an agent on environment: 6/10 Episodes, score: 5                            \n",
      "an agent on environment: 7/10 Episodes, score: 5                            \n",
      "an agent on environment: 8/10 Episodes, score: 4                            \n",
      "an agent on environment: 9/10 Episodes, score: 4                            \n",
      "an agent on environment: 10/10 Episodes, score: 5                           \n",
      "---Time taken for opt-epoch: 37.24807745218277 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 3.8                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.061599021362976104, 'env_max_step': 1500, 'env_rwd_func': 2, 'gae_lambda': 0.8505867906920105, 'learning_rate': 0.009, 'n_steps': 64, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 5                            \n",
      "an agent on environment: 2/10 Episodes, score: 6                             \n",
      "an agent on environment: 3/10 Episodes, score: 5                             \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 5                             \n",
      "an agent on environment: 6/10 Episodes, score: 5                             \n",
      "an agent on environment: 7/10 Episodes, score: 4                             \n",
      "an agent on environment: 8/10 Episodes, score: 3                             \n",
      "an agent on environment: 9/10 Episodes, score: 6                             \n",
      "an agent on environment: 10/10 Episodes, score: 4                            \n",
      "---Time taken for opt-epoch: 37.49870518843333 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 4.6                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.006615827771524075, 'env_max_step': 3500, 'env_rwd_func': 2, 'gae_lambda': 0.1984539813376635, 'learning_rate': 0.0007, 'n_steps': 512, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 4                             \n",
      "an agent on environment: 2/10 Episodes, score: 4                             \n",
      "an agent on environment: 3/10 Episodes, score: 3                             \n",
      "an agent on environment: 4/10 Episodes, score: 4                             \n",
      "an agent on environment: 5/10 Episodes, score: 6                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 4                             \n",
      "an agent on environment: 9/10 Episodes, score: 5                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 37.77194551626841 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 3.9                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.1532653921880386, 'env_max_step': 0, 'env_rwd_func': 3, 'gae_lambda': 0.09345427953592998, 'learning_rate': 0.0007, 'n_steps': 256, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                             \n",
      "an agent on environment: 2/10 Episodes, score: 4                             \n",
      "an agent on environment: 3/10 Episodes, score: 3                             \n",
      "an agent on environment: 4/10 Episodes, score: 7                             \n",
      "an agent on environment: 5/10 Episodes, score: 5                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 9                             \n",
      "an agent on environment: 8/10 Episodes, score: 3                             \n",
      "an agent on environment: 9/10 Episodes, score: 5                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 38.93106208642324 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 4.5                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.010878538602704536, 'env_max_step': 1500, 'env_rwd_func': 2, 'gae_lambda': 0.3119725138420385, 'learning_rate': 0.0006, 'n_steps': 64, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 9                             \n",
      "an agent on environment: 2/10 Episodes, score: 6                             \n",
      "an agent on environment: 3/10 Episodes, score: 7                             \n",
      "an agent on environment: 4/10 Episodes, score: 5                             \n",
      "an agent on environment: 5/10 Episodes, score: 4                             \n",
      "an agent on environment: 6/10 Episodes, score: 4                             \n",
      "an agent on environment: 7/10 Episodes, score: 5                             \n",
      "an agent on environment: 8/10 Episodes, score: 7                             \n",
      "an agent on environment: 9/10 Episodes, score: 6                             \n",
      "an agent on environment: 10/10 Episodes, score: 5                            \n",
      "---Time taken for opt-epoch: 37.757735681533816 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 5.8                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.17973726822265376, 'env_max_step': 0, 'env_rwd_func': 3, 'gae_lambda': 0.5612962875623391, 'learning_rate': 0.0006, 'n_steps': 6, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                             \n",
      "an agent on environment: 2/10 Episodes, score: 4                             \n",
      "an agent on environment: 3/10 Episodes, score: 10                            \n",
      "an agent on environment: 4/10 Episodes, score: 7                             \n",
      "an agent on environment: 5/10 Episodes, score: 3                             \n",
      "an agent on environment: 6/10 Episodes, score: 4                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 5                             \n",
      "an agent on environment: 9/10 Episodes, score: 11                            \n",
      "an agent on environment: 10/10 Episodes, score: 6                            \n",
      "---Time taken for opt-epoch: 37.96811002890269 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 5.6                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.0002704468272470041, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.3530198529897188, 'learning_rate': 0.003, 'n_steps': 4, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 9                             \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 8                             \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 9                             \n",
      "an agent on environment: 6/10 Episodes, score: 11                            \n",
      "an agent on environment: 7/10 Episodes, score: 9                             \n",
      "an agent on environment: 8/10 Episodes, score: 10                            \n",
      "an agent on environment: 9/10 Episodes, score: 9                             \n",
      "an agent on environment: 10/10 Episodes, score: 10                           \n",
      "---Time taken for opt-epoch: 37.432751242319746 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 8.7                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.04443936956730704, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.6624628699023176, 'learning_rate': 0.008, 'n_steps': 256, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 8                             \n",
      "an agent on environment: 2/10 Episodes, score: 8                             \n",
      "an agent on environment: 3/10 Episodes, score: 11                            \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 9                             \n",
      "an agent on environment: 6/10 Episodes, score: 10                            \n",
      "an agent on environment: 7/10 Episodes, score: 8                             \n",
      "an agent on environment: 8/10 Episodes, score: 9                             \n",
      "an agent on environment: 9/10 Episodes, score: 9                             \n",
      "an agent on environment: 10/10 Episodes, score: 10                           \n",
      "---Time taken for opt-epoch: 38.047169188658394 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 8.5                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.028627327469133768, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.3744274750530204, 'learning_rate': 0.0004, 'n_steps': 16, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 10                            \n",
      "an agent on environment: 2/10 Episodes, score: 7                             \n",
      "an agent on environment: 3/10 Episodes, score: 10                            \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 9                             \n",
      "an agent on environment: 6/10 Episodes, score: 4                             \n",
      "an agent on environment: 7/10 Episodes, score: 9                             \n",
      "an agent on environment: 8/10 Episodes, score: 4                             \n",
      "an agent on environment: 9/10 Episodes, score: 9                             \n",
      "an agent on environment: 10/10 Episodes, score: 7                            \n",
      "---Time taken for opt-epoch: 37.63417927821477 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 7.2                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.04890829892895008, 'env_max_step': 1500, 'env_rwd_func': 0, 'gae_lambda': 0.9831936818493684, 'learning_rate': 0.001, 'n_steps': 3, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 10                            \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 9                             \n",
      "an agent on environment: 4/10 Episodes, score: 9                             \n",
      "an agent on environment: 5/10 Episodes, score: 6                             \n",
      "an agent on environment: 6/10 Episodes, score: 10                            \n",
      "an agent on environment: 7/10 Episodes, score: 10                            \n",
      "an agent on environment: 8/10 Episodes, score: 8                             \n",
      "an agent on environment: 9/10 Episodes, score: 9                             \n",
      "an agent on environment: 10/10 Episodes, score: 9                            \n",
      "---Time taken for opt-epoch: 38.46950851281484 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 8.9                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.11176259030724006, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.2642946223966524, 'learning_rate': 0.007, 'n_steps': 256, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 4                             \n",
      "an agent on environment: 2/10 Episodes, score: 5                             \n",
      "an agent on environment: 3/10 Episodes, score: 4                             \n",
      "an agent on environment: 4/10 Episodes, score: 4                             \n",
      "an agent on environment: 5/10 Episodes, score: 3                             \n",
      "an agent on environment: 6/10 Episodes, score: 6                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 4                             \n",
      "an agent on environment: 9/10 Episodes, score: 4                             \n",
      "an agent on environment: 10/10 Episodes, score: 6                            \n",
      "---Time taken for opt-epoch: 38.71855371793111 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 4.3                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.017335725254749457, 'env_max_step': 1500, 'env_rwd_func': 0, 'gae_lambda': 0.01549864255435443, 'learning_rate': 0.003, 'n_steps': 3, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 9                             \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 11                            \n",
      "an agent on environment: 4/10 Episodes, score: 9                             \n",
      "an agent on environment: 5/10 Episodes, score: 11                            \n",
      "an agent on environment: 6/10 Episodes, score: 9                             \n",
      "an agent on environment: 7/10 Episodes, score: 10                            \n",
      "an agent on environment: 8/10 Episodes, score: 12                            \n",
      "an agent on environment: 9/10 Episodes, score: 11                            \n",
      "an agent on environment: 10/10 Episodes, score: 11                           \n",
      "Saving new best model.                                                       \n",
      "Saving new best model's gif.                                                 \n",
      "---Time taken for opt-epoch: 39.60575855573018 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 10.2                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.10167545280031724, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.01210441471627281, 'learning_rate': 0.003, 'n_steps': 3, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                             \n",
      "an agent on environment: 2/10 Episodes, score: 5                             \n",
      "an agent on environment: 3/10 Episodes, score: 6                             \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 8                             \n",
      "an agent on environment: 6/10 Episodes, score: 4                             \n",
      "an agent on environment: 7/10 Episodes, score: 9                             \n",
      "an agent on environment: 8/10 Episodes, score: 4                             \n",
      "an agent on environment: 9/10 Episodes, score: 6                             \n",
      "an agent on environment: 10/10 Episodes, score: 5                            \n",
      "---Time taken for opt-epoch: 37.4257732629776 Minutes                        \n",
      "---Average Episode Reward of opt-epoch: 5.3                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.003780338726125182, 'env_max_step': 1500, 'env_rwd_func': 0, 'gae_lambda': 0.06608925664998803, 'learning_rate': 0.0001, 'n_steps': 3, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 5                             \n",
      "an agent on environment: 2/10 Episodes, score: 3                             \n",
      "an agent on environment: 3/10 Episodes, score: 5                             \n",
      "an agent on environment: 4/10 Episodes, score: 8                             \n",
      "an agent on environment: 5/10 Episodes, score: 9                             \n",
      "an agent on environment: 6/10 Episodes, score: 6                             \n",
      "an agent on environment: 7/10 Episodes, score: 4                             \n",
      "an agent on environment: 8/10 Episodes, score: 4                             \n",
      "an agent on environment: 9/10 Episodes, score: 9                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 37.582437912623085 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 5.6                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.06406371634326524, 'env_max_step': 1500, 'env_rwd_func': 0, 'gae_lambda': 0.2471552118607574, 'learning_rate': 0.003, 'n_steps': 16, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 6                             \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 3                             \n",
      "an agent on environment: 4/10 Episodes, score: 7                             \n",
      "an agent on environment: 5/10 Episodes, score: 6                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 8                             \n",
      "an agent on environment: 8/10 Episodes, score: 10                            \n",
      "an agent on environment: 9/10 Episodes, score: 11                            \n",
      "an agent on environment: 10/10 Episodes, score: 8                            \n",
      "---Time taken for opt-epoch: 37.5348065495491 Minutes                        \n",
      "---Average Episode Reward of opt-epoch: 7.1                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.03968438050698267, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.156525229778124, 'learning_rate': 0.003, 'n_steps': 5, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 10                            \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 9                             \n",
      "an agent on environment: 4/10 Episodes, score: 5                             \n",
      "an agent on environment: 5/10 Episodes, score: 10                            \n",
      "an agent on environment: 6/10 Episodes, score: 9                             \n",
      "an agent on environment: 7/10 Episodes, score: 13                            \n",
      "an agent on environment: 8/10 Episodes, score: 14                            \n",
      "an agent on environment: 9/10 Episodes, score: 9                             \n",
      "an agent on environment: 10/10 Episodes, score: 15                           \n",
      "Saving new best model.                                                       \n",
      "Saving new best model's gif.                                                 \n",
      "---Time taken for opt-epoch: 37.76316306988398 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 10.3                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.03882242747971183, 'env_max_step': 1500, 'env_rwd_func': 1, 'gae_lambda': 0.422792872872145, 'learning_rate': 0.003, 'n_steps': 5, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 5                             \n",
      "an agent on environment: 2/10 Episodes, score: 3                             \n",
      "an agent on environment: 3/10 Episodes, score: 5                             \n",
      "an agent on environment: 4/10 Episodes, score: 7                             \n",
      "an agent on environment: 5/10 Episodes, score: 3                             \n",
      "an agent on environment: 6/10 Episodes, score: 5                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 3                             \n",
      "an agent on environment: 9/10 Episodes, score: 4                             \n",
      "an agent on environment: 10/10 Episodes, score: 8                            \n",
      "---Time taken for opt-epoch: 37.702094395955406 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.6                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.057370293996757915, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.11262908919452357, 'learning_rate': 0.003, 'n_steps': 5, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 5                             \n",
      "an agent on environment: 2/10 Episodes, score: 7                             \n",
      "an agent on environment: 3/10 Episodes, score: 9                             \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 5                             \n",
      "an agent on environment: 6/10 Episodes, score: 9                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 7                             \n",
      "an agent on environment: 9/10 Episodes, score: 9                             \n",
      "an agent on environment: 10/10 Episodes, score: 8                            \n",
      "---Time taken for opt-epoch: 37.247526065508524 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 6.5                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.076306929109379, 'env_max_step': 1500, 'env_rwd_func': 0, 'gae_lambda': 0.20292404083812246, 'learning_rate': 0.008, 'n_steps': 5, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                             \n",
      "an agent on environment: 2/10 Episodes, score: 7                             \n",
      "an agent on environment: 3/10 Episodes, score: 6                             \n",
      "an agent on environment: 4/10 Episodes, score: 6                             \n",
      "an agent on environment: 5/10 Episodes, score: 7                             \n",
      "an agent on environment: 6/10 Episodes, score: 4                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 5                             \n",
      "an agent on environment: 9/10 Episodes, score: 6                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 37.33976316849391 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 5.0                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.016955328265390705, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.01288571174895814, 'learning_rate': 0.001, 'n_steps': 1024, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                             \n",
      "an agent on environment: 2/10 Episodes, score: 3                             \n",
      "an agent on environment: 3/10 Episodes, score: 3                             \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 4                             \n",
      "an agent on environment: 6/10 Episodes, score: 7                             \n",
      "an agent on environment: 7/10 Episodes, score: 4                             \n",
      "an agent on environment: 8/10 Episodes, score: 5                             \n",
      "an agent on environment: 9/10 Episodes, score: 3                             \n",
      "an agent on environment: 10/10 Episodes, score: 5                            \n",
      "---Time taken for opt-epoch: 37.609533576170605 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.0                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.030737946535895146, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.19036733453783086, 'learning_rate': 0.0004, 'n_steps': 3, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 4                             \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 15                            \n",
      "an agent on environment: 4/10 Episodes, score: 8                             \n",
      "an agent on environment: 5/10 Episodes, score: 9                             \n",
      "an agent on environment: 6/10 Episodes, score: 10                            \n",
      "an agent on environment: 7/10 Episodes, score: 4                             \n",
      "an agent on environment: 8/10 Episodes, score: 7                             \n",
      "an agent on environment: 9/10 Episodes, score: 9                             \n",
      "an agent on environment: 10/10 Episodes, score: 14                           \n",
      "---Time taken for opt-epoch: 37.813590029875435 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 8.9                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.03146420028301625, 'env_max_step': 0, 'env_rwd_func': 1, 'gae_lambda': 0.055571024068361834, 'learning_rate': 0.007, 'n_steps': 4, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 7                             \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 4                             \n",
      "an agent on environment: 4/10 Episodes, score: 6                             \n",
      "an agent on environment: 5/10 Episodes, score: 7                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 5                             \n",
      "an agent on environment: 9/10 Episodes, score: 3                             \n",
      "an agent on environment: 10/10 Episodes, score: 7                            \n",
      "---Time taken for opt-epoch: 37.38203999201457 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 5.4                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.09615696740153859, 'env_max_step': 1500, 'env_rwd_func': 0, 'gae_lambda': 0.300204651493271, 'learning_rate': 0.003, 'n_steps': 32, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                             \n",
      "an agent on environment: 2/10 Episodes, score: 5                             \n",
      "an agent on environment: 3/10 Episodes, score: 3                             \n",
      "an agent on environment: 4/10 Episodes, score: 5                             \n",
      "an agent on environment: 5/10 Episodes, score: 5                             \n",
      "an agent on environment: 6/10 Episodes, score: 4                             \n",
      "an agent on environment: 7/10 Episodes, score: 4                             \n",
      "an agent on environment: 8/10 Episodes, score: 3                             \n",
      "an agent on environment: 9/10 Episodes, score: 3                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 37.21801309188207 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 3.8                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.07564678389158179, 'env_max_step': 1500, 'env_rwd_func': 0, 'gae_lambda': 0.5993762787593486, 'learning_rate': 0.01, 'n_steps': 6, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 11                            \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 9                             \n",
      "an agent on environment: 4/10 Episodes, score: 10                            \n",
      "an agent on environment: 5/10 Episodes, score: 9                             \n",
      "an agent on environment: 6/10 Episodes, score: 5                             \n",
      "an agent on environment: 7/10 Episodes, score: 9                             \n",
      "an agent on environment: 8/10 Episodes, score: 9                             \n",
      "an agent on environment: 9/10 Episodes, score: 11                            \n",
      "an agent on environment: 10/10 Episodes, score: 8                            \n",
      "---Time taken for opt-epoch: 38.99977090358734 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 9.0                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.055304162748706566, 'env_max_step': 0, 'env_rwd_func': 1, 'gae_lambda': 0.13809462265016773, 'learning_rate': 0.003, 'n_steps': 5, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 5                             \n",
      "an agent on environment: 2/10 Episodes, score: 4                             \n",
      "an agent on environment: 3/10 Episodes, score: 4                             \n",
      "an agent on environment: 4/10 Episodes, score: 7                             \n",
      "an agent on environment: 5/10 Episodes, score: 4                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 5                             \n",
      "an agent on environment: 9/10 Episodes, score: 4                             \n",
      "an agent on environment: 10/10 Episodes, score: 4                            \n",
      "---Time taken for opt-epoch: 38.863206096490224 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.3                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.16130175278479647, 'env_max_step': 3500, 'env_rwd_func': 2, 'gae_lambda': 0.39823210748350024, 'learning_rate': 0.0001, 'n_steps': 10, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 6                             \n",
      "an agent on environment: 2/10 Episodes, score: 3                             \n",
      "an agent on environment: 3/10 Episodes, score: 3                             \n",
      "an agent on environment: 4/10 Episodes, score: 4                             \n",
      "an agent on environment: 5/10 Episodes, score: 5                             \n",
      "an agent on environment: 6/10 Episodes, score: 5                             \n",
      "an agent on environment: 7/10 Episodes, score: 5                             \n",
      "an agent on environment: 8/10 Episodes, score: 4                             \n",
      "an agent on environment: 9/10 Episodes, score: 3                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 37.283827527364096 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.1                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.1320932299601315, 'env_max_step': 1500, 'env_rwd_func': 0, 'gae_lambda': 0.24842711674949786, 'learning_rate': 0.009, 'n_steps': 1024, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                             \n",
      "an agent on environment: 2/10 Episodes, score: 5                             \n",
      "an agent on environment: 3/10 Episodes, score: 4                             \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 6                             \n",
      "an agent on environment: 6/10 Episodes, score: 4                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 4                             \n",
      "an agent on environment: 9/10 Episodes, score: 7                             \n",
      "an agent on environment: 10/10 Episodes, score: 4                            \n",
      "---Time taken for opt-epoch: 37.69300963083903 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 4.3                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.011085299358648656, 'env_max_step': 0, 'env_rwd_func': 0, 'gae_lambda': 0.4399699709171768, 'learning_rate': 0.003, 'n_steps': 128, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 9                             \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 9                             \n",
      "an agent on environment: 4/10 Episodes, score: 10                            \n",
      "an agent on environment: 5/10 Episodes, score: 9                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 10                            \n",
      "an agent on environment: 8/10 Episodes, score: 12                            \n",
      "an agent on environment: 9/10 Episodes, score: 9                             \n",
      "an agent on environment: 10/10 Episodes, score: 9                            \n",
      "---Time taken for opt-epoch: 38.090195620059966 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 8.9                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.08587805313512019, 'env_max_step': 3500, 'env_rwd_func': 3, 'gae_lambda': 0.010156900888976195, 'learning_rate': 0.0003, 'n_steps': 3, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 5                             \n",
      "an agent on environment: 2/10 Episodes, score: 4                             \n",
      "an agent on environment: 3/10 Episodes, score: 4                             \n",
      "an agent on environment: 4/10 Episodes, score: 1                             \n",
      "an agent on environment: 5/10 Episodes, score: 1                             \n",
      "an agent on environment: 6/10 Episodes, score: 5                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 2                             \n",
      "an agent on environment: 9/10 Episodes, score: 4                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 38.28719615936279 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 3.2                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.03789171886915513, 'env_max_step': 1500, 'env_rwd_func': 1, 'gae_lambda': 0.33327006064359216, 'learning_rate': 0.006, 'n_steps': 10, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 4                             \n",
      "an agent on environment: 2/10 Episodes, score: 3                             \n",
      "an agent on environment: 3/10 Episodes, score: 7                             \n",
      "an agent on environment: 4/10 Episodes, score: 5                             \n",
      "an agent on environment: 5/10 Episodes, score: 4                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 4                             \n",
      "an agent on environment: 9/10 Episodes, score: 3                             \n",
      "an agent on environment: 10/10 Episodes, score: 5                            \n",
      "---Time taken for opt-epoch: 37.172839216391246 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.1                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.04986755911852049, 'env_max_step': 3500, 'env_rwd_func': 2, 'gae_lambda': 0.08331115528106343, 'learning_rate': 0.01, 'n_steps': 32, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                             \n",
      "an agent on environment: 2/10 Episodes, score: 7                             \n",
      "an agent on environment: 3/10 Episodes, score: 3                             \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 3                             \n",
      "an agent on environment: 6/10 Episodes, score: 5                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 4                             \n",
      "an agent on environment: 9/10 Episodes, score: 6                             \n",
      "an agent on environment: 10/10 Episodes, score: 4                            \n",
      "---Time taken for opt-epoch: 37.394354192415875 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.1                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.06970355329327602, 'env_max_step': 1500, 'env_rwd_func': 0, 'gae_lambda': 0.47638298276963026, 'learning_rate': 0.008, 'n_steps': 8, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 9                             \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 12                            \n",
      "an agent on environment: 4/10 Episodes, score: 10                            \n",
      "an agent on environment: 5/10 Episodes, score: 9                             \n",
      "an agent on environment: 6/10 Episodes, score: 9                             \n",
      "an agent on environment: 7/10 Episodes, score: 9                             \n",
      "an agent on environment: 8/10 Episodes, score: 8                             \n",
      "an agent on environment: 9/10 Episodes, score: 9                             \n",
      "an agent on environment: 10/10 Episodes, score: 6                            \n",
      "---Time taken for opt-epoch: 37.46443909406662 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 9.0                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.10637779444108322, 'env_max_step': 0, 'env_rwd_func': 3, 'gae_lambda': 0.16552345272728589, 'learning_rate': 0.007, 'n_steps': 5, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 7                             \n",
      "an agent on environment: 2/10 Episodes, score: 3                             \n",
      "an agent on environment: 3/10 Episodes, score: 3                             \n",
      "an agent on environment: 4/10 Episodes, score: 8                             \n",
      "an agent on environment: 5/10 Episodes, score: 5                             \n",
      "an agent on environment: 6/10 Episodes, score: 9                             \n",
      "an agent on environment: 7/10 Episodes, score: 5                             \n",
      "an agent on environment: 8/10 Episodes, score: 6                             \n",
      "an agent on environment: 9/10 Episodes, score: 5                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 38.1436270793279 Minutes                        \n",
      "---Average Episode Reward of opt-epoch: 5.4                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.11841681999666485, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.7814830284115335, 'learning_rate': 0.0004, 'n_steps': 512, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 5                             \n",
      "an agent on environment: 2/10 Episodes, score: 3                             \n",
      "an agent on environment: 3/10 Episodes, score: 5                             \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 3                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 4                             \n",
      "an agent on environment: 8/10 Episodes, score: 6                             \n",
      "an agent on environment: 9/10 Episodes, score: 3                             \n",
      "an agent on environment: 10/10 Episodes, score: 4                            \n",
      "---Time taken for opt-epoch: 37.738812637329104 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 3.9                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.01803758255084379, 'env_max_step': 1500, 'env_rwd_func': 2, 'gae_lambda': 0.11353275139355672, 'learning_rate': 0.003, 'n_steps': 64, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 4                             \n",
      "an agent on environment: 2/10 Episodes, score: 6                             \n",
      "an agent on environment: 3/10 Episodes, score: 5                             \n",
      "an agent on environment: 4/10 Episodes, score: 10                            \n",
      "an agent on environment: 5/10 Episodes, score: 5                             \n",
      "an agent on environment: 6/10 Episodes, score: 5                             \n",
      "an agent on environment: 7/10 Episodes, score: 13                            \n",
      "an agent on environment: 8/10 Episodes, score: 9                             \n",
      "an agent on environment: 9/10 Episodes, score: 3                             \n",
      "an agent on environment: 10/10 Episodes, score: 9                            \n",
      "---Time taken for opt-epoch: 37.75211264292399 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 6.9                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.08964081111214314, 'env_max_step': 3500, 'env_rwd_func': 1, 'gae_lambda': 0.04641352514861925, 'learning_rate': 0.001, 'n_steps': 128, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 5                             \n",
      "an agent on environment: 2/10 Episodes, score: 5                             \n",
      "an agent on environment: 3/10 Episodes, score: 4                             \n",
      "an agent on environment: 4/10 Episodes, score: 5                             \n",
      "an agent on environment: 5/10 Episodes, score: 3                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 3                             \n",
      "an agent on environment: 9/10 Episodes, score: 3                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 37.961863760153456 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 3.7                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.0013744312713756882, 'env_max_step': 0, 'env_rwd_func': 3, 'gae_lambda': 0.9240447255191522, 'learning_rate': 0.009, 'n_steps': 6, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 0                             \n",
      "an agent on environment: 2/10 Episodes, score: 0                             \n",
      "an agent on environment: 3/10 Episodes, score: 1                             \n",
      "an agent on environment: 4/10 Episodes, score: 0                             \n",
      "an agent on environment: 5/10 Episodes, score: 6                             \n",
      "an agent on environment: 6/10 Episodes, score: 2                             \n",
      "an agent on environment: 7/10 Episodes, score: 0                             \n",
      "an agent on environment: 8/10 Episodes, score: 0                             \n",
      "an agent on environment: 9/10 Episodes, score: 0                             \n",
      "an agent on environment: 10/10 Episodes, score: 0                            \n",
      "---Time taken for opt-epoch: 38.0968980828921 Minutes                        \n",
      "---Average Episode Reward of opt-epoch: 0.9                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.1704265024860269, 'env_max_step': 1500, 'env_rwd_func': 0, 'gae_lambda': 0.5680640661092257, 'learning_rate': 0.0007, 'n_steps': 4, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 4                             \n",
      "an agent on environment: 2/10 Episodes, score: 6                             \n",
      "an agent on environment: 3/10 Episodes, score: 5                             \n",
      "an agent on environment: 4/10 Episodes, score: 4                             \n",
      "an agent on environment: 5/10 Episodes, score: 4                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 4                             \n",
      "an agent on environment: 8/10 Episodes, score: 3                             \n",
      "an agent on environment: 9/10 Episodes, score: 3                             \n",
      "an agent on environment: 10/10 Episodes, score: 4                            \n",
      "---Time taken for opt-epoch: 37.42126463651657 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 4.0                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.022147300997861212, 'env_max_step': 3500, 'env_rwd_func': 2, 'gae_lambda': 0.3010552610418633, 'learning_rate': 0.0003, 'n_steps': 3, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 4                             \n",
      "an agent on environment: 2/10 Episodes, score: 6                             \n",
      "an agent on environment: 3/10 Episodes, score: 5                             \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 6                             \n",
      "an agent on environment: 6/10 Episodes, score: 7                             \n",
      "an agent on environment: 7/10 Episodes, score: 9                             \n",
      "an agent on environment: 8/10 Episodes, score: 8                             \n",
      "an agent on environment: 9/10 Episodes, score: 4                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 37.95351901451747 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 5.5                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.13403830537739028, 'env_max_step': 1500, 'env_rwd_func': 0, 'gae_lambda': 0.21329606733538514, 'learning_rate': 0.006, 'n_steps': 8, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 6                             \n",
      "an agent on environment: 2/10 Episodes, score: 4                             \n",
      "an agent on environment: 3/10 Episodes, score: 6                             \n",
      "an agent on environment: 4/10 Episodes, score: 4                             \n",
      "an agent on environment: 5/10 Episodes, score: 4                             \n",
      "an agent on environment: 6/10 Episodes, score: 4                             \n",
      "an agent on environment: 7/10 Episodes, score: 5                             \n",
      "an agent on environment: 8/10 Episodes, score: 4                             \n",
      "an agent on environment: 9/10 Episodes, score: 5                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 38.29724497795105 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 4.5                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.038577531812007564, 'env_max_step': 3500, 'env_rwd_func': 3, 'gae_lambda': 0.6384868784561403, 'learning_rate': 0.0006, 'n_steps': 5, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 2                             \n",
      "an agent on environment: 2/10 Episodes, score: 8                             \n",
      "an agent on environment: 3/10 Episodes, score: 3                             \n",
      "an agent on environment: 4/10 Episodes, score: 4                             \n",
      "an agent on environment: 5/10 Episodes, score: 9                             \n",
      "an agent on environment: 6/10 Episodes, score: 7                             \n",
      "an agent on environment: 7/10 Episodes, score: 4                             \n",
      "an agent on environment: 8/10 Episodes, score: 3                             \n",
      "an agent on environment: 9/10 Episodes, score: 1                             \n",
      "an agent on environment: 10/10 Episodes, score: 5                            \n",
      "---Time taken for opt-epoch: 41.31322951714198 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 4.6                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.011926145438401484, 'env_max_step': 0, 'env_rwd_func': 0, 'gae_lambda': 0.7233670688599732, 'learning_rate': 0.0001, 'n_steps': 1024, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                             \n",
      "an agent on environment: 2/10 Episodes, score: 3                             \n",
      "an agent on environment: 3/10 Episodes, score: 4                             \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 3                             \n",
      "an agent on environment: 6/10 Episodes, score: 4                             \n",
      "an agent on environment: 7/10 Episodes, score: 6                             \n",
      "an agent on environment: 8/10 Episodes, score: 5                             \n",
      "an agent on environment: 9/10 Episodes, score: 5                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 42.04607326984406 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 3.9                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.05768374117835538, 'env_max_step': 1500, 'env_rwd_func': 1, 'gae_lambda': 0.15592644436794764, 'learning_rate': 0.003, 'n_steps': 512, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                             \n",
      "an agent on environment: 2/10 Episodes, score: 5                             \n",
      "an agent on environment: 3/10 Episodes, score: 4                             \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 3                             \n",
      "an agent on environment: 6/10 Episodes, score: 5                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 6                             \n",
      "an agent on environment: 9/10 Episodes, score: 4                             \n",
      "an agent on environment: 10/10 Episodes, score: 4                            \n",
      "---Time taken for opt-epoch: 38.012385149796806 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.0                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.0782259309600573, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.37632770638366475, 'learning_rate': 0.01, 'n_steps': 16, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 7                             \n",
      "an agent on environment: 2/10 Episodes, score: 6                             \n",
      "an agent on environment: 3/10 Episodes, score: 3                             \n",
      "an agent on environment: 4/10 Episodes, score: 6                             \n",
      "an agent on environment: 5/10 Episodes, score: 5                             \n",
      "an agent on environment: 6/10 Episodes, score: 4                             \n",
      "an agent on environment: 7/10 Episodes, score: 5                             \n",
      "an agent on environment: 8/10 Episodes, score: 5                             \n",
      "an agent on environment: 9/10 Episodes, score: 7                             \n",
      "an agent on environment: 10/10 Episodes, score: 9                            \n",
      "---Time taken for opt-epoch: 38.55314662853877 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 5.7                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.0658463940003213, 'env_max_step': 1500, 'env_rwd_func': 2, 'gae_lambda': 0.5319978739706419, 'learning_rate': 0.003, 'n_steps': 64, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                             \n",
      "an agent on environment: 2/10 Episodes, score: 3                             \n",
      "an agent on environment: 3/10 Episodes, score: 4                             \n",
      "an agent on environment: 4/10 Episodes, score: 6                             \n",
      "an agent on environment: 5/10 Episodes, score: 3                             \n",
      "an agent on environment: 6/10 Episodes, score: 5                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 9                             \n",
      "an agent on environment: 9/10 Episodes, score: 4                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 42.16038854916891 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 4.3                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.0962568531631975, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.2728376499986147, 'learning_rate': 0.0007, 'n_steps': 3, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 5                             \n",
      "an agent on environment: 2/10 Episodes, score: 7                             \n",
      "an agent on environment: 3/10 Episodes, score: 6                             \n",
      "an agent on environment: 4/10 Episodes, score: 5                             \n",
      "an agent on environment: 5/10 Episodes, score: 6                             \n",
      "an agent on environment: 6/10 Episodes, score: 4                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 6                             \n",
      "an agent on environment: 9/10 Episodes, score: 7                             \n",
      "an agent on environment: 10/10 Episodes, score: 4                            \n",
      "---Time taken for opt-epoch: 39.094796148935956 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 5.3                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.045274641952798886, 'env_max_step': 1500, 'env_rwd_func': 3, 'gae_lambda': 0.00212239912328413, 'learning_rate': 0.009, 'n_steps': 10, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 7                             \n",
      "an agent on environment: 2/10 Episodes, score: 4                             \n",
      "an agent on environment: 3/10 Episodes, score: 3                             \n",
      "an agent on environment: 4/10 Episodes, score: 0                             \n",
      "an agent on environment: 5/10 Episodes, score: 1                             \n",
      "an agent on environment: 6/10 Episodes, score: 11                            \n",
      "an agent on environment: 7/10 Episodes, score: 10                            \n",
      "an agent on environment: 8/10 Episodes, score: 4                             \n",
      "an agent on environment: 9/10 Episodes, score: 5                             \n",
      "an agent on environment: 10/10 Episodes, score: 0                            \n",
      "---Time taken for opt-epoch: 38.08471239010493 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 4.5                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.02379125109617202, 'env_max_step': 0, 'env_rwd_func': 0, 'gae_lambda': 0.1253690824258188, 'learning_rate': 0.0004, 'n_steps': 256, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 4                             \n",
      "an agent on environment: 2/10 Episodes, score: 3                             \n",
      "an agent on environment: 3/10 Episodes, score: 3                             \n",
      "an agent on environment: 4/10 Episodes, score: 4                             \n",
      "an agent on environment: 5/10 Episodes, score: 4                             \n",
      "an agent on environment: 6/10 Episodes, score: 4                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 6                             \n",
      "an agent on environment: 9/10 Episodes, score: 3                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 37.8980052113533 Minutes                        \n",
      "---Average Episode Reward of opt-epoch: 3.7                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.1481181789733434, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.08754969870209114, 'learning_rate': 0.008, 'n_steps': 128, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 4                             \n",
      "an agent on environment: 2/10 Episodes, score: 3                             \n",
      "an agent on environment: 3/10 Episodes, score: 3                             \n",
      "an agent on environment: 4/10 Episodes, score: 5                             \n",
      "an agent on environment: 5/10 Episodes, score: 3                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 3                             \n",
      "an agent on environment: 9/10 Episodes, score: 3                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 38.11662423610687 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 3.3                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.006809356226631397, 'env_max_step': 1500, 'env_rwd_func': 1, 'gae_lambda': 0.23433671728068878, 'learning_rate': 0.0006, 'n_steps': 32, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 4                             \n",
      "an agent on environment: 2/10 Episodes, score: 3                             \n",
      "an agent on environment: 3/10 Episodes, score: 4                             \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 4                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 5                             \n",
      "an agent on environment: 8/10 Episodes, score: 6                             \n",
      "an agent on environment: 9/10 Episodes, score: 5                             \n",
      "an agent on environment: 10/10 Episodes, score: 4                            \n",
      "---Time taken for opt-epoch: 39.17234291235606 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 4.1                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.0002675051405930315, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.181974922246779, 'learning_rate': 0.003, 'n_steps': 5, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 10                            \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 10                            \n",
      "an agent on environment: 4/10 Episodes, score: 10                            \n",
      "an agent on environment: 5/10 Episodes, score: 9                             \n",
      "an agent on environment: 6/10 Episodes, score: 9                             \n",
      "an agent on environment: 7/10 Episodes, score: 12                            \n",
      "an agent on environment: 8/10 Episodes, score: 7                             \n",
      "an agent on environment: 9/10 Episodes, score: 9                             \n",
      "an agent on environment: 10/10 Episodes, score: 9                            \n",
      "---Time taken for opt-epoch: 37.369875395298 Minutes                         \n",
      "---Average Episode Reward of opt-epoch: 9.4                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.026099636586225106, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.037813302911236676, 'learning_rate': 0.003, 'n_steps': 16, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 10                            \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 10                            \n",
      "an agent on environment: 4/10 Episodes, score: 11                            \n",
      "an agent on environment: 5/10 Episodes, score: 9                             \n",
      "an agent on environment: 6/10 Episodes, score: 9                             \n",
      "an agent on environment: 7/10 Episodes, score: 9                             \n",
      "an agent on environment: 8/10 Episodes, score: 10                            \n",
      "an agent on environment: 9/10 Episodes, score: 9                             \n",
      "an agent on environment: 10/10 Episodes, score: 11                           \n",
      "---Time taken for opt-epoch: 37.446877340475716 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 9.7                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.035779807294799126, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.037572516868033576, 'learning_rate': 0.003, 'n_steps': 16, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 9                             \n",
      "an agent on environment: 2/10 Episodes, score: 10                            \n",
      "an agent on environment: 3/10 Episodes, score: 9                             \n",
      "an agent on environment: 4/10 Episodes, score: 9                             \n",
      "an agent on environment: 5/10 Episodes, score: 5                             \n",
      "an agent on environment: 6/10 Episodes, score: 11                            \n",
      "an agent on environment: 7/10 Episodes, score: 7                             \n",
      "an agent on environment: 8/10 Episodes, score: 9                             \n",
      "an agent on environment: 9/10 Episodes, score: 10                            \n",
      "an agent on environment: 10/10 Episodes, score: 11                           \n",
      "---Time taken for opt-epoch: 37.522283883889514 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 9.0                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.014722305311628495, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.08179432834573155, 'learning_rate': 0.003, 'n_steps': 16, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 14                            \n",
      "an agent on environment: 2/10 Episodes, score: 12                            \n",
      "an agent on environment: 3/10 Episodes, score: 8                             \n",
      "an agent on environment: 4/10 Episodes, score: 10                            \n",
      "an agent on environment: 5/10 Episodes, score: 10                            \n",
      "an agent on environment: 6/10 Episodes, score: 12                            \n",
      "an agent on environment: 7/10 Episodes, score: 11                            \n",
      "an agent on environment: 8/10 Episodes, score: 11                            \n",
      "an agent on environment: 9/10 Episodes, score: 9                             \n",
      "an agent on environment: 10/10 Episodes, score: 8                            \n",
      "Saving new best model.                                                       \n",
      "Saving new best model's gif.                                                 \n",
      "---Time taken for opt-epoch: 37.61168023745219 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 10.5                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.015485994905501647, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.09761599253725033, 'learning_rate': 0.003, 'n_steps': 16, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 9                             \n",
      "an agent on environment: 2/10 Episodes, score: 12                            \n",
      "an agent on environment: 3/10 Episodes, score: 8                             \n",
      "an agent on environment: 4/10 Episodes, score: 4                             \n",
      "an agent on environment: 5/10 Episodes, score: 8                             \n",
      "an agent on environment: 6/10 Episodes, score: 11                            \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 10                            \n",
      "an agent on environment: 9/10 Episodes, score: 9                             \n",
      "an agent on environment: 10/10 Episodes, score: 8                            \n",
      "---Time taken for opt-epoch: 37.76578252315521 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 8.2                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.05135416708716138, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.22173651902422892, 'learning_rate': 0.003, 'n_steps': 16, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 9                             \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 9                             \n",
      "an agent on environment: 4/10 Episodes, score: 12                            \n",
      "an agent on environment: 5/10 Episodes, score: 10                            \n",
      "an agent on environment: 6/10 Episodes, score: 7                             \n",
      "an agent on environment: 7/10 Episodes, score: 7                             \n",
      "an agent on environment: 8/10 Episodes, score: 9                             \n",
      "an agent on environment: 9/10 Episodes, score: 5                             \n",
      "an agent on environment: 10/10 Episodes, score: 5                            \n",
      "---Time taken for opt-epoch: 38.145084818204246 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 8.2                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.029748940401873158, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.0652697323606426, 'learning_rate': 0.001, 'n_steps': 3, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                             \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 9                             \n",
      "an agent on environment: 4/10 Episodes, score: 8                             \n",
      "an agent on environment: 5/10 Episodes, score: 9                             \n",
      "an agent on environment: 6/10 Episodes, score: 10                            \n",
      "an agent on environment: 7/10 Episodes, score: 10                            \n",
      "an agent on environment: 8/10 Episodes, score: 10                            \n",
      "an agent on environment: 9/10 Episodes, score: 8                             \n",
      "an agent on environment: 10/10 Episodes, score: 9                            \n",
      "---Time taken for opt-epoch: 37.843148120244344 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 8.5                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.006327922431836504, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.27672248798192545, 'learning_rate': 0.007, 'n_steps': 4, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 10                            \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 9                             \n",
      "an agent on environment: 4/10 Episodes, score: 7                             \n",
      "an agent on environment: 5/10 Episodes, score: 10                            \n",
      "an agent on environment: 6/10 Episodes, score: 9                             \n",
      "an agent on environment: 7/10 Episodes, score: 6                             \n",
      "an agent on environment: 8/10 Episodes, score: 9                             \n",
      "an agent on environment: 9/10 Episodes, score: 11                            \n",
      "an agent on environment: 10/10 Episodes, score: 9                            \n",
      "---Time taken for opt-epoch: 37.58719644546509 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 8.9                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.04255840596038595, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.1706237783218813, 'learning_rate': 0.0003, 'n_steps': 6, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                             \n",
      "an agent on environment: 2/10 Episodes, score: 4                             \n",
      "an agent on environment: 3/10 Episodes, score: 5                             \n",
      "an agent on environment: 4/10 Episodes, score: 4                             \n",
      "an agent on environment: 5/10 Episodes, score: 3                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 7                             \n",
      "an agent on environment: 9/10 Episodes, score: 3                             \n",
      "an agent on environment: 10/10 Episodes, score: 7                            \n",
      "---Time taken for opt-epoch: 37.41388426621755 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 4.2                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.18735904781209128, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.0015026089523089287, 'learning_rate': 0.0001, 'n_steps': 5, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 7                             \n",
      "an agent on environment: 2/10 Episodes, score: 5                             \n",
      "an agent on environment: 3/10 Episodes, score: 4                             \n",
      "an agent on environment: 4/10 Episodes, score: 4                             \n",
      "an agent on environment: 5/10 Episodes, score: 5                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 5                             \n",
      "an agent on environment: 8/10 Episodes, score: 7                             \n",
      "an agent on environment: 9/10 Episodes, score: 4                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 38.269132069746654 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.7                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.06373490071474434, 'env_max_step': 3500, 'env_rwd_func': 2, 'gae_lambda': 0.132344641054279, 'learning_rate': 0.006, 'n_steps': 8, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 5                             \n",
      "an agent on environment: 2/10 Episodes, score: 6                             \n",
      "an agent on environment: 3/10 Episodes, score: 3                             \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 8                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 6                             \n",
      "an agent on environment: 8/10 Episodes, score: 9                             \n",
      "an agent on environment: 9/10 Episodes, score: 8                             \n",
      "an agent on environment: 10/10 Episodes, score: 6                            \n",
      "---Time taken for opt-epoch: 37.75363323688507 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 5.7                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.03395502530791286, 'env_max_step': 1500, 'env_rwd_func': 0, 'gae_lambda': 0.3436316038390297, 'learning_rate': 0.003, 'n_steps': 3, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 9                             \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 9                             \n",
      "an agent on environment: 4/10 Episodes, score: 10                            \n",
      "an agent on environment: 5/10 Episodes, score: 11                            \n",
      "an agent on environment: 6/10 Episodes, score: 9                             \n",
      "an agent on environment: 7/10 Episodes, score: 12                            \n",
      "an agent on environment: 8/10 Episodes, score: 9                             \n",
      "an agent on environment: 9/10 Episodes, score: 9                             \n",
      "an agent on environment: 10/10 Episodes, score: 10                           \n",
      "---Time taken for opt-epoch: 39.453278028964995 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 9.7                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.05389599958264754, 'env_max_step': 0, 'env_rwd_func': 3, 'gae_lambda': 0.3278785486035336, 'learning_rate': 0.003, 'n_steps': 1024, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 4                             \n",
      "an agent on environment: 2/10 Episodes, score: 3                             \n",
      "an agent on environment: 3/10 Episodes, score: 3                             \n",
      "an agent on environment: 4/10 Episodes, score: 5                             \n",
      "an agent on environment: 5/10 Episodes, score: 3                             \n",
      "an agent on environment: 6/10 Episodes, score: 4                             \n",
      "an agent on environment: 7/10 Episodes, score: 9                             \n",
      "an agent on environment: 8/10 Episodes, score: 3                             \n",
      "an agent on environment: 9/10 Episodes, score: 3                             \n",
      "an agent on environment: 10/10 Episodes, score: 4                            \n",
      "---Time taken for opt-epoch: 41.121780749162035 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.1                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.01130984377854942, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.3935713208602946, 'learning_rate': 0.01, 'n_steps': 512, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 1                             \n",
      "an agent on environment: 2/10 Episodes, score: 1                             \n",
      "an agent on environment: 3/10 Episodes, score: 1                             \n",
      "an agent on environment: 4/10 Episodes, score: 1                             \n",
      "an agent on environment: 5/10 Episodes, score: 1                             \n",
      "an agent on environment: 6/10 Episodes, score: 1                             \n",
      "an agent on environment: 7/10 Episodes, score: 1                             \n",
      "an agent on environment: 8/10 Episodes, score: 0                             \n",
      "an agent on environment: 9/10 Episodes, score: 0                             \n",
      "an agent on environment: 10/10 Episodes, score: 2                            \n",
      "---Time taken for opt-epoch: 38.531599724292754 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 0.9                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.07092575801464462, 'env_max_step': 1500, 'env_rwd_func': 1, 'gae_lambda': 0.06963738268304079, 'learning_rate': 0.008, 'n_steps': 16, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 4                             \n",
      "an agent on environment: 2/10 Episodes, score: 4                             \n",
      "an agent on environment: 3/10 Episodes, score: 4                             \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 3                             \n",
      "an agent on environment: 6/10 Episodes, score: 8                             \n",
      "an agent on environment: 7/10 Episodes, score: 4                             \n",
      "an agent on environment: 8/10 Episodes, score: 4                             \n",
      "an agent on environment: 9/10 Episodes, score: 3                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 37.52207954327265 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 4.0                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.08065253891045938, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.49670609543795563, 'learning_rate': 0.0007, 'n_steps': 64, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 4                             \n",
      "an agent on environment: 2/10 Episodes, score: 6                             \n",
      "an agent on environment: 3/10 Episodes, score: 6                             \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 6                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 4                             \n",
      "an agent on environment: 8/10 Episodes, score: 3                             \n",
      "an agent on environment: 9/10 Episodes, score: 7                             \n",
      "an agent on environment: 10/10 Episodes, score: 4                            \n",
      "---Time taken for opt-epoch: 37.777970321973164 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.6                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.017996440357748684, 'env_max_step': 1500, 'env_rwd_func': 2, 'gae_lambda': 0.14654863876011934, 'learning_rate': 0.003, 'n_steps': 256, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 7                             \n",
      "an agent on environment: 2/10 Episodes, score: 9                             \n",
      "an agent on environment: 3/10 Episodes, score: 3                             \n",
      "an agent on environment: 4/10 Episodes, score: 10                            \n",
      "an agent on environment: 5/10 Episodes, score: 4                             \n",
      "an agent on environment: 6/10 Episodes, score: 3                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 8                             \n",
      "an agent on environment: 9/10 Episodes, score: 9                             \n",
      "an agent on environment: 10/10 Episodes, score: 5                            \n",
      "---Time taken for opt-epoch: 38.346841887633005 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 6.1                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.02513891357663323, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.10711058887648565, 'learning_rate': 0.0004, 'n_steps': 10, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 9                             \n",
      "an agent on environment: 2/10 Episodes, score: 7                             \n",
      "an agent on environment: 3/10 Episodes, score: 5                             \n",
      "an agent on environment: 4/10 Episodes, score: 9                             \n",
      "an agent on environment: 5/10 Episodes, score: 7                             \n",
      "an agent on environment: 6/10 Episodes, score: 4                             \n",
      "an agent on environment: 7/10 Episodes, score: 5                             \n",
      "an agent on environment: 8/10 Episodes, score: 10                            \n",
      "an agent on environment: 9/10 Episodes, score: 3                             \n",
      "an agent on environment: 10/10 Episodes, score: 3                            \n",
      "---Time taken for opt-epoch: 37.60520821809769 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 6.2                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.060227306993141605, 'env_max_step': 0, 'env_rwd_func': 0, 'gae_lambda': 0.19336112612479794, 'learning_rate': 0.0006, 'n_steps': 5, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 9                             \n",
      "an agent on environment: 2/10 Episodes, score: 11                            \n",
      "an agent on environment: 3/10 Episodes, score: 8                             \n",
      "an agent on environment: 4/10 Episodes, score: 12                            \n",
      "an agent on environment: 5/10 Episodes, score: 9                             \n",
      "an agent on environment: 6/10 Episodes, score: 8                             \n",
      "an agent on environment: 7/10 Episodes, score: 3                             \n",
      "an agent on environment: 8/10 Episodes, score: 10                            \n",
      "an agent on environment: 9/10 Episodes, score: 3                             \n",
      "an agent on environment: 10/10 Episodes, score: 11                           \n",
      "---Time taken for opt-epoch: 37.75662657419841 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 8.4                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.11078151817554698, 'env_max_step': 1500, 'env_rwd_func': 3, 'gae_lambda': 0.2876820203833165, 'learning_rate': 0.001, 'n_steps': 32, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                             \n",
      "an agent on environment: 2/10 Episodes, score: 3                             \n",
      "an agent on environment: 3/10 Episodes, score: 1                             \n",
      "an agent on environment: 4/10 Episodes, score: 5                             \n",
      "an agent on environment: 5/10 Episodes, score: 8                             \n",
      "an agent on environment: 6/10 Episodes, score: 6                             \n",
      "an agent on environment: 7/10 Episodes, score: 6                             \n",
      "an agent on environment: 8/10 Episodes, score: 3                             \n",
      "an agent on environment: 9/10 Episodes, score: 8                             \n",
      "an agent on environment: 10/10 Episodes, score: 6                            \n",
      "---Time taken for opt-epoch: 38.52291206518809 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 4.9                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.004737508316370374, 'env_max_step': 3500, 'env_rwd_func': 1, 'gae_lambda': 0.01885868106444194, 'learning_rate': 0.007, 'n_steps': 3, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                             \n",
      "an agent on environment: 2/10 Episodes, score: 2                             \n",
      "an agent on environment: 3/10 Episodes, score: 5                             \n",
      "an agent on environment: 4/10 Episodes, score: 3                             \n",
      "an agent on environment: 5/10 Episodes, score: 1                             \n",
      "an agent on environment: 6/10 Episodes, score: 9                             \n",
      "an agent on environment: 7/10 Episodes, score: 7                             \n",
      "an agent on environment: 8/10 Episodes, score: 4                             \n",
      "an agent on environment: 9/10 Episodes, score: 5                             \n",
      "an agent on environment: 10/10 Episodes, score: 6                            \n",
      "---Time taken for opt-epoch: 37.913023320833844 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.5                                  \n",
      "---Chosen Hyperparameters of opt-epoch:                                      \n",
      "{'ent_coef': 0.048047550630438404, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.4318081141610267, 'learning_rate': 0.009, 'n_steps': 128, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 9                            \n",
      "an agent on environment: 2/10 Episodes, score: 9                            \n",
      "an agent on environment: 3/10 Episodes, score: 11                           \n",
      "an agent on environment: 4/10 Episodes, score: 2                            \n",
      "an agent on environment: 5/10 Episodes, score: 11                           \n",
      "an agent on environment: 6/10 Episodes, score: 10                           \n",
      "an agent on environment: 7/10 Episodes, score: 14                           \n",
      "an agent on environment: 8/10 Episodes, score: 10                           \n",
      "an agent on environment: 9/10 Episodes, score: 6                            \n",
      "an agent on environment: 10/10 Episodes, score: 9                           \n",
      "---Time taken for opt-epoch: 38.59081318775813 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 9.1                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.08542271928990396, 'env_max_step': 1500, 'env_rwd_func': 0, 'gae_lambda': 0.2585963738498807, 'learning_rate': 0.003, 'n_steps': 4, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 7                            \n",
      "an agent on environment: 2/10 Episodes, score: 5                            \n",
      "an agent on environment: 3/10 Episodes, score: 3                            \n",
      "an agent on environment: 4/10 Episodes, score: 5                            \n",
      "an agent on environment: 5/10 Episodes, score: 4                            \n",
      "an agent on environment: 6/10 Episodes, score: 3                            \n",
      "an agent on environment: 7/10 Episodes, score: 3                            \n",
      "an agent on environment: 8/10 Episodes, score: 3                            \n",
      "an agent on environment: 9/10 Episodes, score: 3                            \n",
      "an agent on environment: 10/10 Episodes, score: 6                           \n",
      "---Time taken for opt-epoch: 37.600753549734755 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 4.2                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.03970153850919144, 'env_max_step': 0, 'env_rwd_func': 2, 'gae_lambda': 0.8067535574953812, 'learning_rate': 0.0003, 'n_steps': 5, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                            \n",
      "an agent on environment: 2/10 Episodes, score: 3                            \n",
      "an agent on environment: 3/10 Episodes, score: 3                            \n",
      "an agent on environment: 4/10 Episodes, score: 5                            \n",
      "an agent on environment: 5/10 Episodes, score: 3                            \n",
      "an agent on environment: 6/10 Episodes, score: 3                            \n",
      "an agent on environment: 7/10 Episodes, score: 7                            \n",
      "an agent on environment: 8/10 Episodes, score: 3                            \n",
      "an agent on environment: 9/10 Episodes, score: 4                            \n",
      "an agent on environment: 10/10 Episodes, score: 5                           \n",
      "---Time taken for opt-epoch: 37.55154957771301 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 3.9                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.00021489848906378756, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.8964411630383772, 'learning_rate': 0.0001, 'n_steps': 6, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 6                            \n",
      "an agent on environment: 2/10 Episodes, score: 7                            \n",
      "an agent on environment: 3/10 Episodes, score: 9                            \n",
      "an agent on environment: 4/10 Episodes, score: 8                            \n",
      "an agent on environment: 5/10 Episodes, score: 6                            \n",
      "an agent on environment: 6/10 Episodes, score: 9                            \n",
      "an agent on environment: 7/10 Episodes, score: 8                            \n",
      "an agent on environment: 8/10 Episodes, score: 4                            \n",
      "an agent on environment: 9/10 Episodes, score: 5                            \n",
      "an agent on environment: 10/10 Episodes, score: 8                           \n",
      "---Time taken for opt-epoch: 37.928152084350586 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 7.0                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.07447544573594443, 'env_max_step': 1500, 'env_rwd_func': 3, 'gae_lambda': 0.4642218243568638, 'learning_rate': 0.003, 'n_steps': 16, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 5                            \n",
      "an agent on environment: 2/10 Episodes, score: 10                           \n",
      "an agent on environment: 3/10 Episodes, score: 5                            \n",
      "an agent on environment: 4/10 Episodes, score: 3                            \n",
      "an agent on environment: 5/10 Episodes, score: 9                            \n",
      "an agent on environment: 6/10 Episodes, score: 4                            \n",
      "an agent on environment: 7/10 Episodes, score: 6                            \n",
      "an agent on environment: 8/10 Episodes, score: 7                            \n",
      "an agent on environment: 9/10 Episodes, score: 3                            \n",
      "an agent on environment: 10/10 Episodes, score: 9                           \n",
      "---Time taken for opt-epoch: 38.323227580388384 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 6.1                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.02925912369970761, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.361520490698663, 'learning_rate': 0.006, 'n_steps': 8, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 9                            \n",
      "an agent on environment: 2/10 Episodes, score: 9                            \n",
      "an agent on environment: 3/10 Episodes, score: 13                           \n",
      "an agent on environment: 4/10 Episodes, score: 6                            \n",
      "an agent on environment: 5/10 Episodes, score: 12                           \n",
      "an agent on environment: 6/10 Episodes, score: 12                           \n",
      "an agent on environment: 7/10 Episodes, score: 11                           \n",
      "an agent on environment: 8/10 Episodes, score: 7                            \n",
      "an agent on environment: 9/10 Episodes, score: 9                            \n",
      "an agent on environment: 10/10 Episodes, score: 11                          \n",
      "---Time taken for opt-epoch: 37.94094513257345 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 9.9                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.09372997389081547, 'env_max_step': 1500, 'env_rwd_func': 1, 'gae_lambda': 0.02600513443449228, 'learning_rate': 0.003, 'n_steps': 1024, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                            \n",
      "an agent on environment: 2/10 Episodes, score: 3                            \n",
      "an agent on environment: 3/10 Episodes, score: 3                            \n",
      "an agent on environment: 4/10 Episodes, score: 5                            \n",
      "an agent on environment: 5/10 Episodes, score: 4                            \n",
      "an agent on environment: 6/10 Episodes, score: 5                            \n",
      "an agent on environment: 7/10 Episodes, score: 3                            \n",
      "an agent on environment: 8/10 Episodes, score: 4                            \n",
      "an agent on environment: 9/10 Episodes, score: 3                            \n",
      "an agent on environment: 10/10 Episodes, score: 7                           \n",
      "---Time taken for opt-epoch: 40.79968063433965 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.0                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.008401531084758393, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.2302593768852726, 'learning_rate': 0.0004, 'n_steps': 512, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                            \n",
      "an agent on environment: 2/10 Episodes, score: 6                            \n",
      "an agent on environment: 3/10 Episodes, score: 6                            \n",
      "an agent on environment: 4/10 Episodes, score: 5                            \n",
      "an agent on environment: 5/10 Episodes, score: 3                            \n",
      "an agent on environment: 6/10 Episodes, score: 3                            \n",
      "an agent on environment: 7/10 Episodes, score: 4                            \n",
      "an agent on environment: 8/10 Episodes, score: 4                            \n",
      "an agent on environment: 9/10 Episodes, score: 3                            \n",
      "an agent on environment: 10/10 Episodes, score: 4                           \n",
      "---Time taken for opt-epoch: 41.60706677834193 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 4.1                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.019948448961115087, 'env_max_step': 0, 'env_rwd_func': 2, 'gae_lambda': 0.07537775817010328, 'learning_rate': 0.01, 'n_steps': 3, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 8                            \n",
      "an agent on environment: 2/10 Episodes, score: 8                            \n",
      "an agent on environment: 3/10 Episodes, score: 10                           \n",
      "an agent on environment: 4/10 Episodes, score: 9                            \n",
      "an agent on environment: 5/10 Episodes, score: 1                            \n",
      "an agent on environment: 6/10 Episodes, score: 8                            \n",
      "an agent on environment: 7/10 Episodes, score: 9                            \n",
      "an agent on environment: 8/10 Episodes, score: 9                            \n",
      "an agent on environment: 9/10 Episodes, score: 9                            \n",
      "an agent on environment: 10/10 Episodes, score: 5                           \n",
      "---Time taken for opt-epoch: 41.386314475536345 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 7.6                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.04157077115499719, 'env_max_step': 1500, 'env_rwd_func': 0, 'gae_lambda': 0.047828777805307976, 'learning_rate': 0.008, 'n_steps': 256, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                            \n",
      "an agent on environment: 2/10 Episodes, score: 3                            \n",
      "an agent on environment: 3/10 Episodes, score: 5                            \n",
      "an agent on environment: 4/10 Episodes, score: 6                            \n",
      "an agent on environment: 5/10 Episodes, score: 4                            \n",
      "an agent on environment: 6/10 Episodes, score: 6                            \n",
      "an agent on environment: 7/10 Episodes, score: 10                           \n",
      "an agent on environment: 8/10 Episodes, score: 5                            \n",
      "an agent on environment: 9/10 Episodes, score: 9                            \n",
      "an agent on environment: 10/10 Episodes, score: 4                           \n",
      "---Time taken for opt-epoch: 41.68602246046066 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 5.5                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.015610026341953713, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.6311502434570961, 'learning_rate': 0.0007, 'n_steps': 64, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 8                            \n",
      "an agent on environment: 2/10 Episodes, score: 5                            \n",
      "an agent on environment: 3/10 Episodes, score: 3                            \n",
      "an agent on environment: 4/10 Episodes, score: 6                            \n",
      "an agent on environment: 5/10 Episodes, score: 3                            \n",
      "an agent on environment: 6/10 Episodes, score: 9                            \n",
      "an agent on environment: 7/10 Episodes, score: 8                            \n",
      "an agent on environment: 8/10 Episodes, score: 9                            \n",
      "an agent on environment: 9/10 Episodes, score: 3                            \n",
      "an agent on environment: 10/10 Episodes, score: 3                           \n",
      "---Time taken for opt-epoch: 38.46547118822733 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 5.7                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.10323778144977389, 'env_max_step': 3500, 'env_rwd_func': 3, 'gae_lambda': 0.3206684735093652, 'learning_rate': 0.003, 'n_steps': 5, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 4                            \n",
      "an agent on environment: 2/10 Episodes, score: 6                            \n",
      "an agent on environment: 3/10 Episodes, score: 3                            \n",
      "an agent on environment: 4/10 Episodes, score: 6                            \n",
      "an agent on environment: 5/10 Episodes, score: 5                            \n",
      "an agent on environment: 6/10 Episodes, score: 4                            \n",
      "an agent on environment: 7/10 Episodes, score: 3                            \n",
      "an agent on environment: 8/10 Episodes, score: 7                            \n",
      "an agent on environment: 9/10 Episodes, score: 3                            \n",
      "an agent on environment: 10/10 Episodes, score: 5                           \n",
      "---Time taken for opt-epoch: 39.0966477115949 Minutes                       \n",
      "---Average Episode Reward of opt-epoch: 4.6                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.11705900902429382, 'env_max_step': 1500, 'env_rwd_func': 1, 'gae_lambda': 0.9938670443761288, 'learning_rate': 0.009, 'n_steps': 10, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 5                            \n",
      "an agent on environment: 2/10 Episodes, score: 6                            \n",
      "an agent on environment: 3/10 Episodes, score: 6                            \n",
      "an agent on environment: 4/10 Episodes, score: 5                            \n",
      "an agent on environment: 5/10 Episodes, score: 6                            \n",
      "an agent on environment: 6/10 Episodes, score: 3                            \n",
      "an agent on environment: 7/10 Episodes, score: 7                            \n",
      "an agent on environment: 8/10 Episodes, score: 3                            \n",
      "an agent on environment: 9/10 Episodes, score: 4                            \n",
      "an agent on environment: 10/10 Episodes, score: 5                           \n",
      "---Time taken for opt-epoch: 37.97865223487218 Minutes                      \n",
      "---Average Episode Reward of opt-epoch: 5.0                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.12846981580754535, 'env_max_step': 0, 'env_rwd_func': 0, 'gae_lambda': 0.41024277218582683, 'learning_rate': 0.0006, 'n_steps': 32, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                            \n",
      "an agent on environment: 2/10 Episodes, score: 3                            \n",
      "an agent on environment: 3/10 Episodes, score: 5                            \n",
      "an agent on environment: 4/10 Episodes, score: 5                            \n",
      "an agent on environment: 5/10 Episodes, score: 3                            \n",
      "an agent on environment: 6/10 Episodes, score: 3                            \n",
      "an agent on environment: 7/10 Episodes, score: 5                            \n",
      "an agent on environment: 8/10 Episodes, score: 4                            \n",
      "an agent on environment: 9/10 Episodes, score: 4                            \n",
      "an agent on environment: 10/10 Episodes, score: 3                           \n",
      "---Time taken for opt-epoch: 37.950525931517284 Minutes                     \n",
      "---Average Episode Reward of opt-epoch: 3.8                                 \n",
      "---Chosen Hyperparameters of opt-epoch:                                     \n",
      "{'ent_coef': 0.13815709855136363, 'env_max_step': 3500, 'env_rwd_func': 0, 'gae_lambda': 0.1213242137332818, 'learning_rate': 0.007, 'n_steps': 16, 'env_num_stack': 4, 'n_envs': 1000, 'gamma': 0.8, 'normalize_advantage': True}\n",
      "an agent on environment: 1/10 Episodes, score: 3                          \n",
      "an agent on environment: 2/10 Episodes, score: 5                          \n",
      "an agent on environment: 3/10 Episodes, score: 5                          \n",
      "an agent on environment: 4/10 Episodes, score: 3                          \n",
      "an agent on environment: 5/10 Episodes, score: 7                          \n",
      "an agent on environment: 6/10 Episodes, score: 6                          \n",
      "an agent on environment: 7/10 Episodes, score: 3                          \n",
      "an agent on environment: 8/10 Episodes, score: 3                          \n",
      "an agent on environment: 9/10 Episodes, score: 3                          \n",
      "an agent on environment: 10/10 Episodes, score: 4                         \n",
      "---Time taken for opt-epoch: 38.708166086673735 Minutes                   \n",
      "---Average Episode Reward of opt-epoch: 4.2                               \n",
      "100%|██████████| 100/100 [63:41:04<00:00, 2292.65s/trial, best loss: -1.05]\n"
     ]
    }
   ],
   "source": [
    "best_avg_rwd = float(\"-inf\")\n",
    "expr_no = 3\n",
    "total_timesteps = 5_000_000\n",
    "fixed_args = {\"env_num_stack\": 4, \"n_envs\": 1000, \"gamma\": 0.80, \"normalize_advantage\": True}\n",
    "\n",
    "best = fmin(objective, get_space(), algo=tpe.suggest, max_evals=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ent_coef': 0.014722305311628495,\n",
       " 'env_max_step': 2,\n",
       " 'env_rwd_func': 0,\n",
       " 'gae_lambda': 0.08179432834573155,\n",
       " 'learning_rate': 6,\n",
       " 'n_steps': 6}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ent_coef': 0.014722305311628495,\n",
       " 'env_max_step': 3500,\n",
       " 'env_rwd_func': 0,\n",
       " 'gae_lambda': 0.08179432834573155,\n",
       " 'learning_rate': 0.003,\n",
       " 'n_steps': 16}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "space_eval(get_space(), best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# {'ent_coef': 0.04728136562858354, 'env_max_step': 3500, 'env_num_stack': 4, 'env_rwd_func': 0, 'gae_lambda': 0.009744980806713911, 'gamma': 0.8606867601900862, 'learning_rate': 0.006, 'max_grad_norm': 13.564322735262257, 'n_steps': 128, 'vf_coef': 0.5746884979668675}\n",
    "# {'ent_coef': 0.048131320125026966, 'env_max_step': 3500, 'env_num_stack': 4, 'env_rwd_func': 3, 'gae_lambda': 0.9650094161425868, 'gamma': 0.7910967341909643, 'learning_rate': 0.0007, 'max_grad_norm': 7.697800855483016, 'n_steps': 8, 'vf_coef': 0.20991523532588405} trin it for more than 5mil\n",
    "# {'ent_coef': 0.013584894198168228, 'env_max_step': 1500, 'env_num_stack': 8, 'env_rwd_func': 2, 'gae_lambda': 0.631485162290132, 'gamma': 0.8164578775162806, 'learning_rate': 0.0009, 'max_grad_norm': 2.8580838035360085, 'n_steps': 5, 'vf_coef': 0.5761365542365204}\n",
    "# {'ent_coef': 0.0032490067827949387, 'env_max_step': 1500, 'env_num_stack': 8, 'env_rwd_func': 2, 'gae_lambda': 0.30487890025338343, 'gamma': 0.8075603609540954, 'learning_rate': 0.0009, 'max_grad_norm': 4.250884836635666, 'n_steps': 5, 'vf_coef': 0.3819627894410967}"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "33ffef3325e3a534c0147e772ed31e50103af8c12a21f8b2f2c8601a52e63a0b"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}