DL_comp4_16.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pygame 1.9.6\n",
      "Hello from the pygame community. https://www.pygame.org/contribute.html\n",
      "couldn't import doomish\n",
      "Couldn't import doom\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Input, Dense\n",
    "import numpy as np\n",
    "from threading import Thread, Lock\n",
    "from multiprocessing import cpu_count\n",
    "import copy\n",
    "\n",
    "import os\n",
    "os.environ[\"SDL_VIDEODRIVER\"] = \"dummy\"  # this line disable pop-out window\n",
    "from ple.games.flappybird import FlappyBird\n",
    "from ple import PLE\n",
    "# default use float32 in conda env\n",
    "# tf.keras.backend.set_floatx('float64')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 Physical GPUs, 1 Logical GPUs\n"
     ]
    }
   ],
   "source": [
    "# set visible GPU\n",
    "gpu_number = 0\n",
    "\n",
    "#set seed gpu_number\n",
    "seed = 2021\n",
    "\n",
    "gamma = 0.99\n",
    "update_interval = 5\n",
    "actor_lr = 0.0005\n",
    "critic_lr = 0.001\n",
    "save_model_episode = 100\n",
    "\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "if gpus:\n",
    "    try:\n",
    "        tf.config.experimental.set_visible_devices(gpus[gpu_number], 'GPU')\n",
    "        # Currently, memory growth needs to be the same across GPUs\n",
    "        for gpu in gpus:\n",
    "            tf.config.experimental.set_memory_growth(gpu, True)\n",
    "        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n",
    "    except RuntimeError as e:\n",
    "        # Memory growth must be set before GPUs have been initialized\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "CUR_EPISODE = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Actor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Actor:\n",
    "    def __init__(self, state_dim, action_dim):\n",
    "        self.state_dim = state_dim\n",
    "        self.action_dim = action_dim\n",
    "        self.model = self.create_model()\n",
    "        self.opt = tf.keras.optimizers.Adam(actor_lr)\n",
    "        self.entropy_beta = 0.01\n",
    "\n",
    "    def create_model(self):\n",
    "        return tf.keras.Sequential([\n",
    "            Input((self.state_dim,)),\n",
    "            Dense(64, activation='relu'),\n",
    "            Dense(8, activation='relu'),\n",
    "            Dense(self.action_dim, activation='softmax')\n",
    "        ])\n",
    "    \n",
    "    def compute_loss(self, actions, logits, advantages):\n",
    "        ce_loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "        actions = tf.cast(actions, tf.int32)\n",
    "        policy_loss = ce_loss(actions, logits, sample_weight=tf.stop_gradient(advantages))\n",
    "\n",
    "        entropy_loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n",
    "        entropy = entropy_loss(logits, logits)\n",
    "\n",
    "        # ppt page48 solution to pitfall:exploration\n",
    "        return policy_loss - self.entropy_beta * entropy \n",
    "\n",
    "    def TA_state(self, game):\n",
    "        bucket_range_per_feature = {\n",
    "            'next_next_pipe_bottom_y': 40,\n",
    "            'next_next_pipe_dist_to_player': 512,\n",
    "            'next_next_pipe_top_y': 40,\n",
    "            'next_pipe_bottom_y': 20,\n",
    "            'next_pipe_dist_to_player': 20,\n",
    "            'next_pipe_top_y': 20,\n",
    "            'player_vel': 4,\n",
    "            'player_y': 16\n",
    "        }\n",
    "        state = copy.deepcopy(game.getGameState())\n",
    "        \n",
    "        state['next_next_pipe_bottom_y'] -= state['player_y']\n",
    "        state['next_next_pipe_top_y'] -= state['player_y']\n",
    "        state['next_pipe_bottom_y'] -= state['player_y']\n",
    "        state['next_pipe_top_y'] -= state['player_y']\n",
    "\n",
    "# =============================================================================\n",
    "#         state_key = [k for k, v in sorted(state.items())]\n",
    "#         for key in state_key:\n",
    "#             state[key] = int(state[key] / bucket_range_per_feature[key])\n",
    "# =============================================================================\n",
    "            \n",
    "        relative_state = list(state.values())\n",
    "\n",
    "\n",
    "        # return the state in tensor type, with batch dimension\n",
    "        relative_state = tf.convert_to_tensor(relative_state, dtype=tf.float32)\n",
    "        relative_state = tf.expand_dims(relative_state, axis=0)\n",
    "        \n",
    "        return relative_state\n",
    "    \n",
    "    def train(self, states, actions, advantages):\n",
    "        with tf.GradientTape() as tape:\n",
    "            logits = self.model(states, training=True)\n",
    "            loss = self.compute_loss(actions, logits, advantages)\n",
    "        grads = tape.gradient(loss, self.model.trainable_variables)\n",
    "        self.opt.apply_gradients(zip(grads, self.model.trainable_variables))\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Critic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Critic:\n",
    "    def __init__(self, state_dim):\n",
    "        self.state_dim = state_dim\n",
    "        self.model = self.create_model()\n",
    "        self.opt = tf.keras.optimizers.Adam(critic_lr)\n",
    "    \n",
    "    def create_model(self):\n",
    "        return tf.keras.Sequential([\n",
    "            Input((self.state_dim,)),\n",
    "            Dense(64, activation='relu'),\n",
    "            Dense(64, activation='relu'),\n",
    "            Dense(8, activation='relu'),\n",
    "            Dense(1, activation='linear')\n",
    "        ])\n",
    "\n",
    "    def compute_loss(self, v_pred, td_targets):\n",
    "        # ppt page47 update fV_pi\n",
    "        mse = tf.keras.losses.MeanSquaredError()\n",
    "        return mse(td_targets, v_pred)\n",
    "\n",
    "    def train(self, states, td_targets):\n",
    "        with tf.GradientTape() as tape:\n",
    "            v_pred = self.model(states, training=True)\n",
    "            assert v_pred.shape == td_targets.shape\n",
    "            loss = self.compute_loss(v_pred, tf.stop_gradient(td_targets))\n",
    "        grads = tape.gradient(loss, self.model.trainable_variables)\n",
    "        self.opt.apply_gradients(zip(grads, self.model.trainable_variables))\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent:\n",
    "    def __init__(self):\n",
    "        game = FlappyBird()\n",
    "        env = PLE(game, fps=30, display_screen=False, rng=seed)  # game environment interface\n",
    "        env.reset_game()\n",
    "\n",
    "        self.state_dim = len(self.TA_state(game)[0])\n",
    "        self.action_dim = len(env.getActionSet()) # number of actions\n",
    "\n",
    "        self.global_actor = Actor(self.state_dim, self.action_dim)\n",
    "        self.global_critic = Critic(self.state_dim)\n",
    "        self.num_workers = cpu_count() # 16 for R7-5800X\n",
    "\n",
    "    def TA_state(self, game):\n",
    "        bucket_range_per_feature = {\n",
    "            'next_next_pipe_bottom_y': 40,\n",
    "            'next_next_pipe_dist_to_player': 512,\n",
    "            'next_next_pipe_top_y': 40,\n",
    "            'next_pipe_bottom_y': 20,\n",
    "            'next_pipe_dist_to_player': 20,\n",
    "            'next_pipe_top_y': 20,\n",
    "            'player_vel': 4,\n",
    "            'player_y': 16\n",
    "        }\n",
    "        state = copy.deepcopy(game.getGameState())\n",
    "        \n",
    "        state['next_next_pipe_bottom_y'] -= state['player_y']\n",
    "        state['next_next_pipe_top_y'] -= state['player_y']\n",
    "        state['next_pipe_bottom_y'] -= state['player_y']\n",
    "        state['next_pipe_top_y'] -= state['player_y']\n",
    "\n",
    "# =============================================================================\n",
    "#         state_key = [k for k, v in sorted(state.items())]\n",
    "#         for key in state_key:\n",
    "#             state[key] = int(state[key] / bucket_range_per_feature[key])\n",
    "# =============================================================================\n",
    "            \n",
    "        relative_state = list(state.values())\n",
    "\n",
    "\n",
    "        # return the state in tensor type, with batch dimension\n",
    "        relative_state = tf.convert_to_tensor(relative_state, dtype=tf.float32)\n",
    "        relative_state = tf.expand_dims(relative_state, axis=0)\n",
    "        \n",
    "        return relative_state\n",
    "\n",
    "    def train(self, max_episodes=20000):\n",
    "        workers = []\n",
    "\n",
    "        for _ in range(self.num_workers):\n",
    "            game = FlappyBird()\n",
    "            env = PLE(game, fps=30, display_screen=False, rng=seed)  # game environment interface\n",
    "            env.reset_game()\n",
    "\n",
    "            workers.append(WorkerAgent(game, env, self.global_actor, self.global_critic, max_episodes))\n",
    "            \n",
    "        for worker in workers:\n",
    "            worker.start()\n",
    "        \n",
    "        for worker in workers:\n",
    "            worker.join()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Worker Thread"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class WorkerAgent(Thread):\n",
    "    def __init__(self, game, env, global_actor, global_critic, max_episodes):\n",
    "        Thread.__init__(self)\n",
    "        self.lock = Lock()\n",
    "        self.game = game\n",
    "        self.env = env\n",
    "        self.state_dim = len(self.TA_state(self.game)[0])\n",
    "        self.action_dim = len(self.env.getActionSet())\n",
    "\n",
    "        self.max_episodes = max_episodes\n",
    "        self.global_actor = global_actor\n",
    "        self.global_critic = global_critic\n",
    "        self.actor = Actor(self.state_dim, self.action_dim)\n",
    "        self.critic = Critic(self.state_dim)\n",
    "\n",
    "        self.actor.model.set_weights(self.global_actor.model.get_weights())\n",
    "        self.critic.model.set_weights(self.global_critic.model.get_weights())\n",
    "\n",
    "    def n_step_td_target(self, rewards, next_v_value, done):\n",
    "        td_targets = np.zeros_like(rewards)\n",
    "        cumulative = 0\n",
    "        if not done:\n",
    "            cumulative = next_v_value # estimate of fVpi(t+1)\n",
    "\n",
    "        for k in reversed(range(0, len(rewards))):\n",
    "            cumulative = gamma * cumulative + rewards[k] # ppt page 47 紅字, estimate fQpi\n",
    "            td_targets[k] = cumulative\n",
    "        return td_targets\n",
    "\n",
    "    def list_to_batch(self, list):\n",
    "        batch = list[0]\n",
    "        for elem in list[1:]:\n",
    "            batch = np.append(batch, elem, axis=0)\n",
    "        return batch\n",
    "\n",
    "    def TA_state(self, game):\n",
    "        bucket_range_per_feature = {\n",
    "            'next_next_pipe_bottom_y': 40,\n",
    "            'next_next_pipe_dist_to_player': 512,\n",
    "            'next_next_pipe_top_y': 40,\n",
    "            'next_pipe_bottom_y': 20,\n",
    "            'next_pipe_dist_to_player': 20,\n",
    "            'next_pipe_top_y': 20,\n",
    "            'player_vel': 4,\n",
    "            'player_y': 16\n",
    "        }\n",
    "        state = copy.deepcopy(game.getGameState())\n",
    "        \n",
    "        state['next_next_pipe_bottom_y'] -= state['player_y']\n",
    "        state['next_next_pipe_top_y'] -= state['player_y']\n",
    "        state['next_pipe_bottom_y'] -= state['player_y']\n",
    "        state['next_pipe_top_y'] -= state['player_y']\n",
    "\n",
    "\n",
    "        state_key = [k for k, v in sorted(state.items())]\n",
    "        for key in state_key:\n",
    "            state[key] = int(state[key] / bucket_range_per_feature[key])\n",
    "\n",
    "        relative_state = list(state.values())\n",
    "\n",
    "\n",
    "        # return the state in tensor type, with batch dimension\n",
    "        relative_state = tf.convert_to_tensor(relative_state, dtype=tf.float32)\n",
    "        relative_state = tf.expand_dims(relative_state, axis=0)\n",
    "        \n",
    "        return relative_state\n",
    "\n",
    "    def train(self):\n",
    "        global CUR_EPISODE\n",
    "\n",
    "        while CUR_EPISODE < self.max_episodes:\n",
    "            state_batch = []\n",
    "            action_batch = []\n",
    "            reward_batch = []\n",
    "            episode_reward, done = 0, False\n",
    "            \n",
    "            # Reset the environment\n",
    "            self.env.reset_game()\n",
    "            state = self.TA_state(self.game)\n",
    "\n",
    "            \n",
    "            while not done:\n",
    "                probs = self.actor.model.predict(\n",
    "                    np.reshape(state, [1, self.state_dim]))\n",
    "                action = np.random.choice(self.action_dim,p=probs[0])\n",
    "                reward = self.env.act(self.env.getActionSet()[action])\n",
    "                done = self.env.game_over()\n",
    "\n",
    "                next_state = self.TA_state(self.game)  # get next state\n",
    "                state = np.reshape(state, [1, self.state_dim])\n",
    "                action = np.reshape(action, [1, 1])\n",
    "                next_state = np.reshape(next_state, [1, self.state_dim])\n",
    "                reward = np.reshape(reward, [1, 1])\n",
    "                \n",
    "                state_batch.append(state)\n",
    "                action_batch.append(action)\n",
    "                reward_batch.append(reward)\n",
    "\n",
    "                if(len(state_batch) >= update_interval or done):\n",
    "                    states = self.list_to_batch(state_batch)\n",
    "                    actions = self.list_to_batch(action_batch)\n",
    "                    rewards = self.list_to_batch(reward_batch)\n",
    "\n",
    "                    next_v_value = self.critic.model.predict(next_state) # fVpi(t+1)\n",
    "                    td_targets = self.n_step_td_target(rewards, next_v_value, done)\n",
    "                    advantages = td_targets - self.critic.model.predict(states)\n",
    "\n",
    "                    with self.lock:\n",
    "                        actor_loss = self.global_actor.train(\n",
    "                            states, actions, advantages)\n",
    "                        critic_loss = self.global_critic.train(\n",
    "                            states, td_targets)\n",
    "\n",
    "                        self.actor.model.set_weights(\n",
    "                            self.global_actor.model.get_weights())\n",
    "                        self.critic.model.set_weights(\n",
    "                            self.global_critic.model.get_weights())\n",
    "\n",
    "                    state_batch = []\n",
    "                    action_batch = []\n",
    "                    reward_batch = []\n",
    "                    # td_target_batch = []\n",
    "                    # advatnage_batch = []\n",
    "\n",
    "                episode_reward += reward[0][0]\n",
    "                state = next_state[0]\n",
    "            \n",
    "            if CUR_EPISODE % save_model_episode == 0:\n",
    "                self.global_actor.model.save(\"models/ep%d\"%CUR_EPISODE)\n",
    "\n",
    "            print('EP{} EpisodeReward={}'.format(CUR_EPISODE, episode_reward))\n",
    "            CUR_EPISODE += 1\n",
    "\n",
    "    def run(self):\n",
    "        self.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Start Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:5 out of the last 5 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CCF0FF558> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:6 out of the last 6 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CCF403948> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:7 out of the last 7 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CD0BF6828> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:8 out of the last 8 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CD072D4C8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:9 out of the last 9 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CCF0F0B88> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:10 out of the last 10 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CC9154318> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CCF0FFDC8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CC9124DC8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CC91243A8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CD06ED288> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CD20E1EE8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CD06EDE58> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:5 out of the last 69 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CE47F6678> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:6 out of the last 70 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CED1129D8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:7 out of the last 71 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CF1704798> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:8 out of the last 72 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CDF324EE8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:9 out of the last 73 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CED1698B8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:10 out of the last 74 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CDF34BF78> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:11 out of the last 75 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CE419BC18> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CBC99DEE8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CF1721B88> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:11 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CE41C30D8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:10 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CF1704A68> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:10 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x0000026CE47F6558> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: models/ep0\\assets\n",
      "EP0 EpisodeReward=-5.0\n",
      "WARNING:tensorflow:5 out of the last 5 calls to <function LayerCall.__call__ at 0x0000026D0FD3F558> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:6 out of the last 6 calls to <function LayerCall.__call__ at 0x0000026D0FD3F558> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:7 out of the last 8 calls to <function LayerCall.__call__ at 0x0000026D0FD3F558> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:8 out of the last 9 calls to <function LayerCall.__call__ at 0x0000026D0FD3FE58> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-16:\n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\threading.py\", line 926, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"<ipython-input-7-c821059f13f0>\", line 135, in run\n",
      "    self.train()\n",
      "  File \"<ipython-input-7-c821059f13f0>\", line 110, in train\n",
      "    states, actions, advantages)\n",
      "  File \"<ipython-input-4-d3d9c2d9ef9b>\", line 63, in train\n",
      "    logits = self.model(states, training=True)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\base_layer.py\", line 1012, in __call__\n",
      "    outputs = call_fn(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\sequential.py\", line 375, in call\n",
      "    return super(Sequential, self).call(inputs, training=training, mask=mask)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\functional.py\", line 425, in call\n",
      "    inputs, training=training, mask=mask)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\functional.py\", line 560, in _run_internal_graph\n",
      "    outputs = node.layer(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\base_layer.py\", line 1012, in __call__\n",
      "    outputs = call_fn(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\utils.py\", line 73, in return_outputs_and_add_losses\n",
      "    outputs, losses = fn(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\save_impl.py\", line 545, in __call__\n",
      "    return super(LayerCall, self).__call__(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 828, in __call__\n",
      "    result = self._call(*args, **kwds)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 862, in _call\n",
      "    results = self._stateful_fn(*args, **kwds)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 2943, in __call__\n",
      "    filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1924, in _call_flat\n",
      "    forward_function, args_with_tangents = forward_backward.forward()\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1448, in forward\n",
      "    self._inference_args, self._input_tangents)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1207, in forward\n",
      "    self._forward_and_backward_functions(inference_args, input_tangents))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1359, in _forward_and_backward_functions\n",
      "    outputs, inference_args, input_tangents)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 910, in _build_functions_for_outputs\n",
      "    src_graph=self._func_graph)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in _GradientsHelper\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 340, in _MaybeCompile\n",
      "    return grad_fn()  # Exit early\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in <lambda>\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 722, in _rewrite_forward_and_call_backward\n",
      "    forward_function, backwards_function = self.forward_backward(len(doutputs))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 631, in forward_backward\n",
      "    forward, backward = self._construct_forward_backward(num_doutputs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 718, in _construct_forward_backward\n",
      "    return forward_function, backward_function\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\contextlib.py\", line 119, in __exit__\n",
      "    next(self.gen)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\framework\\func_graph.py\", line 414, in inner_cm\n",
      "    for fn in self._scope_exit_callbacks:\n",
      "TypeError: 'NoneType' object is not iterable\n",
      "\n",
      "Exception in thread Thread-7:\n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\threading.py\", line 926, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"<ipython-input-7-c821059f13f0>\", line 135, in run\n",
      "    self.train()\n",
      "  File \"<ipython-input-7-c821059f13f0>\", line 110, in train\n",
      "    states, actions, advantages)\n",
      "  File \"<ipython-input-4-d3d9c2d9ef9b>\", line 63, in train\n",
      "    logits = self.model(states, training=True)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\base_layer.py\", line 1012, in __call__\n",
      "    outputs = call_fn(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\sequential.py\", line 375, in call\n",
      "    return super(Sequential, self).call(inputs, training=training, mask=mask)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\functional.py\", line 425, in call\n",
      "    inputs, training=training, mask=mask)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\functional.py\", line 560, in _run_internal_graph\n",
      "    outputs = node.layer(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\base_layer.py\", line 1012, in __call__\n",
      "    outputs = call_fn(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\utils.py\", line 73, in return_outputs_and_add_losses\n",
      "    outputs, losses = fn(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\save_impl.py\", line 545, in __call__\n",
      "    return super(LayerCall, self).__call__(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 828, in __call__\n",
      "    result = self._call(*args, **kwds)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 862, in _call\n",
      "    results = self._stateful_fn(*args, **kwds)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 2943, in __call__\n",
      "    filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1924, in _call_flat\n",
      "    forward_function, args_with_tangents = forward_backward.forward()\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1448, in forward\n",
      "    self._inference_args, self._input_tangents)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1207, in forward\n",
      "    self._forward_and_backward_functions(inference_args, input_tangents))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1359, in _forward_and_backward_functions\n",
      "    outputs, inference_args, input_tangents)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 910, in _build_functions_for_outputs\n",
      "    src_graph=self._func_graph)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in _GradientsHelper\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 340, in _MaybeCompile\n",
      "    return grad_fn()  # Exit early\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in <lambda>\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 722, in _rewrite_forward_and_call_backward\n",
      "    forward_function, backwards_function = self.forward_backward(len(doutputs))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 631, in forward_backward\n",
      "    forward, backward = self._construct_forward_backward(num_doutputs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 679, in _construct_forward_backward\n",
      "    func_graph=backwards_graph)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\framework\\func_graph.py\", line 990, in func_graph_from_py_func\n",
      "    func_outputs = python_func(*func_args, **func_kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 669, in _backprop_function\n",
      "    src_graph=self._func_graph)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in _GradientsHelper\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 340, in _MaybeCompile\n",
      "    return grad_fn()  # Exit early\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in <lambda>\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 722, in _rewrite_forward_and_call_backward\n",
      "    forward_function, backwards_function = self.forward_backward(len(doutputs))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 631, in forward_backward\n",
      "    forward, backward = self._construct_forward_backward(num_doutputs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 718, in _construct_forward_backward\n",
      "    return forward_function, backward_function\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\contextlib.py\", line 119, in __exit__\n",
      "    next(self.gen)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\framework\\func_graph.py\", line 414, in inner_cm\n",
      "    for fn in self._scope_exit_callbacks:\n",
      "TypeError: 'NoneType' object is not iterable\n",
      "\n",
      "Exception in thread Thread-13:\n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\threading.py\", line 926, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"<ipython-input-7-c821059f13f0>\", line 135, in run\n",
      "    self.train()\n",
      "  File \"<ipython-input-7-c821059f13f0>\", line 110, in train\n",
      "    states, actions, advantages)\n",
      "  File \"<ipython-input-4-d3d9c2d9ef9b>\", line 63, in train\n",
      "    logits = self.model(states, training=True)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\base_layer.py\", line 1012, in __call__\n",
      "    outputs = call_fn(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\sequential.py\", line 375, in call\n",
      "    return super(Sequential, self).call(inputs, training=training, mask=mask)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\functional.py\", line 425, in call\n",
      "    inputs, training=training, mask=mask)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\functional.py\", line 560, in _run_internal_graph\n",
      "    outputs = node.layer(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\base_layer.py\", line 1012, in __call__\n",
      "    outputs = call_fn(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\utils.py\", line 73, in return_outputs_and_add_losses\n",
      "    outputs, losses = fn(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\save_impl.py\", line 544, in __call__\n",
      "    self.call_collection.add_trace(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\save_impl.py\", line 423, in add_trace\n",
      "    fn.get_concrete_function(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\save_impl.py\", line 550, in get_concrete_function\n",
      "    return super(LayerCall, self).get_concrete_function(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 1299, in get_concrete_function\n",
      "    concrete = self._get_concrete_function_garbage_collected(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 1205, in _get_concrete_function_garbage_collected\n",
      "    self._initialize(args, kwargs, add_initializers_to=initializers)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 726, in _initialize\n",
      "    *args, **kwds))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 2969, in _get_concrete_function_internal_garbage_collected\n",
      "    graph_function, _ = self._maybe_define_function(args, kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 3361, in _maybe_define_function\n",
      "    graph_function = self._create_graph_function(args, kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 3206, in _create_graph_function\n",
      "    capture_by_value=self._capture_by_value),\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\framework\\func_graph.py\", line 990, in func_graph_from_py_func\n",
      "    func_outputs = python_func(*func_args, **func_kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 634, in wrapped_fn\n",
      "    out = weak_wrapped_fn().__wrapped__(*args, **kwds)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\save_impl.py\", line 527, in wrapper\n",
      "    ret = method(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\save_impl.py\", line 587, in call\n",
      "    return call_and_return_conditional_losses(inputs, *args, **kwargs)[0]\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\save_impl.py\", line 545, in __call__\n",
      "    return super(LayerCall, self).__call__(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 828, in __call__\n",
      "    result = self._call(*args, **kwds)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 862, in _call\n",
      "    results = self._stateful_fn(*args, **kwds)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 2941, in __call__\n",
      "    filtered_flat_args) = self._maybe_define_function(args, kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 3361, in _maybe_define_function\n",
      "    graph_function = self._create_graph_function(args, kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 3206, in _create_graph_function\n",
      "    capture_by_value=self._capture_by_value),\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\framework\\func_graph.py\", line 990, in func_graph_from_py_func\n",
      "    func_outputs = python_func(*func_args, **func_kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 634, in wrapped_fn\n",
      "    out = weak_wrapped_fn().__wrapped__(*args, **kwds)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\save_impl.py\", line 527, in wrapper\n",
      "    ret = method(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\save_impl.py\", line 570, in call_and_return_conditional_losses\n",
      "    call_output = layer_call(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\utils.py\", line 73, in return_outputs_and_add_losses\n",
      "    outputs, losses = fn(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\save_impl.py\", line 545, in __call__\n",
      "    return super(LayerCall, self).__call__(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 828, in __call__\n",
      "    result = self._call(*args, **kwds)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 862, in _call\n",
      "    results = self._stateful_fn(*args, **kwds)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 2943, in __call__\n",
      "    filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1932, in _call_flat\n",
      "    flat_outputs = forward_function.call(ctx, args_with_tangents)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 589, in call\n",
      "    executor_type=executor_type)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\functional_ops.py\", line 1206, in partitioned_call\n",
      "    f.add_to_graph(graph)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 506, in add_to_graph\n",
      "    for f in self.graph._functions.values():\n",
      "RuntimeError: OrderedDict mutated during iteration\n",
      "\n",
      "Exception in thread Thread-21:\n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\framework\\func_graph.py\", line 411, in inner_cm\n",
      "    yield g\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 679, in _construct_forward_backward\n",
      "    func_graph=backwards_graph)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\framework\\func_graph.py\", line 990, in func_graph_from_py_func\n",
      "    func_outputs = python_func(*func_args, **func_kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 669, in _backprop_function\n",
      "    src_graph=self._func_graph)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in _GradientsHelper\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 340, in _MaybeCompile\n",
      "    return grad_fn()  # Exit early\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in <lambda>\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 722, in _rewrite_forward_and_call_backward\n",
      "    forward_function, backwards_function = self.forward_backward(len(doutputs))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 631, in forward_backward\n",
      "    forward, backward = self._construct_forward_backward(num_doutputs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 718, in _construct_forward_backward\n",
      "    return forward_function, backward_function\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\contextlib.py\", line 119, in __exit__\n",
      "    next(self.gen)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\framework\\func_graph.py\", line 414, in inner_cm\n",
      "    for fn in self._scope_exit_callbacks:\n",
      "TypeError: 'NoneType' object is not iterable\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\threading.py\", line 926, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"<ipython-input-7-c821059f13f0>\", line 135, in run\n",
      "    self.train()\n",
      "  File \"<ipython-input-7-c821059f13f0>\", line 110, in train\n",
      "    states, actions, advantages)\n",
      "  File \"<ipython-input-4-d3d9c2d9ef9b>\", line 63, in train\n",
      "    logits = self.model(states, training=True)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\base_layer.py\", line 1012, in __call__\n",
      "    outputs = call_fn(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\sequential.py\", line 375, in call\n",
      "    return super(Sequential, self).call(inputs, training=training, mask=mask)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\functional.py\", line 425, in call\n",
      "    inputs, training=training, mask=mask)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\functional.py\", line 560, in _run_internal_graph\n",
      "    outputs = node.layer(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\base_layer.py\", line 1012, in __call__\n",
      "    outputs = call_fn(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\utils.py\", line 73, in return_outputs_and_add_losses\n",
      "    outputs, losses = fn(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\save_impl.py\", line 545, in __call__\n",
      "    return super(LayerCall, self).__call__(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 828, in __call__\n",
      "    result = self._call(*args, **kwds)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 862, in _call\n",
      "    results = self._stateful_fn(*args, **kwds)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 2943, in __call__\n",
      "    filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1924, in _call_flat\n",
      "    forward_function, args_with_tangents = forward_backward.forward()\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1448, in forward\n",
      "    self._inference_args, self._input_tangents)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1207, in forward\n",
      "    self._forward_and_backward_functions(inference_args, input_tangents))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1359, in _forward_and_backward_functions\n",
      "    outputs, inference_args, input_tangents)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 910, in _build_functions_for_outputs\n",
      "    src_graph=self._func_graph)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in _GradientsHelper\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 340, in _MaybeCompile\n",
      "    return grad_fn()  # Exit early\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in <lambda>\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 722, in _rewrite_forward_and_call_backward\n",
      "    forward_function, backwards_function = self.forward_backward(len(doutputs))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 631, in forward_backward\n",
      "    forward, backward = self._construct_forward_backward(num_doutputs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 679, in _construct_forward_backward\n",
      "    func_graph=backwards_graph)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\framework\\func_graph.py\", line 990, in func_graph_from_py_func\n",
      "    func_outputs = python_func(*func_args, **func_kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 669, in _backprop_function\n",
      "    src_graph=self._func_graph)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in _GradientsHelper\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 340, in _MaybeCompile\n",
      "    return grad_fn()  # Exit early\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in <lambda>\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 722, in _rewrite_forward_and_call_backward\n",
      "    forward_function, backwards_function = self.forward_backward(len(doutputs))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 631, in forward_backward\n",
      "    forward, backward = self._construct_forward_backward(num_doutputs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 718, in _construct_forward_backward\n",
      "    return forward_function, backward_function\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\contextlib.py\", line 130, in __exit__\n",
      "    self.gen.throw(type, value, traceback)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\framework\\func_graph.py\", line 414, in inner_cm\n",
      "    for fn in self._scope_exit_callbacks:\n",
      "TypeError: 'NoneType' object is not iterable\n",
      "\n",
      "Exception in thread Thread-20:\n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\framework\\func_graph.py\", line 411, in inner_cm\n",
      "    yield g\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 679, in _construct_forward_backward\n",
      "    func_graph=backwards_graph)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\framework\\func_graph.py\", line 990, in func_graph_from_py_func\n",
      "    func_outputs = python_func(*func_args, **func_kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 669, in _backprop_function\n",
      "    src_graph=self._func_graph)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in _GradientsHelper\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 340, in _MaybeCompile\n",
      "    return grad_fn()  # Exit early\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in <lambda>\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 722, in _rewrite_forward_and_call_backward\n",
      "    forward_function, backwards_function = self.forward_backward(len(doutputs))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 631, in forward_backward\n",
      "    forward, backward = self._construct_forward_backward(num_doutputs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 718, in _construct_forward_backward\n",
      "    return forward_function, backward_function\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\contextlib.py\", line 119, in __exit__\n",
      "    next(self.gen)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\framework\\func_graph.py\", line 414, in inner_cm\n",
      "    for fn in self._scope_exit_callbacks:\n",
      "TypeError: 'NoneType' object is not iterable\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\threading.py\", line 926, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"<ipython-input-7-c821059f13f0>\", line 135, in run\n",
      "    self.train()\n",
      "  File \"<ipython-input-7-c821059f13f0>\", line 110, in train\n",
      "    states, actions, advantages)\n",
      "  File \"<ipython-input-4-d3d9c2d9ef9b>\", line 63, in train\n",
      "    logits = self.model(states, training=True)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\base_layer.py\", line 1012, in __call__\n",
      "    outputs = call_fn(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\sequential.py\", line 375, in call\n",
      "    return super(Sequential, self).call(inputs, training=training, mask=mask)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\functional.py\", line 425, in call\n",
      "    inputs, training=training, mask=mask)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\functional.py\", line 560, in _run_internal_graph\n",
      "    outputs = node.layer(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\base_layer.py\", line 1012, in __call__\n",
      "    outputs = call_fn(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\utils.py\", line 73, in return_outputs_and_add_losses\n",
      "    outputs, losses = fn(inputs, *args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\keras\\saving\\saved_model\\save_impl.py\", line 545, in __call__\n",
      "    return super(LayerCall, self).__call__(*args, **kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 828, in __call__\n",
      "    result = self._call(*args, **kwds)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 862, in _call\n",
      "    results = self._stateful_fn(*args, **kwds)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 2943, in __call__\n",
      "    filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1924, in _call_flat\n",
      "    forward_function, args_with_tangents = forward_backward.forward()\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1448, in forward\n",
      "    self._inference_args, self._input_tangents)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1207, in forward\n",
      "    self._forward_and_backward_functions(inference_args, input_tangents))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1359, in _forward_and_backward_functions\n",
      "    outputs, inference_args, input_tangents)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 910, in _build_functions_for_outputs\n",
      "    src_graph=self._func_graph)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in _GradientsHelper\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 340, in _MaybeCompile\n",
      "    return grad_fn()  # Exit early\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in <lambda>\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 722, in _rewrite_forward_and_call_backward\n",
      "    forward_function, backwards_function = self.forward_backward(len(doutputs))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 631, in forward_backward\n",
      "    forward, backward = self._construct_forward_backward(num_doutputs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 679, in _construct_forward_backward\n",
      "    func_graph=backwards_graph)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\framework\\func_graph.py\", line 990, in func_graph_from_py_func\n",
      "    func_outputs = python_func(*func_args, **func_kwargs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 669, in _backprop_function\n",
      "    src_graph=self._func_graph)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in _GradientsHelper\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 340, in _MaybeCompile\n",
      "    return grad_fn()  # Exit early\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\ops\\gradients_util.py\", line 684, in <lambda>\n",
      "    lambda: grad_fn(op, *out_grads))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 722, in _rewrite_forward_and_call_backward\n",
      "    forward_function, backwards_function = self.forward_backward(len(doutputs))\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 631, in forward_backward\n",
      "    forward, backward = self._construct_forward_backward(num_doutputs)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\", line 718, in _construct_forward_backward\n",
      "    return forward_function, backward_function\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\contextlib.py\", line 130, in __exit__\n",
      "    self.gen.throw(type, value, traceback)\n",
      "  File \"c:\\users\\user\\anaconda3\\envs\\tf2\\lib\\site-packages\\tensorflow\\python\\framework\\func_graph.py\", line 414, in inner_cm\n",
      "    for fn in self._scope_exit_callbacks:\n",
      "TypeError: 'NoneType' object is not iterable\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:9 out of the last 11 calls to <function LayerCall.__call__ at 0x0000026D0FD3FE58> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:8 out of the last 11 calls to <function LayerCall.__call__ at 0x0000026D0FD3FE58> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:8 out of the last 11 calls to <function LayerCall.__call__ at 0x0000026D0FD3FE58> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "WARNING:tensorflow:7 out of the last 11 calls to <function LayerCall.__call__ at 0x0000026D0FA38DC8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "EP1 EpisodeReward=-5.0\n",
      "EP2 EpisodeReward=-5.0\n",
      "WARNING:tensorflow:8 out of the last 12 calls to <function LayerCall.__call__ at 0x0000026D0FA38DC8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "EP3 EpisodeReward=-5.0\n",
      "WARNING:tensorflow:7 out of the last 12 calls to <function LayerCall.__call__ at 0x0000026CDDB68048> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "EP4 EpisodeReward=-5.0\n",
      "EP5 EpisodeReward=-5.0\n",
      "INFO:tensorflow:Assets written to: models/ep0\\assets\n",
      "EP6 EpisodeReward=-5.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:Found untraced functions such as dense_layer_call_fn, dense_layer_call_fn while saving (showing 2 of 2). These functions will not be directly callable after loading.\n",
      "WARNING:absl:Found untraced functions such as dense_layer_call_fn, dense_layer_call_fn while saving (showing 2 of 2). These functions will not be directly callable after loading.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: models/ep0\\assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: models/ep0\\assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EP7 EpisodeReward=-5.0\n",
      "INFO:tensorflow:Assets written to: models/ep0\\assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: models/ep0\\assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EP8 EpisodeReward=-5.0\n",
      "EP9 EpisodeReward=-5.0\n",
      "EP10 EpisodeReward=-5.0\n",
      "EP11 EpisodeReward=-5.0\n",
      "EP12 EpisodeReward=-5.0\n",
      "EP13 EpisodeReward=-5.0\n",
      "EP14 EpisodeReward=-5.0\n",
      "EP15 EpisodeReward=-5.0\n",
      "EP16 EpisodeReward=-5.0\n",
      "EP17 EpisodeReward=-5.0\n",
      "EP18 EpisodeReward=-5.0\n",
      "EP19 EpisodeReward=-5.0\n",
      "EP20 EpisodeReward=-5.0\n",
      "EP21 EpisodeReward=-5.0\n",
      "EP22 EpisodeReward=-5.0\n",
      "EP23 EpisodeReward=-5.0\n",
      "EP24 EpisodeReward=-5.0\n",
      "EP25 EpisodeReward=-5.0\n",
      "EP26 EpisodeReward=-5.0\n",
      "EP27 EpisodeReward=-5.0\n",
      "EP28 EpisodeReward=-5.0\n",
      "EP29 EpisodeReward=-5.0\n",
      "EP30 EpisodeReward=-5.0\n",
      "EP31 EpisodeReward=-5.0\n",
      "EP32 EpisodeReward=-5.0\n",
      "EP33 EpisodeReward=-5.0\n",
      "EP34 EpisodeReward=-5.0\n",
      "EP35 EpisodeReward=-5.0\n",
      "EP36 EpisodeReward=-5.0\n",
      "EP37 EpisodeReward=-5.0\n",
      "EP38 EpisodeReward=-5.0\n",
      "EP39 EpisodeReward=-5.0\n",
      "EP40 EpisodeReward=-5.0\n",
      "EP41 EpisodeReward=-5.0\n",
      "EP42 EpisodeReward=-5.0\n",
      "EP43 EpisodeReward=-5.0\n",
      "EP44 EpisodeReward=-5.0\n",
      "EP45 EpisodeReward=-5.0\n",
      "EP46 EpisodeReward=-5.0\n",
      "EP47 EpisodeReward=-5.0\n",
      "EP48 EpisodeReward=-5.0\n",
      "EP49 EpisodeReward=-5.0\n",
      "EP50 EpisodeReward=-5.0\n",
      "EP51 EpisodeReward=-5.0\n",
      "EP52 EpisodeReward=-5.0\n",
      "EP53 EpisodeReward=-5.0\n",
      "EP54 EpisodeReward=-5.0\n",
      "EP55 EpisodeReward=-5.0\n",
      "EP56 EpisodeReward=-5.0\n",
      "EP57 EpisodeReward=-5.0\n",
      "EP58 EpisodeReward=-5.0\n",
      "EP59 EpisodeReward=-5.0\n",
      "EP60 EpisodeReward=-5.0\n",
      "EP61 EpisodeReward=-5.0\n",
      "EP62 EpisodeReward=-5.0\n",
      "EP63 EpisodeReward=-5.0\n",
      "EP64 EpisodeReward=-5.0\n",
      "EP65 EpisodeReward=-5.0\n",
      "EP66 EpisodeReward=-5.0\n",
      "EP67 EpisodeReward=-5.0\n",
      "EP68 EpisodeReward=-5.0\n",
      "EP69 EpisodeReward=-5.0\n"
     ]
    }
   ],
   "source": [
    "agent = Agent()\n",
    "agent.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert the model to fit test environment\n",
    "During the training process, we feed in states with bucketing. However in our test environment, we do not feed states with bucketing. Hence, we have to manually adjust the input state to fit out models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Bucket(keras.layers.Layer):\n",
    "    def __init__(self):\n",
    "        super(Bucket, self).__init__()\n",
    "        self.bucket_range_per_feature = {\n",
    "            'next_next_pipe_bottom_y': 40,\n",
    "            'next_next_pipe_dist_to_player': 512,\n",
    "            'next_next_pipe_top_y': 40,\n",
    "            'next_pipe_bottom_y': 20,\n",
    "            'next_pipe_dist_to_player': 20,\n",
    "            'next_pipe_top_y': 20,\n",
    "            'player_vel': 4,\n",
    "            'player_y': 16\n",
    "        }\n",
    "        self.bucket_list = tf.constant([16,4,20,20,20,512,40,40])\n",
    "\n",
    "    def call(self, inputs):\n",
    "        \n",
    "\n",
    "        inputs = tf.cast(tf.cast(tf.math.divide(inputs, tf.cast(self.bucket_list, tf.float32)),tf.int32),tf.float32)\n",
    "        # print(inputs.values())\n",
    "\n",
    "        return inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Actor_save(tf.keras.Model):\n",
    "    def __init__(self, state_dim, action_dim):\n",
    "        super(Actor_save, self).__init__()\n",
    "        self.state_dim = state_dim\n",
    "        self.action_dim = action_dim\n",
    "        self.temp_model = self.create_temp_model()\n",
    "        inf_model = tf.keras.models.load_model(\"C:\\\\Users\\\\User\\\\Desktop\\\\comp4\\\\hand_bucket_model\\\\ep1100\", compile=False) #change the path to your model path\n",
    "        self.mid_model = self.create_model()S\n",
    "        self.mid_model.set_weights(inf_model.get_weights())\n",
    "        self.model = self.final_model()\n",
    "        self.opt = tf.keras.optimizers.Adam(actor_lr)\n",
    "        self.entropy_beta = 0.01\n",
    "\n",
    "\n",
    "    def create_temp_model(self):\n",
    "        input = Input(shape=(self.state_dim,))\n",
    "        output = Bucket()(input)\n",
    "        model = Model(inputs=input,outputs=output)\n",
    "        return model\n",
    "        # return tf.keras.Sequential([\n",
    "        #     Input((self.state_dim,)),\n",
    "        #     Bucket()\n",
    "        # ])\n",
    "    def create_model(self):\n",
    "        return tf.keras.Sequential([\n",
    "            Input(shape=(self.state_dim,)),\n",
    "            Dense(64, activation='relu'),\n",
    "            Dense(8, activation='relu'),\n",
    "            Dense(self.action_dim, activation='softmax')\n",
    "        ])\n",
    "    \n",
    "    def final_model(self):\n",
    "        input = Input(shape=(self.state_dim))\n",
    "        x = self.temp_model(input)\n",
    "        output = self.mid_model(x)\n",
    "        model = Model(input,output)\n",
    "        # return tf.keras.Sequential([\n",
    "        #         Input((self.state_dim,)),\n",
    "        #         self.temp_model(),\n",
    "        #         self.mid_model()\n",
    "        # ])\n",
    "        return model\n",
    "\n",
    "    def compute_loss(self, actions, logits, advantages):\n",
    "        ce_loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "        actions = tf.cast(actions, tf.int32)\n",
    "        policy_loss = ce_loss(actions, logits, sample_weight=tf.stop_gradient(advantages))\n",
    "\n",
    "        entropy_loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n",
    "        entropy = entropy_loss(logits, logits)\n",
    "\n",
    "        # ppt page48 solution to pitfall:exploration\n",
    "        return policy_loss - self.entropy_beta * entropy \n",
    "    def call(self, states):\n",
    "        x = self.model(states)\n",
    "        return x\n",
    "    \n",
    "    def TA_state(self, game):\n",
    "        state = copy.deepcopy(game.getGameState())\n",
    "        \n",
    "        state['next_next_pipe_bottom_y'] -= state['player_y']\n",
    "        state['next_next_pipe_top_y'] -= state['player_y']\n",
    "        state['next_pipe_bottom_y'] -= state['player_y']\n",
    "        state['next_pipe_top_y'] -= state['player_y']\n",
    "            \n",
    "        relative_state = list(state.values())\n",
    "\n",
    "\n",
    "        # return the state in tensor type, with batch dimension\n",
    "        relative_state = tf.convert_to_tensor(relative_state, dtype=tf.float32)\n",
    "        relative_state = tf.expand_dims(relative_state, axis=0)\n",
    "        \n",
    "        return relative_state\n",
    "    \n",
    "    def train(self, states, actions, advantages):\n",
    "        with tf.GradientTape() as tape:\n",
    "            logits = self.model(states, training=True)\n",
    "            loss = self.compute_loss(actions, logits, advantages)\n",
    "        grads = tape.gradient(loss, self.model.trainable_variables)\n",
    "        self.opt.apply_gradients(zip(grads, self.model.trainable_variables))\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "actor_save = Actor_save(8,2)\n",
    "input_array = tf.random.uniform((1,8))\n",
    "input = np.reshape(input_array, [1, 8])\n",
    "out = actor_save(input_array)\n",
    "actor_save.save(\"test/ep1100\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At first, I was considering A2C. In order to fasten the speed of training, I chose to implement A3C using multithreading. I took an online github repository for reference(it was trained for Cartpole).<br>\n",
    "Reference: https://github.com/marload/DeepRL-TensorFlow2 <br><br>\n",
    "Since the A3C model works very well, there is no need to try other models.<br><br>\n",
    "My A3C model has several hyperparameters, including gamma, update interval(determine how often a worker thread update its parameter to global agent), actor learning rate, critic learning rate, entropy beta. After tuing, I set gamma = 0.99, update_interval = 5, actor_lr = 0.0005, critic_lr = 0.001.<br>\n",
    "As for the optimizer, Adam is what I chose.<br><br>\n",
    "In the beginning, I did not put the values of state to the bucket. It turned out that it was not learning. Then I tried to throw the values in the bucket and it made progresses. I think it is beacuse after bucketing, the range of the input state decreases, and it makes the model to learn faster.<br><br>\n",
    "When I finally have to save the model, I discovered that the test environment passed the value without using bucket. Therefore I have to convert my original model to adjust to the test environment requirement. Before pass the state to the network, I first pass the state to a layer Bucket() to do bucketing. It took me several hours to solve this problem.<br><br>\n",
    "In this competition, I make good use of Python's threading to implement A3C. It is more difficult to debug, harder to track the weigths. It was a great experience for me."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}