ping_pong_rl.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0cee6a07",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import gym\n",
    "import matplotlib.pyplot as plt \n",
    "from tensorflow.keras.layers import Dense,Reshape,Conv2D,Flatten,InputLayer\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.optimizers import Adam "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a33f8fc8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\jupyternotebooks\\tfu\\lib\\site-packages\\gym\\envs\\registration.py:506: UserWarning: \u001b[33mWARN: The environment Pong-v0 is out of date. You should consider upgrading to version `v5` with the environment ID `ALE/Pong-v5`.\u001b[0m\n",
      "  f\"The environment {path} is out of date. You should consider \"\n",
      "D:\\jupyternotebooks\\tfu\\lib\\site-packages\\gym\\utils\\seeding.py:139: DeprecationWarning: \u001b[33mWARN: Function `hash_seed(seed, max_bytes)` is marked as deprecated and will be removed in the future. \u001b[0m\n",
      "  \"Function `hash_seed(seed, max_bytes)` is marked as deprecated and will be removed in the future. \"\n",
      "D:\\jupyternotebooks\\tfu\\lib\\site-packages\\gym\\utils\\seeding.py:176: DeprecationWarning: \u001b[33mWARN: Function `_bigint_from_bytes(bytes)` is marked as deprecated and will be removed in the future. \u001b[0m\n",
      "  \"Function `_bigint_from_bytes(bytes)` is marked as deprecated and will be removed in the future. \"\n"
     ]
    }
   ],
   "source": [
    "env=gym.make(\"Pong-v0\")\n",
    "observation=env.reset()\n",
    "prev_input=None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "69d094b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "up_action=2\n",
    "down_action=3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b3093550",
   "metadata": {},
   "outputs": [],
   "source": [
    "gamma=0.99"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "531f039b",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train,y_train,rewards=[],[],[]\n",
    "reward_sum=0\n",
    "episode_num=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "69e886db",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\jupyternotebooks\\tfu\\lib\\site-packages\\gym\\utils\\seeding.py:48: DeprecationWarning: \u001b[33mWARN: Function `rng.randint(low, [high, size, dtype])` is marked as deprecated and will be removed in the future. Please use `rng.integers(low, [high, size, dtype])` instead.\u001b[0m\n",
      "  \"Function `rng.randint(low, [high, size, dtype])` is marked as deprecated \"\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAM4AAAD8CAYAAAA/rZtiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAP7UlEQVR4nO3de4xc5X3G8e+Dr40NvmNcY2MbmQjIxRCLIBFoWhoCVhUDf1DTijgp6oKE2yClrQy0KYoUKU1DqKK2RKZYmIpyaQlgqYbiuqgQKRDbxAFzMdjYLt4Ym6wBG4zX9vrXP867y7DsesfvzPjMDM9HWu2c95wz53fW+3jeOTvzG0UEZnZsTii7ALNW5OCYZXBwzDI4OGYZHByzDA6OWYaGBUfSpZI2SdosaWmjjmNWBjXi7ziShgGvAl8BdgBrgasj4qW6H8ysBI16xDkP2BwRr0fEQeB+YGGDjmV23A1v0P1OB96oWN4BfHGwjSUd9WHv5DEnMGqY6lSaWXXe2Nvzm4iYMtC6RgVnSJI6gA6ACaPFd35n3FDbH4+y+pxz5plMGn/0mip1HzzI0+ufa2BFrevVK89j72mTq95+xL4DfP5f/qeBFVXnxsff3j7YukYFpxOYUbF8ahrrExHLgGUAM8cNj+MdjKFIxz+sbe1YfpYt8GNv1HOctcBcSbMljQQWASsbdCyz464hjzgRcVjSEuC/gGHA8oh4sRHHMitDw57jRMQqYFWj7v9429bZyfZf7+xbnjhuHJ89Y26JFbWuqWu3cMr6rX3Le2dOYuuCc0qs6NiVdnGg1fT0HOHgoUN9y4cOHy6xmtY27FAPI/Z39y0PP3DoKFs3J7/kxiyDg2OWwcExy+DgmGVwcMwyODhmGRwcswwOjlkGB8csg4NjlsEvuanSp35rNJPHj+9bPmns2PKKaXEHJozhndkfvj9s/9Tx5RWTycGp0rQpU5g2ZcA3A9ox2nPmdPacOb3sMmriqZpZBgfHLIOnaoM4eOgQB7q7h94w6T7Yei+NP16Gf3CQEfs+qHr7Ee9X/3Mvi4MziI2vbS67hLYx57ENZZdQd9lTNUkzJD0p6SVJL0r6Vhq/VVKnpA3pa0H9yjVrDrU84hwGvh0Rz0k6EVgvaXVad3tE/LDqe5I4YfiIGkoxO76ygxMRO4Gd6fY+SS9TNCI8ZhNnnc0f3b0mtxSzhvjzyYP3gqvLVTVJs4BzgGfT0BJJz0taLmlCPY5h1kxqDo6kscBDwI0RsRe4AzgdmEfxiHTbIPt1SFonaV1XV1etZZgdVzUFR9IIitDcGxE/BYiIXRHRExFHgDspGrB/TEQsi4j5ETF/0qRJtZRhdtzVclVNwF3AyxHxo4rxaRWbXQFszC/PrDnVclXtAuAa4AVJG9LYzcDVkuYBAWwDrqvhGGZNqZaraj9j4PbYbdO902wwfq2aWQYHxyyDg2OWoSle5Ln311t47G+uLLsMs6o1RXAOd39A19YXyi7DrGqeqpllcHDMMjg4ZhkcHLMMDo5ZBgfHLIODY5bBwTHL4OCYZXBwzDI4OGYZHByzDDW/yFPSNmAf0AMcjoj5kiYCDwCzKN4+fVVEvF3rscyaRb0ecX43IuZFxPy0vBRYExFzgTVp2axtNGqqthBYkW6vAC5v0HHMSlGP4ATwhKT1kjrS2NTUIhfgTWBqHY5j1jTq8Ua2L0VEp6STgdWSXqlcGREhKfrvlELWATBhtK9RWGup+Tc2IjrT993AwxSdO3f1NiZM33cPsF9fJ8+xIwfqMmXWvGptgTsmfcQHksYAl1B07lwJLE6bLQYereU4Zs2m1qnaVODhohsuw4F/i4jHJa0FHpR0LbAduKrG45g1lZqCExGvA58fYLwLuLiW+zZrZn5WbpbBwTHL4OCYZXBwzDI4OGYZHByzDA6OWQYHxyyDg2OWwcExy+DgmGVwcMwyODhmGRwcswwOjlkGB8csg4NjliH7HaCSPk3RrbPXHOA7wHjgT4G30vjNEbEq9zhmzSg7OBGxCZgHIGkY0EnR5eabwO0R8cN6FGjWjOo1VbsY2BIR2+t0f2ZNrV7BWQTcV7G8RNLzkpZLmlCnY5g1jZqDI2kk8DXg39PQHcDpFNO4ncBtg+zXIWmdpHXvHfxYo0+zplaPR5zLgOciYhdAROyKiJ6IOALcSdHZ82PcydNaWT2CczUV07Te1rfJFRSdPc3aSk0NCVPb268A11UM/0DSPIpPMdjWb51ZW6i1k+f7wKR+Y9fUVJFZC/ArB8wyODhmGRwcswwOjlkGB8csg4NjlsHBMcvg4JhlcHDMMjg4ZhkcHLMMDo5ZBgfHLIODY5bBwTHLUNP7ccyaxb7fnsCREcP6lj/11j5G7O9u2PEcHGsL2y75HN0Tx/Ytz/nP55i4aWfDjlfVVC21edotaWPF2ERJqyW9lr5PSOOS9GNJm1OLqHMbVbxZWap9jnM3cGm/saXAmoiYC6xJy1B0vZmbvjoo2kWZtZWqghMRTwF7+g0vBFak2yuAyyvG74nCM8D4fp1vzFpeLVfVpkZE7yTyTWBquj0deKNiux1p7CPckNBaWV0uR0dEULSDOpZ93JDQWlYtwdnVOwVL33en8U5gRsV2p6Yxs7ZRS3BWAovT7cXAoxXjX09X184H3q2Y0pm1har+jiPpPuDLwGRJO4C/Bb4PPCjpWmA7cFXafBWwANgM7Kf4vByztlJVcCLi6kFWXTzAtgHcUEtRZs3Or1Uzy+DgmGVwcMwyODhmGRwcswwOjlkGvx/H2sLcR9YSwz58HBix70BDj+fgWFsY/c7+43o8T9XMMjg4ZhkcHLMMDo5ZBgfHLIODY5bBwTHL0BZ/xxk9aiTjxp7Yt3y4p4eud94pr6AGGTV2PKd85gIOH9hP54Ynyy7nE60tgjP+xJP47Blz+5bf27+fn294p7yCGuTEU2Zx4ZJ/YN/u/3NwSjbkVG2QLp5/L+mV1KnzYUnj0/gsSR9I2pC+ftLA2s1KU81znLv5eBfP1cBnIuJzwKvATRXrtkTEvPR1fX3KNGsuQwZnoC6eEfFERBxOi89QtICyBus51M3enVt5/60dZZfyiVeP5zh/AjxQsTxb0i+BvcBfR8TTA+0kqYOitzQTRvviXjXe3v4yK//ykrLLMGoMjqRbgMPAvWloJzAzIrokfQF4RNLZEbG3/74RsQxYBjBz3HD3wLWWkv1fvaRvAH8A/HFqCUVEdEdEV7q9HtgCnFGHOs2aSlZwJF0K/BXwtYjYXzE+RdKwdHsOxUd9vF6PQs2ayZBTtUG6eN4EjAJWSwJ4Jl1Buwj4rqRDwBHg+ojo//EgZi1vyOAM0sXzrkG2fQh4qNaizJqdL2eZZXBwzDI4OGYZHByzDA6OWQYHxyxDW7wfZ/eePTy9fn3fchzxK3issdoiOEeOHOFA98Gyy7BPEE/VzDI4OGYZHByzDA6OWQYHxyyDg2OWwcExy+DgmGVwcMwy5HbyvFVSZ0XHzgUV626StFnSJklfbVThZmXK7eQJcHtFx85VAJLOAhYBZ6d9/rm3eYdZO8nq5HkUC4H7U5uorcBm4Lwa6jNrSrU8x1mSmq4vlzQhjU0H3qjYZkca+xhJHZLWSVr33kG/mtlaS25w7gBOB+ZRdO+87VjvICKWRcT8iJg/dqQyyzArR1ZwImJXRPRExBHgTj6cjnUCMyo2PTWNmbWV3E6e0yoWrwB6r7itBBZJGiVpNkUnz1/UVqJZ88nt5PllSfOAALYB1wFExIuSHgReomjGfkNE9DSkcrMS1bWTZ9r+e8D3ainKrNn5lQNmGRwcswwOjlkGB8csg4NjlsHBMcvg4JhlcHDMMjg4ZhkcHLMMDo5ZBgfHLIODY5bBwTHL4OCYZXBwzDLkNiR8oKIZ4TZJG9L4LEkfVKz7SQNrNytNNZ8Bejfwj8A9vQMR8Ye9tyXdBrxbsf2WiJhXp/rMmlI1b51+StKsgdZJEnAV8Ht1rsusqdX6HOdCYFdEvFYxNlvSLyX9r6QLa7x/s6ZU68e1Xw3cV7G8E5gZEV2SvgA8IunsiNjbf0dJHUAHwITRvkZhrSX7N1bScOBK4IHesdQzuivdXg9sAc4YaH938rRWVst/9b8PvBIRO3oHJE3p/XQCSXMoGhK+XluJZs2nmsvR9wE/Bz4taYeka9OqRXx0mgZwEfB8ujz9H8D1EVHtJx2YtYzchoRExDcGGHsIeKj2ssyam5+Vm2VwcMwyODhmGRwcswwOjlkGB8csg4NjlsHBMcvg4JhlcHDMMjg4ZhkcHLMMDo5ZBgfHLEOtb52ui7Enz+TCP/tu2WWYfdTj1wy6qimCM3LMSZz2xcvKLsOsap6qmWWo5q3TMyQ9KeklSS9K+lYanyhptaTX0vcJaVySfixps6TnJZ3b6JMwO96qecQ5DHw7Is4CzgdukHQWsBRYExFzgTVpGeAyiiYdcynaP91R96rNSjZkcCJiZ0Q8l27vA14GpgMLgRVpsxXA5en2QuCeKDwDjJc0rd6Fm5XpmJ7jpFa45wDPAlMjYmda9SYwNd2eDrxRsduONGbWNqoOjqSxFB1sbuzfmTMiAohjObCkDknrJK3r6uo6ll3NSldVcCSNoAjNvRHx0zS8q3cKlr7vTuOdwIyK3U9NYx9R2clz0qRJufWblaKaq2oC7gJejogfVaxaCSxOtxcDj1aMfz1dXTsfeLdiSmfWFqr5A+gFwDXAC70fIAXcDHwfeDB19txO8XEfAKuABcBmYD/wzXoWbNYMqunk+TNgsK7oFw+wfQA31FiXWVPzKwfMMjg4ZhkcHLMMDo5ZBgfHLIOKi2AlFyG9BbwP/KbsWupoMu1zPu10LlD9+ZwWEVMGWtEUwQGQtC4i5pddR7200/m007lAfc7HUzWzDA6OWYZmCs6ysguos3Y6n3Y6F6jD+TTNcxyzVtJMjzhmLaP04Ei6VNKm1Nxj6dB7NB9J2yS9IGmDpHVpbMBmJs1I0nJJuyVtrBhr2WYsg5zPrZI607/RBkkLKtbdlM5nk6SvVnWQiCjtCxgGbAHmACOBXwFnlVlT5nlsAyb3G/sBsDTdXgr8Xdl1HqX+i4BzgY1D1U/xlpHHKF4xfz7wbNn1V3k+twJ/McC2Z6Xfu1HA7PT7OGyoY5T9iHMesDkiXo+Ig8D9FM0+2sFgzUyaTkQ8BezpN9yyzVgGOZ/BLATuj4juiNhK8T6y84baqezgtEtjjwCekLReUkcaG6yZSatox2YsS9L0cnnF1DnrfMoOTrv4UkScS9FT7gZJF1WujGJO0LKXL1u9/uQO4HRgHrATuK2WOys7OFU19mh2EdGZvu8GHqZ4qB+smUmrqKkZS7OJiF0R0RMRR4A7+XA6lnU+ZQdnLTBX0mxJI4FFFM0+WoakMZJO7L0NXAJsZPBmJq2irZqx9HsedgXFvxEU57NI0ihJsyk60P5iyDtsgisgC4BXKa5m3FJ2PRn1z6G4KvMr4MXecwAmUbQGfg34b2Bi2bUe5Rzuo5i+HKKY4187WP0UV9P+Kf17vQDML7v+Ks/nX1O9z6ewTKvY/pZ0PpuAy6o5hl85YJah7KmaWUtycMwyODhmGRwcswwOjlkGB8csg4NjlsHBMcvw/5Y2m9RndqP3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(22):\n",
    "    if i>20:\n",
    "        plt.imshow(observation)\n",
    "        plt.show()\n",
    "    observation, _, _, _=env.step(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "06eff14d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(image):\n",
    "    image=image[35:195]\n",
    "    image=image[::2,::2,0]\n",
    "    image[image==144]=0\n",
    "    image[image==109]=0\n",
    "    image[image!=0]=1\n",
    "    return image.astype(np.float64).ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0bda5c01",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD7CAYAAACscuKmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAANjUlEQVR4nO3df6zddX3H8eeLXmqrFAvIavXCoIiI8Qc1xEn0D6dhQUdEEyEYtzBj4B+XQLZFi/8sS2ai/6D9w5g0qOMPJ7IqSjSREWSbiYbxy01t7aitQGuh4oVBBbqUvvfH+bZeKuWee+/5cc/9PB/JyTnfz/ec8/18872v+/1xzvm8U1VIWv5OGHcHJI2GYZcaYdilRhh2qRGGXWqEYZcasaiwJ7kkyY4kO5NsGlSnJA1eFvo5e5IVwP8AFwN7gHuAj1TVtsF1T9KgTC3itW8HdlbVLoAkNwOXAccNexK/wSMNWVXlxdoXcxj/WuCRWdN7ujZJS9Bi9ux9SXINcM2wlyPppS0m7HuBM2ZNT3dtL1BVW4At4GG8NE6LOYy/Bzg3ydlJVgJXArcNpluSBm3Be/aqOpTkr4HbgRXAV6rq5wPrmaSBWvBHbwtamIfx0tAN42q8pAli2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGzBn2JF9Jsj/Jz2a1nZrkjiQPdvenDLebkharnz37PwGXHNO2Cbizqs4F7uymJS1hc4a9qv4DmDmm+TLgpu7xTcAHB9stSYO20HP2dVW1r3v8KLBuQP2RNCSLLv9UVfVSQ0Rb/klaGha6Z38syXqA7n7/8Z5YVVuq6sKqunCBy5I0AAsN+23AVd3jq4DvDKY7koZlzoowSb4OvBt4FfAY8PfAt4FbgDOBh4ArqurYi3gv9l5WhJGG7HgVYSz/JC0zln+SGmfYpUYYdqkRhl1qhGGXGmHYpUYYdqkRhl1qhGGXGmHYpUYYdqkRhl1qhGGXGmHYpUYYdqkRhl1qhGGXGmHYpUYYdqkR/dR6OyPJXUm2Jfl5kmu7duu9SROkn9Fl1wPrq+r+JGuA++iVe/orYKaqPptkE3BKVX1qjvdywElpyBY84GRV7auq+7vHTwPbgddivTdposyr/FOSs4CNwN30We/N8k/S0tD3uPFJTgL+HfhMVX0ryZNVtXbW/Ceq6iXP2z2Ml4ZvUePGJzkR+Cbwtar6Vtfcd703SePXz9X4AF8GtlfVDbNmWe9NmiD9XI1/F/BD4KfA4a750/TO2+dV783DeGn4rPUmNcJab1LjDLvUCMMuNcKwS40w7FIjDLvUCMMuNcKwS40w7FIjDLvUCMMuNcKwS40w7FIjDLvUCMMuNcKwS40w7FIjDLvUiH4GnFyV5D+T/FdX/ukfuvazk9ydZGeSbyRZOfzuSlqofvbsB4H3VNVbgQuAS5K8A/gc8Pmqeh3wBPDxofVS0qL1U/6pqupAN3lidyvgPcDWrt3yT9IS11f5pyQr6BV0fB3wReCXwJNVdah7yh569d9e7LWWf1LTpqameNnLXkavBENPVXHw4EEOHTr0Eq8ccD/6eVJVPQ9ckGQtcCvwhn4XUFVbgC3gUNJq05ve9CYuvvhiTj755KNtMzMz3H777Wzbtm1k/ZhXYceqejLJXcBFwNokU93efRrYO4wOSpMsCeeffz5XX30109PTR9t37tzJrl27Rhr2fq7Gn97t0UmyGriYXtnmu4APd0+z/JN0HFNTU6xatYrVq1cfva1atYoTThjtJ9/97NnXAzd15+0nALdU1XeTbANuTvKPwAP06sFJWqLmDHtV/Te9muzHtu8C3j6MTkkaPL9BJzXCsEuNMOxSIwy71AjDLjXCsEuNMOxSIwy71AjDLjXCsEuNMOxSIwy71AjDLjXCsEuNMOxSIwy71Ih5jUEnaf6eeeYZ9u3b94K2ffv28eyzz460H6ka3YCvji6rFm3YsIGNGzfy8pe//Gjb008/zf3338/DDz888OVVVV6sve+wd2PQ3QvsrapLk5wN3AycRm9M+b+sqv+b4z0Mu5o0e8z4I4a1oz1e2Odzzn4tvVFlj7D8k9SnqvqD26j1FfYk08CfAzd208HyT9JE6XfP/gXgk8Dhbvo05lH+Kcm9Se5dTEclLU4/RSIuBfZX1X0LWUBVbamqC6vqwoW8XtJg9PPR2zuBDyR5P7AKOBnYjOWfpInST8nm66tquqrOAq4EflBVH8XyT9JEWcw36D4F/E2SnfTO4S3/JC1hfqlGWmYG8Tm7pAlm2KVGGHapERP/q7dXv/rVvP71r2f16tVH25599lkefPDBP/ilUatOPPFEzjnnHM4880wOHDjAjh07+O1vfzvubmnEJj7sb3nLW7juuuuYnp4+2vbrX/+azZs3G/bOSSedxIc+9CEuv/xydu7cyQ033GDYGzTxYV+7di3nnXceGzZsONq2Zs0a1q5dO75OLTErVqzgNa95DW9+85tJwpo1a8bdJY2B5+xSIwy71AjD3oDDhw9z4MAB9u/fz8zMDAcPHhx3lzQGE3/Orrk988wzfO9732P37t3MzMywa9eucXdJY2DYG/Dcc8/xox/9iB//+MdUFYcPH577RVp2DHsjDLg8Z5caYdilRhh2qRGGXWqEYZcaYdilRvT10VuSXwFPA88Dh6rqwiSnAt8AzgJ+BVxRVU8Mp5uSFms+e/Y/raoLZo3/vgm4s6rOBe7spiUtUYs5jL+MXtknsPyTtOT1G/YC/jXJfUmu6drWVdWR0SEeBdYNvHeSBqbfr8u+q6r2Jvkj4I4kv5g9s6rqeMNEd/8crnmxeZJGp689e1Xt7e73A7cCbwceS7IeoLvff5zXWutNWgL6Kez4iiRrjjwG/gz4GXAbvbJPMMbyT0d+xXXsbRz1r6WlrJ/D+HXArb2S7EwB/1xV309yD3BLko8DDwFXDK+bx7d79262bt3K6aeffrTt8ccf9zfb0jEmvvzTqlWrWLNmDVNTv/+/9fzzz/PUU0/x3HPPDXpx0pJ3vPJPEx92SS9krTepcYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEX2FPcnaJFuT/CLJ9iQXJTk1yR1JHuzuTxl2ZyUtXL979s3A96vqDcBbge1Y/kmaKHOOQZfklcBPgA0168lJdgDvrqp93bjx/1ZV583xXo5BJw3ZYsagOxv4DfDVJA8kubEbP97yT9IE6SfsU8DbgC9V1UbgdxxzyN7t8Y9b/inJvUnuXWxnJS1cP2HfA+ypqru76a30wm/5J2mCzBn2qnoUeCTJkfPx9wLbWCLlnyT1p68iEUkuAG4EVgK7gI/R+0dxC3AmXfmnqpqZ4328QCcNmRVhpEZYEUZqnGGXGmHYpUYYdqkRhl1qhGGXGmHYpUYYdqkRhl1qhGGXGmHYpUYYdqkRhl1qhGGXGmHYpUYYdqkRhl1qhGGXGjFn2JOcl+Qns25PJbnO8k/SZJnXGHRJVgB7gT8BPgHMVNVnk2wCTqmqT83xesegk4ZsUGPQvRf4ZVU9BFwG3NS13wR8cMG9kzR08w37lcDXu8eWf5ImSN9hT7IS+ADwL8fOs/yTtPTNZ8/+PuD+qnqsm7b8kzRB5hP2j/D7Q3iw/JM0Ufot//QK4GF6Ndr/t2s7Dcs/SUuO5Z+kRlj+SWqcYZcaYdilRhh2qRGGXWqEYZcaYdilRhh2qRGGXWqEYZcaYdilRhh2qRGGXWqEYZcaYdilRhh2qRGGXWqEYZcaYdilRhh2qRGGXWrE1IiX9zjwu+5+OXoVy3PdXK/J8cfHmzHSoaQBkty7XKvDLNd1c72WBw/jpUYYdqkR4wj7ljEsc1SW67q5XsvAyM/ZJY2Hh/FSI0Ya9iSXJNmRZGeSTaNc9iAlOSPJXUm2Jfl5kmu79lOT3JHkwe7+lHH3dSGSrEjyQJLvdtNnJ7m7227fSLJy3H1ciCRrk2xN8osk25NctFy2WT9GFvYkK4AvAu8D3gh8JMkbR7X8ATsE/G1VvRF4B/CJbl02AXdW1bnAnd30JLoW2D5r+nPA56vqdcATwMfH0qvF2wx8v6reALyV3joul202t6oayQ24CLh91vT1wPWjWv6Q1+07wMXADmB917Ye2DHuvi1gXabp/dG/B/guEHpfPJl6se04KTfglcBuuutUs9onfpv1exvlYfxrgUdmTe/p2iZakrOAjcDdwLqq2tfNehRYN65+LcIXgE8Ch7vp04Anq+pQNz2p2+1s4DfAV7tTlBuTvILlsc364gW6RUhyEvBN4Lqqemr2vOrtKibqo44klwL7q+q+cfdlCKaAtwFfqqqN9L62/YJD9kncZvMxyrDvBc6YNT3dtU2kJCfSC/rXqupbXfNjSdZ389cD+8fVvwV6J/CBJL8CbqZ3KL8ZWJvkyO8oJnW77QH2VNXd3fRWeuGf9G3Wt1GG/R7g3O7K7krgSuC2ES5/YJIE+DKwvapumDXrNuCq7vFV9M7lJ0ZVXV9V01V1Fr3t84Oq+ihwF/Dh7mkTt14AVfUo8EiS87qm9wLbmPBtNh8j/VJNkvfTOydcAXylqj4zsoUPUJJ3AT8Efsrvz20/Te+8/RbgTOAh4IqqmhlLJxcpybuBv6uqS5NsoLenPxV4APiLqjo4xu4tSJILgBuBlcAu4GP0dnjLYpvNxW/QSY3wAp3UCMMuNcKwS40w7FIjDLvUCMMuNcKwS40w7FIj/h8b0vTsBVxBsAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "observation_preprocessed=preprocess(observation).reshape(80,80)\n",
    "plt.imshow(observation_preprocessed,cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b8803c73",
   "metadata": {},
   "outputs": [],
   "source": [
    "def discount_rewards(r,gamma):\n",
    "    r=np.array(r)\n",
    "    discounted_r=np.zeros_like(r)\n",
    "    running_add=0\n",
    "    \n",
    "    for t in reversed(range(0,r.size)):\n",
    "        if r[t]!=0:\n",
    "            running_add=0\n",
    "        running_add=running_add*gamma+r[t]\n",
    "        discounted_r[t]=running_add\n",
    "    discounted_r-=np.mean(discounted_r)\n",
    "    discounted_r/=np.std(discounted_r)\n",
    "    return discounted_r\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cec7acbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "model=Sequential()\n",
    "model.add(Dense(units=200,input_dim=80*80,activation='relu',kernel_initializer=\"glorot_uniform\"))\n",
    "model.add(Dense(units=1,activation='sigmoid',kernel_initializer=\"RandomNormal\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3ede2f39",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " dense (Dense)               (None, 200)               1280200   \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 1)                 201       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1,280,401\n",
      "Trainable params: 1,280,401\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c472ea15",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='binary_crossentropy',optimizer=\"adam\",metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2c0d22d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "At the end of the episode 0 the total reward was -29.0\n",
      "4/4 [==============================] - 1s 14ms/step - loss: -6.1833e-04 - accuracy: 0.4977\n",
      "At the end of the episode 1 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: 7.0058e-04 - accuracy: 0.5363\n",
      "At the end of the episode 2 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 9ms/step - loss: 0.0026 - accuracy: 0.5288\n",
      "At the end of the episode 3 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0025 - accuracy: 0.5316\n",
      "At the end of the episode 4 the total reward was -19.0\n",
      "4/4 [==============================] - 0s 9ms/step - loss: -0.0026 - accuracy: 0.5460\n",
      "At the end of the episode 5 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: -6.4581e-04 - accuracy: 0.5314\n",
      "At the end of the episode 6 the total reward was -21.0\n",
      "4/4 [==============================] - 0s 9ms/step - loss: -0.0018 - accuracy: 0.5508\n",
      "At the end of the episode 7 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: 0.0016 - accuracy: 0.5430\n",
      "At the end of the episode 8 the total reward was -18.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: 0.0052 - accuracy: 0.5590\n",
      "At the end of the episode 9 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0014 - accuracy: 0.5418\n",
      "At the end of the episode 10 the total reward was -19.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: -0.0072 - accuracy: 0.5642\n",
      "At the end of the episode 11 the total reward was -19.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0044 - accuracy: 0.5749\n",
      "At the end of the episode 12 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: -0.0048 - accuracy: 0.5555\n",
      "At the end of the episode 13 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 12ms/step - loss: 0.0041 - accuracy: 0.5465\n",
      "At the end of the episode 14 the total reward was -20.0\n",
      "4/4 [==============================] - 0s 10ms/step - loss: -0.0014 - accuracy: 0.5522\n",
      "At the end of the episode 15 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: 0.0074 - accuracy: 0.5692\n",
      "At the end of the episode 16 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: 0.0017 - accuracy: 0.5706\n",
      "At the end of the episode 17 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0074 - accuracy: 0.5913\n",
      "At the end of the episode 18 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0047 - accuracy: 0.5754\n",
      "At the end of the episode 19 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0047 - accuracy: 0.6015\n",
      "At the end of the episode 20 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0150 - accuracy: 0.5754\n",
      "At the end of the episode 21 the total reward was -20.0\n",
      "4/4 [==============================] - 0s 41ms/step - loss: -0.0014 - accuracy: 0.6045\n",
      "At the end of the episode 22 the total reward was -20.0\n",
      "4/4 [==============================] - 0s 9ms/step - loss: -2.5133e-04 - accuracy: 0.5666\n",
      "At the end of the episode 23 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: 0.0045 - accuracy: 0.5869\n",
      "At the end of the episode 24 the total reward was -19.0\n",
      "4/4 [==============================] - 0s 11ms/step - loss: -0.0083 - accuracy: 0.5592\n",
      "At the end of the episode 25 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 8ms/step - loss: 4.6314e-04 - accuracy: 0.5704\n",
      "At the end of the episode 26 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 13ms/step - loss: 0.0042 - accuracy: 0.5939\n",
      "At the end of the episode 27 the total reward was -19.0\n",
      "3/3 [==============================] - 0s 12ms/step - loss: 0.0041 - accuracy: 0.6056\n",
      "At the end of the episode 28 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0027 - accuracy: 0.6080\n",
      "At the end of the episode 29 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 9ms/step - loss: 0.0189 - accuracy: 0.5975\n",
      "At the end of the episode 30 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0059 - accuracy: 0.6046\n",
      "At the end of the episode 31 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 12ms/step - loss: -3.4725e-05 - accuracy: 0.6130\n",
      "At the end of the episode 32 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: 0.0040 - accuracy: 0.5902\n",
      "At the end of the episode 33 the total reward was -21.0\n",
      "4/4 [==============================] - 0s 11ms/step - loss: -0.0011 - accuracy: 0.6215\n",
      "At the end of the episode 34 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 13ms/step - loss: -0.0039 - accuracy: 0.6047\n",
      "At the end of the episode 35 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 9ms/step - loss: 6.4116e-04 - accuracy: 0.6189\n",
      "At the end of the episode 36 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: -0.0042 - accuracy: 0.6101\n",
      "At the end of the episode 37 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0135 - accuracy: 0.6107\n",
      "At the end of the episode 38 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0142 - accuracy: 0.6219\n",
      "At the end of the episode 39 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 16ms/step - loss: -0.0062 - accuracy: 0.6016\n",
      "At the end of the episode 40 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 9ms/step - loss: 0.0173 - accuracy: 0.6511\n",
      "At the end of the episode 41 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: 0.0046 - accuracy: 0.6118\n",
      "At the end of the episode 42 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: 6.1199e-07 - accuracy: 0.6032\n",
      "At the end of the episode 43 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 8ms/step - loss: 0.0061 - accuracy: 0.6132 \n",
      "At the end of the episode 44 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 9ms/step - loss: 0.0135 - accuracy: 0.6134\n",
      "At the end of the episode 45 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 9ms/step - loss: -0.0053 - accuracy: 0.6069\n",
      "At the end of the episode 46 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: 0.0059 - accuracy: 0.6378\n",
      "At the end of the episode 47 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0079 - accuracy: 0.6348\n",
      "At the end of the episode 48 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0205 - accuracy: 0.5973\n",
      "At the end of the episode 49 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0103 - accuracy: 0.6340\n",
      "At the end of the episode 50 the total reward was -21.0\n",
      "4/4 [==============================] - 0s 9ms/step - loss: -0.0065 - accuracy: 0.6232\n",
      "At the end of the episode 51 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0158 - accuracy: 0.6194\n",
      "At the end of the episode 52 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 9ms/step - loss: -0.0028 - accuracy: 0.6148\n",
      "At the end of the episode 53 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 9ms/step - loss: -0.0045 - accuracy: 0.6402\n",
      "At the end of the episode 54 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 9ms/step - loss: 0.0027 - accuracy: 0.6260\n",
      "At the end of the episode 55 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 9ms/step - loss: -6.2726e-04 - accuracy: 0.6382\n",
      "At the end of the episode 56 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 8ms/step - loss: -0.0022 - accuracy: 0.6164\n",
      "At the end of the episode 57 the total reward was -20.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4/4 [==============================] - 0s 9ms/step - loss: -0.0119 - accuracy: 0.6196\n",
      "At the end of the episode 58 the total reward was -21.0\n",
      "4/4 [==============================] - 0s 12ms/step - loss: -0.0038 - accuracy: 0.6539\n",
      "At the end of the episode 59 the total reward was -19.0\n",
      "3/3 [==============================] - 0s 12ms/step - loss: 0.0079 - accuracy: 0.6478\n",
      "At the end of the episode 60 the total reward was -17.0\n",
      "4/4 [==============================] - 0s 9ms/step - loss: 0.0125 - accuracy: 0.6608\n",
      "At the end of the episode 61 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 8ms/step - loss: -0.0101 - accuracy: 0.6350\n",
      "At the end of the episode 62 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: -0.0132 - accuracy: 0.6702\n",
      "At the end of the episode 63 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0142 - accuracy: 0.6299\n",
      "At the end of the episode 64 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 8ms/step - loss: -0.0069 - accuracy: 0.6351\n",
      "At the end of the episode 65 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 8ms/step - loss: -0.0032 - accuracy: 0.6981\n",
      "At the end of the episode 66 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 9ms/step - loss: -0.0213 - accuracy: 0.6723\n",
      "At the end of the episode 67 the total reward was -19.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: -0.0038 - accuracy: 0.6787\n",
      "At the end of the episode 68 the total reward was -19.0\n",
      "3/3 [==============================] - 0s 12ms/step - loss: 0.0066 - accuracy: 0.6768\n",
      "At the end of the episode 69 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: 0.0020 - accuracy: 0.6830\n",
      "At the end of the episode 70 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0032 - accuracy: 0.6830\n",
      "At the end of the episode 71 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: 0.0080 - accuracy: 0.6657\n",
      "At the end of the episode 72 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 9ms/step - loss: 0.0129 - accuracy: 0.6998\n",
      "At the end of the episode 73 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: 0.0132 - accuracy: 0.6723\n",
      "At the end of the episode 74 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: -0.0047 - accuracy: 0.6838\n",
      "At the end of the episode 75 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: 0.0037 - accuracy: 0.6659\n",
      "At the end of the episode 76 the total reward was -19.0\n",
      "4/4 [==============================] - 0s 10ms/step - loss: -0.0127 - accuracy: 0.6837\n",
      "At the end of the episode 77 the total reward was -20.0\n",
      "4/4 [==============================] - 0s 10ms/step - loss: -0.0089 - accuracy: 0.6874\n",
      "At the end of the episode 78 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 9ms/step - loss: -0.0153 - accuracy: 0.6828\n",
      "At the end of the episode 79 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0034 - accuracy: 0.6908\n",
      "At the end of the episode 80 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: 9.2448e-05 - accuracy: 0.6726\n",
      "At the end of the episode 81 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: 0.0137 - accuracy: 0.6897\n",
      "At the end of the episode 82 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: -0.0115 - accuracy: 0.6830\n",
      "At the end of the episode 83 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: -0.0112 - accuracy: 0.7045\n",
      "At the end of the episode 84 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: 3.0227e-04 - accuracy: 0.6941\n",
      "At the end of the episode 85 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: -0.0086 - accuracy: 0.6950\n",
      "At the end of the episode 86 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 13ms/step - loss: 0.0132 - accuracy: 0.6962\n",
      "At the end of the episode 87 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 15ms/step - loss: -0.0036 - accuracy: 0.6885\n",
      "At the end of the episode 88 the total reward was -21.0\n",
      "4/4 [==============================] - 0s 10ms/step - loss: -0.0129 - accuracy: 0.6952\n",
      "At the end of the episode 89 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: -0.0066 - accuracy: 0.6909\n",
      "At the end of the episode 90 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: 0.0162 - accuracy: 0.7072\n",
      "At the end of the episode 91 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 9ms/step - loss: -0.0083 - accuracy: 0.7081\n",
      "At the end of the episode 92 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: 0.0041 - accuracy: 0.7247\n",
      "At the end of the episode 93 the total reward was -18.0\n",
      "4/4 [==============================] - 0s 10ms/step - loss: -0.0047 - accuracy: 0.6920\n",
      "At the end of the episode 94 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: 0.0159 - accuracy: 0.6851\n",
      "At the end of the episode 95 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 11ms/step - loss: 0.0082 - accuracy: 0.7266\n",
      "At the end of the episode 96 the total reward was -20.0\n",
      "4/4 [==============================] - 0s 10ms/step - loss: -0.0034 - accuracy: 0.6994\n",
      "At the end of the episode 97 the total reward was -20.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: 4.8452e-04 - accuracy: 0.7280\n",
      "At the end of the episode 98 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: 0.0065 - accuracy: 0.7188\n",
      "At the end of the episode 99 the total reward was -21.0\n",
      "4/4 [==============================] - 0s 10ms/step - loss: -0.0090 - accuracy: 0.7183\n",
      "At the end of the episode 100 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 13ms/step - loss: 0.0017 - accuracy: 0.7234\n",
      "At the end of the episode 101 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: 0.0069 - accuracy: 0.7208\n",
      "At the end of the episode 102 the total reward was -21.0\n",
      "3/3 [==============================] - 0s 10ms/step - loss: 0.0070 - accuracy: 0.7250\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_5748\\3707595335.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      9\u001b[0m     \u001b[0mprev_input\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcur_input\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m     \u001b[0mprob\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexpand_dims\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mT\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m     \u001b[0maction\u001b[0m\u001b[1;33m=\u001b[0m \u001b[0mup_action\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0muniform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m<\u001b[0m\u001b[0mprob\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mdown_action\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\jupyternotebooks\\tfu\\lib\\site-packages\\keras\\utils\\traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     62\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     63\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 64\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     65\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# pylint: disable=broad-except\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     66\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\jupyternotebooks\\tfu\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, x, batch_size, verbose, steps, callbacks, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m   1959\u001b[0m           \u001b[0muse_multiprocessing\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1960\u001b[0m           \u001b[0mmodel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1961\u001b[1;33m           steps_per_execution=self._steps_per_execution)\n\u001b[0m\u001b[0;32m   1962\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1963\u001b[0m       \u001b[1;31m# Container that configures and calls `tf.keras.Callback`s.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\jupyternotebooks\\tfu\\lib\\site-packages\\keras\\engine\\data_adapter.py\u001b[0m in \u001b[0;36mget_data_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m   1397\u001b[0m   \u001b[1;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"model\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"_cluster_coordinator\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1398\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0m_ClusterCoordinatorDataHandler\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1399\u001b[1;33m   \u001b[1;32mreturn\u001b[0m \u001b[0mDataHandler\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1400\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1401\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\jupyternotebooks\\tfu\\lib\\site-packages\\keras\\engine\\data_adapter.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, x, y, sample_weight, batch_size, steps_per_epoch, initial_epoch, epochs, shuffle, class_weight, max_queue_size, workers, use_multiprocessing, model, steps_per_execution, distribute)\u001b[0m\n\u001b[0;32m   1159\u001b[0m         \u001b[0muse_multiprocessing\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1160\u001b[0m         \u001b[0mdistribution_strategy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdistribute\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_strategy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1161\u001b[1;33m         model=model)\n\u001b[0m\u001b[0;32m   1162\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1163\u001b[0m     \u001b[0mstrategy\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdistribute\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_strategy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\jupyternotebooks\\tfu\\lib\\site-packages\\keras\\engine\\data_adapter.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, x, y, sample_weights, sample_weight_modes, batch_size, epochs, steps, shuffle, **kwargs)\u001b[0m\n\u001b[0;32m    290\u001b[0m     \u001b[1;31m# trigger the next permutation. On the other hand, too many simultaneous\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    291\u001b[0m     \u001b[1;31m# shuffles can contend on a hardware level and degrade all performance.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 292\u001b[1;33m     \u001b[0mindices_dataset\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mindices_dataset\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpermutation\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprefetch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    293\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    294\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mslice_batch_indices\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindices\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\jupyternotebooks\\tfu\\lib\\site-packages\\tensorflow\\python\\data\\ops\\dataset_ops.py\u001b[0m in \u001b[0;36mmap\u001b[1;34m(self, map_func, num_parallel_calls, deterministic, name)\u001b[0m\n\u001b[0;32m   2014\u001b[0m         warnings.warn(\"The `deterministic` argument has no effect unless the \"\n\u001b[0;32m   2015\u001b[0m                       \"`num_parallel_calls` argument is specified.\")\n\u001b[1;32m-> 2016\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mMapDataset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmap_func\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpreserve_cardinality\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2017\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2018\u001b[0m       return ParallelMapDataset(\n",
      "\u001b[1;32mD:\\jupyternotebooks\\tfu\\lib\\site-packages\\tensorflow\\python\\data\\ops\\dataset_ops.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, input_dataset, map_func, use_inter_op_parallelism, preserve_cardinality, use_legacy_function, name)\u001b[0m\n\u001b[0;32m   5203\u001b[0m         \u001b[0muse_inter_op_parallelism\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_use_inter_op_parallelism\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   5204\u001b[0m         \u001b[0mpreserve_cardinality\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_preserve_cardinality\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 5205\u001b[1;33m         **self._common_args)\n\u001b[0m\u001b[0;32m   5206\u001b[0m     \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mMapDataset\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_dataset\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvariant_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   5207\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\jupyternotebooks\\tfu\\lib\\site-packages\\tensorflow\\python\\ops\\gen_dataset_ops.py\u001b[0m in \u001b[0;36mmap_dataset\u001b[1;34m(input_dataset, other_arguments, f, output_types, output_shapes, use_inter_op_parallelism, preserve_cardinality, metadata, name)\u001b[0m\n\u001b[0;32m   3367\u001b[0m         \u001b[1;34m\"output_types\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutput_types\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"output_shapes\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutput_shapes\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3368\u001b[0m         \u001b[1;34m\"use_inter_op_parallelism\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0muse_inter_op_parallelism\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3369\u001b[1;33m         \"preserve_cardinality\", preserve_cardinality, \"metadata\", metadata)\n\u001b[0m\u001b[0;32m   3370\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0m_result\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3371\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0m_core\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "history=[]\n",
    "observation=env.reset()\n",
    "prev_input=None\n",
    "\n",
    "while (True):\n",
    "    cur_input=preprocess(observation)\n",
    "    \n",
    "    x=cur_input-prev_input if prev_input is not None else np.zeros(80*80)\n",
    "    prev_input=cur_input\n",
    "    \n",
    "    prob=model.predict(np.expand_dims(x,axis=1).T)\n",
    "    \n",
    "    action= up_action if np.random.uniform()<prob else down_action\n",
    "    \n",
    "    y=1 if action==2 else 0\n",
    "    \n",
    "    x_train.append(x)\n",
    "    y_train.append(y)\n",
    "    \n",
    "    observation,reward,done,info=env.step(action)\n",
    "    rewards.append(reward)\n",
    "    reward_sum+=reward\n",
    "    \n",
    "    if done:\n",
    "        history.append(reward_sum)\n",
    "        print(\"At the end of the episode {} the total reward was {}\".format(episode_num,reward_sum))\n",
    "        if episode_num>=100 and reward_sum>=-12:\n",
    "            break \n",
    "        else :\n",
    "            \n",
    "            episode_num+=1\n",
    "            \n",
    "            model.fit(x=np.vstack(x_train),y=np.vstack(y_train),verbose=1,sample_weight=discount_rewards(rewards,gamma),batch_size=512)\n",
    "            \n",
    "            x_train,y_train,rewards=[],[],[]\n",
    "            observation=env.reset()\n",
    "            reward_sum=0\n",
    "            prev_input=None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3fcdad75",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: 100eps\\assets\n"
     ]
    }
   ],
   "source": [
    "model.save(\"100eps\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "800286a3",
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[WinError 2] The system cannot find the file specified",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_5748\\3217607521.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     19\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mpyvirtualdisplay\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mDisplay\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 21\u001b[1;33m \u001b[0mdisplay\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mDisplay\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvisible\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1400\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m900\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     22\u001b[0m \u001b[0mdisplay\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\jupyternotebooks\\tfu\\lib\\site-packages\\pyvirtualdisplay\\display.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, backend, visible, size, color_depth, bgcolor, use_xauth, retries, extra_args, manage_global_env, **kwargs)\u001b[0m\n\u001b[0;32m     61\u001b[0m             \u001b[0mextra_args\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mextra_args\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     62\u001b[0m             \u001b[0mmanage_global_env\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmanage_global_env\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 63\u001b[1;33m             \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     64\u001b[0m         )\n\u001b[0;32m     65\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\jupyternotebooks\\tfu\\lib\\site-packages\\pyvirtualdisplay\\xvfb.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, size, color_depth, bgcolor, use_xauth, fbdir, dpi, retries, extra_args, manage_global_env)\u001b[0m\n\u001b[0;32m     48\u001b[0m             \u001b[0mretries\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mretries\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     49\u001b[0m             \u001b[0mextra_args\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mextra_args\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 50\u001b[1;33m             \u001b[0mmanage_global_env\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmanage_global_env\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     51\u001b[0m         )\n\u001b[0;32m     52\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\jupyternotebooks\\tfu\\lib\\site-packages\\pyvirtualdisplay\\abstractdisplay.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, program, use_xauth, retries, extra_args, manage_global_env)\u001b[0m\n\u001b[0;32m     83\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_retries_current\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 85\u001b[1;33m         \u001b[0mhelptext\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_helptext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprogram\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     86\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_has_displayfd\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"-displayfd\"\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mhelptext\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     87\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_has_displayfd\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\jupyternotebooks\\tfu\\lib\\site-packages\\pyvirtualdisplay\\util.py\u001b[0m in \u001b[0;36mget_helptext\u001b[1;34m(program)\u001b[0m\n\u001b[0;32m     15\u001b[0m         \u001b[0mstdout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msubprocess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mPIPE\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m         \u001b[0mstderr\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msubprocess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mPIPE\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m         \u001b[0mshell\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     18\u001b[0m     )\n\u001b[0;32m     19\u001b[0m     \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstderr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommunicate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32md:\\python37\\lib\\subprocess.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)\u001b[0m\n\u001b[0;32m    798\u001b[0m                                 \u001b[0mc2pread\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mc2pwrite\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    799\u001b[0m                                 \u001b[0merrread\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrwrite\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 800\u001b[1;33m                                 restore_signals, start_new_session)\n\u001b[0m\u001b[0;32m    801\u001b[0m         \u001b[1;32mexcept\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    802\u001b[0m             \u001b[1;31m# Cleanup if the child failed starting.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32md:\\python37\\lib\\subprocess.py\u001b[0m in \u001b[0;36m_execute_child\u001b[1;34m(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, unused_restore_signals, unused_start_new_session)\u001b[0m\n\u001b[0;32m   1205\u001b[0m                                          \u001b[0menv\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1206\u001b[0m                                          \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfspath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcwd\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mcwd\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1207\u001b[1;33m                                          startupinfo)\n\u001b[0m\u001b[0;32m   1208\u001b[0m             \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1209\u001b[0m                 \u001b[1;31m# Child is launched. Close the parent's copy of those pipe\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [WinError 2] The system cannot find the file specified"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "from gym import logger as gymlogger\n",
    "# from gym.wrappers import Monitor\n",
    "gymlogger.set_level(40) #error only\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import random\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import math\n",
    "import glob\n",
    "import io\n",
    "import base64\n",
    "from IPython.display import HTML\n",
    "\n",
    "from IPython import display as ipythondisplay\n",
    "\n",
    "\n",
    "from pyvirtualdisplay import Display\n",
    "display = Display(visible=0, size=(1400, 900))\n",
    "display.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3bd116b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tfu",
   "language": "python",
   "name": "tfu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}