CartPole_Mellow_Tensor.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CartPole_Mellow_Tensor.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sayam47/Mellowmax-RL/blob/master/CartPole_Mellow_Tensor.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UKKI09hG41sz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import random\n",
        "import gym\n",
        "import numpy as np\n",
        "from collections import deque\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense\n",
        "from keras.optimizers import Adam\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('error')\n",
        "\n",
        "\n",
        "EPISODES = 5000\n",
        "\n",
        "class DQNAgent:\n",
        "    def __init__(self, state_size, action_size):\n",
        "        self.state_size = state_size\n",
        "        self.action_size = action_size\n",
        "        self.memory = list()\n",
        "        self.gamma = 0.95    # discount rate\n",
        "        self.learning_rate = 0.001\n",
        "        self.model = self._build_model()\n",
        "\n",
        "    def _build_model(self):\n",
        "        model = Sequential()\n",
        "        model.add(Dense(24, input_dim=self.state_size, activation='relu'))\n",
        "        model.add(Dense(24, activation='relu'))\n",
        "        model.add(Dense(self.action_size, activation='linear'))\n",
        "        model.compile(loss='mse',optimizer=Adam(lr=self.learning_rate))\n",
        "        return model\n",
        "\n",
        "    def remember(self, state, action, reward, next_state, done):\n",
        "        self.memory.append((state, action, reward, next_state, done))\n",
        "\n",
        "    def act(self, state):\n",
        "        act_values = self.model.predict(state)[0]\n",
        "        prob=0\n",
        "        try:\n",
        "            prob = np.exp(act_values)/np.sum(np.exp(act_values))\n",
        "        except:\n",
        "            print(act_values)\n",
        "        return np.random.choice(len(prob) , p = prob)\n",
        "\n",
        "    def replay(self, batch_size):\n",
        "        minibatch = [self.memory[-1]]\n",
        "        for state, action, reward, next_state, done in minibatch:\n",
        "            target = reward\n",
        "            if not done:\n",
        "                target = (reward + self.gamma * np.amax(self.model.predict(next_state)[0]))\n",
        "            target_f = self.model.predict(state)\n",
        "            target_f[0][action] = target\n",
        "            self.model.fit(state, target_f, epochs=1, verbose=0)\n",
        "\n",
        "    def load(self, name):\n",
        "        self.model.load_weights(name)\n",
        "\n",
        "    def save(self, name):\n",
        "        self.model.save_weights(name)\n",
        "\n",
        "\n",
        "env = gym.make('CartPole-v0')\n",
        "state_size = env.observation_space.shape[0]\n",
        "action_size = env.action_space.n\n",
        "agent = DQNAgent(state_size, action_size)\n",
        "done = False\n",
        "batch_size = 1\n",
        "all_reward = deque(maxlen=5)\n",
        "for e in range(EPISODES):\n",
        "    state = env.reset()\n",
        "    state = np.reshape(state, [1, state_size])\n",
        "    for time in range(500):\n",
        "        # env.render()\n",
        "        action = agent.act(state)\n",
        "        next_state, reward, done, _ = env.step(action)\n",
        "        reward = reward if not done else -10\n",
        "        next_state = np.reshape(next_state, [1, state_size])\n",
        "        agent.remember(state, action, reward, next_state, done)\n",
        "        state = next_state\n",
        "        if done:\n",
        "            all_reward.append(time+1)\n",
        "            print(\"episode: {}/{}, score: {}, avg: {}\".format(e, EPISODES, time+1, np.mean(all_reward)))\n",
        "            break\n",
        "            \n",
        "        if len(agent.memory) > batch_size:\n",
        "            agent.replay(batch_size)\n"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}