CartPole.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gym\n",
    "import math\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import deque\n",
    "from itertools import count\n",
    "from copy import deepcopy\n",
    "from PIL import Image\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import torchvision.transforms as T\n",
    "\n",
    "\n",
    "env = gym.make('CartPole-v0').unwrapped\n",
    "\n",
    "# set up matplotlib\n",
    "is_ipython = 'inline' in matplotlib.get_backend()\n",
    "if is_ipython:\n",
    "    from IPython import display\n",
    "\n",
    "plt.ion()\n",
    "\n",
    "# if gpu is to be used\n",
    "use_cuda = torch.cuda.is_available()\n",
    "FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor\n",
    "LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor\n",
    "ByteTensor = torch.cuda.ByteTensor if use_cuda else torch.ByteTensor\n",
    "Tensor = FloatTensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Q(S,a)\n",
    "class DQN(nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(DQN, self).__init__()\n",
    "        self.dense1 = nn.Linear(4, 20)\n",
    "        self.dense2 = nn.Linear(20, 2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.dense1(x))\n",
    "        x = self.dense2(x)\n",
    "        return x\n",
    "    \n",
    "model = DQN()\n",
    "# a = Tensor([[1,2,3,4]])\n",
    "# print(a)\n",
    "# a = Variable(a)\n",
    "# a = model(a)\n",
    "# print(a)\n",
    "# print(a.max(1)[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#定义回放缓存\n",
    "memory = deque(maxlen=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#全局参数设置\n",
    "BATCH_SIZE = 32\n",
    "GAMMA = 0.9\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.0001)\n",
    "criterion = nn.MSELoss() #采用最小均方误差是线性回归"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "INITIAL_EPSILON = 0.5\n",
    "FINAL_EPSILON = 0.01\n",
    "\n",
    "epsilon = INITIAL_EPSILON\n",
    "\n",
    "def select_action(state):\n",
    "    \n",
    "    global epsilon\n",
    "    sample = random.random()\n",
    "    \n",
    "    #epsilon -= (INITIAL_EPSILON - FINAL_EPSILON)/10000\n",
    "    \n",
    "    if random.random() <= epsilon:\n",
    "        return random.randint(0,1)\n",
    "    else:\n",
    "        s = Variable(FloatTensor(state.reshape(1,4)),volatile=True)\n",
    "        out = model(s).max(1)[1].data[0]\n",
    "        return out\n",
    "\n",
    "a = select_action(np.ones(4))\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#优化\n",
    "def optimize_model(memory, model, optimizer):\n",
    "    if len(memory) < BATCH_SIZE:\n",
    "        return\n",
    "    batch = random.sample(memory,BATCH_SIZE)\n",
    "    [states, actions, rewards, next_states, dones] = zip(*batch)\n",
    "    \n",
    "    state_batch = Variable(Tensor(states))\n",
    "    action_batch = Variable(LongTensor(actions))\n",
    "    reward_batch = Variable(Tensor(rewards))\n",
    "    next_states_batch = Variable(Tensor(next_states))\n",
    "    \n",
    "    #反向传播时更新参数\n",
    "    state_action_values = model(state_batch).gather(1, action_batch.view(-1,1))\n",
    "    \n",
    "    #仅前向计算，不反向传播\n",
    "    next_states_batch.volatile = True\n",
    "    next_state_values = model(next_states_batch).max(1)[0]\n",
    "    for i in range(BATCH_SIZE):\n",
    "        if dones[i]:\n",
    "            next_state_values.data[i]=0\n",
    "    \n",
    "    # Compute the expected Q values\n",
    "    expected_state_action_values = (next_state_values * GAMMA) + reward_batch\n",
    "    expected_state_action_values.volatile = False\n",
    "    \n",
    "    loss = criterion(state_action_values, expected_state_action_values)    \n",
    "\n",
    "    # Optimize the model\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    for param in model.parameters():\n",
    "        param.grad.data.clamp_(-1, 1)\n",
    "    optimizer.step()\n",
    "    \n",
    "    return 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:  0 Evaluation Average Reward: 171.1\n",
      "episode:  100 Evaluation Average Reward: 341.1\n"
     ]
    }
   ],
   "source": [
    "#尝试次数\n",
    "for episode in range(10000):  \n",
    "    \n",
    "    observation = env.reset()  \n",
    "    state = observation\n",
    "    \n",
    "    #尝试玩一次游戏并优化动作估值函数\n",
    "    for t in count():  \n",
    "        env.render()  \n",
    "        \n",
    "        action = select_action(state) #选择动作\n",
    "        observation,reward,done,info=env.step(action)\n",
    "        next_state = observation\n",
    "        \n",
    "        # Store the transition in memory\n",
    "        memory.append([state, action, reward, next_state, done])\n",
    "        \n",
    "        #下一轮迭代\n",
    "        state = next_state\n",
    "        \n",
    "        #优化Q(S,a)\n",
    "        if len(memory) >= BATCH_SIZE:\n",
    "            optimize_model(memory, model, optimizer)\n",
    "            \n",
    "        if done:              \n",
    "            break  \n",
    "        \n",
    "    #测试动作估值函数\n",
    "    if episode % 100 == 0:\n",
    "        \n",
    "        test_cnt = 10\n",
    "        total_reward = 0\n",
    "        for i in range(test_cnt):\n",
    "            test_state = env.reset()\n",
    "            for j in count():\n",
    "                env.render()\n",
    "                test_action = model(Variable(FloatTensor(test_state.reshape(1,4)),volatile=True)).max(1)[1].data[0]\n",
    "                test_state, test_reward, test_done, _ = env.step(test_action)\n",
    "                total_reward += test_reward\n",
    "                if test_done:\n",
    "                    break\n",
    "        ave_reward = total_reward/test_cnt\n",
    "        print('episode: ',episode,'Evaluation Average Reward:',ave_reward)\n",
    "        if ave_reward >= 300:\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test:  0 Test Reward: 243.0\n",
      "test:  1 Test Reward: 240.0\n",
      "test:  2 Test Reward: 254.0\n",
      "test:  3 Test Reward: 248.0\n",
      "test:  4 Test Reward: 200.0\n",
      "test:  5 Test Reward: 250.0\n",
      "test:  6 Test Reward: 226.0\n",
      "test:  7 Test Reward: 239.0\n",
      "test:  8 Test Reward: 216.0\n",
      "test:  9 Test Reward: 439.0\n",
      "test:  10 Test Reward: 234.0\n",
      "test:  11 Test Reward: 255.0\n",
      "test:  12 Test Reward: 248.0\n",
      "test:  13 Test Reward: 234.0\n",
      "test:  14 Test Reward: 186.0\n",
      "test:  15 Test Reward: 231.0\n",
      "test:  16 Test Reward: 468.0\n",
      "test:  17 Test Reward: 236.0\n"
     ]
    }
   ],
   "source": [
    "#对已训练模型测试表现\n",
    "test_cnt = 30\n",
    "for i in range(test_cnt):\n",
    "    total_reward = 0\n",
    "    test_state = env.reset()\n",
    "    for j in count():\n",
    "        env.render()\n",
    "        test_action = model(Variable(FloatTensor(test_state.reshape(1,4)),volatile=True)).max(1)[1].data[0]\n",
    "        test_state, test_reward, test_done, _ = env.step(test_action)\n",
    "        total_reward += test_reward\n",
    "        if test_done:\n",
    "            break\n",
    "    print('test: ',i,'Test Reward:',total_reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}