policy_iteration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from amalearn.reward import GaussianReward, RewardBase\n",
    "from amalearn.agent import AgentBase\n",
    "from amalearn.environment import EnvironmentBase\n",
    "import numpy as np\n",
    "from math import *\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.stats as stats\n",
    "import gym\n",
    "from random import randrange"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Class rerward:\n",
    "    * get_reward: returns the difference of curent value and value before update\n",
    "    * update: updates the value based on defined probabilties for possible values and save last value.\n",
    "* NArmedBanditEnvironment:\n",
    "    * calculate_reward: updates each company and for each selected company in the selected action, if any, calculates the reward and returns it\n",
    "    * init_p_table: calls claculate_p_tabel: for each possible s,sp,r and a.\n",
    "    * claculate_p_tabel: calculates P(sp,r| s,a)\n",
    "    * claculate_p returns P(sp,r| s,a)\n",
    "* Agent:\n",
    "    * initilizes V(s) =0 for each state and Policy to a random policy.\n",
    "    * policy_evaluation and policy_iteration: implementation of policy iteration steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Reward(RewardBase):\n",
    "    def __init__(self, stay_prob, base, p, min_value, max_value):\n",
    "        super(Reward, self).__init__()\n",
    "        self.inc = p[0]\n",
    "        self.dec = p[1]\n",
    "        self.value = base\n",
    "        self.min = min_value\n",
    "        self.max = max_value\n",
    "        self.stay_prob = stay_prob\n",
    "        self.past_value = 0\n",
    "    \n",
    "    def get_reward(self):\n",
    "        return self.value - self.past_value\n",
    "    \n",
    "    def update(self, base_change):\n",
    "        self.past_value = self.value\n",
    "        rand = random.uniform(0, 1)\n",
    "        if self.value < self.min + 1:\n",
    "            if rand > self.stay_prob:\n",
    "                self.value = self.value + self.base_change\n",
    "        elif self.value > self.max - 1:\n",
    "            if rand  > self.stay_prob:\n",
    "                self.value = self.value - self.base_change\n",
    "        else:\n",
    "            if rand < self.inc:\n",
    "                self.value = self.value + self.base_change\n",
    "            elif rand < self.inc + self.dec:\n",
    "                self.value = self.value - self.base_change\n",
    "\n",
    "class NArmedBanditEnvironment(EnvironmentBase):\n",
    "    def __init__(self, action_count, state_count, id, initial_money, goal_money, base_change, base_value, change_prob, stay_prob, min_value, max_value, terminal, actions, container=None):\n",
    "        state_space = gym.spaces.Discrete(state_count)\n",
    "        action_space = gym.spaces.Discrete(action_count)\n",
    "        super(NArmedBanditEnvironment, self).__init__(action_space, state_space, id, container)\n",
    "        self.state_count = state_count\n",
    "        self.action_count = action_count\n",
    "        self.money = initial_money\n",
    "        self.initial_money = initial_money\n",
    "        self.goal_money = goal_money\n",
    "        self.base_change = base_change\n",
    "        self.stay_prob = stay_prob\n",
    "        self.min_value = min_value\n",
    "        self.max_value = max_value\n",
    "        self.base_value = base_value\n",
    "        self.change_prob = change_prob\n",
    "        self.rewards = [Reward(b, stay_prob, cp, min_value, max_value) for (b, cp) in zip(base_value, change_prob)]\n",
    "        self.actions = actions\n",
    "        self.terminal = terminal\n",
    "        self.state = 0\n",
    "        self.p_table = []\n",
    "    \n",
    "    def calculate_reward(self, action):\n",
    "        for r in self.rewards:\n",
    "            r.update(self.base_change)\n",
    "        change = 0\n",
    "        if action != 0:\n",
    "            for a in self.actions[action]:\n",
    "                change = change + self.rewards[a].get_reward()\n",
    "        return change\n",
    "\n",
    "    def claculate_p_tabel(self, sp, r, s, a):\n",
    "        if abs(sp - s) > 2:\n",
    "            return 0\n",
    "        if abs(r) != self.base_change and r != 0 and abs(r) != 2*self.base_change:\n",
    "            return 0\n",
    "        if sp == s:\n",
    "            if r == 0:\n",
    "                p = 1\n",
    "                for ai in a:\n",
    "                    p= p *(1-self.rewards[ai].inc - self.rewards[ai].dec)\n",
    "                return p\n",
    "            else:\n",
    "                return 0\n",
    "        elif sp == s - 1:\n",
    "            if r == -1*self.base_change:\n",
    "                if len(a) == 0:\n",
    "                    return 0\n",
    "                if len(a)>1:\n",
    "                    return self.rewards[a[0]].dec*(1-self.rewards[a[1]].dec-self.rewards[a[1]].inc)+self.rewards[a[1]].dec*(1-self.rewards[a[0]].dec-self.rewards[a[0]].inc)\n",
    "                else:\n",
    "                    return self.rewards[a[0]].dec\n",
    "            else:\n",
    "                return 0\n",
    "        elif sp == s + 1:           \n",
    "            if r == self.base_change:\n",
    "                if len(a) == 0:\n",
    "                    return 0\n",
    "                if len(a)>1:\n",
    "                    return self.rewards[a[0]].inc*(1-self.rewards[a[1]].dec-self.rewards[a[1]].inc)+self.rewards[a[1]].inc*(1-self.rewards[a[0]].dec-self.rewards[a[0]].inc)\n",
    "                else:\n",
    "                    return self.rewards[a[0]].inc\n",
    "            else:\n",
    "                return 0\n",
    "        elif sp == s - 2:\n",
    "            if r == -2 * self.base_change:\n",
    "                if len(a) == 2:\n",
    "                    return self.rewards[a[0]].dec*self.rewards[a[1]].dec\n",
    "                return 0\n",
    "            else:\n",
    "                return 0            \n",
    "        else:\n",
    "            if r == 2 * self.base_change:\n",
    "                if len(a) == 2:\n",
    "                    return self.rewards[a[0]].inc*self.rewards[a[1]].inc\n",
    "                return 0\n",
    "            else:\n",
    "                return 0\n",
    "    def init_p_table(self):\n",
    "        for a in self.actions:\n",
    "            temp = []\n",
    "            for i in range(self.state_count):\n",
    "                p0 = []\n",
    "                p1 = []\n",
    "                p2 = []\n",
    "                p3 = []\n",
    "                p4 = []\n",
    "                for j in range(self.state_count):\n",
    "                    p0.append(self.claculate_p_tabel(j, -10 , i, a))\n",
    "                    p1.append(self.claculate_p_tabel(j, -5 , i, a))\n",
    "                    p2.append(self.claculate_p_tabel(j, 0 , i, a))\n",
    "                    p3.append(self.claculate_p_tabel(j, 5 , i, a))\n",
    "                    p4.append(self.claculate_p_tabel(j, 10 , i, a))\n",
    "                temp.append([p0,p1,p2,p3,p4])\n",
    "            self.p_table.append(temp)\n",
    "    \n",
    "    def claculate_p(self, sp, r, s, a):\n",
    "        action_index = self.actions.index(a)\n",
    "        return self.p_table[action_index][s][int((r+10)/5)][sp]\n",
    "    \n",
    "    def terminated(self):\n",
    "        if self.state == self.terminal:\n",
    "            return False\n",
    "        return True\n",
    "\n",
    "    def observe(self):\n",
    "        return \n",
    "\n",
    "    def available_actions(self):\n",
    "        return self.action_space.n\n",
    "\n",
    "    def next_state(self, action):\n",
    "        v = self.money + self.rewards[action].get_reward()\n",
    "        if v < 0:\n",
    "            self.state = 0\n",
    "            return\n",
    "        if v == self.goal_money:\n",
    "            self.state = self.terminal\n",
    "            return\n",
    "        self.state = int(v/50) \n",
    "        return #(0-50)(50-100)...(950-1000)+Terminal\n",
    "\n",
    "    def reset(self):\n",
    "        self.rewards = [Reward(b, self.stay_prob, cp, self.min_value, self.max_value) for (b, cp) in zip(self.base_value, self.change_prob)]\n",
    "        self.money = self.initial_money\n",
    "        return\n",
    "\n",
    "    def render(self, mode='human'):\n",
    "        #print('{}:\\taction={}'.format(self.state['length'], self.state['last_action']))\n",
    "        return \n",
    "\n",
    "    def close(self):\n",
    "        return\n",
    "    \n",
    "class Agent(AgentBase):\n",
    "    def __init__(self, id, environment, discount, theta):\n",
    "        #initialize a random policy and V(s) = 0 for each state\n",
    "        self.V = np.zeros(environment.state_count)\n",
    "        self.policy = [randrange(environment.action_count) for i in range(environment.state_count)]\n",
    "        super(Agent, self).__init__(id, environment)\n",
    "        self.discount = discount\n",
    "        self.theta = theta\n",
    "        self.environment.init_p_table()\n",
    "    def policy_evaluation(self):\n",
    "        delta = 0\n",
    "        while True:\n",
    "            for s in range(self.environment.state_count):\n",
    "                vp = self.V[s]\n",
    "                x1 = 0\n",
    "                for sp in range(self.environment.state_count):\n",
    "                    #comment for faster learning with theta <5\n",
    "                    y = s * 5\n",
    "                    x1 = x1 + self.environment.claculate_p(sp, 0, s, self.environment.actions[self.policy[s]])*(0 + y + self.discount*self.V[sp])\n",
    "                    x1 = x1 + self.environment.claculate_p(sp, self.environment.base_change, s, self.environment.actions[self.policy[s]])*(self.environment.base_change + y + self.discount*self.V[sp])\n",
    "                    x1 = x1 + self.environment.claculate_p(sp, -1*self.environment.base_change, s, self.environment.actions[self.policy[s]])*(-1*self.environment.base_change + y + self.discount*self.V[sp])\n",
    "                    x1 = x1 + self.environment.claculate_p(sp, 2*self.environment.base_change, s, self.environment.actions[self.policy[s]])*(2*self.environment.base_change + y + self.discount*self.V[sp])\n",
    "                    x1 = x1 + self.environment.claculate_p(sp, -2*self.environment.base_change, s, self.environment.actions[self.policy[s]])*(-2*self.environment.base_change + y + self.discount*self.V[sp])\n",
    "                    \n",
    "            self.V[s] = x1\n",
    "            delta = max(delta, abs(vp - self.V[s]))            \n",
    "            #print(delta)\n",
    "            if delta < self.theta:\n",
    "                break\n",
    "        return\n",
    "    def policy_iteration(self):\n",
    "        stable = True\n",
    "        for s in range(self.environment.state_count):\n",
    "            old_action = self.policy[s]\n",
    "            temp = []\n",
    "            #comment for faster learning with theta <5\n",
    "            y = s * 5\n",
    "            self.policy[s] = np.argmax([np.sum([self.environment.claculate_p(sp, 0, s, self.environment.actions[i])*(0 + y+ self.discount*self.V[sp]) + self.environment.claculate_p(sp, self.environment.base_change, s, self.environment.actions[i])*(self.environment.base_change + y + self.discount*self.V[sp])+ self.environment.claculate_p(sp, -1*self.environment.base_change, s, self.environment.actions[i])*(-1*self.environment.base_change + y + self.discount*self.V[sp]) + self.environment.claculate_p(sp, 2*self.environment.base_change, s, self.environment.actions[i])*(2*self.environment.base_change + y + self.discount*self.V[sp])+ self.environment.claculate_p(sp, -2*self.environment.base_change, s, self.environment.actions[i])*(-2*self.environment.base_change + y + self.discount*self.V[sp])for sp in range(self.environment.state_count)]) for i in range(self.environment.action_count)])\n",
    "            if old_action != self.policy[s]:\n",
    "                stable = False\n",
    "        return stable\n",
    "    def take_action(self) -> (object, float, bool, object):\n",
    "        index_selected_arm = 0\n",
    "        obs, Ri, d, i = self.environment.step(index_selected_arm)\n",
    "        self.environment.money = self.environment.money + Ri\n",
    "        #self.environment.update_selected_arm(index_selected_arm, Ri)\n",
    "        self.environment.render()\n",
    "        return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7, 20, 5, 20)\n",
      "discount factor of 0.9\n",
      "recommended policy:\n",
      "['B and C', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B and C', 'B', 'nothing']\n",
      "discount factor of 0.8\n",
      "recommended policy:\n",
      "['B and C', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'nothing']\n",
      "discount factor of 0.7\n",
      "recommended policy:\n",
      "['B and C', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'nothing']\n",
      "discount factor of 0.4\n",
      "recommended policy:\n",
      "['B and C', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'nothing']\n",
      "discount factor of 0.1\n",
      "recommended policy:\n",
      "['B and C', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'nothing']\n"
     ]
    }
   ],
   "source": [
    "goal_money = 100\n",
    "initial_money = 20\n",
    "base_change = 5\n",
    "base_value = [5, 15, 5] # 5+2, 10+5, 5+2\n",
    "change_prob = [[0.39, 0.31], [0.15, 0.15], [0.23, 0.64]]\n",
    "action_company = ['nothing','B', 'C', 'D', 'B and C', 'B and D', 'C and D']\n",
    "actions = [[], [0], [1], [2], [0,1], [0,2], [1,2]]\n",
    "stay_prob = 0.25\n",
    "min_value = 5\n",
    "max_value = 50\n",
    "delta = 10 # change to bigger number for faster\n",
    "env = NArmedBanditEnvironment(len(actions), int(goal_money/5), 1, initial_money, goal_money, base_change, base_value, change_prob, stay_prob, min_value, max_value, int(goal_money/5), actions)\n",
    "agent = Agent('1', env, 0.9, delta)\n",
    "print(np.shape(env.p_table))\n",
    "it = 0\n",
    "while True:\n",
    "    agent.policy_evaluation()\n",
    "    if agent.policy_iteration():\n",
    "        break\n",
    "    it = it + 1\n",
    "\n",
    "print(\"discount factor of 0.9\")\n",
    "print(\"recommended policy:\")\n",
    "print([action_company[i] for i in agent.policy])\n",
    "\n",
    "for d in [0.8,0.7,0.4,0.1]:\n",
    "    agent = Agent('1', env, d, delta)\n",
    "    it = 0\n",
    "    while True:\n",
    "        agent.policy_evaluation()\n",
    "        if agent.policy_iteration():\n",
    "            break\n",
    "        it = it + 1\n",
    "    print(\"discount factor of \"+str(d))\n",
    "    print(\"recommended policy:\")\n",
    "    print([action_company[i] for i in agent.policy])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}