3_USD_CAD_train_and_evaluate.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Install TensorTrade"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Collecting tensorforce\n",
      "  Downloading Tensorforce-0.6.3-py3-none-any.whl (300 kB)\n",
      "\u001b[K     |████████████████████████████████| 300 kB 7.7 MB/s \n",
      "\u001b[?25hCollecting tqdm>=4.59.0\n",
      "  Downloading tqdm-4.60.0-py2.py3-none-any.whl (75 kB)\n",
      "\u001b[K     |████████████████████████████████| 75 kB 5.6 MB/s \n",
      "\u001b[?25hCollecting msgpack-numpy>=0.4.7.1\n",
      "  Downloading msgpack_numpy-0.4.7.1-py2.py3-none-any.whl (6.7 kB)\n",
      "Requirement already satisfied: numpy==1.19.5 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorforce) (1.19.5)\n",
      "Requirement already satisfied: matplotlib>=3.3.4 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorforce) (3.4.1)\n",
      "Requirement already satisfied: msgpack>=1.0.2 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorforce) (1.0.2)\n",
      "Requirement already satisfied: tensorflow==2.4.1 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorforce) (2.4.1)\n",
      "Requirement already satisfied: h5py~=2.10.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorforce) (2.10.0)\n",
      "Requirement already satisfied: Pillow==7.2.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorforce) (7.2.0)\n",
      "Requirement already satisfied: gym>=0.18.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorforce) (0.18.0)\n",
      "Requirement already satisfied: typing-extensions~=3.7.4 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorflow==2.4.1->tensorforce) (3.7.4.3)\n",
      "Requirement already satisfied: astunparse~=1.6.3 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorflow==2.4.1->tensorforce) (1.6.3)\n",
      "Requirement already satisfied: keras-preprocessing~=1.1.2 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorflow==2.4.1->tensorforce) (1.1.2)\n",
      "Requirement already satisfied: six~=1.15.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorflow==2.4.1->tensorforce) (1.15.0)\n",
      "Requirement already satisfied: wheel~=0.35 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorflow==2.4.1->tensorforce) (0.36.2)\n",
      "Requirement already satisfied: termcolor~=1.1.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorflow==2.4.1->tensorforce) (1.1.0)\n",
      "Requirement already satisfied: protobuf>=3.9.2 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorflow==2.4.1->tensorforce) (3.15.8)\n",
      "Requirement already satisfied: grpcio~=1.32.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorflow==2.4.1->tensorforce) (1.32.0)\n",
      "Requirement already satisfied: tensorboard~=2.4 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorflow==2.4.1->tensorforce) (2.5.0)\n",
      "Requirement already satisfied: tensorflow-estimator<2.5.0,>=2.4.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorflow==2.4.1->tensorforce) (2.4.0)\n",
      "Requirement already satisfied: flatbuffers~=1.12.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorflow==2.4.1->tensorforce) (1.12)\n",
      "Requirement already satisfied: wrapt~=1.12.1 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorflow==2.4.1->tensorforce) (1.12.1)\n",
      "Requirement already satisfied: gast==0.3.3 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorflow==2.4.1->tensorforce) (0.3.3)\n",
      "Requirement already satisfied: absl-py~=0.10 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorflow==2.4.1->tensorforce) (0.12.0)\n",
      "Requirement already satisfied: google-pasta~=0.2 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorflow==2.4.1->tensorforce) (0.2.0)\n",
      "Requirement already satisfied: opt-einsum~=3.3.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorflow==2.4.1->tensorforce) (3.3.0)\n",
      "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from gym>=0.18.0->tensorforce) (1.5.0)\n",
      "Requirement already satisfied: scipy in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from gym>=0.18.0->tensorforce) (1.6.3)\n",
      "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from gym>=0.18.0->tensorforce) (1.6.0)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from matplotlib>=3.3.4->tensorforce) (2.4.7)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from matplotlib>=3.3.4->tensorforce) (1.3.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from matplotlib>=3.3.4->tensorforce) (0.10.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from matplotlib>=3.3.4->tensorforce) (2.8.1)\n",
      "Requirement already satisfied: future in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from pyglet<=1.5.0,>=1.4.0->gym>=0.18.0->tensorforce) (0.18.2)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (1.8.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (3.3.4)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (52.0.0.post20210125)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (0.6.0)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (0.4.4)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (1.0.1)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (2.25.1)\n",
      "Requirement already satisfied: google-auth<2,>=1.6.3 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (1.29.0)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (4.2.1)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (4.7.2)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (1.3.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (0.4.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (2020.12.5)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (1.26.4)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (4.0.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (2.10)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /Users/yuan/Projects/github/ai-trading/env/lib/python3.8/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow==2.4.1->tensorforce) (3.1.0)\n",
      "Installing collected packages: tqdm, msgpack-numpy, tensorforce\n",
      "Successfully installed msgpack-numpy-0.4.7.1 tensorforce-0.6.3 tqdm-4.60.0\n"
     ]
    }
   ],
   "source": [
    "# !python3 -m pip install git+https://github.com/tensortrade-org/tensortrade.git\n",
    "\n",
    "# !pip install stable_baselines\n",
    "!pip install tensorforce"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup Data Fetching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import tensortrade.env.default as default\n",
    "\n",
    "from tensortrade.data.cdd import CryptoDataDownload\n",
    "from tensortrade.feed.core import Stream, DataFeed\n",
    "from tensortrade.oms.exchanges import Exchange\n",
    "from tensortrade.oms.services.execution.simulated import execute_order\n",
    "from tensortrade.oms.instruments import USD, BTC, ETH\n",
    "from tensortrade.oms.wallets import Wallet, Portfolio\n",
    "from tensortrade.agents import DQNAgent\n",
    "\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "              open         high          low        close        volume\n",
       "count  5000.000000  5000.000000  5000.000000  5000.000000   5000.000000\n",
       "mean      1.318374     1.319967     1.316767     1.318357   7230.880200\n",
       "std       0.036112     0.036399     0.035814     0.036127   7389.247272\n",
       "min       1.215230     1.215970     1.212690     1.213680      1.000000\n",
       "25%       1.299422     1.300977     1.297720     1.299392   2210.750000\n",
       "50%       1.318335     1.319745     1.316970     1.318290   4609.000000\n",
       "75%       1.333775     1.334937     1.332330     1.333712   9545.000000\n",
       "max       1.462440     1.467060     1.454140     1.462470  59501.000000"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>open</th>\n      <th>high</th>\n      <th>low</th>\n      <th>close</th>\n      <th>volume</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>count</th>\n      <td>5000.000000</td>\n      <td>5000.000000</td>\n      <td>5000.000000</td>\n      <td>5000.000000</td>\n      <td>5000.000000</td>\n    </tr>\n    <tr>\n      <th>mean</th>\n      <td>1.318374</td>\n      <td>1.319967</td>\n      <td>1.316767</td>\n      <td>1.318357</td>\n      <td>7230.880200</td>\n    </tr>\n    <tr>\n      <th>std</th>\n      <td>0.036112</td>\n      <td>0.036399</td>\n      <td>0.035814</td>\n      <td>0.036127</td>\n      <td>7389.247272</td>\n    </tr>\n    <tr>\n      <th>min</th>\n      <td>1.215230</td>\n      <td>1.215970</td>\n      <td>1.212690</td>\n      <td>1.213680</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>25%</th>\n      <td>1.299422</td>\n      <td>1.300977</td>\n      <td>1.297720</td>\n      <td>1.299392</td>\n      <td>2210.750000</td>\n    </tr>\n    <tr>\n      <th>50%</th>\n      <td>1.318335</td>\n      <td>1.319745</td>\n      <td>1.316970</td>\n      <td>1.318290</td>\n      <td>4609.000000</td>\n    </tr>\n    <tr>\n      <th>75%</th>\n      <td>1.333775</td>\n      <td>1.334937</td>\n      <td>1.332330</td>\n      <td>1.333712</td>\n      <td>9545.000000</td>\n    </tr>\n    <tr>\n      <th>max</th>\n      <td>1.462440</td>\n      <td>1.467060</td>\n      <td>1.454140</td>\n      <td>1.462470</td>\n      <td>59501.000000</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 2
    }
   ],
   "source": [
    "import pandas as pd\n",
    "def load_csv(filename):\n",
    "    df = pd.read_csv('data/' + filename, skiprows=0)\n",
    "#     df.drop(columns=['symbol', 'volume_btc'], inplace=True)\n",
    "\n",
    "    # Fix timestamp form \"2019-10-17 09-AM\" to \"2019-10-17 09-00-00 AM\"\n",
    "#     df['date'] = df['date'].str[:14] + '00-00 ' + df['date'].str[-2:]\n",
    "\n",
    "    # Convert the date column type from string to datetime for proper sorting.\n",
    "    df['date'] = pd.to_datetime(df['date'])\n",
    "\n",
    "    # Make sure historical prices are sorted chronologically, oldest first.\n",
    "    df.sort_values(by='date', ascending=True, inplace=True)\n",
    "\n",
    "    df.reset_index(drop=True, inplace=True)\n",
    "\n",
    "    # Format timestamps as you want them to appear on the chart buy/sell marks.\n",
    "    df['date'] = df['date'].dt.strftime('%Y-%m-%d %I:%M %p')\n",
    "\n",
    "    return df\n",
    "file_name = 'USD_CAD_H4_2021-05-07.csv'\n",
    "data = load_csv(file_name)\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create features with the feed module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rsi(price: Stream[float], period: float) -> Stream[float]:\n",
    "    r = price.diff()\n",
    "    upside = r.clamp_min(0).abs()\n",
    "    downside = r.clamp_max(0).abs()\n",
    "    rs = upside.ewm(alpha=1 / period).mean() / downside.ewm(alpha=1 / period).mean()\n",
    "    return 100*(1 - (1 + rs) ** -1)\n",
    "\n",
    "\n",
    "def macd(price: Stream[float], fast: float, slow: float, signal: float) -> Stream[float]:\n",
    "    fm = price.ewm(span=fast, adjust=False).mean()\n",
    "    sm = price.ewm(span=slow, adjust=False).mean()\n",
    "    md = fm - sm\n",
    "    signal = md - md.ewm(span=signal, adjust=False).mean()\n",
    "    return signal\n",
    "\n",
    "\n",
    "features = []\n",
    "for c in data.columns[2:]:\n",
    "    s = Stream.source(list(data[c]), dtype=\"float\").rename(data[c].name)\n",
    "    features += [s]\n",
    "\n",
    "cp = Stream.select(features, lambda s: s.name == \"close\")\n",
    "\n",
    "features = [\n",
    "    cp.log().diff().rename(\"lr\"),\n",
    "    rsi(cp, period=20).rename(\"rsi\"),\n",
    "    macd(cp, fast=10, slow=50, signal=5).rename(\"macd\")\n",
    "]\n",
    "\n",
    "feed = DataFeed(features)\n",
    "\n",
    "feed.compile()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "{'lr': nan, 'rsi': nan, 'macd': 0.0}\n{'lr': -0.0021132537317512523, 'rsi': 0.0, 'macd': -0.00025383244206770666}\n{'lr': 0.0005861432733618444, 'rsi': 22.585075537924805, 'macd': -0.00029659793912712514}\n{'lr': 0.003896237122257923, 'rsi': 70.03888631580796, 'macd': 0.00017354293391302488}\n{'lr': 0.0007254090599987661, 'rsi': 73.25880241509162, 'macd': 0.0004950867867173517}\n"
     ]
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    print(feed.next())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup Trading Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "{'internal': {'oanda:/CAD-USD': 1.26479,\n",
       "  'oanda:/CAD:/free': 1000.0,\n",
       "  'oanda:/CAD:/locked': 0.0,\n",
       "  'oanda:/CAD:/total': 1000.0,\n",
       "  'oanda:/USD:/free': 10000.0,\n",
       "  'oanda:/USD:/locked': 0.0,\n",
       "  'oanda:/USD:/total': 10000.0,\n",
       "  'oanda:/USD:/worth': 12647.900000000001,\n",
       "  'net_worth': 13647.900000000001},\n",
       " 'external': {'lr': nan, 'rsi': nan, 'macd': 0.0},\n",
       " 'renderer': {'date': '2018-02-25 10:00 PM',\n",
       "  'open': 1.26496,\n",
       "  'high': 1.2663,\n",
       "  'low': 1.26405,\n",
       "  'close': 1.26479,\n",
       "  'volume': 8890}}"
      ]
     },
     "metadata": {},
     "execution_count": 5
    }
   ],
   "source": [
    "from tensortrade.oms.instruments import Instrument\n",
    "from tensortrade.oms.orders import (\n",
    "    Broker,\n",
    "    Order,\n",
    "    OrderListener,\n",
    "    OrderSpec,\n",
    "    proportion_order,\n",
    "    risk_managed_order,\n",
    "    TradeSide,\n",
    "    TradeType\n",
    ")\n",
    "\n",
    "USD = Instrument(\"USD\", 6, \"U.S. Dollar\")\n",
    "CAD = Instrument(\"CAD\", 6, \"Canadian Dollar\")\n",
    "\n",
    "oanda = Exchange(\"oanda\", service=execute_order)(\n",
    "    Stream.source(list(data[\"close\"]), dtype=\"float\").rename(\"CAD-USD\")\n",
    ")\n",
    "\n",
    "# cash = Wallet(oanda, 10000 * USD)\n",
    "# asset = Wallet(oanda, 1000 * CAD)\n",
    "\n",
    "cash = Wallet(oanda, 1000 * CAD)\n",
    "asset = Wallet(oanda, 10000 * USD)\n",
    "\n",
    "portfolio = Portfolio(CAD, [\n",
    "    cash,\n",
    "    asset\n",
    "])\n",
    "\n",
    "\n",
    "renderer_feed = DataFeed([\n",
    "    Stream.source(list(data[\"date\"])).rename(\"date\"),\n",
    "    Stream.source(list(data[\"open\"]), dtype=\"float\").rename(\"open\"),\n",
    "    Stream.source(list(data[\"high\"]), dtype=\"float\").rename(\"high\"),\n",
    "    Stream.source(list(data[\"low\"]), dtype=\"float\").rename(\"low\"),\n",
    "    Stream.source(list(data[\"close\"]), dtype=\"float\").rename(\"close\"), \n",
    "    Stream.source(list(data[\"volume\"]), dtype=\"float\").rename(\"volume\") \n",
    "])\n",
    "\n",
    "\n",
    "env = default.create(\n",
    "    portfolio=portfolio,\n",
    "    # action_scheme=\"managed-risk\",\n",
    "    # action_scheme=default.actions.ManagedRiskOrders(),\n",
    "    # action_scheme=default.actions.BSH(cash, asset),\n",
    "    action_scheme=default.actions.SimpleOrders(\n",
    "                 trade_type=TradeType.MARKET,\n",
    "                 min_order_pct=0.5),\n",
    "    reward_scheme=default.rewards.SimpleProfit(),\n",
    "    # reward_scheme=\"risk-adjusted\",\n",
    "    feed=feed,\n",
    "    renderer_feed=renderer_feed,\n",
    "    # renderer=default.renderers.PlotlyTradingChart(),\n",
    "    renderer=\"screen-log\",\n",
    "    window_size=20\n",
    ")\n",
    "\n",
    "env.observer.feed.next()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup and Train Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "outputPrepend"
    ]
   },
   "outputs": [],
   "source": [
    "# agent = DQNAgent(env)\n",
    "\n",
    "# agent.train(n_steps=1000, n_episodes=20, save_path=\"agents_forex/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "error",
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'tensorflow.contrib'",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-ee9ddee25945>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mstable_baselines\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcommon\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpolicies\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mMlpLnLstmPolicy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mstable_baselines\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPPO2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPPO2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mpolicy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMlpLnLstmPolicy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Projects/github/ai-trading/env/lib/python3.8/site-packages/stable_baselines/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mstable_baselines\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macer\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mACER\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mstable_baselines\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macktr\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mACKTR\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mstable_baselines\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeepq\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDQN\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mstable_baselines\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mher\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mHER\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mstable_baselines\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mppo2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPPO2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Projects/github/ai-trading/env/lib/python3.8/site-packages/stable_baselines/deepq/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mstable_baselines\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeepq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpolicies\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mMlpPolicy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mCnnPolicy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLnMlpPolicy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLnCnnPolicy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mstable_baselines\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeepq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild_graph\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mbuild_act\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuild_train\u001b[0m  \u001b[0;31m# noqa\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mstable_baselines\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeepq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdqn\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDQN\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mstable_baselines\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcommon\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffers\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mReplayBuffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mPrioritizedReplayBuffer\u001b[0m  \u001b[0;31m# noqa\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Projects/github/ai-trading/env/lib/python3.8/site-packages/stable_baselines/deepq/policies.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtf_layers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mgym\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspaces\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDiscrete\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'tensorflow.contrib'"
     ]
    }
   ],
   "source": [
    "# from stable_baselines.common.policies import MlpLnLstmPolicy\n",
    "# from stable_baselines import PPO2\n",
    "\n",
    "# model = PPO2\n",
    "# policy = MlpLnLstmPolicy\n",
    "# params = { \"learning_rate\": 1e-5 }\n",
    "\n",
    "# agent = model(policy, environment, model_kwargs=params)\n",
    "# agent.train(n_steps=1000, n_episodes=20, save_path=\"agents_forex/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "error",
     "ename": "AttributeError",
     "evalue": "'TradingEnv' object has no attribute 'states'",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-7d5fbc3e520c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     18\u001b[0m agent = Agent.create(spec=agent_spec,\n\u001b[1;32m     19\u001b[0m                         kwargs=dict(network=network_spec,\n\u001b[0;32m---> 20\u001b[0;31m                                     \u001b[0mstates\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstates\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m                                     actions=env.actions))\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'TradingEnv' object has no attribute 'states'"
     ]
    }
   ],
   "source": [
    "# from tensorforce.agents import Agent\n",
    "\n",
    "# agent_spec = {\n",
    "#     \"type\": \"ppo_agent\",\n",
    "#     \"step_optimizer\": {\n",
    "#         \"type\": \"adam\",\n",
    "#         \"learning_rate\": 1e-4\n",
    "#     },\n",
    "#     \"discount\": 0.99,\n",
    "#     \"likelihood_ratio_clipping\": 0.2,\n",
    "# }\n",
    "\n",
    "# network_spec = [\n",
    "#     dict(type='dense', size=64, activation=\"tanh\"),\n",
    "#     dict(type='dense', size=32, activation=\"tanh\")\n",
    "# ]\n",
    "\n",
    "# agent = Agent.create(spec=agent_spec,\n",
    "#                         kwargs=dict(network=network_spec,\n",
    "#                                     states=env.states,\n",
    "#                                     actions=env.actions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "outputPrepend"
    ]
   },
   "outputs": [],
   "source": [
    "# Run until episode ends\n",
    "episode_reward = 0\n",
    "done = False\n",
    "\n",
    "# obs = env.reset()\n",
    "# while not done:\n",
    "#     action = agent.compute_action(obs)\n",
    "#     obs, reward, done, info = env.step(action)\n",
    "#     episode_reward += reward\n",
    "\n",
    "state  = env.reset()\n",
    "# while not done:\n",
    "for _ in range(len(data.index)):\n",
    "    action = agent.get_action(state, threshold=0)\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "    print(info)\n",
    "    episode_reward += reward\n",
    "    state = next_state\n",
    "\n",
    "env.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "performance = pd.DataFrame.from_dict(env.action_scheme.portfolio.performance, orient='index')\n",
    "\n",
    "# copy performance data to clipboard for analysis using Number\n",
    "# performance.to_clipboard(index=True)\n",
    "performance.to_csv('USD_CAD_Performance.csv')\n",
    "performance.plot()\n",
    "\n",
    "from pandasgui import show\n",
    "gui = show(performance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ledger = env.action_scheme.portfolio.ledger.as_frame()\n",
    "# portfolio.ledger.as_frame().head(20)\n",
    "gui = show(ledger)\n",
    "# portfolio.ledger.as_frame().to_clipboard(index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python388jvsc74a57bd01c4c0bf6beae42fd240ca5a341f6bd890b27028c2190f4d96dbd3777b38ff428",
   "display_name": "Python 3.8.8 64-bit (conda)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}