05-binance-ohlcv-1d.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Fetch and Write MarketStore"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "2f68643ba217d353"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import datetime\n",
    "import ccxt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pymarketstore as pymkts\n",
    "import time\n",
    "\n",
    "# Required parameters\n",
    "binanceApiKey = 'your binance API key'\n",
    "binanceSecret = 'your binance secret'\n",
    "\n",
    "endpoint = \"http://localhost:5993/rpc\"  # Your MarketStore Url\n",
    "\n",
    "symbol = 'BTC/USDT'\n",
    "tbk = f'Binance-{symbol.replace(\"/\", \"-\")}-Custom/1Min/OHLCV-1D'  # MarketStore tbk \n",
    "\n",
    "# Optional parameters\n",
    "limit = 365  # The amount of data acquired at one time\n",
    "\n",
    "# init binance with ccxt\n",
    "exchange = ccxt.binance()\n",
    "exchange.apiKey = binanceApiKey\n",
    "exchange.secret = binanceSecret\n",
    "\n",
    "# set end time\n",
    "end_time = datetime.datetime.now()\n",
    "\n",
    "print('------------------ Starting ------------------')\n",
    "\n",
    "# same data flag\n",
    "flag = None  \n",
    "\n",
    "# loop data by limit\n",
    "while True:\n",
    "    # Calculate start time\n",
    "    start_time = end_time - datetime.timedelta(days=limit)\n",
    "\n",
    "    # Convert time to millisecond timestamp\n",
    "    start_timestamp = int(start_time.timestamp() * 1000)\n",
    "    end_timestamp = int(end_time.timestamp() * 1000)\n",
    "\n",
    "    # Fetch historical data\n",
    "    try:\n",
    "        ohlcv = exchange.fetch_ohlcv(symbol, timeframe='1d', since=start_timestamp, limit=limit)\n",
    "    except Exception as e:\n",
    "        print(\"Error fetching ohlcv.Try again in 1 second.\",e)\n",
    "        time.sleep(1)\n",
    "        continue\n",
    "\n",
    "    # Convert timestamp format\n",
    "    for item in ohlcv:\n",
    "        item[0] = pd.to_datetime(item[0], unit='ms').timestamp()\n",
    "\n",
    "    # Format the data from [[1701820800000, 44131.1, 44981.47, 20000.0, 43769.54, 200.52232],[1701907200000, 43770.54, 45649.78, 1000.0, 43270.1, 431.88535]] \n",
    "    # replaced by [ (1701820800000, 44131.1, 44981.47, 20000.0, 43769.54, 200.52232),(1701907200000, 43770.54, 45649.78, 1000.0, 43270.1, 431.88535)]\n",
    "    converted_data = [tuple(item) for item in ohlcv]\n",
    "\n",
    "    # Set dType\n",
    "    dType = [('Epoch', 'i8'), ('open', 'f4'), ('high', 'f4'), ('low', 'f4'), ('close', 'f4'), ('volume', 'f4')]\n",
    "\n",
    "    # Convert NumPy\n",
    "    result = np.array(converted_data, dtype=dType)\n",
    "\n",
    "    # Set MarketStore Url\n",
    "    cli = pymkts.Client(endpoint=endpoint)\n",
    "\n",
    "    # Print write result\n",
    "    response = cli.write(result, tbk)\n",
    "    if response['responses'] is not None:\n",
    "        print(\"Write Error:\", response['responses'])\n",
    "    else:\n",
    "        # Update end time\n",
    "        end_time = start_time\n",
    "        print(f'Write Success: {len(result)} rows to {tbk}.')\n",
    "\n",
    "    if flag == ohlcv[0][0]:\n",
    "        print('------------------ Finished ------------------')\n",
    "        break\n",
    "\n",
    "    flag = ohlcv[0][0]\n",
    "    print(ohlcv[0][0])"
   ],
   "metadata": {
    "collapsed": true
   },
   "id": "initial_id"
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Query From MarketStore"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "427876a3eb0f18dc"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import pymarketstore as pymkts\n",
    "\n",
    "endpoint = \"http://localhost:5993/rpc\"  # Your MarketStore Url\n",
    "\n",
    "symbol = 'BTC/USDT'\n",
    "pymkts.Client(endpoint=endpoint).query(pymkts.Params(symbols=f'Binance-{symbol.replace(\"/\", \"-\")}-Custom', timeframe=\"1Min\", attrgroup=\"OHLCV-1D\")).first().df()"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "cfc19571341cc8e9"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}