Save backtest history in sqlite. (#60)

* Save backtest history in sqlite.
Get backtest history by strategyId.
Get backtest history strategy details by id.
Set backtest strategy as favorite.

* Sqlite Db connection error check added.
symbol column datatype size increase.

* Typo fix.

* Unit tests added for Sqlite DB and Backtest DAO functions.

* Save bt result to db. Replaced meta with executionId. Added function to delete bt.

* BtData: parse btResult

* send bt id as second argument in responce array

* Added endpoint to delete all bts by strategy ID.

* send bt full data on single request

* normalize bt history response data

* send message on success bt saving

* Return error if backtest execution fails.
Changed saveBt callback to async call.
Fixed unit test.

* typo fix

* Call sqlite connection method from server open method

* log sqlite connection error

---------

Co-authored-by: Dmytro Shcherbonos <dmytro.sh@code-care.pro>

lib/cmds/delete_all_bts.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const { deleteAllBts } = require('../db/bt_dao')
+const send = require('../wss/send')
+
+module.exports = async (ds, ws, msg) => {
+  const [, strategyId] = msg
+
+  await deleteAllBts(strategyId)
+  send(ws, ['data.bt.history.all.deleted', strategyId])
+}

lib/cmds/delete_bt.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const { deleteBt } = require('../db/bt_dao')
+const send = require('../wss/send')
+
+module.exports = async (ds, ws, msg) => {
+  const [, executionId] = msg
+
+  await deleteBt(executionId)
+  send(ws, ['data.bt.history.deleted', executionId])
+}

lib/cmds/exec_strategy.js

@@ -23,6 +23,7 @@ const RequestSemaphore = require('../bt/request_semaphore')
 const seedCandlesFactory = require('../bt/seed_candles')
 const generateResults = require('bfx-hf-strategy/lib/util/generate_strategy_results')
 const parseStrategy = require('bfx-hf-strategy/lib/util/parse_strategy')
+const btDao = require('../db/bt_dao')
 
 /**
  * @param {DataServer} ds
@@ -39,7 +40,7 @@ module.exports = async (ds, ws, msg) => {
 
   const [
     // eslint-disable-next-line no-unused-vars
-    exchange, start, end, symbol, timeframe, includeCandles, includeTrades, candleSeed, sync = true, strategyContent, meta, constraints = {}
+    exchange, strategyId, start, end, symbol, timeframe, includeCandles, includeTrades, candleSeed, sync = true, strategyContent, executionId, constraints = {}
   ] = msg[1] || []
   const { capitalAllocation, stopLossPerc, maxDrawdownPerc } = constraints
 
@@ -48,7 +49,7 @@ module.exports = async (ds, ws, msg) => {
     strategy = parseStrategy(strategyContent)
   } catch (e) {
     console.log(e)
-    return send(ws, ['bt.btresult', { error: 'Strategy could not get parsed - parse error' }, meta])
+    return send(ws, ['bt.btresult', { error: 'Strategy could not get parsed - parse error' }, executionId])
   }
 
   const priceFeed = new PriceFeed()
@@ -68,7 +69,7 @@ module.exports = async (ds, ws, msg) => {
       perfManager
     })
   } catch (e) {
-    return send(ws, ['bt.btresult', { error: 'Strategy is invalid' }, meta])
+    return send(ws, ['bt.btresult', { error: 'Strategy is invalid' }, executionId])
   }
 
   const context = new ExecutionContext()
@@ -106,8 +107,10 @@ module.exports = async (ds, ws, msg) => {
 
   ds.activeBacktests.set(strategy.gid, context)
 
+  let executionError
   const reportError = (err) => {
     console.error(err)
+    executionError = err.message
     sendError(ws, { code: 600, res: err.message })
   }
 
@@ -120,7 +123,7 @@ module.exports = async (ds, ws, msg) => {
       const progress = Math.round((mts - start) / (end - start) * 100)
       if (progress > progressSent) {
         progressSent = progress
-        send(ws, ['bt.progress', progress, meta])
+        send(ws, ['bt.progress', progress, executionId])
       }
     }
   }
@@ -148,15 +151,25 @@ module.exports = async (ds, ws, msg) => {
   send(ws, ['bt.started', strategy.gid])
 
   return execOffline(strategy, args)
-    .then((btState = {}) => {
+    .then(async (btState = {}) => {
+      // check if error received during execution
+      if (executionError) {
+        send(ws, ['bt.btresult', { error: executionError }, executionId])
+        return
+      }
+
       const { nCandles, nTrades, strategy = {} } = btState
       const res = generateResults(perfManager, {
         ...strategy,
         nCandles,
         nTrades
       })
 
-      send(ws, ['bt.btresult', res, meta])
+      send(ws, ['bt.btresult', res, executionId])
+
+      // save to bt history db
+      const savedBt = await btDao.saveBt(msg[1], res)
+      send(ws, ['data.bt.saved', executionId, savedBt])
     })
     .catch(reportError)
     .finally(() => {

lib/cmds/get_bt_history.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const { getBtHistory } = require('../db/bt_dao')
+const send = require('../wss/send')
+
+module.exports = async (ds, ws, msg) => {
+  const [, strategyId] = msg
+  const btHistory = await getBtHistory(strategyId)
+
+  send(ws, ['data.bt.history.list', strategyId, btHistory])
+}

lib/cmds/set_bt_favorite.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const { setBtFavorite } = require('../db/bt_dao')
+const send = require('../wss/send')
+
+module.exports = async (ds, ws, msg) => {
+  const [, executionId, isFavorite] = msg
+
+  await setBtFavorite(executionId, isFavorite)
+  send(ws, ['data.bt.history.favorite', executionId, !!isFavorite])
+}

lib/db/bt_dao.js

@@ -0,0 +1,87 @@
+const sqliteDb = require('./sqlite_db')
+
+const saveBt = async (args, btResult) => {
+  const [
+    exchange, strategyId, start, end, symbol, timeframe,
+    includeCandles, includeTrades, candleSeed, sync = true, , executionId,
+    { capitalAllocation, stopLossPerc, maxDrawdownPerc }
+  ] = args
+
+  const isFavorite = 0
+  const timestamp = Date.now()
+
+  const query = 'insert into bt_history (exchange, strategyId, start, end, symbol, timeframe, includeCandles, includeTrades, candleSeed, sync, executionId, capitalAllocation, stopLossPerc, maxDrawdownPerc, isFavorite, timestamp, btResult) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'
+  const values = [
+    exchange, strategyId, start, end, symbol, timeframe,
+    includeCandles, includeTrades, candleSeed, sync, executionId,
+    capitalAllocation, stopLossPerc, maxDrawdownPerc,
+    isFavorite, timestamp, JSON.stringify(btResult)
+  ]
+
+  await sqliteDb.createData(query, values)
+
+  const savedBt = {
+    exchange,
+    strategyId,
+    start,
+    end,
+    symbol,
+    timeframe,
+    includeCandles,
+    includeTrades,
+    candleSeed,
+    sync,
+    executionId,
+    capitalAllocation,
+    stopLossPerc,
+    maxDrawdownPerc,
+    isFavorite: false,
+    timestamp,
+    btResult
+  }
+
+  return savedBt
+}
+
+const getBtHistory = async (strategyId) => {
+  const query = 'SELECT * FROM bt_history where strategyId=?'
+  const values = [strategyId]
+
+  const btHistory = await sqliteDb.queryData(query, values)
+  if (btHistory.length === 0) {
+    return btHistory
+  }
+  const normalizedBtHistory = btHistory.map((bt) => {
+    const parsedResult = bt?.btResult ? JSON.parse(bt.btResult) : {}
+    return {
+      ...bt,
+      btResult: parsedResult,
+      // Transform binary values to boolean type after SQLite
+      includeCandles: !!bt.includeCandles,
+      includeTrades: !!bt.includeTrades,
+      sync: !!bt.sync,
+      isFavorite: !!bt.isFavorite
+    }
+  })
+  return normalizedBtHistory
+}
+
+const setBtFavorite = async (executionId, isFavorite) => {
+  const query = 'update bt_history set isFavorite=? where executionId=?'
+  const values = [isFavorite, executionId]
+  await sqliteDb.executeQuery(query, values)
+}
+
+const deleteBt = async (executionId) => {
+  const query = 'delete from bt_history where executionId=?'
+  const values = [executionId]
+  await sqliteDb.executeQuery(query, values)
+}
+
+const deleteAllBts = async (strategyId) => {
+  const query = 'delete from bt_history where strategyId=?'
+  const values = [strategyId]
+  await sqliteDb.executeQuery(query, values)
+}
+
+module.exports = { saveBt, getBtHistory, setBtFavorite, deleteBt, deleteAllBts }

lib/db/sqlite_db.js

@@ -0,0 +1,67 @@
+'use strict'
+
+const debug = require('debug')('bfx:hf:data-server:db:sqlite_db')
+const Sqlite = require('bfx-facs-db-sqlite')
+let sqliteDb
+
+// connect to db
+const connectDb = async (sqlitePath) => {
+  const opts = { name: 'hf_ds', label: '', dbPathAbsolute: sqlitePath }
+  const sqlite = new Sqlite(this, opts, {})
+
+  return new Promise((resolve, reject) => {
+    sqlite.start(async () => {
+      if (!sqlite.db) reject(new Error('sqlite connection failed'))
+
+      debug('sqlite connected')
+      sqliteDb = sqlite.db
+      // create tables after db connected
+      await createTables()
+
+      resolve(true)
+    })
+  })
+}
+
+const createTables = async () => {
+  const createBTHistory = 'create table if not exists bt_history (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, exchange VARCHAR(20), strategyId VARCHAR(50), start INTEGER, end INTEGER, symbol VARCHAR(20), timeframe VARCHAR(10), includeCandles TINYINT(1), includeTrades TINYINT(1), candleSeed INTEGER, sync TINYINT(1), executionId VARCHAR(50), capitalAllocation DOUBLE, stopLossPerc DOUBLE, maxDrawdownPerc DOUBLE, isFavorite TINYINT(1), timestamp INTEGER, btResult TEXT)'
+  await executeQuery(createBTHistory)
+}
+
+const createData = async (query, values = []) => {
+  await executeQuery(query, values)
+}
+
+const executeQuery = async (query, values = []) => {
+  if (!sqliteDb) return
+
+  return new Promise((resolve, reject) => {
+    sqliteDb.run(query, values, (err, data) => _handleDbCallback(err, data, resolve, reject))
+  })
+}
+
+const getData = async (query, values = []) => {
+  if (!sqliteDb) return
+
+  return new Promise((resolve, reject) => {
+    sqliteDb.get(query, values, (err, data) => _handleDbCallback(err, data, resolve, reject))
+  })
+}
+
+const queryData = async (query, values = []) => {
+  if (!sqliteDb) return
+
+  return new Promise((resolve, reject) => {
+    sqliteDb.all(query, values, (err, data) => _handleDbCallback(err, data, resolve, reject))
+  })
+}
+
+const _handleDbCallback = (error, data, resolve, reject) => {
+  if (error) {
+    console.error(error)
+    reject(new Error(error.toString()))
+  }
+  resolve(data)
+}
+
+module.exports = { connectDb, executeQuery, createData, getData, queryData }

lib/server.js

@@ -4,6 +4,7 @@ const debug = require('debug')('bfx:hf:data-server')
 const _isFunction = require('lodash/isFunction')
 const { nonce } = require('bfx-api-node-util')
 const { Server } = require('ws')
+const sqliteDb = require('./db/sqlite_db')
 
 const { version } = require('../package.json')
 const getCandles = require('./cmds/get_candles')
@@ -16,6 +17,10 @@ const stopBT = require('./cmds/stop_bt')
 const sendError = require('./wss/send_error')
 const send = require('./wss/send')
 const ERRORS = require('./errors')
+const getBtHistory = require('./cmds/get_bt_history')
+const setBtFavorite = require('./cmds/set_bt_favorite')
+const deleteBt = require('./cmds/delete_bt')
+const deleteAllBts = require('./cmds/delete_all_bts')
 
 const COMMANDS = {
   'exec.str': execStr,
@@ -24,18 +29,24 @@ const COMMANDS = {
   'get.candles': getCandles,
   'get.trades': getTrades,
   'submit.bt': submitBT,
-  'stop.bt': stopBT
+  'stop.bt': stopBT,
+  'get.bt.history.list': getBtHistory,
+  'set.bt.history.favorite': setBtFavorite,
+  'delete.bt.history': deleteBt,
+  'delete.bt.history.all': deleteAllBts
 }
 
 class DataServer {
   /**
    * @param {Object} args
    * @param {Object} db - bfx-hf-models DB instance
    * @param {number} port - websocket server port
+   * @param {string} sqlitePath - dir to save sqlite db
    */
   constructor ({
     db,
-    port
+    port,
+    sqlitePath
   } = {}) {
     this.db = db
     this.activeSyncs = [] // sync ranges
@@ -45,12 +56,16 @@ class DataServer {
 
     this.port = port
     this.wss = null
+    this.sqlitePath = sqlitePath
   }
 
   /**
    * Spawns the WebSocket API server; throws an error if it is already open
    */
-  open () {
+  async open () {
+    // connect to sqlite db before starting the ws server
+    await this.connectSqlite()
+
     if (this.wss) {
       throw new Error('already open')
     }
@@ -65,6 +80,21 @@ class DataServer {
     debug('websocket API open on port %d', this.port)
   }
 
+  /**
+   * Connects to sqlite db; throws an error if connection fails
+   */
+  async connectSqlite () {
+    if (!this.sqlitePath) throw new Error('sqlitePath is missing')
+
+    try {
+      // connect to sqlite db
+      await sqliteDb.connectDb(this.sqlitePath)
+    } catch (err) {
+      debug('error connecting to sqlite', err)
+      throw new Error(err.message)
+    }
+  }
+
   /**
    * Closes the WebSocket API server; throws an error if it is not open
    */

lib/util/validate_bt_args.js

@@ -13,7 +13,7 @@ const ERRORS = require('../errors')
  */
 module.exports = (msg = []) => {
   const [
-    , start, end, symbol, tf, includeCandles, includeTrades, candleSeed, sync, , , constraints = {}
+    , , start, end, symbol, tf, includeCandles, includeTrades, candleSeed, sync, , , constraints = {}
   ] = msg[1] || []
   const { capitalAllocation, maxDrawdownPerc, stopLossPerc } = constraints
 

package.json

@@ -38,6 +38,7 @@
   "dependencies": {
     "bfx-api-node-rest": "^4.6.0",
     "bfx-api-node-util": "^1.0.9",
+    "bfx-facs-db-sqlite": "git+https://github.com/bitfinexcom/bfx-facs-db-sqlite.git",
     "bfx-hf-backtest": "git+https://github.com/bitfinexcom/bfx-hf-backtest.git#v2.4.1",
     "bfx-hf-indicators": "git+https://github.com/bitfinexcom/bfx-hf-indicators.git#v2.2.0",
     "bfx-hf-models": "git+https://github.com/bitfinexcom/bfx-hf-models.git#v4.0.0",
@@ -57,6 +58,7 @@
     "mocha": "^6.2.0",
     "sinon": "^14.0.0",
     "standard": "^14.2.0",
-    "ws": "^8.2.1"
+    "ws": "^8.2.1",
+    "proxyquire": "^2.1.3"
   }
 }