Save backtest history in sqlite.

lib/cmds/delete_all_bts.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const { deleteAllBts } = require('../db/bt_dao')
+const send = require('../wss/send')
+
+module.exports = async (ds, ws, msg) => {
+  const [, strategyId] = msg
+
+  await deleteAllBts(strategyId)
+  send(ws, ['data.bt.history.all.deleted', strategyId])
+}

lib/cmds/delete_bt.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const { deleteBt } = require('../db/bt_dao')
+const send = require('../wss/send')
+
+module.exports = async (ds, ws, msg) => {
+  const [, executionId] = msg
+
+  await deleteBt(executionId)
+  send(ws, ['data.bt.history.deleted', executionId])
+}

lib/cmds/exec_strategy.js

@@ -23,6 +23,7 @@ const RequestSemaphore = require('../bt/request_semaphore')
 const seedCandlesFactory = require('../bt/seed_candles')
 const generateResults = require('bfx-hf-strategy/lib/util/generate_strategy_results')
 const parseStrategy = require('bfx-hf-strategy/lib/util/parse_strategy')
+const btDao = require('../db/bt_dao')
 
 /**
  * @param {DataServer} ds
@@ -39,7 +40,7 @@ module.exports = async (ds, ws, msg) => {
 
   const [
     // eslint-disable-next-line no-unused-vars
-    exchange, start, end, symbol, timeframe, includeCandles, includeTrades, candleSeed, sync = true, strategyContent, meta, constraints = {}
+    exchange, strategyId, start, end, symbol, timeframe, includeCandles, includeTrades, candleSeed, sync = true, strategyContent, executionId, constraints = {}
   ] = msg[1] || []
   const { capitalAllocation, stopLossPerc, maxDrawdownPerc } = constraints
 
@@ -48,7 +49,7 @@ module.exports = async (ds, ws, msg) => {
     strategy = parseStrategy(strategyContent)
   } catch (e) {
     console.log(e)
-    return send(ws, ['bt.btresult', { error: 'Strategy could not get parsed - parse error' }, meta])
+    return send(ws, ['bt.btresult', { error: 'Strategy could not get parsed - parse error' }, executionId])
   }
 
   const priceFeed = new PriceFeed()
@@ -68,7 +69,7 @@ module.exports = async (ds, ws, msg) => {
       perfManager
     })
   } catch (e) {
-    return send(ws, ['bt.btresult', { error: 'Strategy is invalid' }, meta])
+    return send(ws, ['bt.btresult', { error: 'Strategy is invalid' }, executionId])
   }
 
   const context = new ExecutionContext()
@@ -106,8 +107,10 @@ module.exports = async (ds, ws, msg) => {
 
   ds.activeBacktests.set(strategy.gid, context)
 
+  let executionError
   const reportError = (err) => {
     console.error(err)
+    executionError = err.message
     sendError(ws, { code: 600, res: err.message })
   }
 
@@ -120,7 +123,7 @@ module.exports = async (ds, ws, msg) => {
       const progress = Math.round((mts - start) / (end - start) * 100)
       if (progress > progressSent) {
         progressSent = progress
-        send(ws, ['bt.progress', progress, meta])
+        send(ws, ['bt.progress', progress, executionId])
       }
     }
   }
@@ -148,15 +151,25 @@ module.exports = async (ds, ws, msg) => {
   send(ws, ['bt.started', strategy.gid])
 
   return execOffline(strategy, args)
-    .then((btState = {}) => {
+    .then(async (btState = {}) => {
+      // check if error received during execution
+      if (executionError) {
+        send(ws, ['bt.btresult', { error: executionError }, executionId])
+        return
+      }
+
       const { nCandles, nTrades, strategy = {} } = btState
       const res = generateResults(perfManager, {
         ...strategy,
         nCandles,
         nTrades
       })
 
-      send(ws, ['bt.btresult', res, meta])
+      send(ws, ['bt.btresult', res, executionId])
+
+      // save to bt history db
+      const savedBt = await btDao.saveBt(msg[1], res)
+      send(ws, ['data.bt.saved', executionId, savedBt])
     })
     .catch(reportError)
     .finally(() => {

lib/cmds/get_bt_history.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const { getBtHistory } = require('../db/bt_dao')
+const send = require('../wss/send')
+
+module.exports = async (ds, ws, msg) => {
+  const [, strategyId] = msg
+  const btHistory = await getBtHistory(strategyId)
+
+  send(ws, ['data.bt.history.list', strategyId, btHistory])
+}

lib/cmds/set_bt_favorite.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const { setBtFavorite } = require('../db/bt_dao')
+const send = require('../wss/send')
+
+module.exports = async (ds, ws, msg) => {
+  const [, executionId, isFavorite] = msg
+
+  await setBtFavorite(executionId, isFavorite)
+  send(ws, ['data.bt.history.favorite', executionId, !!isFavorite])
+}

lib/db/bt_dao.js

@@ -0,0 +1,87 @@
+const sqliteDb = require('./sqlite_db')
+
+const saveBt = async (args, btResult) => {
+  const [
+    exchange, strategyId, start, end, symbol, timeframe,
+    includeCandles, includeTrades, candleSeed, sync = true, , executionId,
+    { capitalAllocation, stopLossPerc, maxDrawdownPerc }
+  ] = args
+
+  const isFavorite = 0
+  const timestamp = Date.now()
+
+  const query = 'insert into bt_history (exchange, strategyId, start, end, symbol, timeframe, includeCandles, includeTrades, candleSeed, sync, executionId, capitalAllocation, stopLossPerc, maxDrawdownPerc, isFavorite, timestamp, btResult) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'
+  const values = [
+    exchange, strategyId, start, end, symbol, timeframe,
+    includeCandles, includeTrades, candleSeed, sync, executionId,
+    capitalAllocation, stopLossPerc, maxDrawdownPerc,
+    isFavorite, timestamp, JSON.stringify(btResult)
+  ]
+
+  await sqliteDb.createData(query, values)
+
+  const savedBt = {
+    exchange,
+    strategyId,
+    start,
+    end,
+    symbol,
+    timeframe,
+    includeCandles,
+    includeTrades,
+    candleSeed,
+    sync,
+    executionId,
+    capitalAllocation,
+    stopLossPerc,
+    maxDrawdownPerc,
+    isFavorite: false,
+    timestamp,
+    btResult
+  }
+
+  return savedBt
+}
+
+const getBtHistory = async (strategyId) => {
+  const query = 'SELECT * FROM bt_history where strategyId=?'
+  const values = [strategyId]
+
+  const btHistory = await sqliteDb.queryData(query, values)
+  if (btHistory.length === 0) {
+    return btHistory
+  }
+  const normilizedBtHistory = btHistory.map((bt) => {
+    const parsedResult = bt?.btResult ? JSON.parse(bt.btResult) : {}
+    return {
+      ...bt,
+      btResult: parsedResult,
+      // Transform binary values to boolean type after SQLite
+      includeCandles: !!bt.includeCandles,
+      includeTrades: !!bt.includeTrades,
+      sync: !!bt.sync,
+      isFavorite: !!bt.isFavorite
+    }
+  })
+  return normilizedBtHistory
+}
+
+const setBtFavorite = async (executionId, isFavorite) => {
+  const query = 'update bt_history set isFavorite=? where executionId=?'
+  const values = [isFavorite, executionId]
+  await sqliteDb.executeQuery(query, values)
+}
+
+const deleteBt = async (executionId) => {
+  const query = 'delete from bt_history where executionId=?'
+  const values = [executionId]
+  await sqliteDb.executeQuery(query, values)
+}
+
+const deleteAllBts = async (strategyId) => {
+  const query = 'delete from bt_history where strategyId=?'
+  const values = [strategyId]
+  await sqliteDb.executeQuery(query, values)
+}
+
+module.exports = { saveBt, getBtHistory, setBtFavorite, deleteBt, deleteAllBts }

lib/db/sqlite_db.js

@@ -0,0 +1,67 @@
+'use strict'
+
+const debug = require('debug')('bfx:hf:data-server:db:sqlite_db')
+const Sqlite = require('bfx-facs-db-sqlite')
+let sqliteDb
+
+// connect to db
+const connectDb = async (sqlitePath) => {
+  const opts = { name: 'hf_ds', label: '', dbPathAbsolute: sqlitePath }
+  const sqlite = new Sqlite(this, opts, {})
+
+  return new Promise((resolve, reject) => {
+    sqlite.start(async () => {
+      if (!sqlite.db) reject(new Error('sqlite connection failed'))
+
+      debug('sqlite connected')
+      sqliteDb = sqlite.db
+      // create tables after db connected
+      await createTables()
+
+      resolve(true)
+    })
+  })
+}
+
+const createTables = async () => {
+  const createBTHistory = 'create table if not exists bt_history (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, exchange VARCHAR(20), strategyId VARCHAR(50), start INTEGER, end INTEGER, symbol VARCHAR(20), timeframe VARCHAR(10), includeCandles TINYINT(1), includeTrades TINYINT(1), candleSeed INTEGER, sync TINYINT(1), executionId VARCHAR(50), capitalAllocation DOUBLE, stopLossPerc DOUBLE, maxDrawdownPerc DOUBLE, isFavorite TINYINT(1), timestamp INTEGER, btResult TEXT)'
+  await executeQuery(createBTHistory)
+}
+
+const createData = async (query, values = []) => {
+  await executeQuery(query, values)
+}
+
+const executeQuery = async (query, values = []) => {
+  if (!sqliteDb) return
+
+  return new Promise((resolve, reject) => {
+    sqliteDb.run(query, values, (err, data) => _handleDbCallback(err, data, resolve, reject))
+  })
+}
+
+const getData = async (query, values = []) => {
+  if (!sqliteDb) return
+
+  return new Promise((resolve, reject) => {
+    sqliteDb.get(query, values, (err, data) => _handleDbCallback(err, data, resolve, reject))
+  })
+}
+
+const queryData = async (query, values = []) => {
+  if (!sqliteDb) return
+
+  return new Promise((resolve, reject) => {
+    sqliteDb.all(query, values, (err, data) => _handleDbCallback(err, data, resolve, reject))
+  })
+}
+
+const _handleDbCallback = (error, data, resolve, reject) => {
+  if (error) {
+    console.error(error)
+    reject(new Error(error.toString()))
+  }
+  resolve(data)
+}
+
+module.exports = { connectDb, executeQuery, createData, getData, queryData }

lib/server.js

@@ -4,6 +4,7 @@ const debug = require('debug')('bfx:hf:data-server')
 const _isFunction = require('lodash/isFunction')
 const { nonce } = require('bfx-api-node-util')
 const { Server } = require('ws')
+const sqliteDb = require('./db/sqlite_db')
 
 const { version } = require('../package.json')
 const getCandles = require('./cmds/get_candles')
@@ -16,6 +17,10 @@ const stopBT = require('./cmds/stop_bt')
 const sendError = require('./wss/send_error')
 const send = require('./wss/send')
 const ERRORS = require('./errors')
+const getBtHistory = require('./cmds/get_bt_history')
+const setBtFavorite = require('./cmds/set_bt_favorite')
+const deleteBt = require('./cmds/delete_bt')
+const deleteAllBts = require('./cmds/delete_all_bts')
 
 const COMMANDS = {
   'exec.str': execStr,
@@ -24,18 +29,24 @@ const COMMANDS = {
   'get.candles': getCandles,
   'get.trades': getTrades,
   'submit.bt': submitBT,
-  'stop.bt': stopBT
+  'stop.bt': stopBT,
+  'get.bt.history.list': getBtHistory,
+  'set.bt.history.favorite': setBtFavorite,
+  'delete.bt.history': deleteBt,
+  'delete.bt.history.all': deleteAllBts
 }
 
 class DataServer {
   /**
    * @param {Object} args
    * @param {Object} db - bfx-hf-models DB instance
    * @param {number} port - websocket server port
+   * @param {string} sqlitePath - dir to save sqlite db
    */
   constructor ({
     db,
-    port
+    port,
+    sqlitePath
   } = {}) {
     this.db = db
     this.activeSyncs = [] // sync ranges
@@ -45,6 +56,8 @@ class DataServer {
 
     this.port = port
     this.wss = null
+
+    if (sqlitePath) sqliteDb.connectDb(sqlitePath)
   }
 
   /**

lib/util/validate_bt_args.js

@@ -13,7 +13,7 @@ const ERRORS = require('../errors')
  */
 module.exports = (msg = []) => {
   const [
-    , start, end, symbol, tf, includeCandles, includeTrades, candleSeed, sync, , , constraints = {}
+    , , start, end, symbol, tf, includeCandles, includeTrades, candleSeed, sync, , , constraints = {}
   ] = msg[1] || []
   const { capitalAllocation, maxDrawdownPerc, stopLossPerc } = constraints
 

package.json

@@ -38,6 +38,7 @@
   "dependencies": {
     "bfx-api-node-rest": "^4.6.0",
     "bfx-api-node-util": "^1.0.9",
+    "bfx-facs-db-sqlite": "git+https://github.com/bitfinexcom/bfx-facs-db-sqlite.git",
     "bfx-hf-backtest": "git+https://github.com/bitfinexcom/bfx-hf-backtest.git#v2.4.1",
     "bfx-hf-indicators": "git+https://github.com/bitfinexcom/bfx-hf-indicators.git#v2.2.0",
     "bfx-hf-models": "git+https://github.com/bitfinexcom/bfx-hf-models.git#v4.0.0",
@@ -57,6 +58,7 @@
     "mocha": "^6.2.0",
     "sinon": "^14.0.0",
     "standard": "^14.2.0",
-    "ws": "^8.2.1"
+    "ws": "^8.2.1",
+    "proxyquire": "^2.1.3"
   }
 }