make binance orderbooks more flexible in in ccxtExchange.go, closes #528

 (#529)

plugins/ccxtExchange.go

@@ -20,6 +20,7 @@ var _ api.Exchange = ccxtExchange{}
 
 // ccxtExchangeSpecificParamFactory knows how to create the exchange-specific params for each exchange
 type ccxtExchangeSpecificParamFactory interface {
+	getParamsForGetOrderBook() map[string]interface{}
 	getParamsForAddOrder(submitMode api.SubmitMode) interface{}
 	getParamsForGetTradeHistory() interface{}
 }
@@ -174,26 +175,54 @@ func (c ccxtExchange) GetAccountBalances(assetList []interface{}) (map[interface
 
 // GetOrderBook impl
 func (c ccxtExchange) GetOrderBook(pair *model.TradingPair, maxCount int32) (*model.OrderBook, error) {
+	maxCountInt := int(maxCount)
+
+	// if the exchange has limitations on how many orders we can fetch, these params will handle it and adjust the fetchLimit accordingly
+	fetchLimit := maxCountInt
+	if c.esParamFactory != nil {
+		orderbookParamsMap := c.esParamFactory.getParamsForGetOrderBook()
+		if orderbookParamsMap != nil {
+			if transformLimitFnResult, ok := orderbookParamsMap["transform_limit"]; ok {
+				transformLimitFn := transformLimitFnResult.(func(limit int) (int, error))
+				newLimit, e := transformLimitFn(int(maxCount))
+				if e != nil {
+					return nil, fmt.Errorf("error while transforming maxCount limit: %s", e)
+				}
+				fetchLimit = newLimit
+			}
+		}
+	}
+
 	pairString, e := pair.ToString(c.assetConverter, c.delimiter)
 	if e != nil {
 		return nil, fmt.Errorf("error converting pair to string: %s", e)
 	}
 
-	limit := int(maxCount)
-	ob, e := c.api.FetchOrderBook(pairString, &limit)
+	ob, e := c.api.FetchOrderBook(pairString, &fetchLimit)
 	if e != nil {
 		return nil, fmt.Errorf("error while fetching orderbook for trading pair '%s': %s", pairString, e)
 	}
-
 	if _, ok := ob["asks"]; !ok {
 		return nil, fmt.Errorf("orderbook did not contain the 'asks' field: %v", ob)
 	}
 	if _, ok := ob["bids"]; !ok {
 		return nil, fmt.Errorf("orderbook did not contain the 'bids' field: %v", ob)
 	}
 
-	asks := c.readOrders(ob["asks"], pair, model.OrderActionSell)
-	bids := c.readOrders(ob["bids"], pair, model.OrderActionBuy)
+	askCcxtOrders := ob["asks"]
+	bidCcxtOrders := ob["bids"]
+	if fetchLimit != maxCountInt {
+		// we may not have fetched all the requested levels because the exchange may not have had that many levels in depth
+		if len(askCcxtOrders) > maxCountInt {
+			askCcxtOrders = askCcxtOrders[:maxCountInt]
+		}
+		if len(bidCcxtOrders) > maxCountInt {
+			bidCcxtOrders = bidCcxtOrders[:maxCountInt]
+		}
+	}
+
+	asks := c.readOrders(askCcxtOrders, pair, model.OrderActionSell)
+	bids := c.readOrders(bidCcxtOrders, pair, model.OrderActionBuy)
 	return model.MakeOrderBook(pair, asks, bids), nil
 }
 

plugins/ccxtExchangeSpecificParamFactory.go

@@ -9,6 +9,10 @@ import (
 
 type ccxtExchangeSpecificParamFactoryCoinbasepro struct{}
 
+func (f *ccxtExchangeSpecificParamFactoryCoinbasepro) getParamsForGetOrderBook() map[string]interface{} {
+	return nil
+}
+
 func (f *ccxtExchangeSpecificParamFactoryCoinbasepro) getParamsForAddOrder(submitMode api.SubmitMode) interface{} {
 	if submitMode == api.SubmitModeMakerOnly {
 		return map[string]interface{}{
@@ -24,7 +28,45 @@ func (f *ccxtExchangeSpecificParamFactoryCoinbasepro) getParamsForGetTradeHistor
 
 var _ ccxtExchangeSpecificParamFactory = &ccxtExchangeSpecificParamFactoryCoinbasepro{}
 
-type ccxtExchangeSpecificParamFactoryBinance struct{}
+type ccxtExchangeSpecificParamFactoryBinance struct {
+	validOrderBookLevels []int
+	lastValidLimit       int
+	cachedResults        map[int]int
+}
+
+func makeCcxtExchangeSpecificParamFactoryBinance() *ccxtExchangeSpecificParamFactoryBinance {
+	validOrderBookLevels := []int{5, 10, 20, 50, 100, 500, 1000, 5000}
+	return &ccxtExchangeSpecificParamFactoryBinance{
+		validOrderBookLevels: validOrderBookLevels,
+		lastValidLimit:       validOrderBookLevels[len(validOrderBookLevels)-1],
+		cachedResults:        map[int]int{},
+	}
+}
+
+func (f *ccxtExchangeSpecificParamFactoryBinance) getParamsForGetOrderBook() map[string]interface{} {
+	return map[string]interface{}{
+		"transform_limit": f.transformLimit,
+	}
+}
+
+func (f *ccxtExchangeSpecificParamFactoryBinance) transformLimit(limit int) (int /*newLimit*/, error) {
+	if newLimit, ok := f.cachedResults[limit]; ok {
+		if newLimit == -1 {
+			return -1, fmt.Errorf("limit requested (%d) is higher than the maximum limit allowed (%d)", limit, f.lastValidLimit)
+		}
+		return newLimit, nil
+	}
+
+	for _, validLimit := range f.validOrderBookLevels {
+		if limit <= validLimit {
+			f.cachedResults[limit] = validLimit
+			return validLimit, nil
+		}
+	}
+
+	f.cachedResults[limit] = -1
+	return -1, fmt.Errorf("limit requested (%d) is higher than the maximum limit allowed (%d)", limit, f.lastValidLimit)
+}
 
 func (f *ccxtExchangeSpecificParamFactoryBinance) getParamsForAddOrder(submitMode api.SubmitMode) interface{} {
 	return nil

plugins/ccxtExchangeSpecificParamFactory_test.go

@@ -0,0 +1,88 @@
+package plugins
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBinanceTransformLimit(t *testing.T) {
+	testCases := []struct {
+		limit        int
+		wantNewLimit int
+		wantError    error
+	}{
+		{
+			limit:        1,
+			wantNewLimit: 5,
+			wantError:    nil,
+		}, {
+			limit:        2,
+			wantNewLimit: 5,
+			wantError:    nil,
+		}, {
+			limit:        3,
+			wantNewLimit: 5,
+			wantError:    nil,
+		}, {
+			limit:        4,
+			wantNewLimit: 5,
+			wantError:    nil,
+		}, {
+			limit:        5,
+			wantNewLimit: 5,
+			wantError:    nil,
+		}, {
+			limit:        6,
+			wantNewLimit: 10,
+			wantError:    nil,
+		}, {
+			limit:        10,
+			wantNewLimit: 10,
+			wantError:    nil,
+		}, {
+			limit:        11,
+			wantNewLimit: 20,
+			wantError:    nil,
+		}, {
+			limit:        21,
+			wantNewLimit: 50,
+			wantError:    nil,
+		}, {
+			limit:        51,
+			wantNewLimit: 100,
+			wantError:    nil,
+		}, {
+			limit:        101,
+			wantNewLimit: 500,
+			wantError:    nil,
+		}, {
+			limit:        501,
+			wantNewLimit: 1000,
+			wantError:    nil,
+		}, {
+			limit:        1001,
+			wantNewLimit: 5000,
+			wantError:    nil,
+		}, {
+			limit:        5001,
+			wantNewLimit: -1,
+			wantError:    fmt.Errorf("limit requested (5001) is higher than the maximum limit allowed (5000)"),
+		},
+	}
+
+	binanceParamFactory := makeCcxtExchangeSpecificParamFactoryBinance()
+	for _, k := range testCases {
+		t.Run(fmt.Sprintf("%d", k.limit), func(t *testing.T) {
+			newLimit, e := binanceParamFactory.transformLimit(k.limit)
+			assert.Equal(t, k.wantNewLimit, newLimit)
+			assert.Equal(t, k.wantError, e)
+
+			// repeat values to test caching
+			newLimit, e = binanceParamFactory.transformLimit(k.limit)
+			assert.Equal(t, k.wantNewLimit, newLimit)
+			assert.Equal(t, k.wantError, e)
+		})
+	}
+}

plugins/ccxtExchange_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/stellar/kelp/model"
 )
 
-var supportedExchanges = []string{"binance", "kraken"}
+var supportedExchanges = []string{"binance", "coinbasepro"}
 var emptyAPIKey = api.ExchangeAPIKey{}
 var emptyParams = api.ExchangeParam{}
 var supportedTradingExchanges = map[string]api.ExchangeAPIKey{
@@ -76,38 +76,45 @@ func TestGetOrderBook_Ccxt(t *testing.T) {
 	}
 
 	for _, exchangeName := range supportedExchanges {
-		t.Run(exchangeName, func(t *testing.T) {
-			testCcxtExchange, e := makeCcxtExchange(
-				exchangeName,
-				testOrderConstraints[exchangeName],
-				[]api.ExchangeAPIKey{emptyAPIKey},
-				[]api.ExchangeParam{emptyParams},
-				[]api.ExchangeHeader{},
-				false,
-				nil,
-			)
-			if !assert.NoError(t, e) {
-				return
-			}
+		var esParamFactory ccxtExchangeSpecificParamFactory
+		if v, ok := ccxtExchangeSpecificParamFactoryMap["ccxt-"+exchangeName]; ok {
+			esParamFactory = v
+		}
 
-			pair := model.TradingPair{Base: model.XLM, Quote: model.BTC}
-			ob, e := testCcxtExchange.GetOrderBook(&pair, 10)
-			if !assert.NoError(t, e) {
-				return
-			}
-			assert.Equal(t, ob.Pair(), &pair)
-
-			assert.True(t, len(ob.Asks()) > 0, fmt.Sprintf("%d", len(ob.Asks())))
-			assert.True(t, len(ob.Bids()) > 0, fmt.Sprintf("%d", len(ob.Bids())))
-			assert.True(t, ob.Asks()[0].OrderAction.IsSell())
-			assert.True(t, ob.Asks()[0].OrderType.IsLimit())
-			assert.True(t, ob.Bids()[0].OrderAction.IsBuy())
-			assert.True(t, ob.Bids()[0].OrderType.IsLimit())
-			assert.True(t, ob.Asks()[0].Price.AsFloat() > 0)
-			assert.True(t, ob.Asks()[0].Volume.AsFloat() > 0)
-			assert.True(t, ob.Bids()[0].Price.AsFloat() > 0)
-			assert.True(t, ob.Bids()[0].Volume.AsFloat() > 0)
-		})
+		for _, obDepth := range []int32{1, 5, 8, 10, 15, 16, 20} {
+			t.Run(fmt.Sprintf("%s_%d", exchangeName, obDepth), func(t *testing.T) {
+				testCcxtExchange, e := makeCcxtExchange(
+					exchangeName,
+					testOrderConstraints[exchangeName],
+					[]api.ExchangeAPIKey{emptyAPIKey},
+					[]api.ExchangeParam{emptyParams},
+					[]api.ExchangeHeader{},
+					false,
+					esParamFactory,
+				)
+				if !assert.NoError(t, e) {
+					return
+				}
+
+				pair := model.TradingPair{Base: model.XLM, Quote: model.BTC}
+				ob, e := testCcxtExchange.GetOrderBook(&pair, obDepth)
+				if !assert.NoError(t, e) {
+					return
+				}
+				assert.Equal(t, ob.Pair(), &pair)
+
+				assert.True(t, len(ob.Asks()) > 0, fmt.Sprintf("%d", len(ob.Asks())))
+				assert.True(t, len(ob.Bids()) > 0, fmt.Sprintf("%d", len(ob.Bids())))
+				assert.True(t, ob.Asks()[0].OrderAction.IsSell())
+				assert.True(t, ob.Asks()[0].OrderType.IsLimit())
+				assert.True(t, ob.Bids()[0].OrderAction.IsBuy())
+				assert.True(t, ob.Bids()[0].OrderType.IsLimit())
+				assert.True(t, ob.Asks()[0].Price.AsFloat() > 0, ob.Asks()[0].Price.AsString())
+				assert.True(t, ob.Asks()[0].Volume.AsFloat() > 0)
+				assert.True(t, ob.Bids()[0].Price.AsFloat() > 0, ob.Bids()[0].Price.AsString())
+				assert.True(t, ob.Bids()[0].Volume.AsFloat() > 0)
+			})
+		}
 	}
 }
 

plugins/factory.go

@@ -41,7 +41,7 @@ type StrategyContainer struct {
 
 var ccxtExchangeSpecificParamFactoryMap = map[string]ccxtExchangeSpecificParamFactory{
 	"ccxt-coinbasepro": &ccxtExchangeSpecificParamFactoryCoinbasepro{},
-	"ccxt-binance":     &ccxtExchangeSpecificParamFactoryBinance{},
+	"ccxt-binance":     makeCcxtExchangeSpecificParamFactoryBinance(),
 }
 
 // strategies is a map of all the strategies available