Skip to content

Commit

Permalink
Exchanges: Add config variable to set bypassing of orderbook verifica…
Browse files Browse the repository at this point in the history
…tion by exchange (thrasher-corp#614)

* Exchanges: Add config variable to set bypassing of orderbook verification

* Exchanges: Consolidate orderbook variables into config struct

* Exchanges: Addr nit; set verification bypass on websocket book implementations
  • Loading branch information
shazbert authored Jan 6, 2021
1 parent 010fab0 commit 7431bf8
Show file tree
Hide file tree
Showing 56 changed files with 291 additions and 128 deletions.
10 changes: 8 additions & 2 deletions cmd/exchange_template/wrapper_file.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ func ({{.Variable}} *{{.CapitalName}}) Setup(exch *config.ExchangeConfig) error

// NOTE: PLEASE ENSURE YOU SET THE ORDERBOOK BUFFER SETTINGS CORRECTLY
{{.Variable}}.Websocket.Orderbook.Setup(
exch.WebsocketOrderbookBufferLimit,
exch.OrderbookConfig.WebsocketBufferLimit,
true,
true,
false,
Expand Down Expand Up @@ -272,7 +272,13 @@ func ({{.Variable}} *{{.CapitalName}}) FetchOrderbook(currency currency.Pair, as

// UpdateOrderbook updates and returns the orderbook for a currency pair
func ({{.Variable}} *{{.CapitalName}}) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) {
book := &orderbook.Base{ExchangeName: {{.Variable}}.Name, Pair: p, AssetType: assetType}
book := &orderbook.Base{
ExchangeName: {{.Variable}}.Name,
Pair: p,
AssetType: assetType,
VerificationBypass: {{.Variable}}.OrderbookVerificationBypass,
}

// NOTE: UPDATE ORDERBOOK EXAMPLE
/*
orderbookNew, err := {{.Variable}}.GetOrderBook(exchange.FormatExchangeCurrency({{.Variable}}.Name, p).String(), 1000)
Expand Down
4 changes: 2 additions & 2 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -1041,12 +1041,12 @@ func (c *Config) CheckExchangeConfigValues() error {
defaultWebsocketTrafficTimeout)
c.Exchanges[i].WebsocketTrafficTimeout = defaultWebsocketTrafficTimeout
}
if c.Exchanges[i].WebsocketOrderbookBufferLimit <= 0 {
if c.Exchanges[i].OrderbookConfig.WebsocketBufferLimit <= 0 {
log.Warnf(log.ConfigMgr,
"Exchange %s Websocket orderbook buffer limit value not set, defaulting to %v.",
c.Exchanges[i].Name,
defaultWebsocketOrderbookBufferLimit)
c.Exchanges[i].WebsocketOrderbookBufferLimit = defaultWebsocketOrderbookBufferLimit
c.Exchanges[i].OrderbookConfig.WebsocketBufferLimit = defaultWebsocketOrderbookBufferLimit
}
err := c.CheckPairConsistency(c.Exchanges[i].Name)
if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1459,7 +1459,7 @@ func TestCheckExchangeConfigValues(t *testing.T) {
// Test websocket and HTTP timeout values
cfg.Exchanges[0].WebsocketResponseMaxLimit = 0
cfg.Exchanges[0].WebsocketResponseCheckTimeout = 0
cfg.Exchanges[0].WebsocketOrderbookBufferLimit = 0
cfg.Exchanges[0].OrderbookConfig.WebsocketBufferLimit = 0
cfg.Exchanges[0].WebsocketTrafficTimeout = 0
cfg.Exchanges[0].HTTPTimeout = 0
err = cfg.CheckExchangeConfigValues()
Expand All @@ -1471,7 +1471,7 @@ func TestCheckExchangeConfigValues(t *testing.T) {
t.Errorf("expected exchange %s to have updated WebsocketResponseMaxLimit value",
cfg.Exchanges[0].Name)
}
if cfg.Exchanges[0].WebsocketOrderbookBufferLimit == 0 {
if cfg.Exchanges[0].OrderbookConfig.WebsocketBufferLimit == 0 {
t.Errorf("expected exchange %s to have updated WebsocketOrderbookBufferLimit value",
cfg.Exchanges[0].Name)
}
Expand Down
42 changes: 24 additions & 18 deletions config/config_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,24 +113,23 @@ type ConnectionMonitorConfig struct {

// ExchangeConfig holds all the information needed for each enabled Exchange.
type ExchangeConfig struct {
Name string `json:"name"`
Enabled bool `json:"enabled"`
Verbose bool `json:"verbose"`
UseSandbox bool `json:"useSandbox,omitempty"`
HTTPTimeout time.Duration `json:"httpTimeout"`
HTTPUserAgent string `json:"httpUserAgent,omitempty"`
HTTPDebugging bool `json:"httpDebugging,omitempty"`
WebsocketResponseCheckTimeout time.Duration `json:"websocketResponseCheckTimeout"`
WebsocketResponseMaxLimit time.Duration `json:"websocketResponseMaxLimit"`
WebsocketTrafficTimeout time.Duration `json:"websocketTrafficTimeout"`
WebsocketOrderbookBufferLimit int `json:"websocketOrderbookBufferLimit"`
WebsocketOrderbookBufferEnabled bool `json:"websocketOrderbookBufferEnabled"`
ProxyAddress string `json:"proxyAddress,omitempty"`
BaseCurrencies currency.Currencies `json:"baseCurrencies"`
CurrencyPairs *currency.PairsManager `json:"currencyPairs"`
API APIConfig `json:"api"`
Features *FeaturesConfig `json:"features"`
BankAccounts []banking.Account `json:"bankAccounts,omitempty"`
Name string `json:"name"`
Enabled bool `json:"enabled"`
Verbose bool `json:"verbose"`
UseSandbox bool `json:"useSandbox,omitempty"`
HTTPTimeout time.Duration `json:"httpTimeout"`
HTTPUserAgent string `json:"httpUserAgent,omitempty"`
HTTPDebugging bool `json:"httpDebugging,omitempty"`
WebsocketResponseCheckTimeout time.Duration `json:"websocketResponseCheckTimeout"`
WebsocketResponseMaxLimit time.Duration `json:"websocketResponseMaxLimit"`
WebsocketTrafficTimeout time.Duration `json:"websocketTrafficTimeout"`
ProxyAddress string `json:"proxyAddress,omitempty"`
BaseCurrencies currency.Currencies `json:"baseCurrencies"`
CurrencyPairs *currency.PairsManager `json:"currencyPairs"`
API APIConfig `json:"api"`
Features *FeaturesConfig `json:"features"`
BankAccounts []banking.Account `json:"bankAccounts,omitempty"`
OrderbookConfig `json:"orderbook"`

// Deprecated settings which will be removed in a future update
AvailablePairs *currency.Pairs `json:"availablePairs,omitempty"`
Expand Down Expand Up @@ -380,3 +379,10 @@ type APIConfig struct {
Credentials APICredentialsConfig `json:"credentials"`
CredentialsValidator *APICredentialsValidatorConfig `json:"credentialsValidator,omitempty"`
}

// OrderbookConfig stores the orderbook configuration variables
type OrderbookConfig struct {
VerificationBypass bool `json:"verificationBypass"`
WebsocketBufferLimit int `json:"websocketBufferLimit"`
WebsocketBufferEnabled bool `json:"websocketBufferEnabled"`
}
2 changes: 1 addition & 1 deletion docs/ADD_NEW_EXCHANGE.md
Original file line number Diff line number Diff line change
Expand Up @@ -1095,7 +1095,7 @@ func (f *FTX) Setup(exch *config.ExchangeConfig) error {
Features: &f.Features.Supports.WebsocketCapabilities, // Defines the capabilities of the websocket outlined in supported features struct. This allows the websocket connection to be flushed appropriately if we have a pair/asset enable/disable change. This is outlined below.

// Orderbook buffer specific variables for processing orderbook updates via websocket feed.
OrderbookBufferLimit: exch.WebsocketOrderbookBufferLimit,
OrderbookBufferLimit: exch.OrderbookConfig.WebsocketBufferLimit,
// Other orderbook buffer vars:
// BufferEnabled bool
// SortBuffer bool
Expand Down
1 change: 1 addition & 0 deletions exchanges/binance/binance_websocket.go
Original file line number Diff line number Diff line change
Expand Up @@ -459,6 +459,7 @@ func (b *Binance) SeedLocalCacheWithBook(p currency.Pair, orderbookNew *OrderBoo
newOrderBook.AssetType = asset.Spot
newOrderBook.ExchangeName = b.Name
newOrderBook.LastUpdateID = orderbookNew.LastUpdateID
newOrderBook.VerificationBypass = b.OrderbookVerificationBypass

return b.Websocket.Orderbook.LoadSnapshot(&newOrderBook)
}
Expand Down
11 changes: 8 additions & 3 deletions exchanges/binance/binance_wrapper.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,8 +182,8 @@ func (b *Binance) Setup(exch *config.ExchangeConfig) error {
UnSubscriber: b.Unsubscribe,
GenerateSubscriptions: b.GenerateSubscriptions,
Features: &b.Features.Supports.WebsocketCapabilities,
OrderbookBufferLimit: exch.WebsocketOrderbookBufferLimit,
BufferEnabled: exch.WebsocketOrderbookBufferEnabled,
OrderbookBufferLimit: exch.OrderbookConfig.WebsocketBufferLimit,
BufferEnabled: exch.OrderbookConfig.WebsocketBufferEnabled,
SortBuffer: true,
SortBufferByUpdateIDs: true,
})
Expand Down Expand Up @@ -401,7 +401,12 @@ func (b *Binance) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderb

// UpdateOrderbook updates and returns the orderbook for a currency pair
func (b *Binance) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) {
book := &orderbook.Base{ExchangeName: b.Name, Pair: p, AssetType: assetType}
book := &orderbook.Base{
ExchangeName: b.Name,
Pair: p,
AssetType: assetType,
VerificationBypass: b.OrderbookVerificationBypass,
}
orderbookNew, err := b.GetOrderBook(OrderBookDataRequestParams{
Symbol: p,
Limit: 1000})
Expand Down
1 change: 1 addition & 0 deletions exchanges/bitfinex/bitfinex_websocket.go
Original file line number Diff line number Diff line change
Expand Up @@ -968,6 +968,7 @@ func (b *Bitfinex) WsInsertSnapshot(p currency.Pair, assetType asset.Item, books
book.NotAggregated = true
book.HasChecksumValidation = true
book.IsFundingRate = fundingRate
book.VerificationBypass = b.OrderbookVerificationBypass
return b.Websocket.Orderbook.LoadSnapshot(&book)
}

Expand Down
14 changes: 8 additions & 6 deletions exchanges/bitfinex/bitfinex_wrapper.go
Original file line number Diff line number Diff line change
Expand Up @@ -198,8 +198,8 @@ func (b *Bitfinex) Setup(exch *config.ExchangeConfig) error {
UnSubscriber: b.Unsubscribe,
GenerateSubscriptions: b.GenerateDefaultSubscriptions,
Features: &b.Features.Supports.WebsocketCapabilities,
OrderbookBufferLimit: exch.WebsocketOrderbookBufferLimit,
BufferEnabled: exch.WebsocketOrderbookBufferEnabled,
OrderbookBufferLimit: exch.OrderbookConfig.WebsocketBufferLimit,
BufferEnabled: exch.OrderbookConfig.WebsocketBufferEnabled,
UpdateEntriesByID: true,
})
if err != nil {
Expand Down Expand Up @@ -387,10 +387,12 @@ func (b *Bitfinex) FetchOrderbook(p currency.Pair, assetType asset.Item) (*order
// UpdateOrderbook updates and returns the orderbook for a currency pair
func (b *Bitfinex) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) {
o := &orderbook.Base{
ExchangeName: b.Name,
Pair: p,
AssetType: assetType,
NotAggregated: true}
ExchangeName: b.Name,
Pair: p,
AssetType: assetType,
NotAggregated: true,
VerificationBypass: b.OrderbookVerificationBypass,
}

fPair, err := b.FormatExchangeCurrency(p, assetType)
if err != nil {
Expand Down
7 changes: 6 additions & 1 deletion exchanges/bitflyer/bitflyer_wrapper.go
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,12 @@ func (b *Bitflyer) FetchOrderbook(p currency.Pair, assetType asset.Item) (*order

// UpdateOrderbook updates and returns the orderbook for a currency pair
func (b *Bitflyer) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) {
book := &orderbook.Base{ExchangeName: b.Name, Pair: p, AssetType: assetType}
book := &orderbook.Base{
ExchangeName: b.Name,
Pair: p,
AssetType: assetType,
VerificationBypass: b.OrderbookVerificationBypass,
}

fPair, err := b.FormatExchangeCurrency(p, assetType)
if err != nil {
Expand Down
7 changes: 6 additions & 1 deletion exchanges/bithumb/bithumb_wrapper.go
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,12 @@ func (b *Bithumb) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderb

// UpdateOrderbook updates and returns the orderbook for a currency pair
func (b *Bithumb) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) {
book := &orderbook.Base{ExchangeName: b.Name, Pair: p, AssetType: assetType}
book := &orderbook.Base{
ExchangeName: b.Name,
Pair: p,
AssetType: assetType,
VerificationBypass: b.OrderbookVerificationBypass,
}
curr := p.Base.String()

orderbookNew, err := b.GetOrderBook(curr)
Expand Down
1 change: 1 addition & 0 deletions exchanges/bitmex/bitmex_websocket.go
Original file line number Diff line number Diff line change
Expand Up @@ -512,6 +512,7 @@ func (b *Bitmex) processOrderbook(data []OrderBookL2, action string, p currency.
book.AssetType = a
book.Pair = p
book.ExchangeName = b.Name
book.VerificationBypass = b.OrderbookVerificationBypass

err := b.Websocket.Orderbook.LoadSnapshot(&book)
if err != nil {
Expand Down
11 changes: 6 additions & 5 deletions exchanges/bitmex/bitmex_wrapper.go
Original file line number Diff line number Diff line change
Expand Up @@ -154,8 +154,8 @@ func (b *Bitmex) Setup(exch *config.ExchangeConfig) error {
UnSubscriber: b.Unsubscribe,
GenerateSubscriptions: b.GenerateDefaultSubscriptions,
Features: &b.Features.Supports.WebsocketCapabilities,
OrderbookBufferLimit: exch.WebsocketOrderbookBufferLimit,
BufferEnabled: exch.WebsocketOrderbookBufferEnabled,
OrderbookBufferLimit: exch.OrderbookConfig.WebsocketBufferLimit,
BufferEnabled: exch.OrderbookConfig.WebsocketBufferEnabled,
UpdateEntriesByID: true,
})
if err != nil {
Expand Down Expand Up @@ -334,9 +334,10 @@ func (b *Bitmex) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbo
// UpdateOrderbook updates and returns the orderbook for a currency pair
func (b *Bitmex) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) {
book := &orderbook.Base{
ExchangeName: b.Name,
Pair: p,
AssetType: assetType,
ExchangeName: b.Name,
Pair: p,
AssetType: assetType,
VerificationBypass: b.OrderbookVerificationBypass,
}

if assetType == asset.Index {
Expand Down
14 changes: 8 additions & 6 deletions exchanges/bitstamp/bitstamp_websocket.go
Original file line number Diff line number Diff line change
Expand Up @@ -239,12 +239,13 @@ func (b *Bitstamp) wsUpdateOrderbook(update websocketOrderBook, p currency.Pair,
bids = append(bids, orderbook.Item{Price: target, Amount: amount})
}
return b.Websocket.Orderbook.LoadSnapshot(&orderbook.Base{
Bids: bids,
Asks: asks,
Pair: p,
LastUpdated: time.Unix(update.Timestamp, 0),
AssetType: assetType,
ExchangeName: b.Name,
Bids: bids,
Asks: asks,
Pair: p,
LastUpdated: time.Unix(update.Timestamp, 0),
AssetType: assetType,
ExchangeName: b.Name,
VerificationBypass: b.OrderbookVerificationBypass,
})
}

Expand Down Expand Up @@ -276,6 +277,7 @@ func (b *Bitstamp) seedOrderBook() error {
newOrderBook.Pair = p[x]
newOrderBook.AssetType = asset.Spot
newOrderBook.ExchangeName = b.Name
newOrderBook.VerificationBypass = b.OrderbookVerificationBypass

err = b.Websocket.Orderbook.LoadSnapshot(&newOrderBook)
if err != nil {
Expand Down
11 changes: 8 additions & 3 deletions exchanges/bitstamp/bitstamp_wrapper.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,8 +164,8 @@ func (b *Bitstamp) Setup(exch *config.ExchangeConfig) error {
UnSubscriber: b.Unsubscribe,
GenerateSubscriptions: b.generateDefaultSubscriptions,
Features: &b.Features.Supports.WebsocketCapabilities,
OrderbookBufferLimit: exch.WebsocketOrderbookBufferLimit,
BufferEnabled: exch.WebsocketOrderbookBufferEnabled,
OrderbookBufferLimit: exch.OrderbookConfig.WebsocketBufferLimit,
BufferEnabled: exch.OrderbookConfig.WebsocketBufferEnabled,
})
if err != nil {
return err
Expand Down Expand Up @@ -316,7 +316,12 @@ func (b *Bitstamp) FetchOrderbook(p currency.Pair, assetType asset.Item) (*order

// UpdateOrderbook updates and returns the orderbook for a currency pair
func (b *Bitstamp) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) {
book := &orderbook.Base{ExchangeName: b.Name, Pair: p, AssetType: assetType}
book := &orderbook.Base{
ExchangeName: b.Name,
Pair: p,
AssetType: assetType,
VerificationBypass: b.OrderbookVerificationBypass,
}
fPair, err := b.FormatExchangeCurrency(p, assetType)
if err != nil {
return book, err
Expand Down
7 changes: 6 additions & 1 deletion exchanges/bittrex/bittrex_wrapper.go
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,12 @@ func (b *Bittrex) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderb

// UpdateOrderbook updates and returns the orderbook for a currency pair
func (b *Bittrex) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) {
book := &orderbook.Base{ExchangeName: b.Name, Pair: p, AssetType: assetType}
book := &orderbook.Base{
ExchangeName: b.Name,
Pair: p,
AssetType: assetType,
VerificationBypass: b.OrderbookVerificationBypass,
}
fpair, err := b.FormatExchangeCurrency(p, assetType)
if err != nil {
return book, err
Expand Down
13 changes: 7 additions & 6 deletions exchanges/btcmarkets/btcmarkets_websocket.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,12 +118,13 @@ func (b *BTCMarkets) wsHandleData(respRaw []byte) error {
}
if ob.Snapshot {
err = b.Websocket.Orderbook.LoadSnapshot(&orderbook.Base{
Pair: p,
Bids: orderbook.SortBids(bids), // Alignment completely out sort is needed
Asks: asks,
LastUpdated: ob.Timestamp,
AssetType: asset.Spot,
ExchangeName: b.Name,
Pair: p,
Bids: orderbook.SortBids(bids), // Alignment completely out sort is needed
Asks: asks,
LastUpdated: ob.Timestamp,
AssetType: asset.Spot,
ExchangeName: b.Name,
VerificationBypass: b.OrderbookVerificationBypass,
})
} else {
err = b.Websocket.Orderbook.Update(&buffer.Update{
Expand Down
13 changes: 8 additions & 5 deletions exchanges/btcmarkets/btcmarkets_wrapper.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,8 +155,8 @@ func (b *BTCMarkets) Setup(exch *config.ExchangeConfig) error {
Subscriber: b.Subscribe,
GenerateSubscriptions: b.generateDefaultSubscriptions,
Features: &b.Features.Supports.WebsocketCapabilities,
OrderbookBufferLimit: exch.WebsocketOrderbookBufferLimit,
BufferEnabled: exch.WebsocketOrderbookBufferEnabled,
OrderbookBufferLimit: exch.OrderbookConfig.WebsocketBufferLimit,
BufferEnabled: exch.OrderbookConfig.WebsocketBufferEnabled,
SortBuffer: true,
})
if err != nil {
Expand Down Expand Up @@ -352,9 +352,12 @@ func (b *BTCMarkets) FetchOrderbook(p currency.Pair, assetType asset.Item) (*ord
// UpdateOrderbook updates and returns the orderbook for a currency pair
func (b *BTCMarkets) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) {
book := &orderbook.Base{
ExchangeName: b.Name,
Pair: p,
AssetType: assetType, NotAggregated: true}
ExchangeName: b.Name,
Pair: p,
AssetType: assetType,
NotAggregated: true,
VerificationBypass: b.OrderbookVerificationBypass,
}

fpair, err := b.FormatExchangeCurrency(p, assetType)
if err != nil {
Expand Down
1 change: 1 addition & 0 deletions exchanges/btse/btse_websocket.go
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,7 @@ func (b *BTSE) wsHandleData(respRaw []byte) error {
newOB.AssetType = a
newOB.ExchangeName = b.Name
orderbook.Reverse(newOB.Asks) // Reverse asks for correct alignment
newOB.VerificationBypass = b.OrderbookVerificationBypass
err = b.Websocket.Orderbook.LoadSnapshot(&newOB)
if err != nil {
return err
Expand Down
11 changes: 8 additions & 3 deletions exchanges/btse/btse_wrapper.go
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,8 @@ func (b *BTSE) Setup(exch *config.ExchangeConfig) error {
UnSubscriber: b.Unsubscribe,
GenerateSubscriptions: b.GenerateDefaultSubscriptions,
Features: &b.Features.Supports.WebsocketCapabilities,
OrderbookBufferLimit: exch.WebsocketOrderbookBufferLimit,
BufferEnabled: exch.WebsocketOrderbookBufferEnabled,
OrderbookBufferLimit: exch.OrderbookConfig.WebsocketBufferLimit,
BufferEnabled: exch.OrderbookConfig.WebsocketBufferEnabled,
})
if err != nil {
return err
Expand Down Expand Up @@ -319,7 +319,12 @@ func (b *BTSE) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbook

// UpdateOrderbook updates and returns the orderbook for a currency pair
func (b *BTSE) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) {
book := &orderbook.Base{ExchangeName: b.Name, Pair: p, AssetType: assetType}
book := &orderbook.Base{
ExchangeName: b.Name,
Pair: p,
AssetType: assetType,
VerificationBypass: b.OrderbookVerificationBypass,
}
fPair, err := b.FormatExchangeCurrency(p, assetType)
if err != nil {
return book, err
Expand Down
Loading

0 comments on commit 7431bf8

Please sign in to comment.