Files
gocryptotrader/engine/routines.go
Ryan O'Hara-Reid eb0571cc9b exchange: binance orderbook fix (#599)
* port orderbook binance management from draft singular asset (spot) processing add additional updates to buffer management

* integrate port

* shifted burden of proof to exchange and remove repairing techniques that obfuscate issues and could caause artifacts

* WIP

* Update exchanges, update tests, update configuration so we can default off on buffer util.

* Add buffer enabled switching to all exchanges and some that are missing, default to off.

* lbtc set not aggregate books

* Addr linter issues

* EOD wip

* optimization and bug fix pass

* clean before test and benchmarking

* add testing/benchmarks to sorting/reversing functions, dropped pointer to slice as we aren't changing slice len or cap

* Add tests and removed ptr for main book as we just ammend amount

* addr exchange test issues

* ci issues

* addr glorious issues

* Addr MCB nits, fixed funding rate book for bitfinex and fixed potential panic on nil book return

* addr linter issues

* updated mistakes

* Fix more tests

* revert bypass

* Addr mcb nits

* fix zero price bug caused by exchange. Filted out bid result rather then unsubscribing. Updated orderbook to L2 so there is no aggregation.

* Allow for zero bid and ask books to be loaded and warn if found.

* remove authentication subscription conflicts as they do not have a channel ID return

* WIP - Batching outbound requests for kraken as they do not give you the partial if you subscribe to do many things.

* finalised outbound request for kraken

* filter zero value due to invalid returned data from exchange, add in max subscription amount and increased outbound batch limit

* expand to max allowed book length & fix issue where they were sending a zero length ask side when we sent a depth of zero

* Updated function comments and added in more realistic book sizing for sort cases

* change map ordering

* amalgamate maps in buffer

* Rm ln

* fix kraken linter issues

* add in buffer initialisation

* increase timout by 30seconds

* Coinbene: Add websocket orderbook length check.

* Engine: Improve switch statement for orderbook summary dissplay.

* Binance: Added tests, remove deadlock

* Exchanges: Change orderbook field -> IsFundingRate

* Orderbook Buffer: Added method to orderbookHolder

* Kraken: removed superfluous integer for sleep

* Bitmex: fixed error return

* cmd/gctcli: force 8 decimal place usage for orderbook streaming

* Kraken: Add checksum and fix bug where we were dropping returned data which was causing artifacts

* Kraken: As per orderbook documentation added in maxdepth field to update to filter depth that goes beyond current scope

* Bitfinex: Tracking down bug on margin-funding, added sequence and checksum validation websocket config on connect (WIP)

* Bitfinex: Complete implementation of checksum

* Bitfinex: Fix funding book insertion and checksum - Dropped updates and deleting items not on book are continuously occuring from stream

* Bitfinex: Fix linter issues

* Bitfinex: Fix even more linter issues.

* Bitmex: Populate orderbook base identification fields to be passed back when error occurrs

* OkGroup: Populate orderbook base identification fields to be passed back when error occurrs

* BTSE: Change string check to 'connect success' to capture multiple user successful strings

* Bitfinex: Updated handling of funding tickers

* Bitfinex: Fix undocumented alignment bug for funding rates

* Bitfinex: Updated error return with more information

* Bitfinex: Change REST fetching to Raw book to keep it in line with websocket implementation. Fix woopsy.

* Localbitcoins: Had to impose a rate limiter to stop errors, fixed return for easier error identification.

* Exchanges: Update failing tests

* LocalBitcoins: Addr nit and bumped time by 1 second for fetching books

* Kraken: Dynamically scale precision based on str return for checksum calculations

* Kraken: Add pair and asset type to validateCRC32 error reponse

* BTSE: Filter out zero amount orderbook price levels in websocket return

* Exchanges: Update orderbook functions to return orderbook base to differentiate errors.

* BTSE: Fix spelling

* Bitmex: Fix error return string

* BTSE: Add orderbook filtering function

* Coinbene: Change wording

* BTSE: Add test for filtering

* Binance: Addr nits, added in variables for buffers and worker amounts and fixed error log messages

* GolangCI: Remove excess 0

* Binance: Reduces double ups on asset and pair in errors

* Binance: Fix error checking
2021-01-04 17:19:55 +11:00

358 lines
9.9 KiB
Go

package engine
import (
"errors"
"fmt"
"strconv"
"strings"
"sync"
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/exchanges/order"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/stats"
"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/log"
)
func printCurrencyFormat(price float64) string {
displaySymbol, err := currency.GetSymbolByCurrencyName(Bot.Config.Currency.FiatDisplayCurrency)
if err != nil {
log.Errorf(log.Global, "Failed to get display symbol: %s\n", err)
}
return fmt.Sprintf("%s%.8f", displaySymbol, price)
}
func printConvertCurrencyFormat(origCurrency currency.Code, origPrice float64) string {
displayCurrency := Bot.Config.Currency.FiatDisplayCurrency
conv, err := currency.ConvertCurrency(origPrice,
origCurrency,
displayCurrency)
if err != nil {
log.Errorf(log.Global, "Failed to convert currency: %s\n", err)
}
displaySymbol, err := currency.GetSymbolByCurrencyName(displayCurrency)
if err != nil {
log.Errorf(log.Global, "Failed to get display symbol: %s\n", err)
}
origSymbol, err := currency.GetSymbolByCurrencyName(origCurrency)
if err != nil {
log.Errorf(log.Global, "Failed to get original currency symbol for %s: %s\n",
origCurrency,
err)
}
return fmt.Sprintf("%s%.2f %s (%s%.2f %s)",
displaySymbol,
conv,
displayCurrency,
origSymbol,
origPrice,
origCurrency,
)
}
func printTickerSummary(result *ticker.Price, protocol string, err error) {
if err != nil {
if err == common.ErrNotYetImplemented {
log.Warnf(log.Ticker, "Failed to get %s ticker. Error: %s\n",
protocol,
err)
return
}
log.Errorf(log.Ticker, "Failed to get %s ticker. Error: %s\n",
protocol,
err)
return
}
stats.Add(result.ExchangeName, result.Pair, result.AssetType, result.Last, result.Volume)
if result.Pair.Quote.IsFiatCurrency() &&
result.Pair.Quote != Bot.Config.Currency.FiatDisplayCurrency {
origCurrency := result.Pair.Quote.Upper()
log.Infof(log.Ticker, "%s %s %s %s: TICKER: Last %s Ask %s Bid %s High %s Low %s Volume %.8f\n",
result.ExchangeName,
protocol,
FormatCurrency(result.Pair),
strings.ToUpper(result.AssetType.String()),
printConvertCurrencyFormat(origCurrency, result.Last),
printConvertCurrencyFormat(origCurrency, result.Ask),
printConvertCurrencyFormat(origCurrency, result.Bid),
printConvertCurrencyFormat(origCurrency, result.High),
printConvertCurrencyFormat(origCurrency, result.Low),
result.Volume)
} else {
if result.Pair.Quote.IsFiatCurrency() &&
result.Pair.Quote == Bot.Config.Currency.FiatDisplayCurrency {
log.Infof(log.Ticker, "%s %s %s %s: TICKER: Last %s Ask %s Bid %s High %s Low %s Volume %.8f\n",
result.ExchangeName,
protocol,
FormatCurrency(result.Pair),
strings.ToUpper(result.AssetType.String()),
printCurrencyFormat(result.Last),
printCurrencyFormat(result.Ask),
printCurrencyFormat(result.Bid),
printCurrencyFormat(result.High),
printCurrencyFormat(result.Low),
result.Volume)
} else {
log.Infof(log.Ticker, "%s %s %s %s: TICKER: Last %.8f Ask %.8f Bid %.8f High %.8f Low %.8f Volume %.8f\n",
result.ExchangeName,
protocol,
FormatCurrency(result.Pair),
strings.ToUpper(result.AssetType.String()),
result.Last,
result.Ask,
result.Bid,
result.High,
result.Low,
result.Volume)
}
}
}
const (
book = "%s %s %s %s: ORDERBOOK: Bids len: %d Amount: %f %s. Total value: %s Asks len: %d Amount: %f %s. Total value: %s\n"
)
func printOrderbookSummary(result *orderbook.Base, protocol string, err error) {
if err != nil {
if result == nil {
log.Errorf(log.OrderBook, "Failed to get %s orderbook. Error: %s\n",
protocol,
err)
return
}
if err == common.ErrNotYetImplemented {
log.Warnf(log.OrderBook, "Failed to get %s orderbook for %s %s %s. Error: %s\n",
protocol,
result.ExchangeName,
result.Pair,
result.AssetType,
err)
return
}
log.Errorf(log.OrderBook, "Failed to get %s orderbook for %s %s %s. Error: %s\n",
protocol,
result.ExchangeName,
result.Pair,
result.AssetType,
err)
return
}
bidsAmount, bidsValue := result.TotalBidsAmount()
asksAmount, asksValue := result.TotalAsksAmount()
var bidValueResult, askValueResult string
switch {
case result.Pair.Quote.IsFiatCurrency() && result.Pair.Quote != Bot.Config.Currency.FiatDisplayCurrency:
origCurrency := result.Pair.Quote.Upper()
bidValueResult = printConvertCurrencyFormat(origCurrency, bidsValue)
askValueResult = printConvertCurrencyFormat(origCurrency, asksValue)
case result.Pair.Quote.IsFiatCurrency() && result.Pair.Quote == Bot.Config.Currency.FiatDisplayCurrency:
bidValueResult = printCurrencyFormat(bidsValue)
askValueResult = printCurrencyFormat(asksValue)
default:
bidValueResult = strconv.FormatFloat(bidsValue, 'f', -1, 64)
askValueResult = strconv.FormatFloat(asksValue, 'f', -1, 64)
}
log.Infof(log.OrderBook, book,
result.ExchangeName,
protocol,
FormatCurrency(result.Pair),
strings.ToUpper(result.AssetType.String()),
len(result.Bids),
bidsAmount,
result.Pair.Base,
bidValueResult,
len(result.Asks),
asksAmount,
result.Pair.Base,
askValueResult,
)
}
func relayWebsocketEvent(result interface{}, event, assetType, exchangeName string) {
evt := WebsocketEvent{
Data: result,
Event: event,
AssetType: assetType,
Exchange: exchangeName,
}
err := BroadcastWebsocketMessage(evt)
if err != nil {
log.Errorf(log.WebsocketMgr, "Failed to broadcast websocket event %v. Error: %s\n",
event, err)
}
}
// WebsocketRoutine Initial routine management system for websocket
func (bot *Engine) WebsocketRoutine() {
if bot.Settings.Verbose {
log.Debugln(log.WebsocketMgr, "Connecting exchange websocket services...")
}
exchanges := bot.GetExchanges()
for i := range exchanges {
go func(i int) {
if exchanges[i].SupportsWebsocket() {
if bot.Settings.Verbose {
log.Debugf(log.WebsocketMgr,
"Exchange %s websocket support: Yes Enabled: %v\n",
exchanges[i].GetName(),
common.IsEnabled(exchanges[i].IsWebsocketEnabled()),
)
}
ws, err := exchanges[i].GetWebsocket()
if err != nil {
log.Errorf(
log.WebsocketMgr,
"Exchange %s GetWebsocket error: %s\n",
exchanges[i].GetName(),
err,
)
return
}
// Exchange sync manager might have already started ws
// service or is in the process of connecting, so check
if ws.IsConnected() || ws.IsConnecting() {
return
}
// Data handler routine
go bot.WebsocketDataReceiver(ws)
if ws.IsEnabled() {
err = ws.Connect()
if err != nil {
log.Errorf(log.WebsocketMgr, "%v\n", err)
}
err = ws.FlushChannels()
if err != nil {
log.Errorf(log.WebsocketMgr, "Failed to subscribe: %v\n", err)
}
}
} else if bot.Settings.Verbose {
log.Debugf(log.WebsocketMgr,
"Exchange %s websocket support: No\n",
exchanges[i].GetName(),
)
}
}(i)
}
}
var shutdowner = make(chan struct{}, 1)
var wg sync.WaitGroup
// WebsocketDataReceiver handles websocket data coming from a websocket feed
// associated with an exchange
func (bot *Engine) WebsocketDataReceiver(ws *stream.Websocket) {
wg.Add(1)
defer wg.Done()
for {
select {
case <-shutdowner:
return
case data := <-ws.ToRoutine:
err := bot.WebsocketDataHandler(ws.GetName(), data)
if err != nil {
log.Error(log.WebsocketMgr, err)
}
}
}
}
// WebsocketDataHandler is a central point for exchange websocket implementations to send
// processed data. WebsocketDataHandler will then pass that to an appropriate handler
func (bot *Engine) WebsocketDataHandler(exchName string, data interface{}) error {
if data == nil {
return fmt.Errorf("routines.go - exchange %s nil data sent to websocket",
exchName)
}
switch d := data.(type) {
case string:
log.Info(log.WebsocketMgr, d)
case error:
return fmt.Errorf("routines.go exchange %s websocket error - %s", exchName, data)
case stream.FundingData:
if bot.Settings.Verbose {
log.Infof(log.WebsocketMgr, "%s websocket %s %s funding updated %+v",
exchName,
FormatCurrency(d.CurrencyPair),
d.AssetType,
d)
}
case *ticker.Price:
if bot.Settings.EnableExchangeSyncManager && bot.ExchangeCurrencyPairManager != nil {
bot.ExchangeCurrencyPairManager.update(exchName,
d.Pair,
d.AssetType,
SyncItemTicker,
nil)
}
err := ticker.ProcessTicker(d)
printTickerSummary(d, "websocket", err)
case stream.KlineData:
if bot.Settings.Verbose {
log.Infof(log.WebsocketMgr, "%s websocket %s %s kline updated %+v",
exchName,
FormatCurrency(d.Pair),
d.AssetType,
d)
}
case *orderbook.Base:
if bot.Settings.EnableExchangeSyncManager && bot.ExchangeCurrencyPairManager != nil {
bot.ExchangeCurrencyPairManager.update(exchName,
d.Pair,
d.AssetType,
SyncItemOrderbook,
nil)
}
printOrderbookSummary(d, "websocket", nil)
case *order.Detail:
if !bot.OrderManager.orderStore.exists(d) {
err := bot.OrderManager.orderStore.Add(d)
if err != nil {
return err
}
} else {
od, err := bot.OrderManager.orderStore.GetByExchangeAndID(d.Exchange, d.ID)
if err != nil {
return err
}
od.UpdateOrderFromDetail(d)
}
case *order.Cancel:
return bot.OrderManager.Cancel(d)
case *order.Modify:
od, err := bot.OrderManager.orderStore.GetByExchangeAndID(d.Exchange, d.ID)
if err != nil {
return err
}
od.UpdateOrderFromModify(d)
case order.ClassificationError:
return errors.New(d.Error())
case stream.UnhandledMessageWarning:
log.Warn(log.WebsocketMgr, d.Message)
default:
if bot.Settings.Verbose {
log.Warnf(log.WebsocketMgr,
"%s websocket Unknown type: %+v",
exchName,
d)
}
}
return nil
}