Websocket orderbook buffering (#333)

* Initial commit setting up a map orderbook system with a buffer. It will write to the buffer, sort apply to main orderbook and then process.

* Moves namespaces again

* Updates orderbook to use a sweet new WebsocketOrderbookUpdate type to handle all updates whether its using ID or not. So good. Adds many tests

* Starting to implement orderbook update handling per exchange. Updates namespaces again. Hopefuylly will find a way to update via ID not timestamp, too many endpoints dont provide update timestamps

* Changes orderbookbuffer to use BufferUpdate type instead of orderbook.Base to achieve more functionality and no need for type conversion functions. Updates tests

* Updates all instances of ws.orderbook.Update. Simplifies some orderbook logic

* Introduces toggleable buffer. Renames orderbooks. Completes implementation for everywhere but OKGroup due to hash calculation

* Implements orderbook update for okgroup, but forgets about the orderbook hash checking

* Fixes okgroup checksum calculation. Fixes linting issue. Removes redundant Kraken tests.

* Introduces sorting toggle and separates from buffer toggle. Uses benchmarks to highlight performance gains

* Fixes Gemini rate limit and parsing. Removes comments and fixes typos

* Fixes bitfinex orderbook processing

* Inbuilt sorting, minor fixes for websocket implementations. Improves test coverage

* Adds surprise LakeBTC websocket support

* Fixes data race

* Fixes rebasing issues due to namespace movements

* Addresses PR nits: moves folder namespace from ws to websocket. Removes line spaces in imports. Fixes lakebtc websocket returns and defer fucntions. Fixes comments

* Adds poloniex orderook sorting support

* Enables bitstamp and hitbtc orderbook sorting. Fixes poloniex's sorting

* Renames namespaces and combines monitor and connection into wshandler. Removes unused SPOT const. Changes how orderbook stuff is loaded. It is done in startup with a setup. Removes exchange name from loadsnapshot as well

* Removes the connection.go from rebasing issues. Removes error response from functions used in goroutines

* Fixes test with exchange name output change

* Fixes issues where copy and paste and replace all were used poorly
This commit is contained in:
Scott
2019-08-13 09:32:59 +10:00
committed by Adrian Gallagher
parent 2078ba907f
commit 0fbf8b172a
110 changed files with 2197 additions and 1909 deletions

View File

@@ -14,7 +14,7 @@ import (
exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
"github.com/thrasher-corp/gocryptotrader/exchanges/request"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
log "github.com/thrasher-corp/gocryptotrader/logger"
)
@@ -41,6 +41,7 @@ const (
// LakeBTC is the overarching type across the LakeBTC package
type LakeBTC struct {
exchange.Base
WebsocketConn
}
// SetDefaults sets LakeBTC defaults
@@ -67,6 +68,10 @@ func (l *LakeBTC) SetDefaults() {
l.APIUrlDefault = lakeBTCAPIURL
l.APIUrl = l.APIUrlDefault
l.Websocket = wshandler.New()
l.Websocket.Functionality = wshandler.WebsocketOrderbookSupported |
wshandler.WebsocketTradeDataSupported |
wshandler.WebsocketSubscribeSupported
l.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit
}
// Setup sets exchange configuration profile
@@ -105,6 +110,25 @@ func (l *LakeBTC) Setup(exch *config.ExchangeConfig) {
if err != nil {
log.Fatal(err)
}
err = l.Websocket.Setup(l.WsConnect,
l.Subscribe,
nil,
exch.Name,
exch.Websocket,
exch.Verbose,
lakeBTCWSURL,
exch.WebsocketURL,
exch.AuthenticatedWebsocketAPISupport)
if err != nil {
log.Fatal(err)
}
l.Websocket.Orderbook.Setup(
exch.WebsocketOrderbookBufferLimit,
false,
false,
false,
false,
exch.Name)
}
}

View File

@@ -1,15 +1,19 @@
package lakebtc
import (
"fmt"
"testing"
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/config"
"github.com/thrasher-corp/gocryptotrader/currency"
exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
)
var l LakeBTC
var setupRan bool
// Please add your own APIkeys to do correct due diligence testing.
const (
@@ -19,21 +23,27 @@ const (
)
func TestSetDefaults(t *testing.T) {
l.SetDefaults()
if !setupRan {
l.SetDefaults()
}
}
func TestSetup(t *testing.T) {
cfg := config.GetConfig()
cfg.LoadConfig("../../testdata/configtest.json")
lakebtcConfig, err := cfg.GetExchangeConfig("LakeBTC")
if err != nil {
t.Error("Test Failed - LakeBTC Setup() init error")
if !setupRan {
cfg := config.GetConfig()
cfg.LoadConfig("../../testdata/configtest.json")
lakebtcConfig, err := cfg.GetExchangeConfig("LakeBTC")
if err != nil {
t.Error("Test Failed - LakeBTC Setup() init error")
}
lakebtcConfig.AuthenticatedAPISupport = true
lakebtcConfig.APIKey = apiKey
lakebtcConfig.APISecret = apiSecret
lakebtcConfig.Websocket = true
l.Setup(&lakebtcConfig)
l.WebsocketURL = lakeBTCWSURL
setupRan = true
}
lakebtcConfig.AuthenticatedAPISupport = true
lakebtcConfig.APIKey = apiKey
lakebtcConfig.APISecret = apiSecret
l.Setup(&lakebtcConfig)
}
func TestGetTradablePairs(t *testing.T) {
@@ -445,3 +455,65 @@ func TestGetDepositAddress(t *testing.T) {
}
}
}
// TestWsConn websocket connection test
func TestWsConn(t *testing.T) {
TestSetDefaults(t)
TestSetup(t)
if !l.Websocket.IsEnabled() {
t.Skip(wshandler.WebsocketNotEnabled)
}
l.Websocket.DataHandler = sharedtestvalues.GetWebsocketInterfaceChannelOverride()
l.Websocket.TrafficAlert = sharedtestvalues.GetWebsocketStructChannelOverride()
err := l.WsConnect()
if err != nil {
t.Fatal(err)
}
}
// TestWsTradeProcessing logic test
func TestWsTradeProcessing(t *testing.T) {
TestSetDefaults(t)
TestSetup(t)
l.Websocket.DataHandler = sharedtestvalues.GetWebsocketInterfaceChannelOverride()
l.Websocket.TrafficAlert = sharedtestvalues.GetWebsocketStructChannelOverride()
json := `{"trades":[{"type":"sell","date":1564985787,"price":"11913.02","amount":"0.49"}]}`
err := l.processTrades(json, "market-btcusd-global")
if err != nil {
t.Error(err)
}
}
// TestWsTickerProcessing logic test
func TestWsTickerProcessing(t *testing.T) {
TestSetDefaults(t)
TestSetup(t)
l.Websocket.DataHandler = sharedtestvalues.GetWebsocketInterfaceChannelOverride()
l.Websocket.TrafficAlert = sharedtestvalues.GetWebsocketStructChannelOverride()
json := `{"btcusd":{"low":"10990.05","high":"11966.24","last":"11903.29","volume":"1803.967079","sell":"11912.39","buy":"11902.2"},"btceur":{"low":"9886.87","high":"10732.72","last":"10691.44","volume":"87.994478","sell":"10711.62","buy":"10691.44"},"btchkd":{"low":null,"high":null,"last":"51776.98","volume":null,"sell":"93307.37","buy":"93177.56"},"btcjpy":{"low":"1176039.0","high":"1272246.0","last":"1265680.0","volume":"129.021421","sell":"1266764.0","buy":"1265680.0"},"btcgbp":{"low":"9157.12","high":"9953.43","last":"9941.28","volume":"10.4997","sell":"10007.89","buy":"9941.28"},"btcaud":{"low":"16102.57","high":"17594.22","last":"17548.16","volume":"7.338316","sell":"17616.67","buy":"17549.69"},"btccad":{"low":"14541.69","high":"15834.87","last":"15763.54","volume":"30.480309","sell":"15793.45","buy":"15756.13"},"btcsgd":{"low":"15133.82","high":"16501.62","last":"16455.53","volume":"4.044026","sell":"16484.37","buy":"16462.18"},"btcchf":{"low":"10800.58","high":"11526.24","last":"11526.24","volume":"0.1765","sell":"11675.34","buy":"11632.02"},"btcnzd":{"low":null,"high":null,"last":"8340.98","volume":null,"sell":"18315.49","buy":"18221.37"},"btcngn":{"low":null,"high":null,"last":"600000.0","volume":null,"sell":null,"buy":null},"eurusd":{"low":"1.1088","high":"1.1138","last":"1.1125","volume":"2680.105249","sell":"1.1142","buy":"1.1121"},"gbpusd":{"low":"1.1934","high":"1.1958","last":"1.1934","volume":"1493.923823","sell":"1.1979","buy":"1.1903"},"usdjpy":{"low":"105.26","high":"107.25","last":"106.33","volume":"114490.2179","sell":"106.34","buy":"106.27"},"usdhkd":{"low":null,"high":null,"last":"7.851","volume":null,"sell":"7.8328","buy":"7.8286"},"usdcad":{"low":"1.3225","high":"1.3272","last":"1.3255","volume":"11033.9877","sell":"1.3258","buy":"1.3238"},"usdsgd":{"low":"1.3776","high":"1.3839","last":"1.3838","volume":"2523.75","sell":"1.3838","buy":"1.3819"},"audusd":{"low":"0.6764","high":"0.6853","last":"0.6771","volume":"5442.608321","sell":"0.6782","buy":"0.6762"},"nzdusd":{"low":null,"high":null,"last":"0.6758","volume":null,"sell":"0.6532","buy":"0.6504"},"usdchf":{"low":"0.9838","high":"0.9838","last":"0.9838","volume":"108.3352","sell":"0.9801","buy":"0.9773"},"usdngn":{"low":null,"high":null,"last":"200.0","volume":null,"sell":null,"buy":null},"ethbtc":{"low":"0.0205","high":"0.025","last":"0.0205","volume":null,"sell":"0.03","buy":"0.0194"},"ltcbtc":{"low":null,"high":null,"last":"0.0114","volume":null,"sell":"0.009","buy":"0.0073"},"bchbtc":{"low":null,"high":null,"last":"0.0544","volume":null,"sell":"0.0322","buy":"0.0274"},"xrpbtc":{"low":"0.000042","high":"0.000042","last":"0.000042","volume":null,"sell":"0.000037","buy":"0.000022"},"baceth":{"low":"0.000035","high":"0.000035","last":"0.000035","volume":null,"sell":"0.0015","buy":null}}`
err := l.processTicker(json)
if err != nil {
t.Error(err)
}
}
func TestGetCurrencyFromChannel(t *testing.T) {
curr := currency.NewPair(currency.LTC, currency.BTC)
result := l.getCurrencyFromChannel(fmt.Sprintf("%v%v%v", marketSubstring, curr, globalSubstring))
if curr != result {
t.Errorf("currency result is not equal. Expected %v", curr)
}
}
// TestWsOrderbookProcessing logic test
func TestWsOrderbookProcessing(t *testing.T) {
TestSetDefaults(t)
TestSetup(t)
l.Websocket.DataHandler = sharedtestvalues.GetWebsocketInterfaceChannelOverride()
l.Websocket.TrafficAlert = sharedtestvalues.GetWebsocketStructChannelOverride()
json := `{"asks":[["11905.66","0.0019"],["11905.73","0.0015"],["11906.43","0.0013"],["11906.62","0.0019"],["11907.25","11.087"],["11907.66","0.0006"],["11907.73","0.3113"],["11907.84","0.0006"],["11908.37","0.0016"],["11908.86","10.3786"],["11909.54","4.2955"],["11910.15","0.0012"],["11910.56","13.5505"],["11911.06","0.0011"],["11911.37","0.0023"]],"bids":[["11905.55","0.0171"],["11904.43","0.0225"],["11903.31","0.0223"],["11902.2","0.0027"],["11901.92","1.002"],["11901.6","0.0015"],["11901.49","0.0012"],["11901.08","0.0227"],["11900.93","0.0009"],["11900.53","1.662"],["11900.08","0.001"],["11900.01","3.6745"],["11899.96","0.003"],["11899.91","0.0006"],["11899.44","0.0013"]]}`
err := l.processOrderbook(json, "market-btcusd-global")
if err != nil {
t.Error(err)
}
}

View File

@@ -1,5 +1,7 @@
package lakebtc
import pusher "github.com/toorop/go-pusher"
// Ticker holds ticker information
type Ticker struct {
Last float64
@@ -112,3 +114,30 @@ type Withdraw struct {
At int64 `json:"at"`
Error string `json:"error"`
}
// WebsocketConn defines a pusher websocket connection
type WebsocketConn struct {
Client *pusher.Client
Ticker chan *pusher.Event
Orderbook chan *pusher.Event
Trade chan *pusher.Event
}
// WsOrderbookUpdate contains orderbook data from websocket
type WsOrderbookUpdate struct {
Asks [][]string `json:"asks"`
Bids [][]string `json:"bids"`
}
// WsTrades contains trade data from websocket
type WsTrades struct {
Trades []WsTrade `json:"trades"`
}
// WsTrade contains individual trade details from websocket
type WsTrade struct {
Type string `json:"type"`
Date int64 `json:"date"`
Price float64 `json:"price,string"`
Amount float64 `json:"amount,string"`
}

View File

@@ -0,0 +1,248 @@
package lakebtc
import (
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/thrasher-corp/gocryptotrader/common"
"github.com/thrasher-corp/gocryptotrader/currency"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
log "github.com/thrasher-corp/gocryptotrader/logger"
"github.com/toorop/go-pusher"
)
const (
lakeBTCWSURL = "ws.lakebtc.com:8085"
marketGlobalEndpoint = "market-global"
marketSubstring = "market-"
globalSubstring = "-global"
volumeString = "volume"
highString = "high"
lowString = "low"
wssSchem = "wss"
)
// WsConnect initiates a new websocket connection
func (l *LakeBTC) WsConnect() error {
if !l.Websocket.IsEnabled() || !l.IsEnabled() {
return errors.New(wshandler.WebsocketNotEnabled)
}
var err error
l.WebsocketConn.Client, err = pusher.NewCustomClient(strings.ToLower(l.Name), lakeBTCWSURL, wssSchem)
if err != nil {
return err
}
err = l.WebsocketConn.Client.Subscribe(marketGlobalEndpoint)
if err != nil {
return err
}
l.GenerateDefaultSubscriptions()
err = l.listenToEndpoints()
if err != nil {
return err
}
go l.wsHandleIncomingData()
return nil
}
func (l *LakeBTC) listenToEndpoints() error {
var err error
l.WebsocketConn.Ticker, err = l.WebsocketConn.Client.Bind("tickers")
if err != nil {
return fmt.Errorf("%s Websocket Bind error: %s", l.GetName(), err)
}
l.WebsocketConn.Orderbook, err = l.WebsocketConn.Client.Bind("update")
if err != nil {
return fmt.Errorf("%s Websocket Bind error: %s", l.GetName(), err)
}
l.WebsocketConn.Trade, err = l.WebsocketConn.Client.Bind("trades")
if err != nil {
return fmt.Errorf("%s Websocket Bind error: %s", l.GetName(), err)
}
return nil
}
// GenerateDefaultSubscriptions Adds default subscriptions to websocket to be handled by ManageSubscriptions()
func (l *LakeBTC) GenerateDefaultSubscriptions() {
var subscriptions []wshandler.WebsocketChannelSubscription
enabledCurrencies := l.GetEnabledCurrencies()
for j := range enabledCurrencies {
enabledCurrencies[j].Delimiter = ""
channel := fmt.Sprintf("%v%v%v", marketSubstring, enabledCurrencies[j].Lower(), globalSubstring)
subscriptions = append(subscriptions, wshandler.WebsocketChannelSubscription{
Channel: channel,
Currency: enabledCurrencies[j],
})
}
l.Websocket.SubscribeToChannels(subscriptions)
}
// Subscribe sends a websocket message to receive data from the channel
func (l *LakeBTC) Subscribe(channelToSubscribe wshandler.WebsocketChannelSubscription) error {
return l.WebsocketConn.Client.Subscribe(channelToSubscribe.Channel)
}
// wsHandleIncomingData services incoming data from the websocket connection
func (l *LakeBTC) wsHandleIncomingData() {
l.Websocket.Wg.Add(1)
defer l.Websocket.Wg.Done()
for {
select {
case <-l.Websocket.ShutdownC:
return
case data := <-l.WebsocketConn.Ticker:
if l.Verbose {
log.Debugf("%v Websocket message received: %v", l.Name, data)
}
l.Websocket.TrafficAlert <- struct{}{}
err := l.processTicker(data.Data)
if err != nil {
l.Websocket.DataHandler <- err
return
}
case data := <-l.WebsocketConn.Trade:
l.Websocket.TrafficAlert <- struct{}{}
if l.Verbose {
log.Debugf("%v Websocket message received: %v", l.Name, data)
}
err := l.processTrades(data.Data, data.Channel)
if err != nil {
l.Websocket.DataHandler <- err
return
}
case data := <-l.WebsocketConn.Orderbook:
l.Websocket.TrafficAlert <- struct{}{}
if l.Verbose {
log.Debugf("%v Websocket message received: %v", l.Name, data)
}
err := l.processOrderbook(data.Data, data.Channel)
if err != nil {
l.Websocket.DataHandler <- err
return
}
}
}
}
func (l *LakeBTC) processTrades(data, channel string) error {
var tradeData WsTrades
err := common.JSONDecode([]byte(data), &tradeData)
if err != nil {
return err
}
curr := l.getCurrencyFromChannel(channel)
for i := 0; i < len(tradeData.Trades); i++ {
l.Websocket.DataHandler <- wshandler.TradeData{
Timestamp: time.Unix(tradeData.Trades[i].Date, 0),
CurrencyPair: curr,
AssetType: orderbook.Spot,
Exchange: l.GetName(),
EventType: orderbook.Spot,
EventTime: tradeData.Trades[i].Date,
Price: tradeData.Trades[i].Price,
Amount: tradeData.Trades[i].Amount,
Side: tradeData.Trades[i].Type,
}
}
return nil
}
func (l *LakeBTC) processOrderbook(obUpdate, channel string) error {
var update WsOrderbookUpdate
err := common.JSONDecode([]byte(obUpdate), &update)
if err != nil {
return err
}
book := orderbook.Base{
Pair: l.getCurrencyFromChannel(channel),
LastUpdated: time.Now(),
AssetType: orderbook.Spot,
ExchangeName: l.Name,
}
for i := 0; i < len(update.Asks); i++ {
var amount, price float64
amount, err = strconv.ParseFloat(update.Asks[i][1], 64)
if err != nil {
l.Websocket.DataHandler <- fmt.Errorf("%v error parsing ticker data 'low' %v", l.Name, update.Asks[i])
continue
}
price, err = strconv.ParseFloat(update.Asks[i][0], 64)
if err != nil {
l.Websocket.DataHandler <- fmt.Errorf("%v error parsing orderbook price %v", l.Name, update.Asks[i])
continue
}
book.Asks = append(book.Asks, orderbook.Item{
Amount: amount,
Price: price,
})
}
for i := 0; i < len(update.Bids); i++ {
var amount, price float64
amount, err = strconv.ParseFloat(update.Bids[i][1], 64)
if err != nil {
l.Websocket.DataHandler <- fmt.Errorf("%v error parsing ticker data 'low' %v", l.Name, update.Bids[i])
continue
}
price, err = strconv.ParseFloat(update.Bids[i][0], 64)
if err != nil {
l.Websocket.DataHandler <- fmt.Errorf("%v error parsing orderbook price %v", l.Name, update.Bids[i])
continue
}
book.Bids = append(book.Bids, orderbook.Item{
Amount: amount,
Price: price,
})
}
return l.Websocket.Orderbook.LoadSnapshot(&book, true)
}
func (l *LakeBTC) getCurrencyFromChannel(channel string) currency.Pair {
curr := strings.Replace(channel, marketSubstring, "", 1)
curr = strings.Replace(curr, globalSubstring, "", 1)
return currency.NewPairFromString(curr)
}
func (l *LakeBTC) processTicker(ticker string) error {
var tUpdate map[string]interface{}
err := common.JSONDecode([]byte(ticker), &tUpdate)
if err != nil {
l.Websocket.DataHandler <- err
return err
}
for k, v := range tUpdate {
tickerData := v.(map[string]interface{})
if tickerData[highString] == nil || tickerData[lowString] == nil || tickerData[volumeString] == nil {
continue
}
high, err := strconv.ParseFloat(tickerData[highString].(string), 64)
if err != nil {
l.Websocket.DataHandler <- fmt.Errorf("%v error parsing ticker data 'high' %v", l.Name, tickerData)
continue
}
low, err := strconv.ParseFloat(tickerData[lowString].(string), 64)
if err != nil {
l.Websocket.DataHandler <- fmt.Errorf("%v error parsing ticker data 'low' %v", l.Name, tickerData)
continue
}
vol, err := strconv.ParseFloat(tickerData[volumeString].(string), 64)
if err != nil {
l.Websocket.DataHandler <- fmt.Errorf("%v error parsing ticker data 'volume' %v", l.Name, tickerData)
continue
}
l.Websocket.DataHandler <- wshandler.TickerData{
Timestamp: time.Now(),
Pair: currency.NewPairFromString(k),
AssetType: orderbook.Spot,
Exchange: l.GetName(),
Quantity: vol,
HighPrice: high,
LowPrice: low,
}
}
return nil
}

View File

@@ -13,7 +13,7 @@ import (
exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
log "github.com/thrasher-corp/gocryptotrader/logger"
)
@@ -268,8 +268,7 @@ func (l *LakeBTC) WithdrawFiatFundsToInternationalBank(withdrawRequest *exchange
// GetWebsocket returns a pointer to the exchange websocket
func (l *LakeBTC) GetWebsocket() (*wshandler.Websocket, error) {
// Documents are too vague to implement
return nil, common.ErrFunctionNotSupported
return l.Websocket, nil
}
// GetFeeByType returns an estimate of fee based on type of transaction