Websocket orderbook buffering (#333)

* Initial commit setting up a map orderbook system with a buffer. It will write to the buffer, sort apply to main orderbook and then process.

* Moves namespaces again

* Updates orderbook to use a sweet new WebsocketOrderbookUpdate type to handle all updates whether its using ID or not. So good. Adds many tests

* Starting to implement orderbook update handling per exchange. Updates namespaces again. Hopefuylly will find a way to update via ID not timestamp, too many endpoints dont provide update timestamps

* Changes orderbookbuffer to use BufferUpdate type instead of orderbook.Base to achieve more functionality and no need for type conversion functions. Updates tests

* Updates all instances of ws.orderbook.Update. Simplifies some orderbook logic

* Introduces toggleable buffer. Renames orderbooks. Completes implementation for everywhere but OKGroup due to hash calculation

* Implements orderbook update for okgroup, but forgets about the orderbook hash checking

* Fixes okgroup checksum calculation. Fixes linting issue. Removes redundant Kraken tests.

* Introduces sorting toggle and separates from buffer toggle. Uses benchmarks to highlight performance gains

* Fixes Gemini rate limit and parsing. Removes comments and fixes typos

* Fixes bitfinex orderbook processing

* Inbuilt sorting, minor fixes for websocket implementations. Improves test coverage

* Adds surprise LakeBTC websocket support

* Fixes data race

* Fixes rebasing issues due to namespace movements

* Addresses PR nits: moves folder namespace from ws to websocket. Removes line spaces in imports. Fixes lakebtc websocket returns and defer fucntions. Fixes comments

* Adds poloniex orderook sorting support

* Enables bitstamp and hitbtc orderbook sorting. Fixes poloniex's sorting

* Renames namespaces and combines monitor and connection into wshandler. Removes unused SPOT const. Changes how orderbook stuff is loaded. It is done in startup with a setup. Removes exchange name from loadsnapshot as well

* Removes the connection.go from rebasing issues. Removes error response from functions used in goroutines

* Fixes test with exchange name output change

* Fixes issues where copy and paste and replace all were used poorly

exchanges/coinut/coinut.go

@@ -12,9 +12,10 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/config"
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )
 
@@ -86,6 +87,7 @@ func (c *COINUT) SetDefaults() {
 		wshandler.WebsocketMessageCorrelationSupported
 	c.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit
 	c.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout
+	c.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit
 }
 
 // Setup sets the current exchange configuration
@@ -147,6 +149,13 @@ func (c *COINUT) Setup(exch *config.ExchangeConfig) {
 			ResponseMaxLimit:     exch.WebsocketResponseMaxLimit,
 			RateLimit:            coinutWebsocketRateLimit,
 		}
+		c.Websocket.Orderbook.Setup(
+			exch.WebsocketOrderbookBufferLimit,
+			true,
+			true,
+			true,
+			false,
+			exch.Name)
 	}
 }
 
@@ -154,7 +163,7 @@ func (c *COINUT) Setup(exch *config.ExchangeConfig) {
 func (c *COINUT) GetInstruments() (Instruments, error) {
 	var result Instruments
 	params := make(map[string]interface{})
-	params["sec_type"] = "SPOT"
+	params["sec_type"] = orderbook.Spot
 
 	return result, c.SendHTTPRequest(coinutInstruments, params, false, &result)
 }

exchanges/coinut/coinut_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 )
 
 var c COINUT

exchanges/coinut/coinut_websocket.go

@@ -12,7 +12,8 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook"
 )
 
 const coinutWebsocketURL = "wss://wsapi.coinut.com"
@@ -137,7 +138,7 @@ func (c *COINUT) wsProcessResponse(resp []byte) {
 		c.Websocket.DataHandler <- wshandler.TickerData{
 			Timestamp:  time.Unix(0, ticker.Timestamp),
 			Exchange:   c.GetName(),
-			AssetType:  "SPOT",
+			AssetType:  orderbook.Spot,
 			HighPrice:  ticker.HighestBuy,
 			LowPrice:   ticker.LowestSell,
 			ClosePrice: ticker.Last,
@@ -159,7 +160,7 @@ func (c *COINUT) wsProcessResponse(resp []byte) {
 		currencyPair := instrumentListByCode[orderbooksnapshot.InstID]
 		c.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{
 			Exchange: c.GetName(),
-			Asset:    "SPOT",
+			Asset:    orderbook.Spot,
 			Pair:     currency.NewPairFromString(currencyPair),
 		}
 	case "inst_order_book_update":
@@ -177,7 +178,7 @@ func (c *COINUT) wsProcessResponse(resp []byte) {
 		currencyPair := instrumentListByCode[orderbookUpdate.InstID]
 		c.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{
 			Exchange: c.GetName(),
-			Asset:    "SPOT",
+			Asset:    orderbook.Spot,
 			Pair:     currency.NewPairFromString(currencyPair),
 		}
 	case "inst_trade":
@@ -199,7 +200,7 @@ func (c *COINUT) wsProcessResponse(resp []byte) {
 		c.Websocket.DataHandler <- wshandler.TradeData{
 			Timestamp:    time.Unix(tradeUpdate.Timestamp, 0),
 			CurrencyPair: currency.NewPairFromString(currencyPair),
-			AssetType:    "SPOT",
+			AssetType:    orderbook.Spot,
 			Exchange:     c.GetName(),
 			Price:        tradeUpdate.Price,
 			Side:         tradeUpdate.Side,
@@ -228,7 +229,7 @@ func (c *COINUT) GetNonce() int64 {
 func (c *COINUT) WsSetInstrumentList() error {
 	request := wsRequest{
 		Request: "inst_list",
-		SecType: "SPOT",
+		SecType: orderbook.Spot,
 		Nonce:   c.WebsocketConn.GenerateMessageID(false),
 	}
 	resp, err := c.WebsocketConn.SendMessageReturnResponse(request.Nonce, request)
@@ -253,18 +254,18 @@ func (c *COINUT) WsSetInstrumentList() error {
 // WsProcessOrderbookSnapshot processes the orderbook snapshot
 func (c *COINUT) WsProcessOrderbookSnapshot(ob *WsOrderbookSnapshot) error {
 	var bids []orderbook.Item
-	for _, bid := range ob.Buy {
+	for i := range ob.Buy {
 		bids = append(bids, orderbook.Item{
-			Amount: bid.Volume,
-			Price:  bid.Price,
+			Amount: ob.Buy[i].Volume,
+			Price:  ob.Buy[i].Price,
 		})
 	}
 
 	var asks []orderbook.Item
-	for _, ask := range ob.Sell {
+	for i := range ob.Sell {
 		asks = append(asks, orderbook.Item{
-			Amount: ask.Volume,
-			Price:  ask.Price,
+			Amount: ob.Sell[i].Volume,
+			Price:  ob.Sell[i].Price,
 		})
 	}
 
@@ -272,32 +273,25 @@ func (c *COINUT) WsProcessOrderbookSnapshot(ob *WsOrderbookSnapshot) error {
 	newOrderBook.Asks = asks
 	newOrderBook.Bids = bids
 	newOrderBook.Pair = currency.NewPairFromString(instrumentListByCode[ob.InstID])
-	newOrderBook.AssetType = "SPOT"
+	newOrderBook.AssetType = orderbook.Spot
 
-	return c.Websocket.Orderbook.LoadSnapshot(&newOrderBook, c.GetName(), false)
+	return c.Websocket.Orderbook.LoadSnapshot(&newOrderBook, false)
 }
 
 // WsProcessOrderbookUpdate process an orderbook update
-func (c *COINUT) WsProcessOrderbookUpdate(ob *WsOrderbookUpdate) error {
-	p := currency.NewPairFromString(instrumentListByCode[ob.InstID])
-
-	if ob.Side == "buy" {
-		return c.Websocket.Orderbook.Update([]orderbook.Item{
-			{Price: ob.Price, Amount: ob.Volume}},
-			nil,
-			p,
-			time.Now(),
-			c.GetName(),
-			"SPOT")
+func (c *COINUT) WsProcessOrderbookUpdate(update *WsOrderbookUpdate) error {
+	p := currency.NewPairFromString(instrumentListByCode[update.InstID])
+	bufferUpdate := &wsorderbook.WebsocketOrderbookUpdate{
+		CurrencyPair: p,
+		UpdateID:     update.TransID,
+		AssetType:    orderbook.Spot,
 	}
-
-	return c.Websocket.Orderbook.Update([]orderbook.Item{
-		{Price: ob.Price, Amount: ob.Volume}},
-		nil,
-		p,
-		time.Now(),
-		c.GetName(),
-		"SPOT")
+	if strings.EqualFold(update.Side, "buy") {
+		bufferUpdate.Bids = []orderbook.Item{{Price: update.Price, Amount: update.Volume}}
+	} else {
+		bufferUpdate.Asks = []orderbook.Item{{Price: update.Price, Amount: update.Volume}}
+	}
+	return c.Websocket.Orderbook.Update(bufferUpdate)
 }
 
 // GenerateDefaultSubscriptions Adds default subscriptions to websocket to be handled by ManageSubscriptions()

exchanges/coinut/coinut_wrapper.go

@@ -13,7 +13,7 @@ import (
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )