Websocket orderbook buffering (#333)

* Initial commit setting up a map orderbook system with a buffer. It will write to the buffer, sort apply to main orderbook and then process.

* Moves namespaces again

* Updates orderbook to use a sweet new WebsocketOrderbookUpdate type to handle all updates whether its using ID or not. So good. Adds many tests

* Starting to implement orderbook update handling per exchange. Updates namespaces again. Hopefuylly will find a way to update via ID not timestamp, too many endpoints dont provide update timestamps

* Changes orderbookbuffer to use BufferUpdate type instead of orderbook.Base to achieve more functionality and no need for type conversion functions. Updates tests

* Updates all instances of ws.orderbook.Update. Simplifies some orderbook logic

* Introduces toggleable buffer. Renames orderbooks. Completes implementation for everywhere but OKGroup due to hash calculation

* Implements orderbook update for okgroup, but forgets about the orderbook hash checking

* Fixes okgroup checksum calculation. Fixes linting issue. Removes redundant Kraken tests.

* Introduces sorting toggle and separates from buffer toggle. Uses benchmarks to highlight performance gains

* Fixes Gemini rate limit and parsing. Removes comments and fixes typos

* Fixes bitfinex orderbook processing

* Inbuilt sorting, minor fixes for websocket implementations. Improves test coverage

* Adds surprise LakeBTC websocket support

* Fixes data race

* Fixes rebasing issues due to namespace movements

* Addresses PR nits: moves folder namespace from ws to websocket. Removes line spaces in imports. Fixes lakebtc websocket returns and defer fucntions. Fixes comments

* Adds poloniex orderook sorting support

* Enables bitstamp and hitbtc orderbook sorting. Fixes poloniex's sorting

* Renames namespaces and combines monitor and connection into wshandler. Removes unused SPOT const. Changes how orderbook stuff is loaded. It is done in startup with a setup. Removes exchange name from loadsnapshot as well

* Removes the connection.go from rebasing issues. Removes error response from functions used in goroutines

* Fixes test with exchange name output change

* Fixes issues where copy and paste and replace all were used poorly

exchanges/hitbtc/hitbtc.go

@@ -15,7 +15,7 @@ import (
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )
 
@@ -89,6 +89,7 @@ func (h *HitBTC) SetDefaults() {
 		wshandler.WebsocketMessageCorrelationSupported
 	h.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit
 	h.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout
+	h.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit
 }
 
 // Setup sets user exchange configuration settings
@@ -150,6 +151,13 @@ func (h *HitBTC) Setup(exch *config.ExchangeConfig) {
 			ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout,
 			ResponseMaxLimit:     exch.WebsocketResponseMaxLimit,
 		}
+		h.Websocket.Orderbook.Setup(
+			exch.WebsocketOrderbookBufferLimit,
+			true,
+			true,
+			true,
+			false,
+			exch.Name)
 	}
 }
 

exchanges/hitbtc/hitbtc_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 )
 
 var h HitBTC

exchanges/hitbtc/hitbtc_websocket.go

@@ -13,7 +13,8 @@ import (
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/nonce"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )
 
@@ -114,7 +115,7 @@ func (h *HitBTC) handleSubscriptionUpdates(resp wshandler.WebsocketResponse, ini
 		}
 		h.Websocket.DataHandler <- wshandler.TickerData{
 			Exchange:  h.GetName(),
-			AssetType: "SPOT",
+			AssetType: orderbook.Spot,
 			Pair:      currency.NewPairFromString(ticker.Params.Symbol),
 			Quantity:  ticker.Params.Volume,
 			Timestamp: ts,
@@ -225,13 +226,13 @@ func (h *HitBTC) WsProcessOrderbookSnapshot(ob WsOrderbook) error {
 	}
 
 	var bids []orderbook.Item
-	for _, bid := range ob.Params.Bid {
-		bids = append(bids, orderbook.Item{Amount: bid.Size, Price: bid.Price})
+	for i := range ob.Params.Bid {
+		bids = append(bids, orderbook.Item{Amount: ob.Params.Bid[i].Size, Price: ob.Params.Bid[i].Price})
 	}
 
 	var asks []orderbook.Item
-	for _, ask := range ob.Params.Ask {
-		asks = append(asks, orderbook.Item{Amount: ask.Size, Price: ask.Price})
+	for i := range ob.Params.Ask {
+		asks = append(asks, orderbook.Item{Amount: ob.Params.Ask[i].Size, Price: ob.Params.Ask[i].Price})
 	}
 
 	p := currency.NewPairFromString(ob.Params.Symbol)
@@ -239,17 +240,17 @@ func (h *HitBTC) WsProcessOrderbookSnapshot(ob WsOrderbook) error {
 	var newOrderBook orderbook.Base
 	newOrderBook.Asks = asks
 	newOrderBook.Bids = bids
-	newOrderBook.AssetType = "SPOT"
+	newOrderBook.AssetType = orderbook.Spot
 	newOrderBook.Pair = p
 
-	err := h.Websocket.Orderbook.LoadSnapshot(&newOrderBook, h.GetName(), false)
+	err := h.Websocket.Orderbook.LoadSnapshot(&newOrderBook, false)
 	if err != nil {
 		return err
 	}
 
 	h.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{
 		Exchange: h.GetName(),
-		Asset:    "SPOT",
+		Asset:    orderbook.Spot,
 		Pair:     p,
 	}
 
@@ -257,30 +258,35 @@ func (h *HitBTC) WsProcessOrderbookSnapshot(ob WsOrderbook) error {
 }
 
 // WsProcessOrderbookUpdate updates a local cache
-func (h *HitBTC) WsProcessOrderbookUpdate(ob WsOrderbook) error {
-	if len(ob.Params.Bid) == 0 && len(ob.Params.Ask) == 0 {
+func (h *HitBTC) WsProcessOrderbookUpdate(update WsOrderbook) error {
+	if len(update.Params.Bid) == 0 && len(update.Params.Ask) == 0 {
 		return errors.New("hitbtc_websocket.go error - no data")
 	}
 
 	var bids, asks []orderbook.Item
-	for _, bid := range ob.Params.Bid {
-		bids = append(bids, orderbook.Item{Price: bid.Price, Amount: bid.Size})
+	for i := range update.Params.Bid {
+		bids = append(bids, orderbook.Item{Price: update.Params.Bid[i].Price, Amount: update.Params.Bid[i].Size})
 	}
 
-	for _, ask := range ob.Params.Ask {
-		asks = append(asks, orderbook.Item{Price: ask.Price, Amount: ask.Size})
+	for i := range update.Params.Ask {
+		asks = append(asks, orderbook.Item{Price: update.Params.Ask[i].Price, Amount: update.Params.Ask[i].Size})
 	}
 
-	p := currency.NewPairFromString(ob.Params.Symbol)
-
-	err := h.Websocket.Orderbook.Update(bids, asks, p, time.Now(), h.GetName(), "SPOT")
+	p := currency.NewPairFromString(update.Params.Symbol)
+	err := h.Websocket.Orderbook.Update(&wsorderbook.WebsocketOrderbookUpdate{
+		Asks:         asks,
+		Bids:         bids,
+		CurrencyPair: p,
+		UpdateID:     update.Params.Sequence,
+		AssetType:    orderbook.Spot,
+	})
 	if err != nil {
 		return err
 	}
 
 	h.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{
 		Exchange: h.GetName(),
-		Asset:    "SPOT",
+		Asset:    orderbook.Spot,
 		Pair:     p,
 	}
 	return nil

exchanges/hitbtc/hitbtc_wrapper.go

@@ -12,7 +12,7 @@ import (
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )