Websocket orderbook buffering (#333)

* Initial commit setting up a map orderbook system with a buffer. It will write to the buffer, sort apply to main orderbook and then process.

* Moves namespaces again

* Updates orderbook to use a sweet new WebsocketOrderbookUpdate type to handle all updates whether its using ID or not. So good. Adds many tests

* Starting to implement orderbook update handling per exchange. Updates namespaces again. Hopefuylly will find a way to update via ID not timestamp, too many endpoints dont provide update timestamps

* Changes orderbookbuffer to use BufferUpdate type instead of orderbook.Base to achieve more functionality and no need for type conversion functions. Updates tests

* Updates all instances of ws.orderbook.Update. Simplifies some orderbook logic

* Introduces toggleable buffer. Renames orderbooks. Completes implementation for everywhere but OKGroup due to hash calculation

* Implements orderbook update for okgroup, but forgets about the orderbook hash checking

* Fixes okgroup checksum calculation. Fixes linting issue. Removes redundant Kraken tests.

* Introduces sorting toggle and separates from buffer toggle. Uses benchmarks to highlight performance gains

* Fixes Gemini rate limit and parsing. Removes comments and fixes typos

* Fixes bitfinex orderbook processing

* Inbuilt sorting, minor fixes for websocket implementations. Improves test coverage

* Adds surprise LakeBTC websocket support

* Fixes data race

* Fixes rebasing issues due to namespace movements

* Addresses PR nits: moves folder namespace from ws to websocket. Removes line spaces in imports. Fixes lakebtc websocket returns and defer fucntions. Fixes comments

* Adds poloniex orderook sorting support

* Enables bitstamp and hitbtc orderbook sorting. Fixes poloniex's sorting

* Renames namespaces and combines monitor and connection into wshandler. Removes unused SPOT const. Changes how orderbook stuff is loaded. It is done in startup with a setup. Removes exchange name from loadsnapshot as well

* Removes the connection.go from rebasing issues. Removes error response from functions used in goroutines

* Fixes test with exchange name output change

* Fixes issues where copy and paste and replace all were used poorly

exchanges/zb/zb.go

@@ -16,7 +16,7 @@ import (
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )
 
@@ -86,6 +86,7 @@ func (z *ZB) SetDefaults() {
 		wshandler.WebsocketMessageCorrelationSupported
 	z.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit
 	z.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout
+	z.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit
 }
 
 // Setup sets user configuration
@@ -148,6 +149,13 @@ func (z *ZB) Setup(exch *config.ExchangeConfig) {
 			ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout,
 			ResponseMaxLimit:     exch.WebsocketResponseMaxLimit,
 		}
+		z.Websocket.Orderbook.Setup(
+			exch.WebsocketOrderbookBufferLimit,
+			false,
+			false,
+			false,
+			false,
+			exch.Name)
 	}
 }
 

exchanges/zb/zb_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/config"
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 )
 
 // Please supply you own test keys here for due diligence testing.

exchanges/zb/zb_websocket.go

@@ -12,7 +12,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )
 
@@ -95,7 +95,7 @@ func (z *ZB) WsHandleData() {
 				z.Websocket.DataHandler <- wshandler.TickerData{
 					Timestamp:  time.Unix(0, ticker.Date),
 					Pair:       currency.NewPairFromString(cPair[0]),
-					AssetType:  "SPOT",
+					AssetType:  orderbook.Spot,
 					Exchange:   z.GetName(),
 					ClosePrice: ticker.Data.Last,
 					HighPrice:  ticker.Data.High,
@@ -111,8 +111,8 @@ func (z *ZB) WsHandleData() {
 				}
 
 				var asks []orderbook.Item
-				for _, askDepth := range depth.Asks {
-					ask := askDepth.([]interface{})
+				for i := range depth.Asks {
+					ask := depth.Asks[i].([]interface{})
 					asks = append(asks, orderbook.Item{
 						Amount: ask[1].(float64),
 						Price:  ask[0].(float64),
@@ -120,8 +120,8 @@ func (z *ZB) WsHandleData() {
 				}
 
 				var bids []orderbook.Item
-				for _, bidDepth := range depth.Bids {
-					bid := bidDepth.([]interface{})
+				for i := range depth.Bids {
+					bid := depth.Bids[i].([]interface{})
 					bids = append(bids, orderbook.Item{
 						Amount: bid[1].(float64),
 						Price:  bid[0].(float64),
@@ -133,11 +133,10 @@ func (z *ZB) WsHandleData() {
 				var newOrderBook orderbook.Base
 				newOrderBook.Asks = asks
 				newOrderBook.Bids = bids
-				newOrderBook.AssetType = "SPOT"
+				newOrderBook.AssetType = orderbook.Spot
 				newOrderBook.Pair = cPair
 
 				err = z.Websocket.Orderbook.LoadSnapshot(&newOrderBook,
-					z.GetName(),
 					true)
 				if err != nil {
 					z.Websocket.DataHandler <- err
@@ -146,7 +145,7 @@ func (z *ZB) WsHandleData() {
 
 				z.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{
 					Pair:     cPair,
-					Asset:    "SPOT",
+					Asset:    orderbook.Spot,
 					Exchange: z.GetName(),
 				}
 
@@ -167,7 +166,7 @@ func (z *ZB) WsHandleData() {
 				z.Websocket.DataHandler <- wshandler.TradeData{
 					Timestamp:    time.Unix(0, t.Date),
 					CurrencyPair: cPair,
-					AssetType:    "SPOT",
+					AssetType:    orderbook.Spot,
 					Exchange:     z.GetName(),
 					EventTime:    t.Date,
 					Price:        t.Price,

exchanges/zb/zb_wrapper.go

@@ -12,7 +12,7 @@ import (
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )