Websocket orderbook buffering (#333)

* Initial commit setting up a map orderbook system with a buffer. It will write to the buffer, sort apply to main orderbook and then process.

* Moves namespaces again

* Updates orderbook to use a sweet new WebsocketOrderbookUpdate type to handle all updates whether its using ID or not. So good. Adds many tests

* Starting to implement orderbook update handling per exchange. Updates namespaces again. Hopefuylly will find a way to update via ID not timestamp, too many endpoints dont provide update timestamps

* Changes orderbookbuffer to use BufferUpdate type instead of orderbook.Base to achieve more functionality and no need for type conversion functions. Updates tests

* Updates all instances of ws.orderbook.Update. Simplifies some orderbook logic

* Introduces toggleable buffer. Renames orderbooks. Completes implementation for everywhere but OKGroup due to hash calculation

* Implements orderbook update for okgroup, but forgets about the orderbook hash checking

* Fixes okgroup checksum calculation. Fixes linting issue. Removes redundant Kraken tests.

* Introduces sorting toggle and separates from buffer toggle. Uses benchmarks to highlight performance gains

* Fixes Gemini rate limit and parsing. Removes comments and fixes typos

* Fixes bitfinex orderbook processing

* Inbuilt sorting, minor fixes for websocket implementations. Improves test coverage

* Adds surprise LakeBTC websocket support

* Fixes data race

* Fixes rebasing issues due to namespace movements

* Addresses PR nits: moves folder namespace from ws to websocket. Removes line spaces in imports. Fixes lakebtc websocket returns and defer fucntions. Fixes comments

* Adds poloniex orderook sorting support

* Enables bitstamp and hitbtc orderbook sorting. Fixes poloniex's sorting

* Renames namespaces and combines monitor and connection into wshandler. Removes unused SPOT const. Changes how orderbook stuff is loaded. It is done in startup with a setup. Removes exchange name from loadsnapshot as well

* Removes the connection.go from rebasing issues. Removes error response from functions used in goroutines

* Fixes test with exchange name output change

* Fixes issues where copy and paste and replace all were used poorly

exchanges/gemini/gemini.go

@@ -14,7 +14,7 @@ import (
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )
 
@@ -116,8 +116,8 @@ func (g *Gemini) SetDefaults() {
 	g.SupportsAutoPairUpdating = true
 	g.SupportsRESTTickerBatching = false
 	g.Requester = request.New(g.Name,
-		request.NewRateLimit(time.Minute, geminiAuthRate),
-		request.NewRateLimit(time.Minute, geminiUnauthRate),
+		request.NewRateLimit(time.Second, geminiAuthRate),
+		request.NewRateLimit(time.Second, geminiUnauthRate),
 		common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 	g.APIUrlDefault = geminiAPIURL
 	g.APIUrl = g.APIUrlDefault
@@ -128,6 +128,7 @@ func (g *Gemini) SetDefaults() {
 		wshandler.WebsocketSequenceNumberSupported
 	g.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit
 	g.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout
+	g.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit
 }
 
 // Setup sets exchange configuration parameters
@@ -186,6 +187,13 @@ func (g *Gemini) Setup(exch *config.ExchangeConfig) {
 		}
 		responseCheckTimeout = exch.WebsocketResponseCheckTimeout
 		responseMaxLimit = exch.WebsocketResponseMaxLimit
+		g.Websocket.Orderbook.Setup(
+			exch.WebsocketOrderbookBufferLimit,
+			true,
+			true,
+			false,
+			false,
+			exch.Name)
 	}
 }
 

exchanges/gemini/gemini_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 )
 
 // Please enter sandbox API keys & assigned roles for better testing procedures

exchanges/gemini/gemini_types.go

@@ -212,7 +212,7 @@ type WsMarketUpdateResponse struct {
 
 // Event defines orderbook and trade data
 type Event struct {
-	Type      string  `json:"change"`
+	Type      string  `json:"type"`
 	Reason    string  `json:"reason"`
 	Price     float64 `json:"price,string"`
 	Delta     float64 `json:"delta,string"`

exchanges/gemini/gemini_websocket.go

@@ -14,7 +14,8 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )
 
@@ -110,9 +111,11 @@ func (g *Gemini) WsSecureSubscribe(dialer *websocket.Dialer, url string) error {
 	headers.Add("Cache-Control", "no-cache")
 
 	g.AuthenticatedWebsocketConn = &wshandler.WebsocketConnection{
-		ExchangeName: g.Name,
-		URL:          endpoint,
-		Verbose:      g.Verbose,
+		ExchangeName:         g.Name,
+		URL:                  endpoint,
+		Verbose:              g.Verbose,
+		ResponseCheckTimeout: responseCheckTimeout,
+		ResponseMaxLimit:     responseMaxLimit,
 	}
 	err = g.AuthenticatedWebsocketConn.Dial(dialer, headers)
 	if err != nil {
@@ -253,86 +256,75 @@ func (g *Gemini) WsHandleData() {
 func (g *Gemini) wsProcessUpdate(result WsMarketUpdateResponse, pair currency.Pair) {
 	if result.Timestamp == 0 && result.TimestampMS == 0 {
 		var bids, asks []orderbook.Item
-		for _, event := range result.Events {
-			if event.Reason != "initial" {
+		for i := range result.Events {
+			if result.Events[i].Reason != "initial" {
 				g.Websocket.DataHandler <- errors.New("gemini_websocket.go orderbook should be snapshot only")
 				continue
 			}
-
-			if event.Side == "ask" {
+			if result.Events[i].Side == "ask" {
 				asks = append(asks, orderbook.Item{
-					Amount: event.Remaining,
-					Price:  event.Price,
+					Amount: result.Events[i].Remaining,
+					Price:  result.Events[i].Price,
 				})
 			} else {
 				bids = append(bids, orderbook.Item{
-					Amount: event.Remaining,
-					Price:  event.Price,
+					Amount: result.Events[i].Remaining,
+					Price:  result.Events[i].Price,
 				})
 			}
 		}
-
 		var newOrderBook orderbook.Base
 		newOrderBook.Asks = asks
 		newOrderBook.Bids = bids
-		newOrderBook.AssetType = "SPOT"
+		newOrderBook.AssetType = orderbook.Spot
 		newOrderBook.Pair = pair
-
 		err := g.Websocket.Orderbook.LoadSnapshot(&newOrderBook,
-			g.GetName(),
 			false)
 		if err != nil {
 			g.Websocket.DataHandler <- err
 			return
 		}
-
 		g.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{Pair: pair,
-			Asset:    "SPOT",
+			Asset:    orderbook.Spot,
 			Exchange: g.GetName()}
 	} else {
-		for _, event := range result.Events {
-			if event.Type == "trade" {
+		var asks, bids []orderbook.Item
+		for i := 0; i < len(result.Events); i++ {
+			if result.Events[i].Type == "trade" {
 				g.Websocket.DataHandler <- wshandler.TradeData{
 					Timestamp:    time.Now(),
 					CurrencyPair: pair,
-					AssetType:    "SPOT",
+					AssetType:    orderbook.Spot,
 					Exchange:     g.Name,
 					EventTime:    result.Timestamp,
-					Price:        event.Price,
-					Amount:       event.Amount,
-					Side:         event.MakerSide,
+					Price:        result.Events[i].Price,
+					Amount:       result.Events[i].Amount,
+					Side:         result.Events[i].MakerSide,
 				}
-
 			} else {
-				var i orderbook.Item
-				i.Amount = event.Remaining
-				i.Price = event.Price
-				if event.Side == "ask" {
-					err := g.Websocket.Orderbook.Update(nil,
-						[]orderbook.Item{i},
-						pair,
-						time.Now(),
-						g.GetName(),
-						"SPOT")
-					if err != nil {
-						g.Websocket.DataHandler <- err
-					}
+				item := orderbook.Item{
+					Amount: result.Events[i].Remaining,
+					Price:  result.Events[i].Price,
+				}
+				if result.Events[i].Side == "ask" {
+					asks = append(asks, item)
 				} else {
-					err := g.Websocket.Orderbook.Update([]orderbook.Item{i},
-						nil,
-						pair,
-						time.Now(),
-						g.GetName(),
-						"SPOT")
-					if err != nil {
-						g.Websocket.DataHandler <- err
-					}
+					bids = append(bids, item)
 				}
 			}
 		}
-
+		err := g.Websocket.Orderbook.Update(&wsorderbook.WebsocketOrderbookUpdate{
+			Asks:         asks,
+			Bids:         bids,
+			CurrencyPair: pair,
+			UpdateTime:   time.Unix(0, result.TimestampMS),
+			AssetType:    orderbook.Spot,
+		})
+		if err != nil {
+			g.Websocket.DataHandler <- fmt.Errorf("%v %v", g.Name, err)
+		}
 		g.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{Pair: pair,
-			Asset:    "SPOT",
+			Asset:    orderbook.Spot,
 			Exchange: g.GetName()}
 	}
 }

exchanges/gemini/gemini_wrapper.go

@@ -14,7 +14,7 @@ import (
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )