Websocket orderbook buffering (#333)

* Initial commit setting up a map orderbook system with a buffer. It will write to the buffer, sort apply to main orderbook and then process.

* Moves namespaces again

* Updates orderbook to use a sweet new WebsocketOrderbookUpdate type to handle all updates whether its using ID or not. So good. Adds many tests

* Starting to implement orderbook update handling per exchange. Updates namespaces again. Hopefuylly will find a way to update via ID not timestamp, too many endpoints dont provide update timestamps

* Changes orderbookbuffer to use BufferUpdate type instead of orderbook.Base to achieve more functionality and no need for type conversion functions. Updates tests

* Updates all instances of ws.orderbook.Update. Simplifies some orderbook logic

* Introduces toggleable buffer. Renames orderbooks. Completes implementation for everywhere but OKGroup due to hash calculation

* Implements orderbook update for okgroup, but forgets about the orderbook hash checking

* Fixes okgroup checksum calculation. Fixes linting issue. Removes redundant Kraken tests.

* Introduces sorting toggle and separates from buffer toggle. Uses benchmarks to highlight performance gains

* Fixes Gemini rate limit and parsing. Removes comments and fixes typos

* Fixes bitfinex orderbook processing

* Inbuilt sorting, minor fixes for websocket implementations. Improves test coverage

* Adds surprise LakeBTC websocket support

* Fixes data race

* Fixes rebasing issues due to namespace movements

* Addresses PR nits: moves folder namespace from ws to websocket. Removes line spaces in imports. Fixes lakebtc websocket returns and defer fucntions. Fixes comments

* Adds poloniex orderook sorting support

* Enables bitstamp and hitbtc orderbook sorting. Fixes poloniex's sorting

* Renames namespaces and combines monitor and connection into wshandler. Removes unused SPOT const. Changes how orderbook stuff is loaded. It is done in startup with a setup. Removes exchange name from loadsnapshot as well

* Removes the connection.go from rebasing issues. Removes error response from functions used in goroutines

* Fixes test with exchange name output change

* Fixes issues where copy and paste and replace all were used poorly

exchanges/coinbasepro/coinbasepro.go

@@ -16,7 +16,7 @@ import (
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )
 
@@ -95,6 +95,7 @@ func (c *CoinbasePro) SetDefaults() {
 		wshandler.WebsocketSequenceNumberSupported
 	c.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit
 	c.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout
+	c.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit
 }
 
 // Setup initialises the exchange parameters with the current configuration
@@ -158,6 +159,13 @@ func (c *CoinbasePro) Setup(exch *config.ExchangeConfig) {
 			ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout,
 			ResponseMaxLimit:     exch.WebsocketResponseMaxLimit,
 		}
+		c.Websocket.Orderbook.Setup(
+			exch.WebsocketOrderbookBufferLimit,
+			true,
+			true,
+			false,
+			false,
+			exch.Name)
 	}
 }
 

exchanges/coinbasepro/coinbasepro_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 )
 
 var c CoinbasePro

exchanges/coinbasepro/coinbasepro_websocket.go

@@ -12,7 +12,8 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook"
 )
 
 const (
@@ -88,7 +89,7 @@ func (c *CoinbasePro) WsHandleData() {
 				c.Websocket.DataHandler <- wshandler.TickerData{
 					Timestamp:  ticker.Time,
 					Pair:       currency.NewPairFromString(ticker.ProductID),
-					AssetType:  "SPOT",
+					AssetType:  orderbook.Spot,
 					Exchange:   c.GetName(),
 					OpenPrice:  ticker.Open24H,
 					HighPrice:  ticker.High24H,
@@ -177,13 +178,13 @@ func (c *CoinbasePro) WsHandleData() {
 // ProcessSnapshot processes the initial orderbook snap shot
 func (c *CoinbasePro) ProcessSnapshot(snapshot *WebsocketOrderbookSnapshot) error {
 	var base orderbook.Base
-	for _, bid := range snapshot.Bids {
-		price, err := strconv.ParseFloat(bid[0].(string), 64)
+	for i := range snapshot.Bids {
+		price, err := strconv.ParseFloat(snapshot.Bids[i][0].(string), 64)
 		if err != nil {
 			return err
 		}
 
-		amount, err := strconv.ParseFloat(bid[1].(string), 64)
+		amount, err := strconv.ParseFloat(snapshot.Bids[i][1].(string), 64)
 		if err != nil {
 			return err
 		}
@@ -192,13 +193,13 @@ func (c *CoinbasePro) ProcessSnapshot(snapshot *WebsocketOrderbookSnapshot) erro
 			orderbook.Item{Price: price, Amount: amount})
 	}
 
-	for _, ask := range snapshot.Asks {
-		price, err := strconv.ParseFloat(ask[0].(string), 64)
+	for i := range snapshot.Asks {
+		price, err := strconv.ParseFloat(snapshot.Asks[i][0].(string), 64)
 		if err != nil {
 			return err
 		}
 
-		amount, err := strconv.ParseFloat(ask[1].(string), 64)
+		amount, err := strconv.ParseFloat(snapshot.Asks[i][1].(string), 64)
 		if err != nil {
 			return err
 		}
@@ -208,17 +209,17 @@ func (c *CoinbasePro) ProcessSnapshot(snapshot *WebsocketOrderbookSnapshot) erro
 	}
 
 	pair := currency.NewPairFromString(snapshot.ProductID)
-	base.AssetType = "SPOT"
+	base.AssetType = orderbook.Spot
 	base.Pair = pair
 
-	err := c.Websocket.Orderbook.LoadSnapshot(&base, c.GetName(), false)
+	err := c.Websocket.Orderbook.LoadSnapshot(&base, false)
 	if err != nil {
 		return err
 	}
 
 	c.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{
 		Pair:     pair,
-		Asset:    "SPOT",
+		Asset:    orderbook.Spot,
 		Exchange: c.GetName(),
 	}
 
@@ -227,33 +228,42 @@ func (c *CoinbasePro) ProcessSnapshot(snapshot *WebsocketOrderbookSnapshot) erro
 
 // ProcessUpdate updates the orderbook local cache
 func (c *CoinbasePro) ProcessUpdate(update WebsocketL2Update) error {
-	var Asks, Bids []orderbook.Item
+	var asks, bids []orderbook.Item
 
-	for _, data := range update.Changes {
-		price, _ := strconv.ParseFloat(data[1].(string), 64)
-		volume, _ := strconv.ParseFloat(data[2].(string), 64)
+	for i := range update.Changes {
+		price, _ := strconv.ParseFloat(update.Changes[i][1].(string), 64)
+		volume, _ := strconv.ParseFloat(update.Changes[i][2].(string), 64)
 
-		if data[0].(string) == "buy" {
-			Bids = append(Bids, orderbook.Item{Price: price, Amount: volume})
+		if update.Changes[i][0].(string) == "buy" {
+			bids = append(bids, orderbook.Item{Price: price, Amount: volume})
 		} else {
-			Asks = append(Asks, orderbook.Item{Price: price, Amount: volume})
+			asks = append(asks, orderbook.Item{Price: price, Amount: volume})
 		}
 	}
 
-	if len(Asks) == 0 && len(Bids) == 0 {
-		return errors.New("coibasepro_websocket.go error - no data in websocket update")
+	if len(asks) == 0 && len(bids) == 0 {
+		return errors.New("coinbasepro_websocket.go error - no data in websocket update")
 	}
 
 	p := currency.NewPairFromString(update.ProductID)
-
-	err := c.Websocket.Orderbook.Update(Bids, Asks, p, time.Now(), c.GetName(), "SPOT")
+	timestamp, err := time.Parse(time.RFC3339, update.Time)
+	if err != nil {
+		return err
+	}
+	err = c.Websocket.Orderbook.Update(&wsorderbook.WebsocketOrderbookUpdate{
+		Bids:         bids,
+		Asks:         asks,
+		CurrencyPair: p,
+		UpdateTime:   timestamp,
+		AssetType:    orderbook.Spot,
+	})
 	if err != nil {
 		return err
 	}
 
 	c.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{
 		Pair:     p,
-		Asset:    "SPOT",
+		Asset:    orderbook.Spot,
 		Exchange: c.GetName(),
 	}
 

exchanges/coinbasepro/coinbasepro_wrapper.go

@@ -12,7 +12,7 @@ import (
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )