Websocket orderbook buffering (#333)

* Initial commit setting up a map orderbook system with a buffer. It will write to the buffer, sort apply to main orderbook and then process.

* Moves namespaces again

* Updates orderbook to use a sweet new WebsocketOrderbookUpdate type to handle all updates whether its using ID or not. So good. Adds many tests

* Starting to implement orderbook update handling per exchange. Updates namespaces again. Hopefuylly will find a way to update via ID not timestamp, too many endpoints dont provide update timestamps

* Changes orderbookbuffer to use BufferUpdate type instead of orderbook.Base to achieve more functionality and no need for type conversion functions. Updates tests

* Updates all instances of ws.orderbook.Update. Simplifies some orderbook logic

* Introduces toggleable buffer. Renames orderbooks. Completes implementation for everywhere but OKGroup due to hash calculation

* Implements orderbook update for okgroup, but forgets about the orderbook hash checking

* Fixes okgroup checksum calculation. Fixes linting issue. Removes redundant Kraken tests.

* Introduces sorting toggle and separates from buffer toggle. Uses benchmarks to highlight performance gains

* Fixes Gemini rate limit and parsing. Removes comments and fixes typos

* Fixes bitfinex orderbook processing

* Inbuilt sorting, minor fixes for websocket implementations. Improves test coverage

* Adds surprise LakeBTC websocket support

* Fixes data race

* Fixes rebasing issues due to namespace movements

* Addresses PR nits: moves folder namespace from ws to websocket. Removes line spaces in imports. Fixes lakebtc websocket returns and defer fucntions. Fixes comments

* Adds poloniex orderook sorting support

* Enables bitstamp and hitbtc orderbook sorting. Fixes poloniex's sorting

* Renames namespaces and combines monitor and connection into wshandler. Removes unused SPOT const. Changes how orderbook stuff is loaded. It is done in startup with a setup. Removes exchange name from loadsnapshot as well

* Removes the connection.go from rebasing issues. Removes error response from functions used in goroutines

* Fixes test with exchange name output change

* Fixes issues where copy and paste and replace all were used poorly

exchanges/okgroup/okgroup.go

@@ -16,7 +16,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/config"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )
 
@@ -160,6 +160,13 @@ func (o *OKGroup) Setup(exch *config.ExchangeConfig) {
 			ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout,
 			ResponseMaxLimit:     exch.WebsocketResponseMaxLimit,
 		}
+		o.Websocket.Orderbook.Setup(
+			exch.WebsocketOrderbookBufferLimit,
+			false,
+			false,
+			false,
+			false,
+			exch.Name)
 	}
 }
 

exchanges/okgroup/okgroup_websocket.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"hash/crc32"
 	"net/http"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -16,7 +15,8 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )
 
@@ -465,7 +465,7 @@ func (o *OKGroup) WsProcessPartialOrderBook(wsEventData *WebsocketDataWrapper, i
 		ExchangeName: o.GetName(),
 	}
 
-	err := o.Websocket.Orderbook.LoadSnapshot(&newOrderBook, o.GetName(), true)
+	err := o.Websocket.Orderbook.LoadSnapshot(&newOrderBook, true)
 	if err != nil {
 		return err
 	}
@@ -480,43 +480,29 @@ func (o *OKGroup) WsProcessPartialOrderBook(wsEventData *WebsocketDataWrapper, i
 // WsProcessUpdateOrderbook updates an existing orderbook using websocket data
 // After merging WS data, it will sort, validate and finally update the existing orderbook
 func (o *OKGroup) WsProcessUpdateOrderbook(wsEventData *WebsocketDataWrapper, instrument currency.Pair, tableName string) error {
-	internalOrderbook, err := o.GetOrderbookEx(instrument, o.GetAssetTypeFromTableName(tableName))
+	update := wsorderbook.WebsocketOrderbookUpdate{
+		AssetType:    orderbook.Spot,
+		CurrencyPair: instrument,
+		UpdateTime:   wsEventData.Timestamp,
+	}
+	update.Asks = o.AppendWsOrderbookItems(wsEventData.Asks)
+	update.Bids = o.AppendWsOrderbookItems(wsEventData.Bids)
+	err := o.Websocket.Orderbook.Update(&update)
 	if err != nil {
-		return errors.New("orderbook nil, could not load existing orderbook")
+		log.Error(err)
 	}
-	if internalOrderbook.LastUpdated.After(wsEventData.Timestamp) {
-		if o.Verbose {
-			log.Errorf("Orderbook update out of order. Existing: %v, Attempted: %v", internalOrderbook.LastUpdated.Unix(), wsEventData.Timestamp.Unix())
-		}
-		return errors.New("updated orderbook is older than existing")
-	}
-	internalOrderbook.Asks = o.WsUpdateOrderbookEntry(wsEventData.Asks, internalOrderbook.Asks)
-	internalOrderbook.Bids = o.WsUpdateOrderbookEntry(wsEventData.Bids, internalOrderbook.Bids)
-	sort.Slice(internalOrderbook.Asks, func(i, j int) bool {
-		return internalOrderbook.Asks[i].Price < internalOrderbook.Asks[j].Price
-	})
-	sort.Slice(internalOrderbook.Bids, func(i, j int) bool {
-		return internalOrderbook.Bids[i].Price > internalOrderbook.Bids[j].Price
-	})
-	checksum := o.CalculateUpdateOrderbookChecksum(&internalOrderbook)
+	updatedOb := o.Websocket.Orderbook.GetOrderbook(instrument, orderbook.Spot)
+	checksum := o.CalculateUpdateOrderbookChecksum(updatedOb)
 	if checksum == wsEventData.Checksum {
 		if o.Verbose {
 			log.Debug("Orderbook valid")
 		}
-		internalOrderbook.LastUpdated = wsEventData.Timestamp
-		if o.Verbose {
-			log.Debug("Internalising orderbook")
-		}
-
-		err := o.Websocket.Orderbook.LoadSnapshot(&internalOrderbook, o.GetName(), true)
-		if err != nil {
-			log.Error(err)
-		}
 		o.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{
 			Exchange: o.GetName(),
 			Asset:    o.GetAssetTypeFromTableName(tableName),
 			Pair:     instrument,
 		}
+
 	} else {
 		if o.Verbose {
 			log.Debug("Orderbook invalid")
@@ -526,35 +512,6 @@ func (o *OKGroup) WsProcessUpdateOrderbook(wsEventData *WebsocketDataWrapper, in
 	return nil
 }
 
-// WsUpdateOrderbookEntry takes WS bid or ask data and merges it with existing orderbook bid or ask data
-func (o *OKGroup) WsUpdateOrderbookEntry(wsEntries [][]interface{}, existingOrderbookEntries []orderbook.Item) []orderbook.Item {
-	for j := range wsEntries {
-		wsEntryPrice, _ := strconv.ParseFloat(wsEntries[j][0].(string), 64)
-		wsEntryAmount, _ := strconv.ParseFloat(wsEntries[j][1].(string), 64)
-		matchFound := false
-		for k := 0; k < len(existingOrderbookEntries); k++ {
-			if existingOrderbookEntries[k].Price != wsEntryPrice {
-				continue
-			}
-			matchFound = true
-			if wsEntryAmount == 0 {
-				existingOrderbookEntries = append(existingOrderbookEntries[:k], existingOrderbookEntries[k+1:]...)
-				k--
-				continue
-			}
-			existingOrderbookEntries[k].Amount = wsEntryAmount
-			continue
-		}
-		if !matchFound {
-			existingOrderbookEntries = append(existingOrderbookEntries, orderbook.Item{
-				Amount: wsEntryAmount,
-				Price:  wsEntryPrice,
-			})
-		}
-	}
-	return existingOrderbookEntries
-}
-
 // CalculatePartialOrderbookChecksum alternates over the first 25 bid and ask entries from websocket data
 // The checksum is made up of the price and the quantity with a semicolon (:) deliminating them
 // This will also work when there are less than 25 entries (for whatever reason)

exchanges/okgroup/okgroup_wrapper.go

@@ -11,7 +11,7 @@ import (
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
 )