btcmarkets: Add websocket orderbook checksum validation (#900)

* btcmarkets: add websocket checksum, fetch different book via REST

* Update exchanges/btcmarkets/btcmarkets_test.go

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

* buffer: add explicit type for buffer related variables and comments, do all checks buffer side and load in setup as per glorious recom.

* buffer: fix tests add error

* buffer: test re-add code cov

* depth/stream/ws: fix tests, change field name to be more specific.

* buffer: rm unused field and small comment fixes

* btcm: remove redundant field

* glorious: nits

* buffer: fix commenting

Co-authored-by: Scott <gloriousCode@users.noreply.github.com>

exchanges/stream/buffer/buffer.go

@@ -17,6 +17,7 @@ const packageError = "websocket orderbook buffer error: %w"
 
 var (
 	errExchangeConfigNil            = errors.New("exchange config is nil")
+	errBufferConfigNil              = errors.New("buffer config is nil")
 	errUnsetDataHandler             = errors.New("datahandler unset")
 	errIssueBufferEnabledButNoLimit = errors.New("buffer enabled but no limit set")
 	errUpdateIsNil                  = errors.New("update is nil")
@@ -26,35 +27,43 @@ var (
 )
 
 // Setup sets private variables
-func (w *Orderbook) Setup(cfg *config.Exchange, sortBuffer, sortBufferByUpdateIDs, updateEntriesByID bool, dataHandler chan interface{}) error {
-	if cfg == nil { // exchange config fields are checked in stream package
+func (w *Orderbook) Setup(exchangeConfig *config.Exchange, c *Config, dataHandler chan<- interface{}) error {
+	if exchangeConfig == nil { // exchange config fields are checked in stream package
 		// prior to calling this, so further checks are not needed.
 		return fmt.Errorf(packageError, errExchangeConfigNil)
 	}
+	if c == nil {
+		return fmt.Errorf(packageError, errBufferConfigNil)
+	}
 	if dataHandler == nil {
 		return fmt.Errorf(packageError, errUnsetDataHandler)
 	}
-	if cfg.Orderbook.WebsocketBufferEnabled &&
-		cfg.Orderbook.WebsocketBufferLimit < 1 {
+	if exchangeConfig.Orderbook.WebsocketBufferEnabled &&
+		exchangeConfig.Orderbook.WebsocketBufferLimit < 1 {
 		return fmt.Errorf(packageError, errIssueBufferEnabledButNoLimit)
 	}
 
-	w.bufferEnabled = cfg.Orderbook.WebsocketBufferEnabled
-	w.obBufferLimit = cfg.Orderbook.WebsocketBufferLimit
-	w.sortBuffer = sortBuffer
-	w.sortBufferByUpdateIDs = sortBufferByUpdateIDs
-	w.updateEntriesByID = updateEntriesByID
-	w.exchangeName = cfg.Name
+	// NOTE: These variables are set by config.json under "orderbook" for each
+	// individual exchange.
+	w.bufferEnabled = exchangeConfig.Orderbook.WebsocketBufferEnabled
+	w.obBufferLimit = exchangeConfig.Orderbook.WebsocketBufferLimit
+
+	w.sortBuffer = c.SortBuffer
+	w.sortBufferByUpdateIDs = c.SortBufferByUpdateIDs
+	w.updateEntriesByID = c.UpdateEntriesByID
+	w.exchangeName = exchangeConfig.Name
 	w.dataHandler = dataHandler
 	w.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder)
-	w.verbose = cfg.Verbose
+	w.verbose = exchangeConfig.Verbose
 
 	// set default publish period if missing
 	orderbookPublishPeriod := config.DefaultOrderbookPublishPeriod
-	if cfg.Orderbook.PublishPeriod != nil {
-		orderbookPublishPeriod = *cfg.Orderbook.PublishPeriod
+	if exchangeConfig.Orderbook.PublishPeriod != nil {
+		orderbookPublishPeriod = *exchangeConfig.Orderbook.PublishPeriod
 	}
 	w.publishPeriod = orderbookPublishPeriod
+	w.updateIDProgression = c.UpdateIDProgression
+	w.checksum = c.Checksum
 	return nil
 }
 
@@ -86,6 +95,18 @@ func (w *Orderbook) Update(u *Update) error {
 			u.Asset)
 	}
 
+	// out of order update ID can be skipped
+	if w.updateIDProgression && u.UpdateID <= book.updateID {
+		if w.verbose {
+			log.Warnf(log.WebsocketMgr,
+				"Exchange %s CurrencyPair: %s AssetType: %s out of order websocket update received",
+				w.exchangeName,
+				u.Pair,
+				u.Asset)
+		}
+		return nil
+	}
+
 	// Checks for when the rest protocol overwrites a streaming dominated book
 	// will stop updating book via incremental updates. This occurs because our
 	// sync manager (engine/sync.go) timer has elapsed for streaming. Usually
@@ -200,6 +221,13 @@ func (w *Orderbook) processObUpdate(o *orderbookHolder, u *Update) error {
 		return o.updateByIDAndAction(u)
 	}
 	o.updateByPrice(u)
+	if w.checksum != nil {
+		err := w.checksum(o.ob.Retrieve(), u.Checksum)
+		if err != nil {
+			return err
+		}
+		o.updateID = u.UpdateID
+	}
 	return nil
 }
 
@@ -281,14 +309,15 @@ func (w *Orderbook) LoadSnapshot(book *orderbook.Base) error {
 		m2[book.Asset] = holder
 	}
 
+	holder.updateID = book.LastUpdateID
+
 	// Checks if book can deploy to linked list
 	err := book.Verify()
 	if err != nil {
 		return err
 	}
 
-	holder.ob.LoadSnapshot(
-		book.Bids,
+	holder.ob.LoadSnapshot(book.Bids,
 		book.Asks,
 		book.LastUpdateID,
 		book.LastUpdated,