Websocket: Restructure files and types (#1859)

* Websocket: Rename stream package

* Websocket: Rename Websocket to Manager

* Websocket: Replace explicit errs with common.NilGuard

* Websocket: Move websocket_types.go to types.go

* Websocket: Minor field comment and alignment in types

* Webosocket: Rename WebsocketConnection to Connection

* Alphapoint: Make gorilla ws import explicit

Just to avoid confusion with our own packages.

* Websocket: Move stream_match to match

* Websocket: Move websocket_connection to connection

* Websocket: Move websocket.go to manager.go

* Websocket: Break out all subscription methods into subscriptions.go

* Websocket: Move connection type into its file

* Websocket: Remove PositionUpdated

Type is not used anywhere

* Kraken: Use local constant for pong

Was the only use of websocket.Pong and doesn't really feel right to
represent kraken's api resp in one of our packages

* Websocket: Move connection sub-types to connection package

* Websocket: Move manager types into manager

* Websocket: Move ConnectionWrapper into manager

* Websocket: Move websocket_test to manager_test

* Websocket: Privatise connectionWrapper

* Websocket: Remaining types into types.go

These really belong somewhere else mostly, but this will do for now

* Websocket: Tidy up connection method vars

* Gofumpt: Moving package imports around

* Websocket: Rename errDuplicateConnectionSetup

* Websocket: Fix duplicate import of gws

* Websocket: Fix gofumpt -extra

* Websocket: Standardise import of gws across other pkgs

* Kraken: Remove unused sub conf consts

These were replaced by the generic Levels and Depth fields on all subs

* Websocket: Privitise ConnectioWrapper fields

* Websocket: inline single use var WebsocketNotAuthenticatedUsingRest

* Websocket: Move documentation to template

* Bithumb: Assertify TestWsHandleData

internal/exchange/websocket/buffer/buffer.go

@@ -0,0 +1,344 @@
+package buffer
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+
+	"github.com/thrasher-corp/gocryptotrader/common/key"
+	"github.com/thrasher-corp/gocryptotrader/config"
+	"github.com/thrasher-corp/gocryptotrader/currency"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
+	"github.com/thrasher-corp/gocryptotrader/log"
+)
+
+const packageError = "websocket orderbook buffer error: %w"
+
+var (
+	errExchangeConfigNil            = errors.New("exchange config is nil")
+	errBufferConfigNil              = errors.New("buffer config is nil")
+	errUnsetDataHandler             = errors.New("datahandler unset")
+	errIssueBufferEnabledButNoLimit = errors.New("buffer enabled but no limit set")
+	errUpdateIsNil                  = errors.New("update is nil")
+	errUpdateNoTargets              = errors.New("update bid/ask targets cannot be nil")
+	errDepthNotFound                = errors.New("orderbook depth not found")
+	errRESTOverwrite                = errors.New("orderbook has been overwritten by REST protocol")
+	errInvalidAction                = errors.New("invalid action")
+	errAmendFailure                 = errors.New("orderbook amend update failure")
+	errDeleteFailure                = errors.New("orderbook delete update failure")
+	errInsertFailure                = errors.New("orderbook insert update failure")
+	errUpdateInsertFailure          = errors.New("orderbook update/insert update failure")
+	errRESTTimerLapse               = errors.New("rest sync timer lapse with active websocket connection")
+	errOrderbookFlushed             = errors.New("orderbook flushed")
+)
+
+// Setup sets private variables
+func (w *Orderbook) Setup(exchangeConfig *config.Exchange, c *Config, dataHandler chan<- any) error {
+	if exchangeConfig == nil { // exchange config fields are checked in websocket package prior to calling this, so further checks are not needed
+		return fmt.Errorf(packageError, errExchangeConfigNil)
+	}
+	if c == nil {
+		return fmt.Errorf(packageError, errBufferConfigNil)
+	}
+	if dataHandler == nil {
+		return fmt.Errorf(packageError, errUnsetDataHandler)
+	}
+	if exchangeConfig.Orderbook.WebsocketBufferEnabled &&
+		exchangeConfig.Orderbook.WebsocketBufferLimit < 1 {
+		return fmt.Errorf(packageError, errIssueBufferEnabledButNoLimit)
+	}
+
+	// NOTE: These variables are set by config.json under "orderbook" for each individual exchange
+	w.bufferEnabled = exchangeConfig.Orderbook.WebsocketBufferEnabled
+	w.obBufferLimit = exchangeConfig.Orderbook.WebsocketBufferLimit
+
+	w.sortBuffer = c.SortBuffer
+	w.sortBufferByUpdateIDs = c.SortBufferByUpdateIDs
+	w.updateEntriesByID = c.UpdateEntriesByID
+	w.exchangeName = exchangeConfig.Name
+	w.dataHandler = dataHandler
+	w.ob = make(map[key.PairAsset]*orderbookHolder)
+	w.verbose = exchangeConfig.Verbose
+	w.updateIDProgression = c.UpdateIDProgression
+	w.checksum = c.Checksum
+	return nil
+}
+
+// validate validates update against setup values
+func (w *Orderbook) validate(u *orderbook.Update) error {
+	if u == nil {
+		return fmt.Errorf(packageError, errUpdateIsNil)
+	}
+	if len(u.Bids) == 0 && len(u.Asks) == 0 {
+		return fmt.Errorf(packageError, errUpdateNoTargets)
+	}
+	return nil
+}
+
+// Update updates a stored pointer to an orderbook.Depth struct containing a
+// bid and ask Tranches, this switches between the usage of a buffered update
+func (w *Orderbook) Update(u *orderbook.Update) error {
+	if err := w.validate(u); err != nil {
+		return err
+	}
+	w.mtx.Lock()
+	defer w.mtx.Unlock()
+	book, ok := w.ob[key.PairAsset{Base: u.Pair.Base.Item, Quote: u.Pair.Quote.Item, Asset: u.Asset}]
+	if !ok {
+		return fmt.Errorf("%w for Exchange %s CurrencyPair: %s AssetType: %s",
+			errDepthNotFound,
+			w.exchangeName,
+			u.Pair,
+			u.Asset)
+	}
+
+	// out of order update ID can be skipped
+	if w.updateIDProgression && u.UpdateID <= book.updateID {
+		if w.verbose {
+			log.Warnf(log.WebsocketMgr,
+				"Exchange %s CurrencyPair: %s AssetType: %s out of order websocket update received",
+				w.exchangeName,
+				u.Pair,
+				u.Asset)
+		}
+		return nil
+	}
+
+	// Checks for when the rest protocol overwrites a streaming dominated book
+	// will stop updating book via incremental updates. This occurs because our
+	// sync manager (engine/sync.go) timer has elapsed for streaming. Usually
+	// because the book is highly illiquid.
+	isREST, err := book.ob.IsRESTSnapshot()
+	if err != nil {
+		if !errors.Is(err, orderbook.ErrOrderbookInvalid) {
+			return err
+		}
+		// In the event a checksum or processing error invalidates the book, all
+		// updates that could be stored in the websocket buffer, skip applying
+		// until a new snapshot comes through.
+		if w.verbose {
+			log.Warnf(log.WebsocketMgr,
+				"Exchange %s CurrencyPair: %s AssetType: %s underlying book is invalid, cannot apply update.",
+				w.exchangeName,
+				u.Pair,
+				u.Asset)
+		}
+		return nil
+	}
+
+	if isREST {
+		if w.verbose {
+			log.Warnf(log.WebsocketMgr,
+				"%s for Exchange %s CurrencyPair: %s AssetType: %s consider extending synctimeoutwebsocket",
+				errRESTOverwrite,
+				w.exchangeName,
+				u.Pair,
+				u.Asset)
+		}
+		// Instance of illiquidity, this signal notifies that there is websocket
+		// activity. We can invalidate the book and request a new snapshot. All
+		// further updates through the websocket should be caught above in the
+		// IsRestSnapshot() call.
+		return book.ob.Invalidate(errRESTTimerLapse)
+	}
+
+	if w.bufferEnabled {
+		var processed bool
+		processed, err = w.processBufferUpdate(book, u)
+		if err != nil {
+			return err
+		}
+
+		if !processed {
+			return nil
+		}
+	} else {
+		err = w.processObUpdate(book, u)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Publish all state changes, disregarding verbosity or sync requirements.
+	book.ob.Publish()
+	w.dataHandler <- book.ob
+	return nil
+}
+
+// processBufferUpdate stores update into buffer, when buffer at capacity as
+// defined by w.obBufferLimit it well then sort and apply updates.
+func (w *Orderbook) processBufferUpdate(o *orderbookHolder, u *orderbook.Update) (bool, error) {
+	*o.buffer = append(*o.buffer, *u)
+	if len(*o.buffer) < w.obBufferLimit {
+		return false, nil
+	}
+
+	if w.sortBuffer {
+		// sort by last updated to ensure each update is in order
+		if w.sortBufferByUpdateIDs {
+			sort.Slice(*o.buffer, func(i, j int) bool {
+				return (*o.buffer)[i].UpdateID < (*o.buffer)[j].UpdateID
+			})
+		} else {
+			sort.Slice(*o.buffer, func(i, j int) bool {
+				return (*o.buffer)[i].UpdateTime.Before((*o.buffer)[j].UpdateTime)
+			})
+		}
+	}
+	for i := range *o.buffer {
+		err := w.processObUpdate(o, &(*o.buffer)[i])
+		if err != nil {
+			return false, err
+		}
+	}
+	// clear buffer of old updates
+	*o.buffer = nil
+	return true, nil
+}
+
+// processObUpdate processes updates either by its corresponding id or by price level
+func (w *Orderbook) processObUpdate(o *orderbookHolder, u *orderbook.Update) error {
+	// Both update methods require post processing to ensure the orderbook is in a valid state.
+	if w.updateEntriesByID {
+		if err := o.updateByIDAndAction(u); err != nil {
+			return err
+		}
+	} else {
+		if err := o.updateByPrice(u); err != nil {
+			return err
+		}
+	}
+	if w.checksum != nil {
+		compare, err := o.ob.Retrieve()
+		if err != nil {
+			return err
+		}
+		err = w.checksum(compare, u.Checksum)
+		if err != nil {
+			return o.ob.Invalidate(err)
+		}
+		o.updateID = u.UpdateID
+	} else if o.ob.VerifyOrderbook() {
+		compare, err := o.ob.Retrieve()
+		if err != nil {
+			return err
+		}
+		err = compare.Verify()
+		if err != nil {
+			return o.ob.Invalidate(err)
+		}
+	}
+	return nil
+}
+
+// updateByPrice amends amount if match occurs by price, deletes if amount is
+// zero or less and inserts if not found.
+func (o *orderbookHolder) updateByPrice(updts *orderbook.Update) error {
+	return o.ob.UpdateBidAskByPrice(updts)
+}
+
+// updateByIDAndAction will receive an action to execute against the orderbook
+// it will then match by IDs instead of price to perform the action
+func (o *orderbookHolder) updateByIDAndAction(updts *orderbook.Update) error {
+	switch updts.Action {
+	case orderbook.Amend:
+		err := o.ob.UpdateBidAskByID(updts)
+		if err != nil {
+			return fmt.Errorf("%w %w", errAmendFailure, err)
+		}
+	case orderbook.Delete:
+		// edge case for Bitfinex as their streaming endpoint duplicates deletes
+		bypassErr := o.ob.GetName() == "Bitfinex" && o.ob.IsFundingRate()
+		err := o.ob.DeleteBidAskByID(updts, bypassErr)
+		if err != nil {
+			return fmt.Errorf("%w %w", errDeleteFailure, err)
+		}
+	case orderbook.Insert:
+		err := o.ob.InsertBidAskByID(updts)
+		if err != nil {
+			return fmt.Errorf("%w %w", errInsertFailure, err)
+		}
+	case orderbook.UpdateInsert:
+		err := o.ob.UpdateInsertByID(updts)
+		if err != nil {
+			return fmt.Errorf("%w %w", errUpdateInsertFailure, err)
+		}
+	default:
+		return fmt.Errorf("%w [%d]", errInvalidAction, updts.Action)
+	}
+	return nil
+}
+
+// LoadSnapshot loads initial snapshot of orderbook data from websocket
+func (w *Orderbook) LoadSnapshot(book *orderbook.Base) error {
+	// Checks if book can deploy to depth
+	err := book.Verify()
+	if err != nil {
+		return err
+	}
+
+	w.mtx.Lock()
+	defer w.mtx.Unlock()
+	holder, ok := w.ob[key.PairAsset{Base: book.Pair.Base.Item, Quote: book.Pair.Quote.Item, Asset: book.Asset}]
+	if !ok {
+		// Associate orderbook pointer with local exchange depth map
+		var depth *orderbook.Depth
+		depth, err = orderbook.DeployDepth(book.Exchange, book.Pair, book.Asset)
+		if err != nil {
+			return err
+		}
+		depth.AssignOptions(book)
+		buffer := make([]orderbook.Update, w.obBufferLimit)
+
+		holder = &orderbookHolder{ob: depth, buffer: &buffer}
+		w.ob[key.PairAsset{Base: book.Pair.Base.Item, Quote: book.Pair.Quote.Item, Asset: book.Asset}] = holder
+	}
+
+	holder.updateID = book.LastUpdateID
+
+	err = holder.ob.LoadSnapshot(book.Bids, book.Asks, book.LastUpdateID, book.LastUpdated, book.UpdatePushedAt, false)
+	if err != nil {
+		return err
+	}
+
+	holder.ob.Publish()
+	w.dataHandler <- holder.ob
+	return nil
+}
+
+// GetOrderbook returns an orderbook copy as orderbook.Base
+func (w *Orderbook) GetOrderbook(p currency.Pair, a asset.Item) (*orderbook.Base, error) {
+	w.mtx.Lock()
+	defer w.mtx.Unlock()
+	book, ok := w.ob[key.PairAsset{Base: p.Base.Item, Quote: p.Quote.Item, Asset: a}]
+	if !ok {
+		return nil, fmt.Errorf("%s %s %s %w", w.exchangeName, p, a, errDepthNotFound)
+	}
+	return book.ob.Retrieve()
+}
+
+// FlushBuffer flushes w.ob data to be garbage collected and refreshed when a
+// connection is lost and reconnected
+func (w *Orderbook) FlushBuffer() {
+	w.mtx.Lock()
+	w.ob = make(map[key.PairAsset]*orderbookHolder)
+	w.mtx.Unlock()
+}
+
+// FlushOrderbook flushes independent orderbook
+func (w *Orderbook) FlushOrderbook(p currency.Pair, a asset.Item) error {
+	w.mtx.Lock()
+	defer w.mtx.Unlock()
+	book, ok := w.ob[key.PairAsset{Base: p.Base.Item, Quote: p.Quote.Item, Asset: a}]
+	if !ok {
+		return fmt.Errorf("cannot flush orderbook %s %s %s %w",
+			w.exchangeName,
+			p,
+			a,
+			errDepthNotFound)
+	}
+	// error not needed in this return
+	_ = book.ob.Invalidate(errOrderbookFlushed)
+	return nil
+}

internal/exchange/websocket/buffer/buffer_test.go

@@ -0,0 +1,992 @@
+package buffer
+
+import (
+	"errors"
+	"math/rand"
+	"slices"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/thrasher-corp/gocryptotrader/common"
+	"github.com/thrasher-corp/gocryptotrader/common/key"
+	"github.com/thrasher-corp/gocryptotrader/config"
+	"github.com/thrasher-corp/gocryptotrader/currency"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
+)
+
+var (
+	itemArray = [][]orderbook.Tranche{
+		{{Price: 1000, Amount: 1, ID: 1000}},
+		{{Price: 2000, Amount: 1, ID: 2000}},
+		{{Price: 3000, Amount: 1, ID: 3000}},
+		{{Price: 3000, Amount: 2, ID: 4000}},
+		{{Price: 4000, Amount: 0, ID: 6000}},
+		{{Price: 5000, Amount: 1, ID: 5000}},
+	}
+	offset = common.Counter{}
+)
+
+const exchangeName = "exchangeTest"
+
+// getExclusivePair returns a currency pair with a unique ID for testing as books are centralised and changes will affect other tests
+func getExclusivePair() (currency.Pair, error) {
+	return currency.NewPairFromStrings(currency.BTC.String(), currency.USDT.String()+strconv.FormatInt(offset.IncrementAndGet(), 10))
+}
+
+func createSnapshot(pair currency.Pair, bookVerifiy ...bool) (holder *Orderbook, asks, bids orderbook.Tranches, err error) {
+	asks = orderbook.Tranches{{Price: 4000, Amount: 1, ID: 6}}
+	bids = orderbook.Tranches{{Price: 4000, Amount: 1, ID: 6}}
+
+	book := &orderbook.Base{
+		Exchange:         exchangeName,
+		Asks:             asks,
+		Bids:             bids,
+		Asset:            asset.Spot,
+		Pair:             pair,
+		PriceDuplication: true,
+		LastUpdated:      time.Now(),
+		VerifyOrderbook:  len(bookVerifiy) > 0 && bookVerifiy[0],
+	}
+
+	newBook := make(map[key.PairAsset]*orderbookHolder)
+
+	ch := make(chan any)
+	go func(<-chan any) { // reader
+		for range ch {
+			continue
+		}
+	}(ch)
+	holder = &Orderbook{
+		exchangeName: exchangeName,
+		dataHandler:  ch,
+		ob:           newBook,
+	}
+	err = holder.LoadSnapshot(book)
+	return holder, asks, bids, err
+}
+
+func bidAskGenerator() []orderbook.Tranche {
+	response := make([]orderbook.Tranche, 100)
+	for i := range 100 {
+		price := float64(rand.Intn(1000)) //nolint:gosec // no need to import crypo/rand for testing
+		if price == 0 {
+			price = 1
+		}
+		response[i] = orderbook.Tranche{
+			Amount: float64(rand.Intn(10)), //nolint:gosec // no need to import crypo/rand for testing
+			Price:  price,
+			ID:     int64(i),
+		}
+	}
+	return response
+}
+
+func BenchmarkUpdateBidsByPrice(b *testing.B) {
+	cp, err := getExclusivePair()
+	require.NoError(b, err)
+
+	ob, _, _, err := createSnapshot(cp)
+	require.NoError(b, err)
+
+	for b.Loop() {
+		bidAsks := bidAskGenerator()
+		update := &orderbook.Update{
+			Bids:       bidAsks,
+			Asks:       bidAsks,
+			Pair:       cp,
+			UpdateTime: time.Now(),
+			Asset:      asset.Spot,
+		}
+		holder := ob.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}]
+		require.NoError(b, holder.updateByPrice(update))
+	}
+}
+
+func BenchmarkUpdateAsksByPrice(b *testing.B) {
+	cp, err := getExclusivePair()
+	require.NoError(b, err)
+
+	ob, _, _, err := createSnapshot(cp)
+	require.NoError(b, err)
+
+	for b.Loop() {
+		bidAsks := bidAskGenerator()
+		update := &orderbook.Update{
+			Bids:       bidAsks,
+			Asks:       bidAsks,
+			Pair:       cp,
+			UpdateTime: time.Now(),
+			Asset:      asset.Spot,
+		}
+		holder := ob.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}]
+		require.NoError(b, holder.updateByPrice(update))
+	}
+}
+
+// BenchmarkBufferPerformance demonstrates buffer more performant than multi
+// process calls
+// 890016	      1688 ns/op	     416 B/op	       3 allocs/op
+func BenchmarkBufferPerformance(b *testing.B) {
+	cp, err := getExclusivePair()
+	require.NoError(b, err)
+
+	holder, asks, bids, err := createSnapshot(cp)
+	require.NoError(b, err)
+
+	holder.bufferEnabled = true
+	update := &orderbook.Update{
+		Bids:       bids,
+		Asks:       asks,
+		Pair:       cp,
+		UpdateTime: time.Now(),
+		Asset:      asset.Spot,
+	}
+	for b.Loop() {
+		randomIndex := rand.Intn(4) //nolint:gosec // no need to import crypo/rand for testing
+		update.Asks = itemArray[randomIndex]
+		update.Bids = itemArray[randomIndex]
+		require.NoError(b, holder.Update(update))
+	}
+}
+
+// BenchmarkBufferSortingPerformance benchmark
+//
+//	613964	      2093 ns/op	     440 B/op	       4 allocs/op
+func BenchmarkBufferSortingPerformance(b *testing.B) {
+	cp, err := getExclusivePair()
+	require.NoError(b, err)
+
+	holder, asks, bids, err := createSnapshot(cp)
+	require.NoError(b, err)
+
+	holder.bufferEnabled = true
+	holder.sortBuffer = true
+	update := &orderbook.Update{
+		Bids:       bids,
+		Asks:       asks,
+		Pair:       cp,
+		UpdateTime: time.Now(),
+		Asset:      asset.Spot,
+	}
+	for b.Loop() {
+		randomIndex := rand.Intn(4) //nolint:gosec // no need to import crypo/rand for testing
+		update.Asks = itemArray[randomIndex]
+		update.Bids = itemArray[randomIndex]
+		require.NoError(b, holder.Update(update))
+	}
+}
+
+// BenchmarkBufferSortingPerformance benchmark
+// 914500	      1599 ns/op	     440 B/op	       4 allocs/op
+func BenchmarkBufferSortingByIDPerformance(b *testing.B) {
+	cp, err := getExclusivePair()
+	require.NoError(b, err)
+
+	holder, asks, bids, err := createSnapshot(cp)
+	require.NoError(b, err)
+
+	holder.bufferEnabled = true
+	holder.sortBuffer = true
+	holder.sortBufferByUpdateIDs = true
+	update := &orderbook.Update{
+		Bids:       bids,
+		Asks:       asks,
+		Pair:       cp,
+		UpdateTime: time.Now(),
+		Asset:      asset.Spot,
+	}
+
+	for b.Loop() {
+		randomIndex := rand.Intn(4) //nolint:gosec // no need to import crypo/rand for testing
+		update.Asks = itemArray[randomIndex]
+		update.Bids = itemArray[randomIndex]
+		require.NoError(b, holder.Update(update))
+	}
+}
+
+// BenchmarkNoBufferPerformance demonstrates orderbook process more performant
+// than buffer
+//   122659	     12792 ns/op	     972 B/op	       7 allocs/op PRIOR
+//  1225924	      1028 ns/op	     240 B/op	       2 allocs/op CURRENT
+
+func BenchmarkNoBufferPerformance(b *testing.B) {
+	cp, err := getExclusivePair()
+	require.NoError(b, err)
+
+	obl, asks, bids, err := createSnapshot(cp)
+	require.NoError(b, err)
+
+	update := &orderbook.Update{
+		Bids:       bids,
+		Asks:       asks,
+		Pair:       cp,
+		UpdateTime: time.Now(),
+		Asset:      asset.Spot,
+	}
+
+	for b.Loop() {
+		randomIndex := rand.Intn(4) //nolint:gosec // no need to import crypo/rand for testing
+		update.Asks = itemArray[randomIndex]
+		update.Bids = itemArray[randomIndex]
+		require.NoError(b, obl.Update(update))
+	}
+}
+
+func TestUpdates(t *testing.T) {
+	t.Parallel()
+	cp, err := getExclusivePair()
+	require.NoError(t, err)
+
+	holder, _, _, err := createSnapshot(cp)
+	require.NoError(t, err)
+
+	book := holder.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}]
+	err = book.updateByPrice(&orderbook.Update{
+		Bids:       itemArray[5],
+		Asks:       itemArray[5],
+		Pair:       cp,
+		UpdateTime: time.Now(),
+		Asset:      asset.Spot,
+	})
+	assert.NoError(t, err)
+
+	err = book.updateByPrice(&orderbook.Update{
+		Bids:       itemArray[0],
+		Asks:       itemArray[0],
+		Pair:       cp,
+		UpdateTime: time.Now(),
+		Asset:      asset.Spot,
+	})
+	assert.NoError(t, err)
+
+	askLen, err := book.ob.GetAskLength()
+	require.NoError(t, err)
+	assert.Equal(t, 3, askLen)
+}
+
+// TestHittingTheBuffer logic test
+func TestHittingTheBuffer(t *testing.T) {
+	t.Parallel()
+	cp, err := getExclusivePair()
+	require.NoError(t, err)
+
+	holder, _, _, err := createSnapshot(cp)
+	require.NoError(t, err)
+
+	holder.bufferEnabled = true
+	holder.obBufferLimit = 5
+	for i := range itemArray {
+		asks := itemArray[i]
+		bids := itemArray[i]
+		err = holder.Update(&orderbook.Update{
+			Bids:       bids,
+			Asks:       asks,
+			Pair:       cp,
+			UpdateTime: time.Now(),
+			Asset:      asset.Spot,
+		})
+		require.NoError(t, err)
+	}
+
+	book := holder.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}]
+	askLen, err := book.ob.GetAskLength()
+	require.NoError(t, err)
+	assert.Equal(t, 3, askLen)
+
+	bidLen, err := book.ob.GetBidLength()
+	require.NoError(t, err)
+	assert.Equal(t, 3, bidLen)
+}
+
+// TestInsertWithIDs logic test
+func TestInsertWithIDs(t *testing.T) {
+	t.Parallel()
+	cp, err := getExclusivePair()
+	require.NoError(t, err)
+
+	holder, _, _, err := createSnapshot(cp)
+	require.NoError(t, err)
+
+	holder.bufferEnabled = true
+	holder.updateEntriesByID = true
+	holder.obBufferLimit = 5
+	for i := range itemArray {
+		asks := itemArray[i]
+		if asks[0].Amount <= 0 {
+			continue
+		}
+		bids := itemArray[i]
+		err = holder.Update(&orderbook.Update{
+			Bids:       bids,
+			Asks:       asks,
+			Pair:       cp,
+			UpdateTime: time.Now(),
+			Asset:      asset.Spot,
+			Action:     orderbook.UpdateInsert,
+		})
+		require.NoError(t, err)
+	}
+
+	book := holder.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}]
+	askLen, err := book.ob.GetAskLength()
+	require.NoError(t, err)
+	assert.Equal(t, 6, askLen)
+
+	bidLen, err := book.ob.GetBidLength()
+	require.NoError(t, err)
+	assert.Equal(t, 6, bidLen)
+
+	cp, err = getExclusivePair()
+	require.NoError(t, err)
+
+	holder, _, _, err = createSnapshot(cp, true)
+	require.NoError(t, err)
+
+	holder.checksum = nil
+	holder.updateIDProgression = false
+	err = holder.Update(&orderbook.Update{
+		UpdateTime: time.Now(),
+		Asset:      asset.Spot,
+		Asks:       []orderbook.Tranche{{Price: 999999}},
+		Pair:       cp,
+	})
+	require.NoError(t, err)
+}
+
+// TestSortIDs logic test
+func TestSortIDs(t *testing.T) {
+	t.Parallel()
+	cp, err := getExclusivePair()
+	require.NoError(t, err)
+
+	holder, _, _, err := createSnapshot(cp)
+	require.NoError(t, err)
+
+	holder.bufferEnabled = true
+	holder.sortBufferByUpdateIDs = true
+	holder.sortBuffer = true
+	holder.obBufferLimit = 5
+	for i := range itemArray {
+		asks := itemArray[i]
+		bids := itemArray[i]
+		err = holder.Update(&orderbook.Update{
+			Bids:       bids,
+			Asks:       asks,
+			Pair:       cp,
+			UpdateID:   int64(i),
+			Asset:      asset.Spot,
+			UpdateTime: time.Now(),
+		})
+		require.NoError(t, err)
+	}
+	book := holder.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}]
+	askLen, err := book.ob.GetAskLength()
+	require.NoError(t, err)
+	assert.Equal(t, 3, askLen)
+
+	bidLen, err := book.ob.GetBidLength()
+	require.NoError(t, err)
+	assert.Equal(t, 3, bidLen)
+}
+
+// TestOutOfOrderIDs logic test
+func TestOutOfOrderIDs(t *testing.T) {
+	t.Parallel()
+	cp, err := getExclusivePair()
+	require.NoError(t, err)
+
+	holder, _, _, err := createSnapshot(cp)
+	require.NoError(t, err)
+
+	outOFOrderIDs := []int64{2, 1, 5, 3, 4, 6, 7}
+	assert.Equal(t, 1000., itemArray[0][0].Price)
+
+	holder.bufferEnabled = true
+	holder.sortBuffer = true
+	holder.obBufferLimit = 5
+	for i := range itemArray {
+		asks := itemArray[i]
+		err = holder.Update(&orderbook.Update{
+			Asks:       asks,
+			Pair:       cp,
+			UpdateID:   outOFOrderIDs[i],
+			Asset:      asset.Spot,
+			UpdateTime: time.Now(),
+		})
+		require.NoError(t, err)
+	}
+	book := holder.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}]
+	cpy, err := book.ob.Retrieve()
+	require.NoError(t, err)
+	// Index 1 since index 0 is price 7000
+	assert.Equal(t, 2000., cpy.Asks[1].Price)
+}
+
+func TestOrderbookLastUpdateID(t *testing.T) {
+	t.Parallel()
+	cp, err := getExclusivePair()
+	require.NoError(t, err)
+
+	holder, _, _, err := createSnapshot(cp)
+	require.NoError(t, err)
+
+	assert.Equal(t, 1000., itemArray[0][0].Price)
+
+	holder.checksum = func(*orderbook.Base, uint32) error { return errors.New("testerino") }
+
+	// this update invalidates the book
+	err = holder.Update(&orderbook.Update{
+		Asks:       []orderbook.Tranche{{Price: 999999}},
+		Pair:       cp,
+		UpdateID:   -1,
+		Asset:      asset.Spot,
+		UpdateTime: time.Now(),
+	})
+	require.ErrorIs(t, err, orderbook.ErrOrderbookInvalid)
+
+	cp, err = getExclusivePair()
+	require.NoError(t, err)
+
+	holder, _, _, err = createSnapshot(cp)
+	require.NoError(t, err)
+
+	holder.checksum = func(*orderbook.Base, uint32) error { return nil }
+	holder.updateIDProgression = true
+
+	for i := range itemArray {
+		asks := itemArray[i]
+		err = holder.Update(&orderbook.Update{
+			Asks:       asks,
+			Pair:       cp,
+			UpdateID:   int64(i) + 1,
+			Asset:      asset.Spot,
+			UpdateTime: time.Now(),
+		})
+		require.NoError(t, err)
+	}
+
+	// out of order
+	err = holder.Update(&orderbook.Update{
+		Asks:     []orderbook.Tranche{{Price: 999999}},
+		Pair:     cp,
+		UpdateID: 1,
+		Asset:    asset.Spot,
+	})
+	require.NoError(t, err)
+
+	ob, err := holder.GetOrderbook(cp, asset.Spot)
+	require.NoError(t, err)
+	assert.Equal(t, int64(len(itemArray)), ob.LastUpdateID)
+}
+
+// TestRunUpdateWithoutSnapshot logic test
+func TestRunUpdateWithoutSnapshot(t *testing.T) {
+	t.Parallel()
+	cp, err := getExclusivePair()
+	require.NoError(t, err)
+
+	var holder Orderbook
+	asks := []orderbook.Tranche{{Price: 4000, Amount: 1, ID: 8}}
+	bids := []orderbook.Tranche{{Price: 5999, Amount: 1, ID: 8}, {Price: 4000, Amount: 1, ID: 9}}
+	holder.exchangeName = exchangeName
+	err = holder.Update(&orderbook.Update{
+		Bids:       bids,
+		Asks:       asks,
+		Pair:       cp,
+		UpdateTime: time.Now(),
+		Asset:      asset.Spot,
+	})
+	require.ErrorIs(t, err, errDepthNotFound)
+}
+
+// TestRunUpdateWithoutAnyUpdates logic test
+func TestRunUpdateWithoutAnyUpdates(t *testing.T) {
+	t.Parallel()
+	cp, err := getExclusivePair()
+	require.NoError(t, err)
+
+	var obl Orderbook
+	obl.exchangeName = exchangeName
+	err = obl.Update(&orderbook.Update{
+		Bids:       []orderbook.Tranche{},
+		Asks:       []orderbook.Tranche{},
+		Pair:       cp,
+		UpdateTime: time.Now(),
+		Asset:      asset.Spot,
+	})
+	require.ErrorIs(t, err, errUpdateNoTargets)
+}
+
+// TestRunSnapshotWithNoData logic test
+func TestRunSnapshotWithNoData(t *testing.T) {
+	t.Parallel()
+	cp, err := getExclusivePair()
+	require.NoError(t, err)
+
+	var obl Orderbook
+	obl.ob = make(map[key.PairAsset]*orderbookHolder)
+	obl.dataHandler = make(chan any, 1)
+	var snapShot1 orderbook.Base
+	snapShot1.Asset = asset.Spot
+	snapShot1.Pair = cp
+	snapShot1.Exchange = "test"
+	obl.exchangeName = "test"
+	snapShot1.LastUpdated = time.Now()
+	require.NoError(t, obl.LoadSnapshot(&snapShot1))
+}
+
+// TestLoadSnapshot logic test
+func TestLoadSnapshot(t *testing.T) {
+	t.Parallel()
+	cp, err := getExclusivePair()
+	require.NoError(t, err)
+
+	var obl Orderbook
+	obl.dataHandler = make(chan any, 100)
+	obl.ob = make(map[key.PairAsset]*orderbookHolder)
+	var snapShot1 orderbook.Base
+	snapShot1.Exchange = "SnapshotWithOverride"
+	asks := []orderbook.Tranche{{Price: 4000, Amount: 1, ID: 8}}
+	bids := []orderbook.Tranche{{Price: 4000, Amount: 1, ID: 9}}
+	snapShot1.Asks = asks
+	snapShot1.Bids = bids
+	snapShot1.Asset = asset.Spot
+	snapShot1.Pair = cp
+	snapShot1.LastUpdated = time.Now()
+	require.NoError(t, obl.LoadSnapshot(&snapShot1))
+}
+
+// TestFlushBuffer logic test
+func TestFlushBuffer(t *testing.T) {
+	t.Parallel()
+	cp, err := getExclusivePair()
+	require.NoError(t, err)
+
+	obl, _, _, err := createSnapshot(cp)
+	require.NoError(t, err)
+
+	assert.NotEmpty(t, obl.ob)
+	obl.FlushBuffer()
+	assert.Empty(t, obl.ob)
+}
+
+// TestInsertingSnapShots logic test
+func TestInsertingSnapShots(t *testing.T) {
+	t.Parallel()
+	cp, err := getExclusivePair()
+	require.NoError(t, err)
+
+	var holder Orderbook
+	holder.dataHandler = make(chan any, 100)
+	holder.ob = make(map[key.PairAsset]*orderbookHolder)
+	var snapShot1 orderbook.Base
+	snapShot1.Exchange = "WSORDERBOOKTEST1"
+	asks := []orderbook.Tranche{
+		{Price: 6000, Amount: 1, ID: 1},
+		{Price: 6001, Amount: 0.5, ID: 2},
+		{Price: 6002, Amount: 2, ID: 3},
+		{Price: 6003, Amount: 3, ID: 4},
+		{Price: 6004, Amount: 5, ID: 5},
+		{Price: 6005, Amount: 2, ID: 6},
+		{Price: 6006, Amount: 1.5, ID: 7},
+		{Price: 6007, Amount: 0.5, ID: 8},
+		{Price: 6008, Amount: 23, ID: 9},
+		{Price: 6009, Amount: 9, ID: 10},
+		{Price: 6010, Amount: 7, ID: 11},
+	}
+
+	bids := []orderbook.Tranche{
+		{Price: 5999, Amount: 1, ID: 12},
+		{Price: 5998, Amount: 0.5, ID: 13},
+		{Price: 5997, Amount: 2, ID: 14},
+		{Price: 5996, Amount: 3, ID: 15},
+		{Price: 5995, Amount: 5, ID: 16},
+		{Price: 5994, Amount: 2, ID: 17},
+		{Price: 5993, Amount: 1.5, ID: 18},
+		{Price: 5992, Amount: 0.5, ID: 19},
+		{Price: 5991, Amount: 23, ID: 20},
+		{Price: 5990, Amount: 9, ID: 21},
+		{Price: 5989, Amount: 7, ID: 22},
+	}
+
+	snapShot1.Asks = asks
+	snapShot1.Bids = bids
+	snapShot1.Asset = asset.Spot
+	snapShot1.Pair = cp
+	snapShot1.LastUpdated = time.Now()
+	require.NoError(t, holder.LoadSnapshot(&snapShot1))
+
+	var snapShot2 orderbook.Base
+	snapShot2.Exchange = "WSORDERBOOKTEST2"
+	asks = []orderbook.Tranche{
+		{Price: 51, Amount: 1, ID: 1},
+		{Price: 52, Amount: 0.5, ID: 2},
+		{Price: 53, Amount: 2, ID: 3},
+		{Price: 54, Amount: 3, ID: 4},
+		{Price: 55, Amount: 5, ID: 5},
+		{Price: 56, Amount: 2, ID: 6},
+		{Price: 57, Amount: 1.5, ID: 7},
+		{Price: 58, Amount: 0.5, ID: 8},
+		{Price: 59, Amount: 23, ID: 9},
+		{Price: 50, Amount: 9, ID: 10},
+		{Price: 60, Amount: 7, ID: 11},
+	}
+
+	bids = []orderbook.Tranche{
+		{Price: 49, Amount: 1, ID: 12},
+		{Price: 48, Amount: 0.5, ID: 13},
+		{Price: 47, Amount: 2, ID: 14},
+		{Price: 46, Amount: 3, ID: 15},
+		{Price: 45, Amount: 5, ID: 16},
+		{Price: 44, Amount: 2, ID: 17},
+		{Price: 43, Amount: 1.5, ID: 18},
+		{Price: 42, Amount: 0.5, ID: 19},
+		{Price: 41, Amount: 23, ID: 20},
+		{Price: 40, Amount: 9, ID: 21},
+		{Price: 39, Amount: 7, ID: 22},
+	}
+
+	snapShot2.Asks = asks
+	snapShot2.Asks.SortAsks()
+	snapShot2.Bids = bids
+	snapShot2.Bids.SortBids()
+	snapShot2.Asset = asset.Spot
+	snapShot2.Pair, err = getExclusivePair()
+	require.NoError(t, err)
+
+	snapShot2.LastUpdated = time.Now()
+	require.NoError(t, holder.LoadSnapshot(&snapShot2))
+
+	var snapShot3 orderbook.Base
+	snapShot3.Exchange = "WSORDERBOOKTEST3"
+	asks = []orderbook.Tranche{
+		{Price: 511, Amount: 1, ID: 1},
+		{Price: 52, Amount: 0.5, ID: 2},
+		{Price: 53, Amount: 2, ID: 3},
+		{Price: 54, Amount: 3, ID: 4},
+		{Price: 55, Amount: 5, ID: 5},
+		{Price: 56, Amount: 2, ID: 6},
+		{Price: 57, Amount: 1.5, ID: 7},
+		{Price: 58, Amount: 0.5, ID: 8},
+		{Price: 59, Amount: 23, ID: 9},
+		{Price: 50, Amount: 9, ID: 10},
+		{Price: 60, Amount: 7, ID: 11},
+	}
+
+	bids = []orderbook.Tranche{
+		{Price: 49, Amount: 1, ID: 12},
+		{Price: 48, Amount: 0.5, ID: 13},
+		{Price: 47, Amount: 2, ID: 14},
+		{Price: 46, Amount: 3, ID: 15},
+		{Price: 45, Amount: 5, ID: 16},
+		{Price: 44, Amount: 2, ID: 17},
+		{Price: 43, Amount: 1.5, ID: 18},
+		{Price: 42, Amount: 0.5, ID: 19},
+		{Price: 41, Amount: 23, ID: 20},
+		{Price: 40, Amount: 9, ID: 21},
+		{Price: 39, Amount: 7, ID: 22},
+	}
+
+	snapShot3.Asks = asks
+	snapShot3.Asks.SortAsks()
+	snapShot3.Bids = bids
+	snapShot3.Bids.SortBids()
+	snapShot3.Asset = asset.Futures
+	snapShot3.Pair, err = getExclusivePair()
+	require.NoError(t, err)
+
+	snapShot3.LastUpdated = time.Now()
+	require.NoError(t, holder.LoadSnapshot(&snapShot3))
+
+	ob, err := holder.GetOrderbook(snapShot1.Pair, snapShot1.Asset)
+	require.NoError(t, err)
+	assert.Equal(t, snapShot1.Asks[0], ob.Asks[0])
+
+	ob, err = holder.GetOrderbook(snapShot2.Pair, snapShot2.Asset)
+	require.NoError(t, err)
+	assert.Equal(t, snapShot2.Asks[0], ob.Asks[0])
+
+	ob, err = holder.GetOrderbook(snapShot3.Pair, snapShot3.Asset)
+	require.NoError(t, err)
+	assert.Equal(t, snapShot3.Asks[0], ob.Asks[0])
+}
+
+func TestGetOrderbook(t *testing.T) {
+	t.Parallel()
+	cp, err := getExclusivePair()
+	require.NoError(t, err)
+
+	holder, _, _, err := createSnapshot(cp)
+	require.NoError(t, err)
+
+	ob, err := holder.GetOrderbook(cp, asset.Spot)
+	require.NoError(t, err)
+
+	bufferOb := holder.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}]
+	b, err := bufferOb.ob.Retrieve()
+	require.NoError(t, err)
+
+	askLen, err := bufferOb.ob.GetAskLength()
+	require.NoError(t, err)
+
+	bidLen, err := bufferOb.ob.GetBidLength()
+	require.NoError(t, err)
+
+	if askLen != len(ob.Asks) ||
+		bidLen != len(ob.Bids) ||
+		b.Asset != ob.Asset ||
+		b.Exchange != ob.Exchange ||
+		b.LastUpdateID != ob.LastUpdateID ||
+		b.PriceDuplication != ob.PriceDuplication ||
+		b.Pair != ob.Pair {
+		t.Fatal("data on both books should be the same")
+	}
+}
+
+func TestSetup(t *testing.T) {
+	t.Parallel()
+	w := Orderbook{}
+	err := w.Setup(nil, nil, nil)
+	require.ErrorIs(t, err, errExchangeConfigNil)
+
+	exchangeConfig := &config.Exchange{}
+	err = w.Setup(exchangeConfig, nil, nil)
+	require.ErrorIs(t, err, errBufferConfigNil)
+
+	bufferConf := &Config{}
+	err = w.Setup(exchangeConfig, bufferConf, nil)
+	require.ErrorIs(t, err, errUnsetDataHandler)
+
+	exchangeConfig.Orderbook.WebsocketBufferEnabled = true
+	err = w.Setup(exchangeConfig, bufferConf, make(chan any))
+	require.ErrorIs(t, err, errIssueBufferEnabledButNoLimit)
+
+	exchangeConfig.Orderbook.WebsocketBufferLimit = 1337
+	exchangeConfig.Orderbook.WebsocketBufferEnabled = true
+	exchangeConfig.Name = "test"
+	bufferConf.SortBuffer = true
+	bufferConf.SortBufferByUpdateIDs = true
+	bufferConf.UpdateEntriesByID = true
+	err = w.Setup(exchangeConfig, bufferConf, make(chan any))
+	require.NoError(t, err)
+
+	if w.obBufferLimit != 1337 ||
+		!w.bufferEnabled ||
+		!w.sortBuffer ||
+		!w.sortBufferByUpdateIDs ||
+		!w.updateEntriesByID ||
+		w.exchangeName != "test" {
+		t.Errorf("Setup incorrectly loaded %s", w.exchangeName)
+	}
+}
+
+func TestValidate(t *testing.T) {
+	t.Parallel()
+	w := Orderbook{}
+	err := w.validate(nil)
+	require.ErrorIs(t, err, errUpdateIsNil)
+	err = w.validate(&orderbook.Update{})
+	require.ErrorIs(t, err, errUpdateNoTargets)
+}
+
+func TestEnsureMultipleUpdatesViaPrice(t *testing.T) {
+	t.Parallel()
+	cp, err := getExclusivePair()
+	require.NoError(t, err)
+
+	holder, _, _, err := createSnapshot(cp)
+	require.NoError(t, err)
+
+	asks := bidAskGenerator()
+	book := holder.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}]
+	err = book.updateByPrice(&orderbook.Update{
+		Bids:       asks,
+		Asks:       asks,
+		Pair:       cp,
+		UpdateTime: time.Now(),
+		Asset:      asset.Spot,
+	})
+	require.NoError(t, err)
+
+	askLen, err := book.ob.GetAskLength()
+	require.NoError(t, err)
+	assert.LessOrEqual(t, 3, askLen)
+}
+
+func deploySliceOrdered(size int) orderbook.Tranches {
+	items := make([]orderbook.Tranche, size)
+	for i := range size {
+		items[i] = orderbook.Tranche{Amount: 1, Price: rand.Float64() + float64(i), ID: rand.Int63()} //nolint:gosec // Not needed for tests
+	}
+	return items
+}
+
+func TestUpdateByIDAndAction(t *testing.T) {
+	t.Parallel()
+	cp, err := getExclusivePair()
+	require.NoError(t, err)
+
+	asks := deploySliceOrdered(100)
+	bids := slices.Clone(asks)
+	bids.Reverse()
+
+	book, err := orderbook.DeployDepth("test", cp, asset.Spot)
+	require.NoError(t, err)
+
+	err = book.LoadSnapshot(slices.Clone(bids), slices.Clone(asks), 0, time.Now(), time.Now(), true)
+	require.NoError(t, err)
+
+	ob, err := book.Retrieve()
+	require.NoError(t, err)
+
+	require.NoError(t, ob.Verify())
+
+	holder := orderbookHolder{ob: book}
+	err = holder.updateByIDAndAction(&orderbook.Update{})
+	require.ErrorIs(t, err, errInvalidAction)
+
+	err = holder.updateByIDAndAction(&orderbook.Update{
+		Action: orderbook.Amend,
+		Bids:   []orderbook.Tranche{{Price: 100, ID: 6969}},
+	})
+	require.ErrorIs(t, err, errAmendFailure)
+
+	err = book.LoadSnapshot(slices.Clone(bids), slices.Clone(asks), 0, time.Now(), time.Now(), true)
+	require.NoError(t, err)
+
+	// append to slice
+	err = holder.updateByIDAndAction(&orderbook.Update{
+		Action:     orderbook.UpdateInsert,
+		Bids:       []orderbook.Tranche{{Price: 0, ID: 1337, Amount: 1}},
+		Asks:       []orderbook.Tranche{{Price: 100, ID: 1337, Amount: 1}},
+		UpdateTime: time.Now(),
+	})
+	require.NoError(t, err)
+
+	cpy, err := book.Retrieve()
+	require.NoError(t, err)
+	require.Equal(t, 0., cpy.Bids[len(cpy.Bids)-1].Price)
+	require.Equal(t, 100., cpy.Asks[len(cpy.Asks)-1].Price)
+
+	// Change amount
+	err = holder.updateByIDAndAction(&orderbook.Update{
+		Action:     orderbook.UpdateInsert,
+		Bids:       []orderbook.Tranche{{Price: 0, ID: 1337, Amount: 100}},
+		Asks:       []orderbook.Tranche{{Price: 100, ID: 1337, Amount: 100}},
+		UpdateTime: time.Now(),
+	})
+	require.NoError(t, err)
+
+	cpy, err = book.Retrieve()
+	require.NoError(t, err)
+	require.Equal(t, 100., cpy.Bids[len(cpy.Bids)-1].Amount)
+	require.Equal(t, 100., cpy.Asks[len(cpy.Asks)-1].Amount)
+
+	// Change price level
+	err = holder.updateByIDAndAction(&orderbook.Update{
+		Action:     orderbook.UpdateInsert,
+		Bids:       []orderbook.Tranche{{Price: 100, ID: 1337, Amount: 99}},
+		Asks:       []orderbook.Tranche{{Price: 0, ID: 1337, Amount: 99}},
+		UpdateTime: time.Now(),
+	})
+	require.NoError(t, err)
+
+	cpy, err = book.Retrieve()
+	require.NoError(t, err)
+
+	require.Equal(t, 99., cpy.Bids[0].Amount)
+	require.Equal(t, 100., cpy.Bids[0].Price)
+	require.Equal(t, 99., cpy.Asks[0].Amount)
+	require.Equal(t, 0., cpy.Asks[0].Price)
+
+	err = book.LoadSnapshot(slices.Clone(bids), slices.Clone(asks), 0, time.Now(), time.Now(), true)
+	require.NoError(t, err)
+	// Delete - not found
+	err = holder.updateByIDAndAction(&orderbook.Update{
+		Action: orderbook.Delete,
+		Asks:   []orderbook.Tranche{{Price: 0, ID: 1337, Amount: 99}},
+	})
+	require.ErrorIs(t, err, errDeleteFailure)
+
+	err = book.LoadSnapshot(slices.Clone(bids), slices.Clone(asks), 0, time.Now(), time.Now(), true)
+	require.NoError(t, err)
+
+	// Delete - found
+	err = holder.updateByIDAndAction(&orderbook.Update{
+		Action:     orderbook.Delete,
+		Asks:       []orderbook.Tranche{asks[0]},
+		UpdateTime: time.Now(),
+	})
+	require.NoError(t, err)
+
+	askLen, err := book.GetAskLength()
+	require.NoError(t, err)
+	require.Equal(t, 99, askLen)
+
+	// Apply update
+	err = holder.updateByIDAndAction(&orderbook.Update{
+		Action: orderbook.Amend,
+		Asks:   []orderbook.Tranche{{ID: 123456}},
+	})
+	require.ErrorIs(t, err, errAmendFailure)
+
+	err = book.LoadSnapshot(slices.Clone(bids), slices.Clone(bids), 0, time.Now(), time.Now(), true)
+	require.NoError(t, err)
+
+	ob, err = book.Retrieve()
+	require.NoError(t, err)
+	require.NotEmpty(t, ob.Asks)
+	require.NotEmpty(t, ob.Bids)
+
+	update := ob.Asks[0]
+	update.Amount = 1337
+
+	err = holder.updateByIDAndAction(&orderbook.Update{
+		Action:     orderbook.Amend,
+		Asks:       []orderbook.Tranche{update},
+		UpdateTime: time.Now(),
+	})
+	require.NoError(t, err)
+
+	ob, err = book.Retrieve()
+	require.NoError(t, err)
+	require.Equal(t, 1337., ob.Asks[0].Amount)
+}
+
+func TestFlushOrderbook(t *testing.T) {
+	t.Parallel()
+	cp, err := getExclusivePair()
+	require.NoError(t, err)
+
+	w := &Orderbook{}
+	err = w.Setup(&config.Exchange{Name: "test"}, &Config{}, make(chan any, 2))
+	require.NoError(t, err)
+
+	var snapShot1 orderbook.Base
+	snapShot1.Exchange = "Snapshooooot"
+	asks := []orderbook.Tranche{{Price: 4000, Amount: 1, ID: 8}}
+	bids := []orderbook.Tranche{{Price: 4000, Amount: 1, ID: 9}}
+	snapShot1.Asks = asks
+	snapShot1.Bids = bids
+	snapShot1.Asset = asset.Spot
+	snapShot1.Pair = cp
+	snapShot1.LastUpdated = time.Now()
+
+	err = w.FlushOrderbook(cp, asset.Spot)
+	if err == nil {
+		t.Fatal("book not loaded error cannot be nil")
+	}
+
+	_, err = w.GetOrderbook(cp, asset.Spot)
+	require.ErrorIs(t, err, errDepthNotFound)
+
+	require.NoError(t, w.LoadSnapshot(&snapShot1))
+	require.NoError(t, w.FlushOrderbook(cp, asset.Spot))
+
+	_, err = w.GetOrderbook(cp, asset.Spot)
+	require.ErrorIs(t, err, orderbook.ErrOrderbookInvalid)
+}

internal/exchange/websocket/buffer/buffer_types.go

@@ -0,0 +1,59 @@
+package buffer
+
+import (
+	"sync"
+
+	"github.com/thrasher-corp/gocryptotrader/common/key"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
+)
+
+// Config defines the configuration variables for the websocket buffer; snapshot
+// and incremental update orderbook processing.
+type Config struct {
+	// SortBuffer enables a websocket to sort incoming updates before processing.
+	SortBuffer bool
+	// SortBufferByUpdateIDs allows the sorting of the buffered updates by their
+	// corresponding update IDs.
+	SortBufferByUpdateIDs bool
+	// UpdateEntriesByID will match by IDs instead of price to perform the an
+	// action. e.g. update, delete, insert.
+	UpdateEntriesByID bool
+	// UpdateIDProgression requires that the new update ID be greater than the
+	// prior ID. This will skip processing and not error.
+	UpdateIDProgression bool
+	// Checksum is a package defined checksum calculation for updated books.
+	Checksum func(state *orderbook.Base, checksum uint32) error
+}
+
+// Orderbook defines a local cache of orderbooks for amending, appending
+// and deleting changes and updates the main store for a stream
+type Orderbook struct {
+	ob                    map[key.PairAsset]*orderbookHolder
+	obBufferLimit         int
+	bufferEnabled         bool
+	sortBuffer            bool
+	sortBufferByUpdateIDs bool // When timestamps aren't provided, an id can help sort
+	updateEntriesByID     bool // Use the update IDs to match ob entries
+	exchangeName          string
+	dataHandler           chan<- any
+	verbose               bool
+
+	// updateIDProgression requires that the new update ID be greater than the
+	// prior ID. This will skip processing and not error.
+	updateIDProgression bool
+	// checksum is a package defined checksum calculation for updated books.
+	checksum func(state *orderbook.Base, checksum uint32) error
+	// TODO: sync.RWMutex. For the moment we process the orderbook in a single
+	// thread. In future when there are workers directly involved this can be
+	// can be improved with RW mechanics which will allow updates to occur at
+	// the same time on different books.
+	mtx sync.Mutex
+}
+
+// orderbookHolder defines a store of pending updates and a pointer to the
+// orderbook depth
+type orderbookHolder struct {
+	ob       *orderbook.Depth
+	buffer   *[]orderbook.Update
+	updateID int64
+}