From c892f492a994f6dad2543ce0cce51dbb5605ebe6 Mon Sep 17 00:00:00 2001 From: Ryan O'Hara-Reid Date: Wed, 18 Jun 2025 16:19:58 +1000 Subject: [PATCH] buffer/orderbook: shift orderbook update logic from buffer package to orderbook package (#1908) * buffer/orderbook: shift orderbook update logic from buffer package to orderbook package * Update exchanges/orderbook/depth.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * linter: fixes * spelling: fix * samboss: add in some todos * sammy nit: add unlock on error * sammy nits: rm ptr to slice field buffer in orderbookHolder * sammy nits: Add more coverage bro * sammy nits: even more coverage * gk: nits on commentary * gk: nits change sort.Slice to slices.SortFunc * gk: fix commentary on buffer clearing * gk: nits fin * linter: fix * Update exchange/websocket/buffer/buffer.go Co-authored-by: Gareth Kirwan * Update exchange/websocket/buffer/buffer.go Co-authored-by: Gareth Kirwan * Update exchanges/orderbook/tranches.go Co-authored-by: Gareth Kirwan * Update exchanges/orderbook/orderbook.go Co-authored-by: Gareth Kirwan * Update exchange/websocket/buffer/buffer_test.go Co-authored-by: Gareth Kirwan * Update exchange/websocket/buffer/buffer_test.go Co-authored-by: Gareth Kirwan * Update exchanges/orderbook/incremental_updates.go Co-authored-by: Gareth Kirwan * gk: refresh action types and names * gk nits: consolidate error vars and naming * gk nits: more name changes * gk nits; buffer tests update * gk nits: error var names change * linter: FIX * it gets inlined but there is an alloc * rn field in TODO * Update exchanges/binance/binance_websocket.go Co-authored-by: Adrian Gallagher * Update exchanges/binance/binance_websocket.go Co-authored-by: Adrian Gallagher * orderbook: shift verify/validate funcs to validate.go and rn Verify() -> Validate() * orderbook: validate even in presence of checksum and allow cowboy mode * buffer; fix test * kraken: fix futures orderbook by reversing incoming bids * okx: change default spread pair * Update exchanges/orderbook/validate.go Co-authored-by: Gareth Kirwan * Update exchanges/orderbook/validate.go Co-authored-by: Gareth Kirwan * Update exchanges/orderbook/validate.go Co-authored-by: Gareth Kirwan * Update exchanges/orderbook/validate.go Co-authored-by: Gareth Kirwan * Update exchanges/orderbook/validate.go Co-authored-by: Gareth Kirwan * gk: initial nits * rn fields V(v)erifyorderbook to V(v)alidateOrderbook * buffer/orderbook: nilguard in validate and change method receiver w -> o --------- Co-authored-by: Ryan O'Hara-Reid Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Gareth Kirwan Co-authored-by: Adrian Gallagher --- cmd/exchange_template/wrapper_file.tmpl | 2 +- engine/rpcserver_test.go | 60 +-- exchange/websocket/buffer/buffer.go | 361 +++++----------- exchange/websocket/buffer/buffer_test.go | 409 ++++-------------- exchange/websocket/buffer/buffer_types.go | 25 +- exchanges/binance/binance_websocket.go | 39 +- exchanges/binance/binance_wrapper.go | 8 +- exchanges/binanceus/binanceus_websocket.go | 39 +- exchanges/binanceus/binanceus_wrapper.go | 8 +- exchanges/bitfinex/bitfinex_websocket.go | 8 +- exchanges/bitfinex/bitfinex_wrapper.go | 14 +- exchanges/bitflyer/bitflyer_wrapper.go | 8 +- exchanges/bithumb/bithumb_wrapper.go | 8 +- exchanges/bithumb/bithumb_ws_orderbook.go | 25 +- exchanges/bitmex/bitmex_test.go | 8 +- exchanges/bitmex/bitmex_websocket.go | 12 +- exchanges/bitmex/bitmex_wrapper.go | 12 +- exchanges/bitstamp/bitstamp_websocket.go | 28 +- exchanges/bitstamp/bitstamp_wrapper.go | 8 +- exchanges/btcmarkets/btcmarkets_test.go | 15 +- exchanges/btcmarkets/btcmarkets_websocket.go | 55 +-- exchanges/btcmarkets/btcmarkets_wrapper.go | 14 +- exchanges/btse/btse_websocket.go | 2 +- exchanges/btse/btse_wrapper.go | 8 +- exchanges/bybit/bybit_wrapper.go | 12 +- .../coinbasepro/coinbasepro_websocket.go | 14 +- exchanges/coinbasepro/coinbasepro_wrapper.go | 8 +- exchanges/coinut/coinut_websocket.go | 2 +- exchanges/coinut/coinut_wrapper.go | 8 +- exchanges/deribit/deribit_websocket.go | 16 +- exchanges/deribit/deribit_wrapper.go | 8 +- exchanges/exchange.go | 2 +- exchanges/exchange_types.go | 4 +- exchanges/exmo/exmo_wrapper.go | 16 +- exchanges/gateio/gateio_websocket_futures.go | 28 +- exchanges/gateio/gateio_websocket_option.go | 28 +- exchanges/gateio/gateio_wrapper.go | 14 +- exchanges/gateio/ws_ob_update_manager.go | 7 +- exchanges/gateio/ws_ob_update_manager_test.go | 5 +- exchanges/gemini/gemini_websocket.go | 2 +- exchanges/gemini/gemini_wrapper.go | 8 +- exchanges/hitbtc/hitbtc_websocket.go | 2 +- exchanges/hitbtc/hitbtc_wrapper.go | 8 +- exchanges/huobi/huobi_websocket.go | 2 +- exchanges/huobi/huobi_wrapper.go | 8 +- exchanges/kraken/kraken_websocket.go | 2 +- exchanges/kraken/kraken_wrapper.go | 9 +- exchanges/kucoin/kucoin_websocket.go | 69 ++- exchanges/kucoin/kucoin_wrapper.go | 13 +- exchanges/lbank/lbank_wrapper.go | 12 +- exchanges/okx/okx_test.go | 8 +- exchanges/okx/okx_websocket.go | 74 ++-- exchanges/okx/okx_wrapper.go | 18 +- exchanges/orderbook/depth.go | 173 ++------ exchanges/orderbook/depth_test.go | 288 ++---------- exchanges/orderbook/incremental_updates.go | 249 +++++++++++ .../orderbook/incremental_updates_test.go | 374 ++++++++++++++++ exchanges/orderbook/levels.go | 2 +- exchanges/orderbook/levels_test.go | 14 +- exchanges/orderbook/orderbook.go | 104 +---- exchanges/orderbook/orderbook_test.go | 87 ++-- exchanges/orderbook/orderbook_types.go | 42 +- exchanges/orderbook/validate.go | 96 ++++ exchanges/poloniex/poloniex_websocket.go | 2 +- exchanges/poloniex/poloniex_wrapper.go | 16 +- exchanges/yobit/yobit_wrapper.go | 8 +- 66 files changed, 1376 insertions(+), 1662 deletions(-) create mode 100644 exchanges/orderbook/incremental_updates.go create mode 100644 exchanges/orderbook/incremental_updates_test.go create mode 100644 exchanges/orderbook/validate.go diff --git a/cmd/exchange_template/wrapper_file.tmpl b/cmd/exchange_template/wrapper_file.tmpl index 9b027f6f..ee54d65d 100644 --- a/cmd/exchange_template/wrapper_file.tmpl +++ b/cmd/exchange_template/wrapper_file.tmpl @@ -255,7 +255,7 @@ func ({{.Variable}} *{{.CapitalName}}) UpdateOrderbook(ctx context.Context, pair Exchange: {{.Variable}}.Name, Pair: pair, Asset: assetType, - VerifyOrderbook: {{.Variable}}.CanVerifyOrderbook, + ValidateOrderbook: {{.Variable}}.ValidateOrderbook, } // NOTE: UPDATE ORDERBOOK EXAMPLE diff --git a/engine/rpcserver_test.go b/engine/rpcserver_test.go index a130a0bd..d92eb9d9 100644 --- a/engine/rpcserver_test.go +++ b/engine/rpcserver_test.go @@ -3081,18 +3081,14 @@ func TestGetOrderbookMovement(t *testing.T) { t.Parallel() em := NewExchangeManager() exch, err := em.NewExchangeByName("binance") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err, "NewExchangeByName must not error") + exch.SetDefaults() b := exch.GetBase() b.Name = fakeExchangeName b.Enabled = true - cp, err := currency.NewPairFromString("btc-metal") - if err != nil { - t.Fatal(err) - } + cp := currency.NewPairWithDelimiter("btc", "metal", "-") b.CurrencyPairs.Pairs = make(map[asset.Item]*currency.PairStore) b.CurrencyPairs.Pairs[asset.Spot] = ¤cy.PairStore{ @@ -3134,9 +3130,7 @@ func TestGetOrderbookMovement(t *testing.T) { } depth, err := orderbook.DeployDepth(req.Exchange, currency.NewPair(currency.BTC, currency.METAL), asset.Spot) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err, "orderbook.DeployDepth must not error") bid := []orderbook.Level{ {Price: 10, Amount: 1}, @@ -3150,10 +3144,8 @@ func TestGetOrderbookMovement(t *testing.T) { {Price: 13, Amount: 1}, {Price: 14, Amount: 1}, } - err = depth.LoadSnapshot(bid, ask, 0, time.Now(), time.Now(), true) - if err != nil { - t.Fatal(err) - } + err = depth.LoadSnapshot(&orderbook.Book{Bids: bid, Asks: ask, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) + require.NoError(t, err, "depth.LoadSnapshot must not error") _, err = s.GetOrderbookMovement(t.Context(), req) if err.Error() != "quote amount invalid" { @@ -3182,18 +3174,14 @@ func TestGetOrderbookAmountByNominal(t *testing.T) { t.Parallel() em := NewExchangeManager() exch, err := em.NewExchangeByName("binance") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err, "NewExchangeByName must not error") + exch.SetDefaults() b := exch.GetBase() b.Name = fakeExchangeName b.Enabled = true - cp, err := currency.NewPairFromString("btc-meme") - if err != nil { - t.Fatal(err) - } + cp := currency.NewPairWithDelimiter("btc", "meme", "-") b.CurrencyPairs.Pairs = make(map[asset.Item]*currency.PairStore) b.CurrencyPairs.Pairs[asset.Spot] = ¤cy.PairStore{ @@ -3235,9 +3223,7 @@ func TestGetOrderbookAmountByNominal(t *testing.T) { } depth, err := orderbook.DeployDepth(req.Exchange, currency.NewPair(currency.BTC, currency.MEME), asset.Spot) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err, "orderbook.DeployDepth must not error") bid := []orderbook.Level{ {Price: 10, Amount: 1}, @@ -3251,10 +3237,8 @@ func TestGetOrderbookAmountByNominal(t *testing.T) { {Price: 13, Amount: 1}, {Price: 14, Amount: 1}, } - err = depth.LoadSnapshot(bid, ask, 0, time.Now(), time.Now(), true) - if err != nil { - t.Fatal(err) - } + err = depth.LoadSnapshot(&orderbook.Book{Bids: bid, Asks: ask, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) + require.NoError(t, err, "depth.LoadSnapshot must not error") nominal, err := s.GetOrderbookAmountByNominal(t.Context(), req) require.NoError(t, err) @@ -3276,18 +3260,14 @@ func TestGetOrderbookAmountByImpact(t *testing.T) { t.Parallel() em := NewExchangeManager() exch, err := em.NewExchangeByName("binance") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err, "NewExchangeByName must not error") + exch.SetDefaults() b := exch.GetBase() b.Name = fakeExchangeName b.Enabled = true - cp, err := currency.NewPairFromString("btc-mad") - if err != nil { - t.Fatal(err) - } + cp := currency.NewPairWithDelimiter("btc", "mad", "-") b.CurrencyPairs.Pairs = make(map[asset.Item]*currency.PairStore) b.CurrencyPairs.Pairs[asset.Spot] = ¤cy.PairStore{ @@ -3329,9 +3309,7 @@ func TestGetOrderbookAmountByImpact(t *testing.T) { } depth, err := orderbook.DeployDepth(req.Exchange, currency.NewPair(currency.BTC, currency.MAD), asset.Spot) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err, "orderbook.DeployDepth must not error") bid := []orderbook.Level{ {Price: 10, Amount: 1}, @@ -3345,10 +3323,8 @@ func TestGetOrderbookAmountByImpact(t *testing.T) { {Price: 13, Amount: 1}, {Price: 14, Amount: 1}, } - err = depth.LoadSnapshot(bid, ask, 0, time.Now(), time.Now(), true) - if err != nil { - t.Fatal(err) - } + err = depth.LoadSnapshot(&orderbook.Book{Bids: bid, Asks: ask, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) + require.NoError(t, err, "depth.LoadSnapshot must not error") req.ImpactPercentage = 9.090909090909092 impact, err := s.GetOrderbookAmountByImpact(t.Context(), req) diff --git a/exchange/websocket/buffer/buffer.go b/exchange/websocket/buffer/buffer.go index 63f8136b..527bf211 100644 --- a/exchange/websocket/buffer/buffer.go +++ b/exchange/websocket/buffer/buffer.go @@ -1,44 +1,30 @@ package buffer import ( + "cmp" "errors" "fmt" - "sort" + "slices" "github.com/thrasher-corp/gocryptotrader/common/key" "github.com/thrasher-corp/gocryptotrader/config" "github.com/thrasher-corp/gocryptotrader/currency" "github.com/thrasher-corp/gocryptotrader/exchanges/asset" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" - "github.com/thrasher-corp/gocryptotrader/log" ) const packageError = "websocket orderbook buffer error: %w" -// Public err vars -var ( - ErrDepthNotFound = errors.New("orderbook depth not found") -) - var ( errExchangeConfigNil = errors.New("exchange config is nil") errBufferConfigNil = errors.New("buffer config is nil") errUnsetDataHandler = errors.New("datahandler unset") errIssueBufferEnabledButNoLimit = errors.New("buffer enabled but no limit set") - errUpdateIsNil = errors.New("update is nil") - errUpdateNoTargets = errors.New("update bid/ask targets cannot be nil") - errRESTOverwrite = errors.New("orderbook has been overwritten by REST protocol") - errInvalidAction = errors.New("invalid action") - errAmendFailure = errors.New("orderbook amend update failure") - errDeleteFailure = errors.New("orderbook delete update failure") - errInsertFailure = errors.New("orderbook insert update failure") - errUpdateInsertFailure = errors.New("orderbook update/insert update failure") - errRESTTimerLapse = errors.New("rest sync timer lapse with active websocket connection") errOrderbookFlushed = errors.New("orderbook flushed") ) // Setup sets private variables -func (w *Orderbook) Setup(exchangeConfig *config.Exchange, c *Config, dataHandler chan<- any) error { +func (o *Orderbook) Setup(exchangeConfig *config.Exchange, c *Config, dataHandler chan<- any) error { if exchangeConfig == nil { // exchange config fields are checked in websocket package prior to calling this, so further checks are not needed return fmt.Errorf(packageError, errExchangeConfigNil) } @@ -54,314 +40,165 @@ func (w *Orderbook) Setup(exchangeConfig *config.Exchange, c *Config, dataHandle } // NOTE: These variables are set by config.json under "orderbook" for each individual exchange - w.bufferEnabled = exchangeConfig.Orderbook.WebsocketBufferEnabled - w.obBufferLimit = exchangeConfig.Orderbook.WebsocketBufferLimit + o.bufferEnabled = exchangeConfig.Orderbook.WebsocketBufferEnabled + o.obBufferLimit = exchangeConfig.Orderbook.WebsocketBufferLimit - w.sortBuffer = c.SortBuffer - w.sortBufferByUpdateIDs = c.SortBufferByUpdateIDs - w.updateEntriesByID = c.UpdateEntriesByID - w.exchangeName = exchangeConfig.Name - w.dataHandler = dataHandler - w.ob = make(map[key.PairAsset]*orderbookHolder) - w.verbose = exchangeConfig.Verbose - w.updateIDProgression = c.UpdateIDProgression - w.checksum = c.Checksum + o.sortBuffer = c.SortBuffer + o.sortBufferByUpdateIDs = c.SortBufferByUpdateIDs + o.exchangeName = exchangeConfig.Name + o.dataHandler = dataHandler + o.ob = make(map[key.PairAsset]*orderbookHolder) + o.verbose = exchangeConfig.Verbose return nil } -// validate validates update against setup values -func (w *Orderbook) validate(u *orderbook.Update) error { - if u == nil { - return fmt.Errorf(packageError, errUpdateIsNil) - } - if len(u.Bids) == 0 && len(u.Asks) == 0 && !u.AllowEmpty { - return fmt.Errorf(packageError, errUpdateNoTargets) - } - return nil -} - -// Update updates a stored pointer to an orderbook.Depth struct containing -// bid and ask levels, this switches between the usage of a buffered update -func (w *Orderbook) Update(u *orderbook.Update) error { - if err := w.validate(u); err != nil { +// LoadSnapshot loads initial snapshot of orderbook data from websocket +func (o *Orderbook) LoadSnapshot(book *orderbook.Book) error { + if err := book.Validate(); err != nil { return err } - w.mtx.Lock() - defer w.mtx.Unlock() - book, ok := w.ob[key.PairAsset{Base: u.Pair.Base.Item, Quote: u.Pair.Quote.Item, Asset: u.Asset}] + + o.m.RLock() + holder, ok := o.ob[key.PairAsset{Base: book.Pair.Base.Item, Quote: book.Pair.Quote.Item, Asset: book.Asset}] + o.m.RUnlock() if !ok { - return fmt.Errorf("%w for Exchange %s CurrencyPair: %s AssetType: %s", - ErrDepthNotFound, - w.exchangeName, - u.Pair, - u.Asset) - } - - // out of order update ID can be skipped - if w.updateIDProgression && u.UpdateID <= book.updateID { - if w.verbose { - log.Warnf(log.WebsocketMgr, - "Exchange %s CurrencyPair: %s AssetType: %s out of order websocket update received", - w.exchangeName, - u.Pair, - u.Asset) - } - return nil - } - - // Checks for when the rest protocol overwrites a streaming dominated book - // will stop updating book via incremental updates. This occurs because our - // sync manager (engine/sync.go) timer has elapsed for streaming. Usually - // because the book is highly illiquid. - isREST, err := book.ob.IsRESTSnapshot() - if err != nil { - if !errors.Is(err, orderbook.ErrOrderbookInvalid) { - return err - } - // In the event a checksum or processing error invalidates the book, all - // updates that could be stored in the websocket buffer, skip applying - // until a new snapshot comes through. - if w.verbose { - log.Warnf(log.WebsocketMgr, - "Exchange %s CurrencyPair: %s AssetType: %s underlying book is invalid, cannot apply update.", - w.exchangeName, - u.Pair, - u.Asset) - } - return nil - } - - if isREST { - if w.verbose { - log.Warnf(log.WebsocketMgr, - "%s for Exchange %s CurrencyPair: %s AssetType: %s consider extending synctimeoutwebsocket", - errRESTOverwrite, - w.exchangeName, - u.Pair, - u.Asset) - } - // Instance of illiquidity, this signal notifies that there is websocket - // activity. We can invalidate the book and request a new snapshot. All - // further updates through the websocket should be caught above in the - // IsRestSnapshot() call. - return book.ob.Invalidate(errRESTTimerLapse) - } - - if w.bufferEnabled { - var processed bool - processed, err = w.processBufferUpdate(book, u) + o.m.Lock() + // Associate orderbook pointer with local exchange depth map + depth, err := orderbook.DeployDepth(book.Exchange, book.Pair, book.Asset) if err != nil { + o.m.Unlock() return err } + depth.AssignOptions(book) + holder = &orderbookHolder{ob: depth, buffer: make([]orderbook.Update, 0, o.obBufferLimit)} + o.ob[key.PairAsset{Base: book.Pair.Base.Item, Quote: book.Pair.Quote.Item, Asset: book.Asset}] = holder + o.m.Unlock() + } - if !processed { - return nil + book.RestSnapshot = false + if err := holder.ob.LoadSnapshot(book); err != nil { + return err + } + + holder.ob.Publish() + o.dataHandler <- holder.ob + return nil +} + +// Update updates a stored pointer to an orderbook.Depth struct containing bid and ask Tranches, this switches between +// the usage of a buffered update +func (o *Orderbook) Update(u *orderbook.Update) error { + o.m.RLock() + holder, ok := o.ob[key.PairAsset{Base: u.Pair.Base.Item, Quote: u.Pair.Quote.Item, Asset: u.Asset}] + o.m.RUnlock() + if !ok { + return fmt.Errorf("%w for Exchange %s CurrencyPair: %s AssetType: %s", orderbook.ErrDepthNotFound, o.exchangeName, u.Pair, u.Asset) + } + + if o.bufferEnabled { + if processed, err := o.processBufferUpdate(holder, u); err != nil || !processed { + return err } } else { - err = w.processObUpdate(book, u) - if err != nil { + if err := holder.ob.ProcessUpdate(u); err != nil { return err } } // Publish all state changes, disregarding verbosity or sync requirements. - book.ob.Publish() - w.dataHandler <- book.ob + holder.ob.Publish() + o.dataHandler <- holder.ob return nil } // processBufferUpdate stores update into buffer, when buffer at capacity as -// defined by w.obBufferLimit it well then sort and apply updates. -func (w *Orderbook) processBufferUpdate(o *orderbookHolder, u *orderbook.Update) (bool, error) { - *o.buffer = append(*o.buffer, *u) - if len(*o.buffer) < w.obBufferLimit { +// defined by o.obBufferLimit it well then sort and apply updates. +func (o *Orderbook) processBufferUpdate(holder *orderbookHolder, u *orderbook.Update) (bool, error) { + holder.buffer = append(holder.buffer, *u) + if len(holder.buffer) < o.obBufferLimit { return false, nil } - if w.sortBuffer { + if o.sortBuffer { // sort by last updated to ensure each update is in order - if w.sortBufferByUpdateIDs { - sort.Slice(*o.buffer, func(i, j int) bool { - return (*o.buffer)[i].UpdateID < (*o.buffer)[j].UpdateID + if o.sortBufferByUpdateIDs { + slices.SortFunc(holder.buffer, func(a, b orderbook.Update) int { + return cmp.Compare(a.UpdateID, b.UpdateID) }) } else { - sort.Slice(*o.buffer, func(i, j int) bool { - return (*o.buffer)[i].UpdateTime.Before((*o.buffer)[j].UpdateTime) + slices.SortFunc(holder.buffer, func(a, b orderbook.Update) int { + return a.UpdateTime.Compare(b.UpdateTime) }) } } - for i := range *o.buffer { - err := w.processObUpdate(o, &(*o.buffer)[i]) - if err != nil { + + // Always empty the buffer after processing, even if there's an error + defer func() { holder.buffer = holder.buffer[:0] }() + + for i := range holder.buffer { + if err := holder.ob.ProcessUpdate(&holder.buffer[i]); err != nil { return false, err } } - // clear buffer of old updates - *o.buffer = nil + return true, nil } -// processObUpdate processes updates either by its corresponding id or by price level -func (w *Orderbook) processObUpdate(o *orderbookHolder, u *orderbook.Update) error { - // Both update methods require post processing to ensure the orderbook is in a valid state. - if w.updateEntriesByID { - if err := o.updateByIDAndAction(u); err != nil { - return err - } - } else { - if err := o.updateByPrice(u); err != nil { - return err - } - } - if w.checksum != nil { - compare, err := o.ob.Retrieve() - if err != nil { - return err - } - err = w.checksum(compare, u.Checksum) - if err != nil { - return o.ob.Invalidate(err) - } - o.updateID = u.UpdateID - } else if o.ob.VerifyOrderbook() { - compare, err := o.ob.Retrieve() - if err != nil { - return err - } - err = compare.Verify() - if err != nil { - return o.ob.Invalidate(err) - } - } - return nil -} - -// updateByPrice amends amount if match occurs by price, deletes if amount is -// zero or less and inserts if not found. -func (o *orderbookHolder) updateByPrice(updts *orderbook.Update) error { - return o.ob.UpdateBidAskByPrice(updts) -} - -// updateByIDAndAction will receive an action to execute against the orderbook -// it will then match by IDs instead of price to perform the action -func (o *orderbookHolder) updateByIDAndAction(updts *orderbook.Update) error { - switch updts.Action { - case orderbook.Amend: - err := o.ob.UpdateBidAskByID(updts) - if err != nil { - return fmt.Errorf("%w %w", errAmendFailure, err) - } - case orderbook.Delete: - // edge case for Bitfinex as their streaming endpoint duplicates deletes - bypassErr := o.ob.GetName() == "Bitfinex" && o.ob.IsFundingRate() - err := o.ob.DeleteBidAskByID(updts, bypassErr) - if err != nil { - return fmt.Errorf("%w %w", errDeleteFailure, err) - } - case orderbook.Insert: - err := o.ob.InsertBidAskByID(updts) - if err != nil { - return fmt.Errorf("%w %w", errInsertFailure, err) - } - case orderbook.UpdateInsert: - err := o.ob.UpdateInsertByID(updts) - if err != nil { - return fmt.Errorf("%w %w", errUpdateInsertFailure, err) - } - default: - return fmt.Errorf("%w [%d]", errInvalidAction, updts.Action) - } - return nil -} - -// LoadSnapshot loads initial snapshot of orderbook data from websocket -func (w *Orderbook) LoadSnapshot(book *orderbook.Book) error { - // Checks if book can deploy to depth - err := book.Verify() - if err != nil { - return err - } - - w.mtx.Lock() - defer w.mtx.Unlock() - holder, ok := w.ob[key.PairAsset{Base: book.Pair.Base.Item, Quote: book.Pair.Quote.Item, Asset: book.Asset}] - if !ok { - // Associate orderbook pointer with local exchange depth map - var depth *orderbook.Depth - depth, err = orderbook.DeployDepth(book.Exchange, book.Pair, book.Asset) - if err != nil { - return err - } - depth.AssignOptions(book) - buffer := make([]orderbook.Update, w.obBufferLimit) - - holder = &orderbookHolder{ob: depth, buffer: &buffer} - w.ob[key.PairAsset{Base: book.Pair.Base.Item, Quote: book.Pair.Quote.Item, Asset: book.Asset}] = holder - } - - holder.updateID = book.LastUpdateID - - err = holder.ob.LoadSnapshot(book.Bids, book.Asks, book.LastUpdateID, book.LastUpdated, book.LastPushed, false) - if err != nil { - return err - } - - holder.ob.Publish() - w.dataHandler <- holder.ob - return nil -} - // GetOrderbook returns an orderbook copy as orderbook.Book -func (w *Orderbook) GetOrderbook(p currency.Pair, a asset.Item) (*orderbook.Book, error) { +func (o *Orderbook) GetOrderbook(p currency.Pair, a asset.Item) (*orderbook.Book, error) { if p.IsEmpty() { return nil, currency.ErrCurrencyPairEmpty } if !a.IsValid() { return nil, asset.ErrInvalidAsset } - w.mtx.Lock() - defer w.mtx.Unlock() - book, ok := w.ob[key.PairAsset{Base: p.Base.Item, Quote: p.Quote.Item, Asset: a}] + o.m.RLock() + holder, ok := o.ob[key.PairAsset{Base: p.Base.Item, Quote: p.Quote.Item, Asset: a}] + o.m.RUnlock() if !ok { - return nil, fmt.Errorf("%s %w: %s.%s", w.exchangeName, ErrDepthNotFound, a, p) + return nil, fmt.Errorf("%s %w: %s.%s", o.exchangeName, orderbook.ErrDepthNotFound, a, p) } - return book.ob.Retrieve() + return holder.ob.Retrieve() } // LastUpdateID returns the last update ID of the orderbook -func (w *Orderbook) LastUpdateID(p currency.Pair, a asset.Item) (int64, error) { +func (o *Orderbook) LastUpdateID(p currency.Pair, a asset.Item) (int64, error) { if p.IsEmpty() { return 0, currency.ErrCurrencyPairEmpty } if !a.IsValid() { return 0, asset.ErrInvalidAsset } - w.mtx.Lock() - defer w.mtx.Unlock() - book, ok := w.ob[key.PairAsset{Base: p.Base.Item, Quote: p.Quote.Item, Asset: a}] + o.m.RLock() + book, ok := o.ob[key.PairAsset{Base: p.Base.Item, Quote: p.Quote.Item, Asset: a}] + o.m.RUnlock() if !ok { - return 0, fmt.Errorf("%s %w: %s.%s", w.exchangeName, ErrDepthNotFound, a, p) + return 0, fmt.Errorf("%s %w: %s.%s", o.exchangeName, orderbook.ErrDepthNotFound, a, p) } return book.ob.LastUpdateID() } -// FlushBuffer flushes w.ob data to be garbage collected and refreshed when a -// connection is lost and reconnected -func (w *Orderbook) FlushBuffer() { - w.mtx.Lock() - w.ob = make(map[key.PairAsset]*orderbookHolder) - w.mtx.Unlock() +// FlushBuffer flushes individual orderbook buffers while keeping the orderbook lookups intact and ready for new updates +// when a connection is re-established. +func (o *Orderbook) FlushBuffer() { + o.m.Lock() + for _, holder := range o.ob { + holder.buffer = holder.buffer[:0] + } + o.m.Unlock() } -// FlushOrderbook flushes independent orderbook -func (w *Orderbook) FlushOrderbook(p currency.Pair, a asset.Item) error { - w.mtx.Lock() - defer w.mtx.Unlock() - book, ok := w.ob[key.PairAsset{Base: p.Base.Item, Quote: p.Quote.Item, Asset: a}] +// InvalidateOrderbook invalidates the orderbook so no trading can occur on potential corrupted data +// TODO: Add in reason for invalidation for debugging purposes. +func (o *Orderbook) InvalidateOrderbook(p currency.Pair, a asset.Item) error { + o.m.RLock() + holder, ok := o.ob[key.PairAsset{Base: p.Base.Item, Quote: p.Quote.Item, Asset: a}] + o.m.RUnlock() if !ok { - return fmt.Errorf("cannot flush orderbook %s %s %s %w", w.exchangeName, p, a, ErrDepthNotFound) + return fmt.Errorf("cannot invalidate orderbook %s %s %s %w", o.exchangeName, p, a, orderbook.ErrDepthNotFound) } - // error not needed in this return - _ = book.ob.Invalidate(errOrderbookFlushed) + // Invalidate returns a formatted version of the error it's passed + // In this context we don't need that, since this method only returns an error if it cannot invalidate + _ = holder.ob.Invalidate(errOrderbookFlushed) return nil } diff --git a/exchange/websocket/buffer/buffer_test.go b/exchange/websocket/buffer/buffer_test.go index 7ae3136c..d6a0390c 100644 --- a/exchange/websocket/buffer/buffer_test.go +++ b/exchange/websocket/buffer/buffer_test.go @@ -1,9 +1,7 @@ package buffer import ( - "errors" "math/rand" - "slices" "strconv" "testing" "time" @@ -37,20 +35,20 @@ func getExclusivePair() (currency.Pair, error) { return currency.NewPairFromStrings(currency.BTC.String(), currency.USDT.String()+strconv.FormatInt(offset.IncrementAndGet(), 10)) } -func createSnapshot(pair currency.Pair, bookVerifiy ...bool) (holder *Orderbook, asks, bids orderbook.Levels, err error) { +func createSnapshot(pair currency.Pair) (holder *Orderbook, asks, bids orderbook.Levels, err error) { asks = orderbook.Levels{{Price: 4000, Amount: 1, ID: 6}} bids = orderbook.Levels{{Price: 4000, Amount: 1, ID: 6}} book := &orderbook.Book{ - Exchange: exchangeName, - Asks: asks, - Bids: bids, - Asset: asset.Spot, - Pair: pair, - PriceDuplication: true, - LastUpdated: time.Now(), - VerifyOrderbook: len(bookVerifiy) > 0 && bookVerifiy[0], - LastUpdateID: 69420, + Exchange: exchangeName, + Asks: asks, + Bids: bids, + Asset: asset.Spot, + Pair: pair, + PriceDuplication: true, + LastUpdated: time.Now(), + LastUpdateID: 69420, + ValidateOrderbook: true, } newBook := make(map[key.PairAsset]*orderbookHolder) @@ -70,64 +68,6 @@ func createSnapshot(pair currency.Pair, bookVerifiy ...bool) (holder *Orderbook, return holder, asks, bids, err } -func bidAskGenerator() []orderbook.Level { - response := make([]orderbook.Level, 100) - for i := range 100 { - price := float64(rand.Intn(1000)) //nolint:gosec // no need to import crypo/rand for testing - if price == 0 { - price = 1 - } - response[i] = orderbook.Level{ - Amount: float64(rand.Intn(10)), //nolint:gosec // no need to import crypo/rand for testing - Price: price, - ID: int64(i), - } - } - return response -} - -func BenchmarkUpdateBidsByPrice(b *testing.B) { - cp, err := getExclusivePair() - require.NoError(b, err) - - ob, _, _, err := createSnapshot(cp) - require.NoError(b, err) - - for b.Loop() { - bidAsks := bidAskGenerator() - update := &orderbook.Update{ - Bids: bidAsks, - Asks: bidAsks, - Pair: cp, - UpdateTime: time.Now(), - Asset: asset.Spot, - } - holder := ob.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}] - require.NoError(b, holder.updateByPrice(update)) - } -} - -func BenchmarkUpdateAsksByPrice(b *testing.B) { - cp, err := getExclusivePair() - require.NoError(b, err) - - ob, _, _, err := createSnapshot(cp) - require.NoError(b, err) - - for b.Loop() { - bidAsks := bidAskGenerator() - update := &orderbook.Update{ - Bids: bidAsks, - Asks: bidAsks, - Pair: cp, - UpdateTime: time.Now(), - Asset: asset.Spot, - } - holder := ob.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}] - require.NoError(b, holder.updateByPrice(update)) - } -} - // BenchmarkBufferPerformance demonstrates buffer more performant than multi // process calls // 890016 1688 ns/op 416 B/op 3 allocs/op @@ -237,38 +177,6 @@ func BenchmarkNoBufferPerformance(b *testing.B) { } } -func TestUpdates(t *testing.T) { - t.Parallel() - cp, err := getExclusivePair() - require.NoError(t, err) - - holder, _, _, err := createSnapshot(cp) - require.NoError(t, err) - - book := holder.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}] - err = book.updateByPrice(&orderbook.Update{ - Bids: itemArray[5], - Asks: itemArray[5], - Pair: cp, - UpdateTime: time.Now(), - Asset: asset.Spot, - }) - assert.NoError(t, err) - - err = book.updateByPrice(&orderbook.Update{ - Bids: itemArray[0], - Asks: itemArray[0], - Pair: cp, - UpdateTime: time.Now(), - Asset: asset.Spot, - }) - assert.NoError(t, err) - - askLen, err := book.ob.GetAskLength() - require.NoError(t, err) - assert.Equal(t, 3, askLen) -} - // TestHittingTheBuffer logic test func TestHittingTheBuffer(t *testing.T) { t.Parallel() @@ -313,7 +221,6 @@ func TestInsertWithIDs(t *testing.T) { require.NoError(t, err) holder.bufferEnabled = true - holder.updateEntriesByID = true holder.obBufferLimit = 5 for i := range itemArray { asks := itemArray[i] @@ -327,7 +234,7 @@ func TestInsertWithIDs(t *testing.T) { Pair: cp, UpdateTime: time.Now(), Asset: asset.Spot, - Action: orderbook.UpdateInsert, + Action: orderbook.UpdateOrInsertAction, }) require.NoError(t, err) } @@ -341,21 +248,13 @@ func TestInsertWithIDs(t *testing.T) { require.NoError(t, err) assert.Equal(t, 6, bidLen) - cp, err = getExclusivePair() - require.NoError(t, err) - - holder, _, _, err = createSnapshot(cp, true) - require.NoError(t, err) - - holder.checksum = nil - holder.updateIDProgression = false + holder.obBufferLimit = 1 err = holder.Update(&orderbook.Update{ UpdateTime: time.Now(), Asset: asset.Spot, - Asks: []orderbook.Level{{Price: 999999}}, Pair: cp, }) - require.NoError(t, err) + assert.ErrorIs(t, err, orderbook.ErrEmptyUpdate) } // TestSortIDs logic test @@ -437,15 +336,15 @@ func TestOrderbookLastUpdateID(t *testing.T) { assert.Equal(t, 1000., itemArray[0][0].Price) - holder.checksum = func(*orderbook.Book, uint32) error { return errors.New("testerino") } - // this update invalidates the book err = holder.Update(&orderbook.Update{ - Asks: []orderbook.Level{{Price: 999999}}, - Pair: cp, - UpdateID: -1, - Asset: asset.Spot, - UpdateTime: time.Now(), + Asks: orderbook.Levels{{Price: 999999}}, + Pair: cp, + UpdateID: -1, + Asset: asset.Spot, + UpdateTime: time.Now(), + ExpectedChecksum: 1337, + GenerateChecksum: func(*orderbook.Book) uint32 { return 1336 }, }) require.ErrorIs(t, err, orderbook.ErrOrderbookInvalid) @@ -455,33 +354,34 @@ func TestOrderbookLastUpdateID(t *testing.T) { holder, _, _, err = createSnapshot(cp) require.NoError(t, err) - holder.checksum = func(*orderbook.Book, uint32) error { return nil } - holder.updateIDProgression = true - for i := range itemArray { asks := itemArray[i] err = holder.Update(&orderbook.Update{ - Asks: asks, - Pair: cp, - UpdateID: int64(i) + 1 + 69420, - Asset: asset.Spot, - UpdateTime: time.Now(), + Asks: asks, + Pair: cp, + UpdateID: int64(i) + 1 + 69420, + Asset: asset.Spot, + UpdateTime: time.Now(), + SkipOutOfOrderLastUpdateID: true, + ExpectedChecksum: 1337, + GenerateChecksum: func(*orderbook.Book) uint32 { return 1337 }, }) require.NoError(t, err) } // out of order err = holder.Update(&orderbook.Update{ - Asks: []orderbook.Level{{Price: 999999}}, - Pair: cp, - UpdateID: 1, - Asset: asset.Spot, + Asks: orderbook.Levels{{Price: 999999}}, + Pair: cp, + UpdateID: 1, + Asset: asset.Spot, + SkipOutOfOrderLastUpdateID: true, }) - require.NoError(t, err) + require.NoError(t, err, "Out of sequence Update must not error") ob, err := holder.GetOrderbook(cp, asset.Spot) - require.NoError(t, err) - assert.Equal(t, int64(len(itemArray)+69420), ob.LastUpdateID) + require.NoError(t, err, "GetOrderbook must not error") + assert.Equal(t, int64(len(itemArray)+69420), ob.LastUpdateID, "Out of sequence Update should not change LastUpdateID") } // TestRunUpdateWithoutSnapshot logic test @@ -501,7 +401,7 @@ func TestRunUpdateWithoutSnapshot(t *testing.T) { UpdateTime: time.Now(), Asset: asset.Spot, }) - require.ErrorIs(t, err, ErrDepthNotFound) + require.ErrorIs(t, err, orderbook.ErrDepthNotFound) } // TestRunUpdateWithoutAnyUpdates logic test @@ -510,16 +410,18 @@ func TestRunUpdateWithoutAnyUpdates(t *testing.T) { cp, err := getExclusivePair() require.NoError(t, err) - var obl Orderbook - obl.exchangeName = exchangeName - err = obl.Update(&orderbook.Update{ - Bids: []orderbook.Level{}, - Asks: []orderbook.Level{}, + holder, _, _, err := createSnapshot(cp) + require.NoError(t, err) + + holder.exchangeName = exchangeName + err = holder.Update(&orderbook.Update{ + Bids: orderbook.Levels{}, + Asks: orderbook.Levels{}, Pair: cp, UpdateTime: time.Now(), Asset: asset.Spot, }) - require.ErrorIs(t, err, errUpdateNoTargets) + require.ErrorIs(t, err, orderbook.ErrEmptyUpdate) } // TestRunSnapshotWithNoData logic test @@ -549,6 +451,16 @@ func TestLoadSnapshot(t *testing.T) { var obl Orderbook obl.dataHandler = make(chan any, 100) obl.ob = make(map[key.PairAsset]*orderbookHolder) + + err = obl.LoadSnapshot(&orderbook.Book{Asks: orderbook.Levels{{Amount: 1}}, ValidateOrderbook: true}) + require.ErrorIs(t, err, orderbook.ErrPriceZero) + + err = obl.LoadSnapshot(&orderbook.Book{Asks: orderbook.Levels{{Amount: 1}}}) + require.ErrorIs(t, err, orderbook.ErrExchangeNameEmpty) + + err = obl.LoadSnapshot(&orderbook.Book{Asks: orderbook.Levels{{Amount: 1}}, Exchange: "test", Pair: cp, Asset: asset.Spot}) + require.ErrorIs(t, err, orderbook.ErrLastUpdatedNotSet) + var snapShot1 orderbook.Book snapShot1.Exchange = "SnapshotWithOverride" asks := []orderbook.Level{{Price: 4000, Amount: 1, ID: 8}} @@ -568,11 +480,19 @@ func TestFlushBuffer(t *testing.T) { require.NoError(t, err) obl, _, _, err := createSnapshot(cp) - require.NoError(t, err) + require.NoError(t, err, "createSnapshot must not error") + require.NotEmpty(t, obl.ob, "createSnapshot must not return empty") + + k := key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot} + holder, ok := obl.ob[k] + require.Truef(t, ok, "createSnapshot must return a orderbook for %v", k) + + holder.buffer = make([]orderbook.Update, 0, 10) + holder.buffer = append(holder.buffer, orderbook.Update{}) - assert.NotEmpty(t, obl.ob) obl.FlushBuffer() - assert.Empty(t, obl.ob) + assert.Empty(t, holder.buffer, "FlushBuffer should empty buffer") + assert.Equal(t, 10, cap(holder.buffer), "FlushBuffer should leave the buffer cap to avoid reallocs") } // TestInsertingSnapShots logic test @@ -767,7 +687,7 @@ func TestLastUpdateID(t *testing.T) { require.ErrorIs(t, err, asset.ErrInvalidAsset) _, err = holder.LastUpdateID(cp, asset.FutureCombo) - require.ErrorIs(t, err, ErrDepthNotFound) + require.ErrorIs(t, err, orderbook.ErrDepthNotFound) ob, err := holder.LastUpdateID(cp, asset.Spot) require.NoError(t, err) @@ -797,194 +717,17 @@ func TestSetup(t *testing.T) { exchangeConfig.Name = "test" bufferConf.SortBuffer = true bufferConf.SortBufferByUpdateIDs = true - bufferConf.UpdateEntriesByID = true err = w.Setup(exchangeConfig, bufferConf, make(chan any)) require.NoError(t, err) - if w.obBufferLimit != 1337 || - !w.bufferEnabled || - !w.sortBuffer || - !w.sortBufferByUpdateIDs || - !w.updateEntriesByID || - w.exchangeName != "test" { - t.Errorf("Setup incorrectly loaded %s", w.exchangeName) - } + require.Equal(t, 1337, w.obBufferLimit) + require.True(t, w.bufferEnabled) + require.True(t, w.sortBuffer) + require.True(t, w.sortBufferByUpdateIDs) + require.Equal(t, "test", w.exchangeName) } -func TestValidate(t *testing.T) { - t.Parallel() - w := Orderbook{} - err := w.validate(nil) - require.ErrorIs(t, err, errUpdateIsNil) - err = w.validate(&orderbook.Update{}) - require.ErrorIs(t, err, errUpdateNoTargets) -} - -func TestEnsureMultipleUpdatesViaPrice(t *testing.T) { - t.Parallel() - cp, err := getExclusivePair() - require.NoError(t, err) - - holder, _, _, err := createSnapshot(cp) - require.NoError(t, err) - - asks := bidAskGenerator() - book := holder.ob[key.PairAsset{Base: cp.Base.Item, Quote: cp.Quote.Item, Asset: asset.Spot}] - err = book.updateByPrice(&orderbook.Update{ - Bids: asks, - Asks: asks, - Pair: cp, - UpdateTime: time.Now(), - Asset: asset.Spot, - }) - require.NoError(t, err) - - askLen, err := book.ob.GetAskLength() - require.NoError(t, err) - assert.LessOrEqual(t, 3, askLen) -} - -func deploySliceOrdered(size int) orderbook.Levels { - items := make([]orderbook.Level, size) - for i := range size { - items[i] = orderbook.Level{Amount: 1, Price: rand.Float64() + float64(i), ID: rand.Int63()} //nolint:gosec // Not needed for tests - } - return items -} - -func TestUpdateByIDAndAction(t *testing.T) { - t.Parallel() - cp, err := getExclusivePair() - require.NoError(t, err) - - asks := deploySliceOrdered(100) - bids := slices.Clone(asks) - bids.Reverse() - - book, err := orderbook.DeployDepth("test", cp, asset.Spot) - require.NoError(t, err) - - err = book.LoadSnapshot(slices.Clone(bids), slices.Clone(asks), 0, time.Now(), time.Now(), true) - require.NoError(t, err) - - ob, err := book.Retrieve() - require.NoError(t, err) - - require.NoError(t, ob.Verify()) - - holder := orderbookHolder{ob: book} - err = holder.updateByIDAndAction(&orderbook.Update{}) - require.ErrorIs(t, err, errInvalidAction) - - err = holder.updateByIDAndAction(&orderbook.Update{ - Action: orderbook.Amend, - Bids: []orderbook.Level{{Price: 100, ID: 6969}}, - }) - require.ErrorIs(t, err, errAmendFailure) - - err = book.LoadSnapshot(slices.Clone(bids), slices.Clone(asks), 0, time.Now(), time.Now(), true) - require.NoError(t, err) - - // append to slice - err = holder.updateByIDAndAction(&orderbook.Update{ - Action: orderbook.UpdateInsert, - Bids: []orderbook.Level{{Price: 0, ID: 1337, Amount: 1}}, - Asks: []orderbook.Level{{Price: 100, ID: 1337, Amount: 1}}, - UpdateTime: time.Now(), - }) - require.NoError(t, err) - - cpy, err := book.Retrieve() - require.NoError(t, err) - require.Equal(t, 0., cpy.Bids[len(cpy.Bids)-1].Price) - require.Equal(t, 100., cpy.Asks[len(cpy.Asks)-1].Price) - - // Change amount - err = holder.updateByIDAndAction(&orderbook.Update{ - Action: orderbook.UpdateInsert, - Bids: []orderbook.Level{{Price: 0, ID: 1337, Amount: 100}}, - Asks: []orderbook.Level{{Price: 100, ID: 1337, Amount: 100}}, - UpdateTime: time.Now(), - }) - require.NoError(t, err) - - cpy, err = book.Retrieve() - require.NoError(t, err) - require.Equal(t, 100., cpy.Bids[len(cpy.Bids)-1].Amount) - require.Equal(t, 100., cpy.Asks[len(cpy.Asks)-1].Amount) - - // Change price level - err = holder.updateByIDAndAction(&orderbook.Update{ - Action: orderbook.UpdateInsert, - Bids: []orderbook.Level{{Price: 100, ID: 1337, Amount: 99}}, - Asks: []orderbook.Level{{Price: 0, ID: 1337, Amount: 99}}, - UpdateTime: time.Now(), - }) - require.NoError(t, err) - - cpy, err = book.Retrieve() - require.NoError(t, err) - - require.Equal(t, 99., cpy.Bids[0].Amount) - require.Equal(t, 100., cpy.Bids[0].Price) - require.Equal(t, 99., cpy.Asks[0].Amount) - require.Equal(t, 0., cpy.Asks[0].Price) - - err = book.LoadSnapshot(slices.Clone(bids), slices.Clone(asks), 0, time.Now(), time.Now(), true) - require.NoError(t, err) - // Delete - not found - err = holder.updateByIDAndAction(&orderbook.Update{ - Action: orderbook.Delete, - Asks: []orderbook.Level{{Price: 0, ID: 1337, Amount: 99}}, - }) - require.ErrorIs(t, err, errDeleteFailure) - - err = book.LoadSnapshot(slices.Clone(bids), slices.Clone(asks), 0, time.Now(), time.Now(), true) - require.NoError(t, err) - - // Delete - found - err = holder.updateByIDAndAction(&orderbook.Update{ - Action: orderbook.Delete, - Asks: []orderbook.Level{asks[0]}, - UpdateTime: time.Now(), - }) - require.NoError(t, err) - - askLen, err := book.GetAskLength() - require.NoError(t, err) - require.Equal(t, 99, askLen) - - // Apply update - err = holder.updateByIDAndAction(&orderbook.Update{ - Action: orderbook.Amend, - Asks: []orderbook.Level{{ID: 123456}}, - }) - require.ErrorIs(t, err, errAmendFailure) - - err = book.LoadSnapshot(slices.Clone(bids), slices.Clone(bids), 0, time.Now(), time.Now(), true) - require.NoError(t, err) - - ob, err = book.Retrieve() - require.NoError(t, err) - require.NotEmpty(t, ob.Asks) - require.NotEmpty(t, ob.Bids) - - update := ob.Asks[0] - update.Amount = 1337 - - err = holder.updateByIDAndAction(&orderbook.Update{ - Action: orderbook.Amend, - Asks: []orderbook.Level{update}, - UpdateTime: time.Now(), - }) - require.NoError(t, err) - - ob, err = book.Retrieve() - require.NoError(t, err) - require.Equal(t, 1337., ob.Asks[0].Amount) -} - -func TestFlushOrderbook(t *testing.T) { +func TestInvalidateOrderbook(t *testing.T) { t.Parallel() cp, err := getExclusivePair() require.NoError(t, err) @@ -1003,16 +746,16 @@ func TestFlushOrderbook(t *testing.T) { snapShot1.Pair = cp snapShot1.LastUpdated = time.Now() - err = w.FlushOrderbook(cp, asset.Spot) + err = w.InvalidateOrderbook(cp, asset.Spot) if err == nil { t.Fatal("book not loaded error cannot be nil") } _, err = w.GetOrderbook(cp, asset.Spot) - require.ErrorIs(t, err, ErrDepthNotFound) + require.ErrorIs(t, err, orderbook.ErrDepthNotFound) require.NoError(t, w.LoadSnapshot(&snapShot1)) - require.NoError(t, w.FlushOrderbook(cp, asset.Spot)) + require.NoError(t, w.InvalidateOrderbook(cp, asset.Spot)) _, err = w.GetOrderbook(cp, asset.Spot) require.ErrorIs(t, err, orderbook.ErrOrderbookInvalid) diff --git a/exchange/websocket/buffer/buffer_types.go b/exchange/websocket/buffer/buffer_types.go index 6d4deace..142f06da 100644 --- a/exchange/websocket/buffer/buffer_types.go +++ b/exchange/websocket/buffer/buffer_types.go @@ -15,14 +15,6 @@ type Config struct { // SortBufferByUpdateIDs allows the sorting of the buffered updates by their // corresponding update IDs. SortBufferByUpdateIDs bool - // UpdateEntriesByID will match by IDs instead of price to perform the an - // action. e.g. update, delete, insert. - UpdateEntriesByID bool - // UpdateIDProgression requires that the new update ID be greater than the - // prior ID. This will skip processing and not error. - UpdateIDProgression bool - // Checksum is a package defined checksum calculation for updated books. - Checksum func(state *orderbook.Book, checksum uint32) error } // Orderbook defines a local cache of orderbooks for amending, appending @@ -33,27 +25,16 @@ type Orderbook struct { bufferEnabled bool sortBuffer bool sortBufferByUpdateIDs bool // When timestamps aren't provided, an id can help sort - updateEntriesByID bool // Use the update IDs to match ob entries exchangeName string dataHandler chan<- any verbose bool - // updateIDProgression requires that the new update ID be greater than the - // prior ID. This will skip processing and not error. - updateIDProgression bool - // checksum is a package defined checksum calculation for updated books. - checksum func(state *orderbook.Book, checksum uint32) error - // TODO: sync.RWMutex. For the moment we process the orderbook in a single - // thread. In future when there are workers directly involved this can be - // can be improved with RW mechanics which will allow updates to occur at - // the same time on different books. - mtx sync.Mutex + m sync.RWMutex } // orderbookHolder defines a store of pending updates and a pointer to the // orderbook depth type orderbookHolder struct { - ob *orderbook.Depth - buffer *[]orderbook.Update - updateID int64 + ob *orderbook.Depth + buffer []orderbook.Update } diff --git a/exchanges/binance/binance_websocket.go b/exchanges/binance/binance_websocket.go index 1137eb37..84f8dd96 100644 --- a/exchanges/binance/binance_websocket.go +++ b/exchanges/binance/binance_websocket.go @@ -463,14 +463,14 @@ func (b *Binance) SeedLocalCache(ctx context.Context, p currency.Pair) error { // SeedLocalCacheWithBook seeds the local orderbook cache func (b *Binance) SeedLocalCacheWithBook(p currency.Pair, orderbookNew *OrderBook) error { newOrderBook := orderbook.Book{ - Pair: p, - Asset: asset.Spot, - Exchange: b.Name, - LastUpdateID: orderbookNew.LastUpdateID, - VerifyOrderbook: b.CanVerifyOrderbook, - Bids: make(orderbook.Levels, len(orderbookNew.Bids)), - Asks: make(orderbook.Levels, len(orderbookNew.Asks)), - LastUpdated: time.Now(), // Time not provided in REST book. + Pair: p, + Asset: asset.Spot, + Exchange: b.Name, + LastUpdateID: orderbookNew.LastUpdateID, + ValidateOrderbook: b.ValidateOrderbook, + Bids: make(orderbook.Levels, len(orderbookNew.Bids)), + Asks: make(orderbook.Levels, len(orderbookNew.Asks)), + LastUpdated: time.Now(), // Time not provided in REST book. } for i := range orderbookNew.Bids { newOrderBook.Bids[i] = orderbook.Level{ @@ -504,7 +504,7 @@ func (b *Binance) UpdateLocalBuffer(wsdp *WebsocketDepthStream) (bool, error) { err = b.applyBufferUpdate(pair) if err != nil { - b.flushAndCleanup(pair) + b.invalidateAndCleanupOrderbook(pair) } return false, err @@ -745,26 +745,19 @@ func (b *Binance) processJob(ctx context.Context, p currency.Pair) error { // new update to initiate this. err = b.applyBufferUpdate(p) if err != nil { - b.flushAndCleanup(p) + b.invalidateAndCleanupOrderbook(p) return err } return nil } -// flushAndCleanup flushes orderbook and clean local cache -func (b *Binance) flushAndCleanup(p currency.Pair) { - errClean := b.Websocket.Orderbook.FlushOrderbook(p, asset.Spot) - if errClean != nil { - log.Errorf(log.WebsocketMgr, - "%s flushing websocket error: %v", - b.Name, - errClean) +// invalidateAndCleanupOrderbook invalidaates orderbook and cleans local cache +func (b *Binance) invalidateAndCleanupOrderbook(p currency.Pair) { + if err := b.Websocket.Orderbook.InvalidateOrderbook(p, asset.Spot); err != nil { + log.Errorf(log.WebsocketMgr, "%s error invalidating websocket orderbook: %v", b.Name, err) } - errClean = b.obm.cleanup(p) - if errClean != nil { - log.Errorf(log.WebsocketMgr, "%s cleanup websocket error: %v", - b.Name, - errClean) + if err := b.obm.cleanup(p); err != nil { + log.Errorf(log.WebsocketMgr, "%s error during websocket orderbook cleanup: %v", b.Name, err) } } diff --git a/exchanges/binance/binance_wrapper.go b/exchanges/binance/binance_wrapper.go index 3ecf34b4..e7de0ba3 100644 --- a/exchanges/binance/binance_wrapper.go +++ b/exchanges/binance/binance_wrapper.go @@ -521,10 +521,10 @@ func (b *Binance) UpdateOrderbook(ctx context.Context, p currency.Pair, assetTyp return nil, err } book := &orderbook.Book{ - Exchange: b.Name, - Pair: p, - Asset: assetType, - VerifyOrderbook: b.CanVerifyOrderbook, + Exchange: b.Name, + Pair: p, + Asset: assetType, + ValidateOrderbook: b.ValidateOrderbook, } var orderbookNew *OrderBook var err error diff --git a/exchanges/binanceus/binanceus_websocket.go b/exchanges/binanceus/binanceus_websocket.go index f3967e0b..a4869d02 100644 --- a/exchanges/binanceus/binanceus_websocket.go +++ b/exchanges/binanceus/binanceus_websocket.go @@ -520,7 +520,7 @@ func (bi *Binanceus) UpdateLocalBuffer(wsdp *WebsocketDepthStream) (bool, error) err = bi.applyBufferUpdate(currencyPair) if err != nil { - bi.flushAndCleanup(currencyPair) + bi.invalidateAndCleanupOrderbook(currencyPair) } return false, err @@ -786,7 +786,7 @@ func (bi *Binanceus) processJob(ctx context.Context, p currency.Pair) error { // new update to initiate this. err = bi.applyBufferUpdate(p) if err != nil { - bi.flushAndCleanup(p) + bi.invalidateAndCleanupOrderbook(p) return err } return nil @@ -808,14 +808,14 @@ func (bi *Binanceus) SeedLocalCache(ctx context.Context, p currency.Pair) error // SeedLocalCacheWithBook seeds the local orderbook cache func (bi *Binanceus) SeedLocalCacheWithBook(p currency.Pair, orderbookNew *OrderBook) error { newOrderBook := orderbook.Book{ - Pair: p, - Asset: asset.Spot, - Exchange: bi.Name, - LastUpdateID: orderbookNew.LastUpdateID, - VerifyOrderbook: bi.CanVerifyOrderbook, - Bids: make(orderbook.Levels, len(orderbookNew.Bids)), - Asks: make(orderbook.Levels, len(orderbookNew.Asks)), - LastUpdated: time.Now(), // Time not provided in REST book. + Pair: p, + Asset: asset.Spot, + Exchange: bi.Name, + LastUpdateID: orderbookNew.LastUpdateID, + ValidateOrderbook: bi.ValidateOrderbook, + Bids: make(orderbook.Levels, len(orderbookNew.Bids)), + Asks: make(orderbook.Levels, len(orderbookNew.Asks)), + LastUpdated: time.Now(), // Time not provided in REST book. } for i := range orderbookNew.Bids { newOrderBook.Bids[i] = orderbook.Level{ @@ -858,20 +858,13 @@ func (o *orderbookManager) handleFetchingBook(pair currency.Pair) (fetching, nee return false, false, nil } -// flushAndCleanup flushes orderbook and clean local cache -func (bi *Binanceus) flushAndCleanup(p currency.Pair) { - errClean := bi.Websocket.Orderbook.FlushOrderbook(p, asset.Spot) - if errClean != nil { - log.Errorf(log.WebsocketMgr, - "%s flushing websocket error: %v", - bi.Name, - errClean) +// invalidateAndCleanupOrderbook invalidaates orderbook and cleans local cache +func (bi *Binanceus) invalidateAndCleanupOrderbook(p currency.Pair) { + if err := bi.Websocket.Orderbook.InvalidateOrderbook(p, asset.Spot); err != nil { + log.Errorf(log.WebsocketMgr, "%s invalidate orderbook websocket error: %v", bi.Name, err) } - errClean = bi.obm.cleanup(p) - if errClean != nil { - log.Errorf(log.WebsocketMgr, "%s cleanup websocket error: %v", - bi.Name, - errClean) + if err := bi.obm.cleanup(p); err != nil { + log.Errorf(log.WebsocketMgr, "%s cleanup websocket error: %v", bi.Name, err) } } diff --git a/exchanges/binanceus/binanceus_wrapper.go b/exchanges/binanceus/binanceus_wrapper.go index 751fd257..d865632f 100644 --- a/exchanges/binanceus/binanceus_wrapper.go +++ b/exchanges/binanceus/binanceus_wrapper.go @@ -319,10 +319,10 @@ func (bi *Binanceus) UpdateOrderbook(ctx context.Context, pair currency.Pair, as return nil, err } book := &orderbook.Book{ - Exchange: bi.Name, - Pair: pair, - Asset: assetType, - VerifyOrderbook: bi.CanVerifyOrderbook, + Exchange: bi.Name, + Pair: pair, + Asset: assetType, + ValidateOrderbook: bi.ValidateOrderbook, } orderbookNew, err := bi.GetOrderBookDepth(ctx, &OrderBookDataRequestParams{ diff --git a/exchanges/bitfinex/bitfinex_websocket.go b/exchanges/bitfinex/bitfinex_websocket.go index d2b8f017..7aacac26 100644 --- a/exchanges/bitfinex/bitfinex_websocket.go +++ b/exchanges/bitfinex/bitfinex_websocket.go @@ -1562,7 +1562,7 @@ func (b *Bitfinex) WsInsertSnapshot(p currency.Pair, assetType asset.Item, books book.Exchange = b.Name book.PriceDuplication = true book.IsFundingRate = fundingRate - book.VerifyOrderbook = b.CanVerifyOrderbook + book.ValidateOrderbook = b.ValidateOrderbook book.LastUpdated = time.Now() // Not included in snapshot return b.Websocket.Orderbook.LoadSnapshot(&book) } @@ -1593,7 +1593,7 @@ func (b *Bitfinex) WsUpdateOrderbook(c *subscription.Subscription, p currency.Pa } if book[i].Price > 0 { - orderbookUpdate.Action = orderbook.UpdateInsert + orderbookUpdate.Action = orderbook.UpdateOrInsertAction if fundingRate { if book[i].Amount < 0 { item.Amount *= -1 @@ -1610,7 +1610,7 @@ func (b *Bitfinex) WsUpdateOrderbook(c *subscription.Subscription, p currency.Pa } } } else { - orderbookUpdate.Action = orderbook.Delete + orderbookUpdate.Action = orderbook.DeleteAction if fundingRate { if book[i].Amount == 1 { // delete bid @@ -1678,7 +1678,7 @@ func (b *Bitfinex) resubOrderbook(c *subscription.Subscription) error { if len(c.Pairs) != 1 { return subscription.ErrNotSinglePair } - if err := b.Websocket.Orderbook.FlushOrderbook(c.Pairs[0], c.Asset); err != nil { + if err := b.Websocket.Orderbook.InvalidateOrderbook(c.Pairs[0], c.Asset); err != nil { // Non-fatal error log.Errorf(log.ExchangeSys, "%s error flushing orderbook: %v", b.Name, err) } diff --git a/exchanges/bitfinex/bitfinex_wrapper.go b/exchanges/bitfinex/bitfinex_wrapper.go index 134c95ac..e1a0e3b9 100644 --- a/exchanges/bitfinex/bitfinex_wrapper.go +++ b/exchanges/bitfinex/bitfinex_wrapper.go @@ -15,7 +15,6 @@ import ( "github.com/thrasher-corp/gocryptotrader/config" "github.com/thrasher-corp/gocryptotrader/currency" "github.com/thrasher-corp/gocryptotrader/exchange/websocket" - "github.com/thrasher-corp/gocryptotrader/exchange/websocket/buffer" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/account" "github.com/thrasher-corp/gocryptotrader/exchanges/asset" @@ -202,9 +201,6 @@ func (b *Bitfinex) Setup(exch *config.Exchange) error { Unsubscriber: b.Unsubscribe, GenerateSubscriptions: b.generateSubscriptions, Features: &b.Features.Supports.WebsocketCapabilities, - OrderbookBufferConfig: buffer.Config{ - UpdateEntriesByID: true, - }, }) if err != nil { return err @@ -340,11 +336,11 @@ func (b *Bitfinex) UpdateOrderbook(ctx context.Context, p currency.Pair, assetTy return nil, err } o := &orderbook.Book{ - Exchange: b.Name, - Pair: p, - Asset: assetType, - PriceDuplication: true, - VerifyOrderbook: b.CanVerifyOrderbook, + Exchange: b.Name, + Pair: p, + Asset: assetType, + PriceDuplication: true, + ValidateOrderbook: b.ValidateOrderbook, } fPair, err := b.FormatExchangeCurrency(p, assetType) diff --git a/exchanges/bitflyer/bitflyer_wrapper.go b/exchanges/bitflyer/bitflyer_wrapper.go index 15614131..d7cbc52b 100644 --- a/exchanges/bitflyer/bitflyer_wrapper.go +++ b/exchanges/bitflyer/bitflyer_wrapper.go @@ -201,10 +201,10 @@ func (b *Bitflyer) UpdateOrderbook(ctx context.Context, p currency.Pair, assetTy return nil, err } book := &orderbook.Book{ - Exchange: b.Name, - Pair: p, - Asset: assetType, - VerifyOrderbook: b.CanVerifyOrderbook, + Exchange: b.Name, + Pair: p, + Asset: assetType, + ValidateOrderbook: b.ValidateOrderbook, } fPair, err := b.FormatExchangeCurrency(p, assetType) diff --git a/exchanges/bithumb/bithumb_wrapper.go b/exchanges/bithumb/bithumb_wrapper.go index adcdfadd..0b5ff4b7 100644 --- a/exchanges/bithumb/bithumb_wrapper.go +++ b/exchanges/bithumb/bithumb_wrapper.go @@ -264,10 +264,10 @@ func (b *Bithumb) UpdateOrderbook(ctx context.Context, p currency.Pair, assetTyp return nil, err } book := &orderbook.Book{ - Exchange: b.Name, - Pair: p, - Asset: assetType, - VerifyOrderbook: b.CanVerifyOrderbook, + Exchange: b.Name, + Pair: p, + Asset: assetType, + ValidateOrderbook: b.ValidateOrderbook, } curr := p.Base.String() diff --git a/exchanges/bithumb/bithumb_ws_orderbook.go b/exchanges/bithumb/bithumb_ws_orderbook.go index f23005ef..d4e8d762 100644 --- a/exchanges/bithumb/bithumb_ws_orderbook.go +++ b/exchanges/bithumb/bithumb_ws_orderbook.go @@ -60,7 +60,7 @@ func (b *Bithumb) UpdateLocalBuffer(wsdp *WsOrderbooks) (bool, error) { err = b.applyBufferUpdate(wsdp.List[0].Symbol) if err != nil { - b.flushAndCleanup(wsdp.List[0].Symbol) + b.invalidateAndCleanupOrderbook(wsdp.List[0].Symbol) } return false, err } @@ -145,26 +145,19 @@ func (b *Bithumb) processJob(ctx context.Context, p currency.Pair) error { // new update to initiate this. err = b.applyBufferUpdate(p) if err != nil { - b.flushAndCleanup(p) + b.invalidateAndCleanupOrderbook(p) return err } return nil } -// flushAndCleanup flushes orderbook and clean local cache -func (b *Bithumb) flushAndCleanup(p currency.Pair) { - errClean := b.Websocket.Orderbook.FlushOrderbook(p, asset.Spot) - if errClean != nil { - log.Errorf(log.WebsocketMgr, - "%s flushing websocket error: %v", - b.Name, - errClean) +// invalidateAndCleanupOrderbook invalidates orderbook and cleans local cache +func (b *Bithumb) invalidateAndCleanupOrderbook(p currency.Pair) { + if err := b.Websocket.Orderbook.InvalidateOrderbook(p, asset.Spot); err != nil { + log.Errorf(log.WebsocketMgr, "%s invalidate orderbook websocket error: %v", b.Name, err) } - errClean = b.obm.cleanup(p) - if errClean != nil { - log.Errorf(log.WebsocketMgr, "%s cleanup websocket error: %v", - b.Name, - errClean) + if err := b.obm.cleanup(p); err != nil { + log.Errorf(log.WebsocketMgr, "%s cleanup websocket error: %v", b.Name, err) } } @@ -435,7 +428,7 @@ func (b *Bithumb) SeedLocalCacheWithBook(p currency.Pair, o *Orderbook) error { newOrderBook.Asset = asset.Spot newOrderBook.Exchange = b.Name newOrderBook.LastUpdated = time.UnixMilli(o.Data.Timestamp) - newOrderBook.VerifyOrderbook = b.CanVerifyOrderbook + newOrderBook.ValidateOrderbook = b.ValidateOrderbook return b.Websocket.Orderbook.LoadSnapshot(&newOrderBook) } diff --git a/exchanges/bitmex/bitmex_test.go b/exchanges/bitmex/bitmex_test.go index f5e26052..f644500d 100644 --- a/exchanges/bitmex/bitmex_test.go +++ b/exchanges/bitmex/bitmex_test.go @@ -939,19 +939,19 @@ func TestGetActionFromString(t *testing.T) { action, err := b.GetActionFromString("update") require.NoError(t, err) - assert.Equal(t, orderbook.Amend, action) + assert.Equal(t, orderbook.UpdateAction, action) action, err = b.GetActionFromString("delete") require.NoError(t, err) - assert.Equal(t, orderbook.Delete, action) + assert.Equal(t, orderbook.DeleteAction, action) action, err = b.GetActionFromString("insert") require.NoError(t, err) - assert.Equal(t, orderbook.Insert, action) + assert.Equal(t, orderbook.InsertAction, action) action, err = b.GetActionFromString("update/insert") require.NoError(t, err) - assert.Equal(t, orderbook.UpdateInsert, action) + assert.Equal(t, orderbook.UpdateOrInsertAction, action) } func TestGetAccountFundingHistory(t *testing.T) { diff --git a/exchanges/bitmex/bitmex_websocket.go b/exchanges/bitmex/bitmex_websocket.go index 24a3a0ea..9575e927 100644 --- a/exchanges/bitmex/bitmex_websocket.go +++ b/exchanges/bitmex/bitmex_websocket.go @@ -425,7 +425,7 @@ func (b *Bitmex) processOrderbook(data []OrderBookL2, action string, p currency. book.Asset = a book.Pair = p book.Exchange = b.Name - book.VerifyOrderbook = b.CanVerifyOrderbook + book.ValidateOrderbook = b.ValidateOrderbook book.LastUpdated = data[0].Timestamp err := b.Websocket.Orderbook.LoadSnapshot(&book) @@ -607,16 +607,16 @@ func (b *Bitmex) websocketSendAuth(ctx context.Context) error { } // GetActionFromString matches a string action to an internal action. -func (b *Bitmex) GetActionFromString(s string) (orderbook.Action, error) { +func (b *Bitmex) GetActionFromString(s string) (orderbook.ActionType, error) { switch s { case "update": - return orderbook.Amend, nil + return orderbook.UpdateAction, nil case "delete": - return orderbook.Delete, nil + return orderbook.DeleteAction, nil case "insert": - return orderbook.Insert, nil + return orderbook.InsertAction, nil case "update/insert": - return orderbook.UpdateInsert, nil + return orderbook.UpdateOrInsertAction, nil } return 0, fmt.Errorf("%s %w", s, orderbook.ErrInvalidAction) } diff --git a/exchanges/bitmex/bitmex_wrapper.go b/exchanges/bitmex/bitmex_wrapper.go index 867d9d4b..752f4cc3 100644 --- a/exchanges/bitmex/bitmex_wrapper.go +++ b/exchanges/bitmex/bitmex_wrapper.go @@ -16,7 +16,6 @@ import ( "github.com/thrasher-corp/gocryptotrader/config" "github.com/thrasher-corp/gocryptotrader/currency" "github.com/thrasher-corp/gocryptotrader/exchange/websocket" - "github.com/thrasher-corp/gocryptotrader/exchange/websocket/buffer" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/account" "github.com/thrasher-corp/gocryptotrader/exchanges/asset" @@ -174,9 +173,6 @@ func (b *Bitmex) Setup(exch *config.Exchange) error { Unsubscriber: b.Unsubscribe, GenerateSubscriptions: b.generateSubscriptions, Features: &b.Features.Supports.WebsocketCapabilities, - OrderbookBufferConfig: buffer.Config{ - UpdateEntriesByID: true, - }, }) if err != nil { return err @@ -395,10 +391,10 @@ func (b *Bitmex) UpdateOrderbook(ctx context.Context, p currency.Pair, assetType return nil, err } book := &orderbook.Book{ - Exchange: b.Name, - Pair: p, - Asset: assetType, - VerifyOrderbook: b.CanVerifyOrderbook, + Exchange: b.Name, + Pair: p, + Asset: assetType, + ValidateOrderbook: b.ValidateOrderbook, } if assetType == asset.Index { diff --git a/exchanges/bitstamp/bitstamp_websocket.go b/exchanges/bitstamp/bitstamp_websocket.go index 6ca74d08..6683307a 100644 --- a/exchanges/bitstamp/bitstamp_websocket.go +++ b/exchanges/bitstamp/bitstamp_websocket.go @@ -300,13 +300,13 @@ func (b *Bitstamp) handleWSOrderbook(msg []byte) error { } obUpdate := &orderbook.Book{ - Bids: make(orderbook.Levels, len(wsOrderBookResp.Data.Bids)), - Asks: make(orderbook.Levels, len(wsOrderBookResp.Data.Asks)), - Pair: p, - LastUpdated: wsOrderBookResp.Data.Microtimestamp.Time(), - Asset: asset.Spot, - Exchange: b.Name, - VerifyOrderbook: b.CanVerifyOrderbook, + Bids: make(orderbook.Levels, len(wsOrderBookResp.Data.Bids)), + Asks: make(orderbook.Levels, len(wsOrderBookResp.Data.Asks)), + Pair: p, + LastUpdated: wsOrderBookResp.Data.Microtimestamp.Time(), + Asset: asset.Spot, + Exchange: b.Name, + ValidateOrderbook: b.ValidateOrderbook, } for i := range wsOrderBookResp.Data.Asks { @@ -338,13 +338,13 @@ func (b *Bitstamp) seedOrderBook(ctx context.Context) error { } newOrderBook := &orderbook.Book{ - Pair: p[x], - Asset: asset.Spot, - Exchange: b.Name, - VerifyOrderbook: b.CanVerifyOrderbook, - Bids: make(orderbook.Levels, len(orderbookSeed.Bids)), - Asks: make(orderbook.Levels, len(orderbookSeed.Asks)), - LastUpdated: orderbookSeed.Timestamp, + Pair: p[x], + Asset: asset.Spot, + Exchange: b.Name, + ValidateOrderbook: b.ValidateOrderbook, + Bids: make(orderbook.Levels, len(orderbookSeed.Bids)), + Asks: make(orderbook.Levels, len(orderbookSeed.Asks)), + LastUpdated: orderbookSeed.Timestamp, } for i := range orderbookSeed.Asks { diff --git a/exchanges/bitstamp/bitstamp_wrapper.go b/exchanges/bitstamp/bitstamp_wrapper.go index 83e9f686..3418acbf 100644 --- a/exchanges/bitstamp/bitstamp_wrapper.go +++ b/exchanges/bitstamp/bitstamp_wrapper.go @@ -296,10 +296,10 @@ func (b *Bitstamp) UpdateOrderbook(ctx context.Context, p currency.Pair, assetTy return nil, err } book := &orderbook.Book{ - Exchange: b.Name, - Pair: p, - Asset: assetType, - VerifyOrderbook: b.CanVerifyOrderbook, + Exchange: b.Name, + Pair: p, + Asset: assetType, + ValidateOrderbook: b.ValidateOrderbook, } fPair, err := b.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/btcmarkets/btcmarkets_test.go b/exchanges/btcmarkets/btcmarkets_test.go index 08b9b992..ea7ff11e 100644 --- a/exchanges/btcmarkets/btcmarkets_test.go +++ b/exchanges/btcmarkets/btcmarkets_test.go @@ -772,26 +772,19 @@ func TestGetHistoricTrades(t *testing.T) { assert.ErrorIs(t, err, common.ErrFunctionNotSupported) } -func TestChecksum(t *testing.T) { +func TestOrderbookChecksum(t *testing.T) { b := &orderbook.Book{ - Asks: []orderbook.Level{ + Asks: orderbook.Levels{ {Price: 0.3965, Amount: 44149.815}, {Price: 0.3967, Amount: 16000.0}, }, - Bids: []orderbook.Level{ + Bids: orderbook.Levels{ {Price: 0.396, Amount: 51.0}, {Price: 0.396, Amount: 25.0}, {Price: 0.3958, Amount: 18570.0}, }, } - - expecting := uint32(3802968298) - err := checksum(b, expecting) - if err != nil { - t.Fatal(err) - } - err = checksum(b, uint32(1223123)) - assert.ErrorIs(t, err, errChecksumFailure) + require.Equal(t, uint32(3802968298), orderbookChecksum(b)) } func TestTrim(t *testing.T) { diff --git a/exchanges/btcmarkets/btcmarkets_websocket.go b/exchanges/btcmarkets/btcmarkets_websocket.go index 5d75c1bd..37363048 100644 --- a/exchanges/btcmarkets/btcmarkets_websocket.go +++ b/exchanges/btcmarkets/btcmarkets_websocket.go @@ -29,11 +29,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/types" ) -const ( - btcMarketsWSURL = "wss://socket.btcmarkets.net/v2" -) - -var errChecksumFailure = errors.New("crc32 checksum failure") +const btcMarketsWSURL = "wss://socket.btcmarkets.net/v2" var defaultSubscriptions = subscription.List{ {Enabled: true, Asset: asset.Spot, Channel: subscription.TickerChannel}, @@ -125,24 +121,26 @@ func (b *BTCMarkets) wsHandleData(ctx context.Context, respRaw []byte) error { if ob.Snapshot { err = b.Websocket.Orderbook.LoadSnapshot(&orderbook.Book{ - Pair: ob.Currency, - Bids: orderbook.Levels(ob.Bids), - Asks: orderbook.Levels(ob.Asks), - LastUpdated: ob.Timestamp, - LastUpdateID: ob.SnapshotID, - Asset: asset.Spot, - Exchange: b.Name, - VerifyOrderbook: b.CanVerifyOrderbook, + Pair: ob.Currency, + Bids: orderbook.Levels(ob.Bids), + Asks: orderbook.Levels(ob.Asks), + LastUpdated: ob.Timestamp, + LastUpdateID: ob.SnapshotID, + Asset: asset.Spot, + Exchange: b.Name, + ValidateOrderbook: b.ValidateOrderbook, }) } else { err = b.Websocket.Orderbook.Update(&orderbook.Update{ - UpdateTime: ob.Timestamp, - UpdateID: ob.SnapshotID, - Asset: asset.Spot, - Bids: orderbook.Levels(ob.Bids), - Asks: orderbook.Levels(ob.Asks), - Pair: ob.Currency, - Checksum: ob.Checksum, + UpdateTime: ob.Timestamp, + UpdateID: ob.SnapshotID, + Asset: asset.Spot, + Bids: orderbook.Levels(ob.Bids), + Asks: orderbook.Levels(ob.Asks), + Pair: ob.Currency, + ExpectedChecksum: ob.Checksum, + GenerateChecksum: orderbookChecksum, + SkipOutOfOrderLastUpdateID: true, }) } if err != nil { @@ -420,20 +418,9 @@ func (b *BTCMarkets) ReSubscribeSpecificOrderbook(pair currency.Pair) error { return b.Subscribe(sub) } -// checksum provides assurance on current in memory liquidity -func checksum(ob *orderbook.Book, checksum uint32) error { - check := crc32.ChecksumIEEE([]byte(concatOrderbookLiquidity(ob.Bids) + concatOrderbookLiquidity(ob.Asks))) - if check != checksum { - return fmt.Errorf("%s %s %s ID: %v expected: %v but received: %v %w", - ob.Exchange, - ob.Pair, - ob.Asset, - ob.LastUpdateID, - checksum, - check, - errChecksumFailure) - } - return nil +// orderbookChecksum calculates a checksum for the orderbook liquidity +func orderbookChecksum(ob *orderbook.Book) uint32 { + return crc32.ChecksumIEEE([]byte(concatOrderbookLiquidity(ob.Bids) + concatOrderbookLiquidity(ob.Asks))) } // concatOrderbookLiquidity concatenates price and amounts together for checksum processing diff --git a/exchanges/btcmarkets/btcmarkets_wrapper.go b/exchanges/btcmarkets/btcmarkets_wrapper.go index cf9a7445..a7dd46d7 100644 --- a/exchanges/btcmarkets/btcmarkets_wrapper.go +++ b/exchanges/btcmarkets/btcmarkets_wrapper.go @@ -161,9 +161,7 @@ func (b *BTCMarkets) Setup(exch *config.Exchange) error { GenerateSubscriptions: b.generateSubscriptions, Features: &b.Features.Supports.WebsocketCapabilities, OrderbookBufferConfig: buffer.Config{ - SortBuffer: true, - UpdateIDProgression: true, - Checksum: checksum, + SortBuffer: true, }, }) if err != nil { @@ -259,11 +257,11 @@ func (b *BTCMarkets) UpdateOrderbook(ctx context.Context, p currency.Pair, asset } book := &orderbook.Book{ - Exchange: b.Name, - Pair: p, - Asset: assetType, - PriceDuplication: true, - VerifyOrderbook: b.CanVerifyOrderbook, + Exchange: b.Name, + Pair: p, + Asset: assetType, + PriceDuplication: true, + ValidateOrderbook: b.ValidateOrderbook, } fPair, err := b.FormatExchangeCurrency(p, assetType) diff --git a/exchanges/btse/btse_websocket.go b/exchanges/btse/btse_websocket.go index 515673a6..a8c03039 100644 --- a/exchanges/btse/btse_websocket.go +++ b/exchanges/btse/btse_websocket.go @@ -343,7 +343,7 @@ func (b *BTSE) wsHandleData(_ context.Context, respRaw []byte) error { newOB.Asset = a newOB.Exchange = b.Name newOB.Asks.Reverse() // Reverse asks for correct alignment - newOB.VerifyOrderbook = b.CanVerifyOrderbook + newOB.ValidateOrderbook = b.ValidateOrderbook newOB.LastUpdated = time.Now() // NOTE: Temp to fix test. err = b.Websocket.Orderbook.LoadSnapshot(&newOB) if err != nil { diff --git a/exchanges/btse/btse_wrapper.go b/exchanges/btse/btse_wrapper.go index f2fde559..ee9171ce 100644 --- a/exchanges/btse/btse_wrapper.go +++ b/exchanges/btse/btse_wrapper.go @@ -304,10 +304,10 @@ func (b *BTSE) UpdateOrderbook(ctx context.Context, p currency.Pair, assetType a return nil, err } book := &orderbook.Book{ - Exchange: b.Name, - Pair: p, - Asset: assetType, - VerifyOrderbook: b.CanVerifyOrderbook, + Exchange: b.Name, + Pair: p, + Asset: assetType, + ValidateOrderbook: b.ValidateOrderbook, } fPair, err := b.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/bybit/bybit_wrapper.go b/exchanges/bybit/bybit_wrapper.go index 6389a185..7f572d67 100644 --- a/exchanges/bybit/bybit_wrapper.go +++ b/exchanges/bybit/bybit_wrapper.go @@ -507,12 +507,12 @@ func (by *Bybit) UpdateOrderbook(ctx context.Context, p currency.Pair, assetType return nil, err } book := &orderbook.Book{ - Exchange: by.Name, - Pair: p, - Asset: assetType, - VerifyOrderbook: by.CanVerifyOrderbook, - Bids: make([]orderbook.Level, len(orderbookNew.Bids)), - Asks: make([]orderbook.Level, len(orderbookNew.Asks)), + Exchange: by.Name, + Pair: p, + Asset: assetType, + ValidateOrderbook: by.ValidateOrderbook, + Bids: make([]orderbook.Level, len(orderbookNew.Bids)), + Asks: make([]orderbook.Level, len(orderbookNew.Asks)), } for x := range orderbookNew.Bids { book.Bids[x] = orderbook.Level{ diff --git a/exchanges/coinbasepro/coinbasepro_websocket.go b/exchanges/coinbasepro/coinbasepro_websocket.go index 2e0af901..2b92db7d 100644 --- a/exchanges/coinbasepro/coinbasepro_websocket.go +++ b/exchanges/coinbasepro/coinbasepro_websocket.go @@ -291,13 +291,13 @@ func (c *CoinbasePro) ProcessSnapshot(snapshot *WebsocketOrderbookSnapshot) erro } ob := &orderbook.Book{ - Pair: pair, - Bids: make(orderbook.Levels, len(snapshot.Bids)), - Asks: make(orderbook.Levels, len(snapshot.Asks)), - Asset: asset.Spot, - Exchange: c.Name, - VerifyOrderbook: c.CanVerifyOrderbook, - LastUpdated: snapshot.Time, + Pair: pair, + Bids: make(orderbook.Levels, len(snapshot.Bids)), + Asks: make(orderbook.Levels, len(snapshot.Asks)), + Asset: asset.Spot, + Exchange: c.Name, + ValidateOrderbook: c.ValidateOrderbook, + LastUpdated: snapshot.Time, } for i := range snapshot.Bids { diff --git a/exchanges/coinbasepro/coinbasepro_wrapper.go b/exchanges/coinbasepro/coinbasepro_wrapper.go index 939540dc..eeb3c996 100644 --- a/exchanges/coinbasepro/coinbasepro_wrapper.go +++ b/exchanges/coinbasepro/coinbasepro_wrapper.go @@ -308,10 +308,10 @@ func (c *CoinbasePro) UpdateOrderbook(ctx context.Context, p currency.Pair, asse return nil, err } book := &orderbook.Book{ - Exchange: c.Name, - Pair: p, - Asset: assetType, - VerifyOrderbook: c.CanVerifyOrderbook, + Exchange: c.Name, + Pair: p, + Asset: assetType, + ValidateOrderbook: c.ValidateOrderbook, } fPair, err := c.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/coinut/coinut_websocket.go b/exchanges/coinut/coinut_websocket.go index 2a9eb649..21dc0186 100644 --- a/exchanges/coinut/coinut_websocket.go +++ b/exchanges/coinut/coinut_websocket.go @@ -517,7 +517,7 @@ func (c *COINUT) WsProcessOrderbookSnapshot(ob *WsOrderbookSnapshot) error { var newOrderBook orderbook.Book newOrderBook.Asks = asks newOrderBook.Bids = bids - newOrderBook.VerifyOrderbook = c.CanVerifyOrderbook + newOrderBook.ValidateOrderbook = c.ValidateOrderbook pairs, err := c.GetEnabledPairs(asset.Spot) if err != nil { diff --git a/exchanges/coinut/coinut_wrapper.go b/exchanges/coinut/coinut_wrapper.go index ee85909d..5306efc4 100644 --- a/exchanges/coinut/coinut_wrapper.go +++ b/exchanges/coinut/coinut_wrapper.go @@ -355,10 +355,10 @@ func (c *COINUT) UpdateOrderbook(ctx context.Context, p currency.Pair, assetType return nil, err } book := &orderbook.Book{ - Exchange: c.Name, - Pair: p, - Asset: assetType, - VerifyOrderbook: c.CanVerifyOrderbook, + Exchange: c.Name, + Pair: p, + Asset: assetType, + ValidateOrderbook: c.ValidateOrderbook, } err := c.loadInstrumentsIfNotLoaded(ctx) if err != nil { diff --git a/exchanges/deribit/deribit_websocket.go b/exchanges/deribit/deribit_websocket.go index 0f38bd04..70730f0a 100644 --- a/exchanges/deribit/deribit_websocket.go +++ b/exchanges/deribit/deribit_websocket.go @@ -698,14 +698,14 @@ func (d *Deribit) processOrderbook(respRaw []byte, channels []string) error { switch orderbookData.Type { case "snapshot": return d.Websocket.Orderbook.LoadSnapshot(&orderbook.Book{ - Exchange: d.Name, - VerifyOrderbook: d.CanVerifyOrderbook, - LastUpdated: orderbookData.Timestamp.Time(), - Pair: cp, - Asks: asks, - Bids: bids, - Asset: a, - LastUpdateID: orderbookData.ChangeID, + Exchange: d.Name, + ValidateOrderbook: d.ValidateOrderbook, + LastUpdated: orderbookData.Timestamp.Time(), + Pair: cp, + Asks: asks, + Bids: bids, + Asset: a, + LastUpdateID: orderbookData.ChangeID, }) case "change": return d.Websocket.Orderbook.Update(&orderbook.Update{ diff --git a/exchanges/deribit/deribit_wrapper.go b/exchanges/deribit/deribit_wrapper.go index d0c01a3e..874aebae 100644 --- a/exchanges/deribit/deribit_wrapper.go +++ b/exchanges/deribit/deribit_wrapper.go @@ -304,10 +304,10 @@ func (d *Deribit) UpdateOrderbook(ctx context.Context, p currency.Pair, assetTyp return nil, err } book := &orderbook.Book{ - Exchange: d.Name, - Pair: p, - Asset: assetType, - VerifyOrderbook: d.CanVerifyOrderbook, + Exchange: d.Name, + Pair: p, + Asset: assetType, + ValidateOrderbook: d.ValidateOrderbook, } book.Asks = make(orderbook.Levels, 0, len(obData.Asks)) for x := range obData.Asks { diff --git a/exchanges/exchange.go b/exchanges/exchange.go index 396c9448..c702e4d0 100644 --- a/exchanges/exchange.go +++ b/exchanges/exchange.go @@ -597,7 +597,7 @@ func (b *Base) SetupDefaults(exch *config.Exchange) error { log.Warnf(log.ExchangeSys, "%s orderbook verification has been bypassed via config.", b.Name) } - b.CanVerifyOrderbook = !exch.Orderbook.VerificationBypass + b.ValidateOrderbook = !exch.Orderbook.VerificationBypass b.States = currencystate.NewCurrencyStates() return nil diff --git a/exchanges/exchange_types.go b/exchanges/exchange_types.go index 011382cb..ae1a2971 100644 --- a/exchanges/exchange_types.go +++ b/exchanges/exchange_types.go @@ -250,10 +250,10 @@ type Base struct { *request.Requester Config *config.Exchange settingsMutex sync.RWMutex - // CanVerifyOrderbook determines if the orderbook verification can be bypassed, + // ValidateOrderbook determines if the orderbook verification can be bypassed, // increasing potential update speed but decreasing confidence in orderbook // integrity. - CanVerifyOrderbook bool + ValidateOrderbook bool order.ExecutionLimits AssetWebsocketSupport diff --git a/exchanges/exmo/exmo_wrapper.go b/exchanges/exmo/exmo_wrapper.go index cd667fa8..022132cf 100644 --- a/exchanges/exmo/exmo_wrapper.go +++ b/exchanges/exmo/exmo_wrapper.go @@ -211,10 +211,10 @@ func (e *EXMO) UpdateOrderbook(ctx context.Context, p currency.Pair, assetType a return nil, err } callingBook := &orderbook.Book{ - Exchange: e.Name, - Pair: p, - Asset: assetType, - VerifyOrderbook: e.CanVerifyOrderbook, + Exchange: e.Name, + Pair: p, + Asset: assetType, + ValidateOrderbook: e.ValidateOrderbook, } enabledPairs, err := e.GetEnabledPairs(assetType) if err != nil { @@ -233,10 +233,10 @@ func (e *EXMO) UpdateOrderbook(ctx context.Context, p currency.Pair, assetType a for i := range enabledPairs { book := &orderbook.Book{ - Exchange: e.Name, - Pair: enabledPairs[i], - Asset: assetType, - VerifyOrderbook: e.CanVerifyOrderbook, + Exchange: e.Name, + Pair: enabledPairs[i], + Asset: assetType, + ValidateOrderbook: e.ValidateOrderbook, } curr, err := e.FormatExchangeCurrency(enabledPairs[i], assetType) diff --git a/exchanges/gateio/gateio_websocket_futures.go b/exchanges/gateio/gateio_websocket_futures.go index 67a539e8..6fa029b3 100644 --- a/exchanges/gateio/gateio_websocket_futures.go +++ b/exchanges/gateio/gateio_websocket_futures.go @@ -442,12 +442,12 @@ func (g *Gateio) processFuturesOrderbookSnapshot(event string, incoming []byte, return err } base := orderbook.Book{ - Asset: assetType, - Exchange: g.Name, - Pair: data.Contract, - LastUpdated: data.Timestamp.Time(), - LastPushed: lastPushed, - VerifyOrderbook: g.CanVerifyOrderbook, + Asset: assetType, + Exchange: g.Name, + Pair: data.Contract, + LastUpdated: data.Timestamp.Time(), + LastPushed: lastPushed, + ValidateOrderbook: g.ValidateOrderbook, } base.Asks = make([]orderbook.Level, len(data.Asks)) for x := range data.Asks { @@ -496,14 +496,14 @@ func (g *Gateio) processFuturesOrderbookSnapshot(event string, incoming []byte, return err } err = g.Websocket.Orderbook.LoadSnapshot(&orderbook.Book{ - Asks: ab[0], - Bids: ab[1], - Asset: assetType, - Exchange: g.Name, - Pair: currencyPair, - LastUpdated: lastPushed, - LastPushed: lastPushed, - VerifyOrderbook: g.CanVerifyOrderbook, + Asks: ab[0], + Bids: ab[1], + Asset: assetType, + Exchange: g.Name, + Pair: currencyPair, + LastUpdated: lastPushed, + LastPushed: lastPushed, + ValidateOrderbook: g.ValidateOrderbook, }) if err != nil { return err diff --git a/exchanges/gateio/gateio_websocket_option.go b/exchanges/gateio/gateio_websocket_option.go index 2b219a07..e3c29f93 100644 --- a/exchanges/gateio/gateio_websocket_option.go +++ b/exchanges/gateio/gateio_websocket_option.go @@ -534,12 +534,12 @@ func (g *Gateio) processOptionsOrderbookSnapshotPushData(event string, incoming return err } base := orderbook.Book{ - Asset: asset.Options, - Exchange: g.Name, - Pair: data.Contract, - LastUpdated: data.Timestamp.Time(), - LastPushed: lastPushed, - VerifyOrderbook: g.CanVerifyOrderbook, + Asset: asset.Options, + Exchange: g.Name, + Pair: data.Contract, + LastUpdated: data.Timestamp.Time(), + LastPushed: lastPushed, + ValidateOrderbook: g.ValidateOrderbook, } base.Asks = make([]orderbook.Level, len(data.Asks)) for x := range data.Asks { @@ -586,14 +586,14 @@ func (g *Gateio) processOptionsOrderbookSnapshotPushData(event string, incoming return err } err = g.Websocket.Orderbook.LoadSnapshot(&orderbook.Book{ - Asks: ab[0], - Bids: ab[1], - Asset: asset.Options, - Exchange: g.Name, - Pair: currencyPair, - LastUpdated: lastPushed, - LastPushed: lastPushed, - VerifyOrderbook: g.CanVerifyOrderbook, + Asks: ab[0], + Bids: ab[1], + Asset: asset.Options, + Exchange: g.Name, + Pair: currencyPair, + LastUpdated: lastPushed, + LastPushed: lastPushed, + ValidateOrderbook: g.ValidateOrderbook, }) if err != nil { return err diff --git a/exchanges/gateio/gateio_wrapper.go b/exchanges/gateio/gateio_wrapper.go index 5ad159df..c14e3009 100644 --- a/exchanges/gateio/gateio_wrapper.go +++ b/exchanges/gateio/gateio_wrapper.go @@ -689,13 +689,13 @@ func (g *Gateio) UpdateOrderbookWithLimit(ctx context.Context, p currency.Pair, return nil, err } book := &orderbook.Book{ - Exchange: g.Name, - Asset: a, - VerifyOrderbook: g.CanVerifyOrderbook, - Pair: p.Upper(), - LastUpdateID: o.ID, - LastUpdated: o.Update.Time(), - LastPushed: o.Current.Time(), + Exchange: g.Name, + Asset: a, + ValidateOrderbook: g.ValidateOrderbook, + Pair: p.Upper(), + LastUpdateID: o.ID, + LastUpdated: o.Update.Time(), + LastPushed: o.Current.Time(), } book.Bids = make(orderbook.Levels, len(o.Bids)) for x := range o.Bids { diff --git a/exchanges/gateio/ws_ob_update_manager.go b/exchanges/gateio/ws_ob_update_manager.go index 9fd8c725..b6ee2ab6 100644 --- a/exchanges/gateio/ws_ob_update_manager.go +++ b/exchanges/gateio/ws_ob_update_manager.go @@ -9,7 +9,6 @@ import ( "github.com/thrasher-corp/gocryptotrader/common/key" "github.com/thrasher-corp/gocryptotrader/currency" - "github.com/thrasher-corp/gocryptotrader/exchange/websocket/buffer" "github.com/thrasher-corp/gocryptotrader/exchanges/asset" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/subscription" @@ -52,7 +51,7 @@ func (m *wsOBUpdateManager) ProcessOrderbookUpdate(ctx context.Context, g *Gatei } lastUpdateID, err := g.Websocket.Orderbook.LastUpdateID(update.Pair, update.Asset) - if err != nil && !errors.Is(err, buffer.ErrDepthNotFound) { + if err != nil && !errors.Is(err, orderbook.ErrDepthNotFound) { return err } @@ -60,8 +59,8 @@ func (m *wsOBUpdateManager) ProcessOrderbookUpdate(ctx context.Context, g *Gatei return applyOrderbookUpdate(g, update) } - // Orderbook is behind notifications, flush store to prevent trading on stale data - if err := g.Websocket.Orderbook.FlushOrderbook(update.Pair, update.Asset); err != nil && !errors.Is(err, buffer.ErrDepthNotFound) { + // Orderbook is behind notifications, therefore Invalidate store + if err := g.Websocket.Orderbook.InvalidateOrderbook(update.Pair, update.Asset); err != nil && !errors.Is(err, orderbook.ErrDepthNotFound) { return err } diff --git a/exchanges/gateio/ws_ob_update_manager_test.go b/exchanges/gateio/ws_ob_update_manager_test.go index 0464188e..b5f401af 100644 --- a/exchanges/gateio/ws_ob_update_manager_test.go +++ b/exchanges/gateio/ws_ob_update_manager_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/thrasher-corp/gocryptotrader/currency" - "github.com/thrasher-corp/gocryptotrader/exchange/websocket/buffer" "github.com/thrasher-corp/gocryptotrader/exchanges/asset" "github.com/thrasher-corp/gocryptotrader/exchanges/kline" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" @@ -193,11 +192,11 @@ func TestApplyOrderbookUpdate(t *testing.T) { } err := applyOrderbookUpdate(g, update) - require.ErrorIs(t, err, buffer.ErrDepthNotFound) + require.ErrorIs(t, err, orderbook.ErrDepthNotFound) update.Asset = asset.Spot err = applyOrderbookUpdate(g, update) - require.ErrorIs(t, err, buffer.ErrDepthNotFound) + require.ErrorIs(t, err, orderbook.ErrDepthNotFound) update.Pair = currency.NewPair(currency.BABY, currency.BABYDOGE) err = applyOrderbookUpdate(g, update) diff --git a/exchanges/gemini/gemini_websocket.go b/exchanges/gemini/gemini_websocket.go index 76803bd7..037df166 100644 --- a/exchanges/gemini/gemini_websocket.go +++ b/exchanges/gemini/gemini_websocket.go @@ -515,7 +515,7 @@ func (g *Gemini) wsProcessUpdate(result *wsL2MarketData) error { newOrderBook.Asset = asset.Spot newOrderBook.Pair = pair newOrderBook.Exchange = g.Name - newOrderBook.VerifyOrderbook = g.CanVerifyOrderbook + newOrderBook.ValidateOrderbook = g.ValidateOrderbook newOrderBook.LastUpdated = time.Now() // No time is sent err := g.Websocket.Orderbook.LoadSnapshot(&newOrderBook) if err != nil { diff --git a/exchanges/gemini/gemini_wrapper.go b/exchanges/gemini/gemini_wrapper.go index 5461b106..cbd3ccd6 100644 --- a/exchanges/gemini/gemini_wrapper.go +++ b/exchanges/gemini/gemini_wrapper.go @@ -295,10 +295,10 @@ func (g *Gemini) UpdateOrderbook(ctx context.Context, p currency.Pair, assetType return nil, err } book := &orderbook.Book{ - Exchange: g.Name, - Pair: p, - Asset: assetType, - VerifyOrderbook: g.CanVerifyOrderbook, + Exchange: g.Name, + Pair: p, + Asset: assetType, + ValidateOrderbook: g.ValidateOrderbook, } fPair, err := g.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/hitbtc/hitbtc_websocket.go b/exchanges/hitbtc/hitbtc_websocket.go index ee2ca1f7..95d2e665 100644 --- a/exchanges/hitbtc/hitbtc_websocket.go +++ b/exchanges/hitbtc/hitbtc_websocket.go @@ -355,7 +355,7 @@ func (h *HitBTC) WsProcessOrderbookSnapshot(ob *WsOrderbook) error { newOrderBook.Asset = asset.Spot newOrderBook.Pair = p newOrderBook.Exchange = h.Name - newOrderBook.VerifyOrderbook = h.CanVerifyOrderbook + newOrderBook.ValidateOrderbook = h.ValidateOrderbook newOrderBook.LastUpdated = ob.Params.Timestamp return h.Websocket.Orderbook.LoadSnapshot(&newOrderBook) diff --git a/exchanges/hitbtc/hitbtc_wrapper.go b/exchanges/hitbtc/hitbtc_wrapper.go index cfe5ce2b..6da0bb9e 100644 --- a/exchanges/hitbtc/hitbtc_wrapper.go +++ b/exchanges/hitbtc/hitbtc_wrapper.go @@ -269,10 +269,10 @@ func (h *HitBTC) UpdateOrderbook(ctx context.Context, c currency.Pair, assetType return nil, err } book := &orderbook.Book{ - Exchange: h.Name, - Pair: c, - Asset: assetType, - VerifyOrderbook: h.CanVerifyOrderbook, + Exchange: h.Name, + Pair: c, + Asset: assetType, + ValidateOrderbook: h.ValidateOrderbook, } fPair, err := h.FormatExchangeCurrency(c, assetType) if err != nil { diff --git a/exchanges/huobi/huobi_websocket.go b/exchanges/huobi/huobi_websocket.go index 21b3b348..cf7f23dc 100644 --- a/exchanges/huobi/huobi_websocket.go +++ b/exchanges/huobi/huobi_websocket.go @@ -334,7 +334,7 @@ func (h *HUOBI) wsHandleOrderbookMsg(s *subscription.Subscription, respRaw []byt newOrderBook.Pair = s.Pairs[0] newOrderBook.Asset = asset.Spot newOrderBook.Exchange = h.Name - newOrderBook.VerifyOrderbook = h.CanVerifyOrderbook + newOrderBook.ValidateOrderbook = h.ValidateOrderbook newOrderBook.LastUpdated = update.Timestamp.Time() return h.Websocket.Orderbook.LoadSnapshot(&newOrderBook) diff --git a/exchanges/huobi/huobi_wrapper.go b/exchanges/huobi/huobi_wrapper.go index ced06e1e..e520ea8d 100644 --- a/exchanges/huobi/huobi_wrapper.go +++ b/exchanges/huobi/huobi_wrapper.go @@ -555,10 +555,10 @@ func (h *HUOBI) UpdateOrderbook(ctx context.Context, p currency.Pair, assetType return nil, fmt.Errorf("%w %v", asset.ErrNotSupported, assetType) } book := &orderbook.Book{ - Exchange: h.Name, - Pair: p, - Asset: assetType, - VerifyOrderbook: h.CanVerifyOrderbook, + Exchange: h.Name, + Pair: p, + Asset: assetType, + ValidateOrderbook: h.ValidateOrderbook, } var err error switch assetType { diff --git a/exchanges/kraken/kraken_websocket.go b/exchanges/kraken/kraken_websocket.go index 9da5ffff..fd38b718 100644 --- a/exchanges/kraken/kraken_websocket.go +++ b/exchanges/kraken/kraken_websocket.go @@ -670,7 +670,7 @@ func (k *Kraken) wsProcessOrderBookPartial(pair currency.Pair, askData, bidData base := orderbook.Book{ Pair: pair, Asset: asset.Spot, - VerifyOrderbook: k.CanVerifyOrderbook, + ValidateOrderbook: k.ValidateOrderbook, Bids: make(orderbook.Levels, len(bidData)), Asks: make(orderbook.Levels, len(askData)), MaxDepth: levels, diff --git a/exchanges/kraken/kraken_wrapper.go b/exchanges/kraken/kraken_wrapper.go index 39184671..b8624381 100644 --- a/exchanges/kraken/kraken_wrapper.go +++ b/exchanges/kraken/kraken_wrapper.go @@ -459,10 +459,10 @@ func (k *Kraken) UpdateOrderbook(ctx context.Context, p currency.Pair, assetType return nil, err } book := &orderbook.Book{ - Exchange: k.Name, - Pair: p, - Asset: assetType, - VerifyOrderbook: k.CanVerifyOrderbook, + Exchange: k.Name, + Pair: p, + Asset: assetType, + ValidateOrderbook: k.ValidateOrderbook, } var err error switch assetType { @@ -506,6 +506,7 @@ func (k *Kraken) UpdateOrderbook(ctx context.Context, p currency.Pair, assetType Amount: futuresOB.Orderbook.Bids[y][1], } } + book.Bids.Reverse() default: return book, fmt.Errorf("%w %v", asset.ErrNotSupported, assetType) } diff --git a/exchanges/kucoin/kucoin_websocket.go b/exchanges/kucoin/kucoin_websocket.go index 1f711b8b..c5a25048 100644 --- a/exchanges/kucoin/kucoin_websocket.go +++ b/exchanges/kucoin/kucoin_websocket.go @@ -546,12 +546,13 @@ func (ku *Kucoin) processFuturesOrderbookSnapshot(respData []byte, instrument st return err } return ku.Websocket.Orderbook.Update(&orderbook.Update{ - UpdateID: resp.Sequence, - UpdateTime: resp.Timestamp.Time(), - Asset: asset.Futures, - Bids: resp.Bids, - Asks: resp.Asks, - Pair: cp, + UpdateID: resp.Sequence, + UpdateTime: resp.Timestamp.Time(), + Asset: asset.Futures, + Bids: resp.Bids, + Asks: resp.Asks, + Pair: cp, + SkipOutOfOrderLastUpdateID: true, }) } @@ -922,9 +923,7 @@ func (ku *Kucoin) updateLocalBuffer(wsdp *WsOrderbook, assetType asset.Item) (bo return false, err } - currencyPair, err := currency.NewPairFromFormattedPairs(wsdp.Symbol, - enabledPairs, - format) + currencyPair, err := currency.NewPairFromFormattedPairs(wsdp.Symbol, enabledPairs, format) if err != nil { return false, err } @@ -939,7 +938,7 @@ func (ku *Kucoin) updateLocalBuffer(wsdp *WsOrderbook, assetType asset.Item) (bo err = ku.applyBufferUpdate(currencyPair, assetType) if err != nil { - ku.FlushAndCleanup(currencyPair, assetType) + ku.invalidateAndCleanupOrderbook(currencyPair, assetType) } return false, err @@ -1182,12 +1181,13 @@ func (ku *Kucoin) processOrderbookUpdate(cp currency.Pair, a asset.Item, ws *WsO } return ku.Websocket.Orderbook.Update(&orderbook.Update{ - Bids: updateBid, - Asks: updateAsk, - Pair: cp, - UpdateID: ws.SequenceEnd, - UpdateTime: ws.TimeMS.Time(), - Asset: a, + Bids: updateBid, + Asks: updateAsk, + Pair: cp, + UpdateID: ws.SequenceEnd, + UpdateTime: ws.TimeMS.Time(), + Asset: a, + SkipOutOfOrderLastUpdateID: true, }) } @@ -1294,14 +1294,14 @@ func (ku *Kucoin) SeedLocalCache(ctx context.Context, p currency.Pair, assetType // SeedLocalCacheWithBook seeds the local orderbook cache func (ku *Kucoin) SeedLocalCacheWithBook(p currency.Pair, orderbookNew *Orderbook, assetType asset.Item) error { newOrderBook := orderbook.Book{ - Pair: p, - Asset: assetType, - Exchange: ku.Name, - LastUpdated: time.Now(), - LastUpdateID: orderbookNew.Sequence, - VerifyOrderbook: ku.CanVerifyOrderbook, - Bids: make(orderbook.Levels, len(orderbookNew.Bids)), - Asks: make(orderbook.Levels, len(orderbookNew.Asks)), + Pair: p, + Asset: assetType, + Exchange: ku.Name, + LastUpdated: time.Now(), + LastUpdateID: orderbookNew.Sequence, + ValidateOrderbook: ku.ValidateOrderbook, + Bids: make(orderbook.Levels, len(orderbookNew.Bids)), + Asks: make(orderbook.Levels, len(orderbookNew.Asks)), } for i := range orderbookNew.Bids { newOrderBook.Bids[i] = orderbook.Level{ @@ -1339,26 +1339,19 @@ func (ku *Kucoin) processJob(ctx context.Context, p currency.Pair, assetType ass // new update to initiate this. err = ku.applyBufferUpdate(p, assetType) if err != nil { - ku.FlushAndCleanup(p, assetType) + ku.invalidateAndCleanupOrderbook(p, assetType) return err } return nil } -// FlushAndCleanup flushes orderbook and clean local cache -func (ku *Kucoin) FlushAndCleanup(p currency.Pair, assetType asset.Item) { - errClean := ku.Websocket.Orderbook.FlushOrderbook(p, assetType) - if errClean != nil { - log.Errorf(log.WebsocketMgr, - "%s flushing websocket error: %v", - ku.Name, - errClean) +// invalidateAndCleanupOrderbook invalidates orderbook and cleans local cache +func (ku *Kucoin) invalidateAndCleanupOrderbook(p currency.Pair, assetType asset.Item) { + if err := ku.Websocket.Orderbook.InvalidateOrderbook(p, assetType); err != nil { + log.Errorf(log.WebsocketMgr, "%s invalidate websocket error: %v", ku.Name, err) } - errClean = ku.obm.Cleanup(p, assetType) - if errClean != nil { - log.Errorf(log.WebsocketMgr, "%s cleanup websocket error: %v", - ku.Name, - errClean) + if err := ku.obm.Cleanup(p, assetType); err != nil { + log.Errorf(log.WebsocketMgr, "%s cleanup websocket error: %v", ku.Name, err) } } diff --git a/exchanges/kucoin/kucoin_wrapper.go b/exchanges/kucoin/kucoin_wrapper.go index ea487f8a..bb95ea63 100644 --- a/exchanges/kucoin/kucoin_wrapper.go +++ b/exchanges/kucoin/kucoin_wrapper.go @@ -204,7 +204,6 @@ func (ku *Kucoin) Setup(exch *config.Exchange) error { OrderbookBufferConfig: buffer.Config{ SortBuffer: true, SortBufferByUpdateIDs: true, - UpdateIDProgression: true, }, TradeFeed: ku.Features.Enabled.TradeFeed, }) @@ -395,12 +394,12 @@ func (ku *Kucoin) UpdateOrderbook(ctx context.Context, pair currency.Pair, asset } book := &orderbook.Book{ - Exchange: ku.Name, - Pair: pair, - Asset: assetType, - VerifyOrderbook: ku.CanVerifyOrderbook, - Asks: ordBook.Asks, - Bids: ordBook.Bids, + Exchange: ku.Name, + Pair: pair, + Asset: assetType, + ValidateOrderbook: ku.ValidateOrderbook, + Asks: ordBook.Asks, + Bids: ordBook.Bids, } err = book.Process() if err != nil { diff --git a/exchanges/lbank/lbank_wrapper.go b/exchanges/lbank/lbank_wrapper.go index 7dfdbb95..6a79a223 100644 --- a/exchanges/lbank/lbank_wrapper.go +++ b/exchanges/lbank/lbank_wrapper.go @@ -212,12 +212,12 @@ func (l *Lbank) UpdateOrderbook(ctx context.Context, p currency.Pair, assetType } book := &orderbook.Book{ - Exchange: l.Name, - Pair: p, - Asset: assetType, - VerifyOrderbook: l.CanVerifyOrderbook, - Asks: make(orderbook.Levels, len(d.Data.Asks)), - Bids: make(orderbook.Levels, len(d.Data.Bids)), + Exchange: l.Name, + Pair: p, + Asset: assetType, + ValidateOrderbook: l.ValidateOrderbook, + Asks: make(orderbook.Levels, len(d.Data.Asks)), + Bids: make(orderbook.Levels, len(d.Data.Bids)), } for i := range d.Data.Asks { diff --git a/exchanges/okx/okx_test.go b/exchanges/okx/okx_test.go index b9e3e912..934e87ae 100644 --- a/exchanges/okx/okx_test.go +++ b/exchanges/okx/okx_test.go @@ -56,7 +56,7 @@ var ( mainPair = currency.NewPairWithDelimiter("BTC", "USDT", "-") // Is used for spot, margin symbols and underlying contracts optionsPair = currency.NewPairWithDelimiter("BTC", "USD", "-") perpetualSwapPair = currency.NewPairWithDelimiter("BTC", "USDT-SWAP", "-") - spreadPair = currency.NewPairWithDelimiter("BTC-USDT-SWAP", "BTC-USD-SWAP", "_") + spreadPair = currency.NewPairWithDelimiter("BTC-USDT", "BTC-USDT-SWAP", "_") ) func TestMain(m *testing.M) { @@ -3903,14 +3903,12 @@ func TestGetHistoricCandlesExtended(t *testing.T) { assert.NotNil(t, result) } -func TestCalculateUpdateOrderbookChecksum(t *testing.T) { +func TestGenerateOrderbookChecksum(t *testing.T) { t.Parallel() var orderbookBase orderbook.Book err := json.Unmarshal([]byte(calculateOrderbookChecksumUpdateOrderbookJSON), &orderbookBase) require.NoError(t, err) - - err = ok.CalculateUpdateOrderbookChecksum(&orderbookBase, 2832680552) - assert.NoError(t, err) + require.Equal(t, uint32(2832680552), generateOrderbookChecksum(&orderbookBase)) } func TestOrderPushData(t *testing.T) { diff --git a/exchanges/okx/okx_websocket.go b/exchanges/okx/okx_websocket.go index a00b453b..6e946c94 100644 --- a/exchanges/okx/okx_websocket.go +++ b/exchanges/okx/okx_websocket.go @@ -838,13 +838,13 @@ func (ok *Okx) wsProcessSpreadOrderbook(respRaw []byte) error { } for x := range extractedResponse.Data { err = ok.Websocket.Orderbook.LoadSnapshot(&orderbook.Book{ - Asset: asset.Spread, - Asks: extractedResponse.Data[x].Asks, - Bids: extractedResponse.Data[x].Bids, - LastUpdated: resp.Data[x].Timestamp.Time(), - Pair: pair, - Exchange: ok.Name, - VerifyOrderbook: ok.CanVerifyOrderbook, + Asset: asset.Spread, + Asks: extractedResponse.Data[x].Asks, + Bids: extractedResponse.Data[x].Bids, + LastUpdated: resp.Data[x].Timestamp.Time(), + Pair: pair, + Exchange: ok.Name, + ValidateOrderbook: ok.ValidateOrderbook, }) if err != nil { return err @@ -884,13 +884,13 @@ func (ok *Okx) wsProcessOrderbook5(data []byte) error { for x := range assets { err = ok.Websocket.Orderbook.LoadSnapshot(&orderbook.Book{ - Asset: assets[x], - Asks: asks, - Bids: bids, - LastUpdated: resp.Data[0].Timestamp.Time(), - Pair: resp.Argument.InstrumentID, - Exchange: ok.Name, - VerifyOrderbook: ok.CanVerifyOrderbook, + Asset: assets[x], + Asks: asks, + Bids: bids, + LastUpdated: resp.Data[0].Timestamp.Time(), + Pair: resp.Argument.InstrumentID, + Exchange: ok.Name, + ValidateOrderbook: ok.ValidateOrderbook, }) if err != nil { return err @@ -1017,13 +1017,13 @@ func (ok *Okx) WsProcessSnapshotOrderBook(data *WsOrderBookData, pair currency.P } for i := range assets { newOrderBook := orderbook.Book{ - Asset: assets[i], - Asks: asks, - Bids: bids, - LastUpdated: data.Timestamp.Time(), - Pair: pair, - Exchange: ok.Name, - VerifyOrderbook: ok.CanVerifyOrderbook, + Asset: assets[i], + Asks: asks, + Bids: bids, + LastUpdated: data.Timestamp.Time(), + Pair: pair, + Exchange: ok.Name, + ValidateOrderbook: ok.ValidateOrderbook, } err = ok.Websocket.Orderbook.LoadSnapshot(&newOrderBook) if err != nil { @@ -1037,25 +1037,24 @@ func (ok *Okx) WsProcessSnapshotOrderBook(data *WsOrderBookData, pair currency.P // After merging WS data, it will sort, validate and finally update the existing // orderbook func (ok *Okx) WsProcessUpdateOrderbook(data *WsOrderBookData, pair currency.Pair, assets []asset.Item) error { - update := orderbook.Update{ - Pair: pair, - UpdateTime: data.Timestamp.Time(), - } - var err error - update.Asks, err = ok.AppendWsOrderbookItems(data.Asks) + asks, err := ok.AppendWsOrderbookItems(data.Asks) if err != nil { return err } - update.Bids, err = ok.AppendWsOrderbookItems(data.Bids) + bids, err := ok.AppendWsOrderbookItems(data.Bids) if err != nil { return err } - update.Checksum = uint32(data.Checksum) //nolint:gosec // Requires type casting for i := range assets { - ob := update - ob.Asset = assets[i] - err = ok.Websocket.Orderbook.Update(&ob) - if err != nil { + if err := ok.Websocket.Orderbook.Update(&orderbook.Update{ + Pair: pair, + Asset: assets[i], + UpdateTime: data.Timestamp.Time(), + GenerateChecksum: generateOrderbookChecksum, + ExpectedChecksum: uint32(data.Checksum), //nolint:gosec // Requires type casting + Asks: asks, + Bids: bids, + }); err != nil { return err } } @@ -1071,12 +1070,12 @@ func (ok *Okx) AppendWsOrderbookItems(entries [][4]types.Number) (orderbook.Leve return items, nil } -// CalculateUpdateOrderbookChecksum alternates over the first 25 bid and ask +// generateOrderbookChecksum alternates over the first 25 bid and ask // entries of a merged orderbook. The checksum is made up of the price and the // quantity with a semicolon (:) deliminating them. This will also work when // there are less than 25 entries (for whatever reason) // eg Bid:Ask:Bid:Ask:Ask:Ask -func (ok *Okx) CalculateUpdateOrderbookChecksum(orderbookData *orderbook.Book, checksumVal uint32) error { +func generateOrderbookChecksum(orderbookData *orderbook.Book) uint32 { var checksum strings.Builder for i := range allowableIterations { if len(orderbookData.Bids)-1 >= i { @@ -1091,10 +1090,7 @@ func (ok *Okx) CalculateUpdateOrderbookChecksum(orderbookData *orderbook.Book, c } } checksumStr := strings.TrimSuffix(checksum.String(), wsOrderbookChecksumDelimiter) - if crc32.ChecksumIEEE([]byte(checksumStr)) != checksumVal { - return fmt.Errorf("%s order book update checksum failed for pair %v", ok.Name, orderbookData.Pair) - } - return nil + return crc32.ChecksumIEEE([]byte(checksumStr)) } // CalculateOrderbookChecksum alternates over the first 25 bid and ask entries from websocket data. diff --git a/exchanges/okx/okx_wrapper.go b/exchanges/okx/okx_wrapper.go index 530f60f7..61aba259 100644 --- a/exchanges/okx/okx_wrapper.go +++ b/exchanges/okx/okx_wrapper.go @@ -16,7 +16,6 @@ import ( "github.com/thrasher-corp/gocryptotrader/config" "github.com/thrasher-corp/gocryptotrader/currency" "github.com/thrasher-corp/gocryptotrader/exchange/websocket" - "github.com/thrasher-corp/gocryptotrader/exchange/websocket/buffer" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/account" "github.com/thrasher-corp/gocryptotrader/exchanges/asset" @@ -214,7 +213,6 @@ func (ok *Okx) Setup(exch *config.Exchange) error { GenerateSubscriptions: ok.generateSubscriptions, Features: &ok.Features.Supports.WebsocketCapabilities, MaxWebsocketSubscriptionsPerConnection: 240, - OrderbookBufferConfig: buffer.Config{Checksum: ok.CalculateUpdateOrderbookChecksum}, RateLimitDefinitions: rateLimits, }); err != nil { return err @@ -535,10 +533,10 @@ func (ok *Okx) UpdateOrderbook(ctx context.Context, pair currency.Pair, assetTyp } for y := range spreadOrderbook { book := &orderbook.Book{ - Exchange: ok.Name, - Pair: pair, - Asset: assetType, - VerifyOrderbook: ok.CanVerifyOrderbook, + Exchange: ok.Name, + Pair: pair, + Asset: assetType, + ValidateOrderbook: ok.ValidateOrderbook, } book.Bids = make(orderbook.Levels, 0, len(spreadOrderbook[y].Bids)) for b := range spreadOrderbook[y].Bids { @@ -584,10 +582,10 @@ func (ok *Okx) UpdateOrderbook(ctx context.Context, pair currency.Pair, assetTyp } instrumentID = pairFormat.Format(pair) book := &orderbook.Book{ - Exchange: ok.Name, - Pair: pair, - Asset: assetType, - VerifyOrderbook: ok.CanVerifyOrderbook, + Exchange: ok.Name, + Pair: pair, + Asset: assetType, + ValidateOrderbook: ok.ValidateOrderbook, } var orderBookD *OrderBookResponseDetail orderBookD, err = ok.GetOrderBookDepth(ctx, instrumentID, 400) diff --git a/exchanges/orderbook/depth.go b/exchanges/orderbook/depth.go index c2bc09e1..a96b6a55 100644 --- a/exchanges/orderbook/depth.go +++ b/exchanges/orderbook/depth.go @@ -16,17 +16,15 @@ import ( "github.com/thrasher-corp/gocryptotrader/log" ) +// Public errors var ( - // ErrOrderbookInvalid defines an error for when the orderbook is invalid and - // should not be trusted - ErrOrderbookInvalid = errors.New("orderbook data integrity compromised") - // ErrInvalidAction defines and error when an action is invalid - ErrInvalidAction = errors.New("invalid action") - - errLastUpdatedNotSet = errors.New("last updated not set") - errInvalidBookDepth = errors.New("invalid book depth") + ErrOrderbookInvalid = errors.New("orderbook data integrity compromised") + ErrInvalidAction = errors.New("invalid action") + ErrLastUpdatedNotSet = errors.New("last updated not set") ) +var errInvalidBookDepth = errors.New("invalid book depth") + // Outbound restricts outbound usage of depth. NOTE: Type assert to // *orderbook.Depth. type Outbound interface { @@ -83,7 +81,7 @@ func (d *Depth) Retrieve() (*Book, error) { LastUpdateID: d.lastUpdateID, PriceDuplication: d.priceDuplication, IsFundingRate: d.isFundingRate, - VerifyOrderbook: d.verifyOrderbook, + ValidateOrderbook: d.validateOrderbook, MaxDepth: d.maxDepth, ChecksumStringRequired: d.checksumStringRequired, RestSnapshot: d.restSnapshot, @@ -92,26 +90,33 @@ func (d *Depth) Retrieve() (*Book, error) { } // LoadSnapshot flushes the bids and asks with a snapshot -func (d *Depth) LoadSnapshot(bids, asks []Level, lastUpdateID int64, lastUpdated, lastPushed time.Time, updateByREST bool) error { +func (d *Depth) LoadSnapshot(incoming *Book) error { d.m.Lock() defer d.m.Unlock() - if lastUpdated.IsZero() { - return fmt.Errorf("%s %s %s %w", d.exchange, d.pair, d.asset, errLastUpdatedNotSet) + if incoming.LastUpdated.IsZero() { + return fmt.Errorf("error loading orderbook snapshot: %s %s %s - %w", d.exchange, d.pair, d.asset, ErrLastUpdatedNotSet) } - d.lastUpdateID = lastUpdateID - d.lastUpdated = lastUpdated - d.lastPushed = lastPushed + d.lastUpdateID = incoming.LastUpdateID + d.lastUpdated = incoming.LastUpdated + d.lastPushed = incoming.LastPushed d.insertedAt = time.Now() - d.restSnapshot = updateByREST - d.bidLevels.load(bids) - d.askLevels.load(asks) + d.restSnapshot = incoming.RestSnapshot + d.bidLevels.load(incoming.Bids) + d.askLevels.load(incoming.Asks) d.validationError = nil d.Alert() return nil } -// invalidate flushes all values back to zero so as to not allow strategy -// traversal on compromised data. NOTE: This requires locking. +// Invalidate initialises the Depth, with a error to explain why it was invalid +func (d *Depth) Invalidate(withReason error) error { + d.m.Lock() + defer d.m.Unlock() + return d.invalidate(withReason) +} + +// invalidate initialises the Depth, with a error to explain why it was invalid +// NOTE: This requires locking. func (d *Depth) invalidate(withReason error) error { d.lastUpdateID = 0 d.lastUpdated = time.Time{} @@ -122,14 +127,6 @@ func (d *Depth) invalidate(withReason error) error { return d.validationError } -// Invalidate flushes all values back to zero so as to not allow strategy -// traversal on compromised data. -func (d *Depth) Invalidate(withReason error) error { - d.m.Lock() - defer d.m.Unlock() - return d.invalidate(withReason) -} - // IsValid returns if the underlying book is valid. func (d *Depth) IsValid() bool { d.m.RLock() @@ -137,118 +134,6 @@ func (d *Depth) IsValid() bool { return d.validationError == nil } -// UpdateBidAskByPrice updates the bid and ask spread by supplied updates, this -// will trim total length of depth level to a specified supplied number -func (d *Depth) UpdateBidAskByPrice(update *Update) error { - d.m.Lock() - defer d.m.Unlock() - if update.UpdateTime.IsZero() { - return fmt.Errorf("%s %s %s %w", d.exchange, d.pair, d.asset, errLastUpdatedNotSet) - } - if len(update.Bids) != 0 { - d.bidLevels.updateInsertByPrice(update.Bids, d.options.maxDepth) - } - if len(update.Asks) != 0 { - d.askLevels.updateInsertByPrice(update.Asks, d.options.maxDepth) - } - d.updateAndAlert(update) - return nil -} - -// UpdateBidAskByID amends details by ID -func (d *Depth) UpdateBidAskByID(update *Update) error { - d.m.Lock() - defer d.m.Unlock() - - if update.UpdateTime.IsZero() { - return fmt.Errorf("%s %s %s %w", d.exchange, d.pair, d.asset, errLastUpdatedNotSet) - } - - if len(update.Bids) != 0 { - err := d.bidLevels.updateByID(update.Bids) - if err != nil { - return d.invalidate(err) - } - } - if len(update.Asks) != 0 { - err := d.askLevels.updateByID(update.Asks) - if err != nil { - return d.invalidate(err) - } - } - d.updateAndAlert(update) - return nil -} - -// DeleteBidAskByID deletes a price level by ID -func (d *Depth) DeleteBidAskByID(update *Update, bypassErr bool) error { - d.m.Lock() - defer d.m.Unlock() - if update.UpdateTime.IsZero() { - return fmt.Errorf("%s %s %s %w", d.exchange, d.pair, d.asset, errLastUpdatedNotSet) - } - if len(update.Bids) != 0 { - err := d.bidLevels.deleteByID(update.Bids, bypassErr) - if err != nil { - return d.invalidate(err) - } - } - if len(update.Asks) != 0 { - err := d.askLevels.deleteByID(update.Asks, bypassErr) - if err != nil { - return d.invalidate(err) - } - } - d.updateAndAlert(update) - return nil -} - -// InsertBidAskByID inserts new updates -func (d *Depth) InsertBidAskByID(update *Update) error { - d.m.Lock() - defer d.m.Unlock() - if update.UpdateTime.IsZero() { - return fmt.Errorf("%s %s %s %w", d.exchange, d.pair, d.asset, errLastUpdatedNotSet) - } - if len(update.Bids) != 0 { - err := d.bidLevels.insertUpdates(update.Bids) - if err != nil { - return d.invalidate(err) - } - } - if len(update.Asks) != 0 { - err := d.askLevels.insertUpdates(update.Asks) - if err != nil { - return d.invalidate(err) - } - } - d.updateAndAlert(update) - return nil -} - -// UpdateInsertByID updates or inserts by ID at current price level. -func (d *Depth) UpdateInsertByID(update *Update) error { - d.m.Lock() - defer d.m.Unlock() - if update.UpdateTime.IsZero() { - return fmt.Errorf("%s %s %s %w", d.exchange, d.pair, d.asset, errLastUpdatedNotSet) - } - if len(update.Bids) != 0 { - err := d.bidLevels.updateInsertByID(update.Bids) - if err != nil { - return d.invalidate(err) - } - } - if len(update.Asks) != 0 { - err := d.askLevels.updateInsertByID(update.Asks) - if err != nil { - return d.invalidate(err) - } - } - d.updateAndAlert(update) - return nil -} - // AssignOptions assigns the initial options for the depth instance func (d *Depth) AssignOptions(b *Book) { d.m.Lock() @@ -260,7 +145,7 @@ func (d *Depth) AssignOptions(b *Book) { lastUpdateID: b.LastUpdateID, priceDuplication: b.PriceDuplication, isFundingRate: b.IsFundingRate, - verifyOrderbook: b.VerifyOrderbook, + validateOrderbook: b.ValidateOrderbook, restSnapshot: b.RestSnapshot, idAligned: b.IDAlignment, maxDepth: b.MaxDepth, @@ -303,11 +188,11 @@ func (d *Depth) IsFundingRate() bool { return d.isFundingRate } -// VerifyOrderbook returns if the verify orderbook option is set -func (d *Depth) VerifyOrderbook() bool { +// ValidateOrderbook returns if the verify orderbook option is set +func (d *Depth) ValidateOrderbook() bool { d.m.RLock() defer d.m.RUnlock() - return d.verifyOrderbook + return d.validateOrderbook } // GetAskLength returns length of asks diff --git a/exchanges/orderbook/depth_test.go b/exchanges/orderbook/depth_test.go index 22d1336a..88890cdc 100644 --- a/exchanges/orderbook/depth_test.go +++ b/exchanges/orderbook/depth_test.go @@ -30,7 +30,7 @@ func TestGetLength(t *testing.T) { _, err = d.GetAskLength() assert.ErrorIs(t, err, ErrOrderbookInvalid, "GetAskLength should error with invalid depth") - err = d.LoadSnapshot([]Level{{Price: 1337}}, nil, 0, time.Now(), time.Now(), true) + err = d.LoadSnapshot(&Book{Bids: Levels{{Price: 1337}}, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") askLen, err := d.GetAskLength() @@ -50,7 +50,7 @@ func TestGetLength(t *testing.T) { _, err = d.GetBidLength() assert.ErrorIs(t, err, ErrOrderbookInvalid, "GetBidLength should error with invalid depth") - err = d.LoadSnapshot(nil, []Level{{Price: 1337}}, 0, time.Now(), time.Now(), true) + err = d.LoadSnapshot(&Book{Asks: Levels{{Price: 1337}}, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") bidLen, err := d.GetBidLength() @@ -79,7 +79,7 @@ func TestRetrieve(t *testing.T) { lastUpdateID: 1337, priceDuplication: true, isFundingRate: true, - verifyOrderbook: true, + validateOrderbook: true, restSnapshot: true, idAligned: true, maxDepth: 10, @@ -107,7 +107,7 @@ func TestRetrieve(t *testing.T) { assert.EqualValues(t, 1337, ob.LastUpdateID, "Should have correct LastUpdateID") assert.True(t, ob.PriceDuplication, "Should have correct PriceDuplication") assert.True(t, ob.IsFundingRate, "Should have correct IsFundingRate") - assert.True(t, ob.VerifyOrderbook, "Should have correct VerifyOrderbook") + assert.True(t, ob.ValidateOrderbook, "Should have correct ValidateOrderbook") assert.True(t, ob.RestSnapshot, "Should have correct RestSnapshot") assert.True(t, ob.IDAlignment, "Should have correct IDAligned") assert.Equal(t, 10, ob.MaxDepth, "Should have correct MaxDepth") @@ -159,10 +159,10 @@ func TestTotalAmounts(t *testing.T) { func TestLoadSnapshot(t *testing.T) { t.Parallel() d := NewDepth(id) - err := d.LoadSnapshot(Levels{{Price: 1337, Amount: 1}}, Levels{{Price: 1337, Amount: 10}}, 0, time.Time{}, time.Now(), false) - assert.ErrorIs(t, err, errLastUpdatedNotSet, "LoadSnapshot should error correctly") + err := d.LoadSnapshot(&Book{Bids: Levels{{Price: 1337, Amount: 1}}, Asks: Levels{{Price: 1337, Amount: 10}}, LastPushed: time.Now()}) + assert.ErrorIs(t, err, ErrLastUpdatedNotSet, "LoadSnapshot should error correctly") - err = d.LoadSnapshot(Levels{{Price: 1337, Amount: 2}}, Levels{{Price: 1338, Amount: 10}}, 0, time.Now(), time.Now(), false) + err = d.LoadSnapshot(&Book{Bids: Levels{{Price: 1337, Amount: 2}}, Asks: Levels{{Price: 1338, Amount: 10}}, LastUpdated: time.Now(), LastPushed: time.Now()}) assert.NoError(t, err, "LoadSnapshot should not error") ob, err := d.Retrieve() @@ -181,7 +181,7 @@ func TestInvalidate(t *testing.T) { d.pair = currency.NewPair(currency.BTC, currency.WABI) d.asset = asset.Spot - err := d.LoadSnapshot(Levels{{Price: 1337, Amount: 1}}, Levels{{Price: 1337, Amount: 10}}, 0, time.Now(), time.Now(), false) + err := d.LoadSnapshot(&Book{Bids: Levels{{Price: 1337, Amount: 1}}, Asks: Levels{{Price: 1337, Amount: 10}}, LastUpdated: time.Now(), LastPushed: time.Now()}) assert.NoError(t, err, "LoadSnapshot should not error") ob, err := d.Retrieve() @@ -206,248 +206,22 @@ func TestInvalidate(t *testing.T) { assert.Empty(t, ob.Bids, "Orderbook Bids should be flushed") } -func TestUpdateBidAskByPrice(t *testing.T) { - t.Parallel() - d := NewDepth(id) - err := d.LoadSnapshot(Levels{{Price: 1337, Amount: 1, ID: 1}}, Levels{{Price: 1338, Amount: 10, ID: 2}}, 0, time.Now(), time.Now(), false) - assert.NoError(t, err, "LoadSnapshot should not error") - - err = d.UpdateBidAskByPrice(&Update{}) - assert.ErrorIs(t, err, errLastUpdatedNotSet, "UpdateBidAskByPrice should error correctly") - - err = d.UpdateBidAskByPrice(&Update{UpdateTime: time.Now()}) - assert.NoError(t, err, "UpdateBidAskByPrice should not error") - - updates := &Update{ - Bids: Levels{{Price: 1337, Amount: 2, ID: 1}}, - Asks: Levels{{Price: 1338, Amount: 3, ID: 2}}, - UpdateID: 1, - UpdateTime: time.Now(), - } - err = d.UpdateBidAskByPrice(updates) - assert.NoError(t, err, "UpdateBidAskByPrice should not error") - - ob, err := d.Retrieve() - assert.NoError(t, err, "Retrieve should not error") - assert.Equal(t, 3.0, ob.Asks[0].Amount, "Asks amount should be correct") - assert.Equal(t, 2.0, ob.Bids[0].Amount, "Bids amount should be correct") - - updates = &Update{ - Bids: Levels{{Price: 1337, Amount: 0, ID: 1}}, - Asks: Levels{{Price: 1338, Amount: 0, ID: 2}}, - UpdateID: 2, - UpdateTime: time.Now(), - } - err = d.UpdateBidAskByPrice(updates) - assert.NoError(t, err, "UpdateBidAskByPrice should not error") - - askLen, err := d.GetAskLength() - assert.NoError(t, err, "GetAskLength should not error") - assert.Zero(t, askLen, "Ask Length should be correct") - - bidLen, err := d.GetBidLength() - assert.NoError(t, err, "GetBidLength should not error") - assert.Zero(t, bidLen, "Bid Length should be correct") -} - -func TestDeleteBidAskByID(t *testing.T) { - t.Parallel() - d := NewDepth(id) - err := d.LoadSnapshot(Levels{{Price: 1337, Amount: 1, ID: 1}}, Levels{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Now(), time.Now(), false) - assert.NoError(t, err, "LoadSnapshot should not error") - - updates := &Update{ - Bids: Levels{{Price: 1337, Amount: 2, ID: 1}}, - Asks: Levels{{Price: 1337, Amount: 2, ID: 2}}, - } - - err = d.DeleteBidAskByID(updates, false) - assert.ErrorIs(t, err, errLastUpdatedNotSet, "DeleteBidAskByID should error correctly") - - updates.UpdateTime = time.Now() - err = d.DeleteBidAskByID(updates, false) - assert.NoError(t, err, "DeleteBidAskByID should not error") - - ob, err := d.Retrieve() - assert.NoError(t, err, "Retrieve should not error") - assert.Empty(t, ob.Asks, "Asks should be empty") - assert.Empty(t, ob.Bids, "Bids should be empty") - - updates = &Update{ - Bids: Levels{{Price: 1337, Amount: 2, ID: 1}}, - UpdateTime: time.Now(), - } - err = d.DeleteBidAskByID(updates, false) - assert.ErrorIs(t, err, errIDCannotBeMatched, "DeleteBidAskByID should error correctly") - - updates = &Update{ - Asks: Levels{{Price: 1337, Amount: 2, ID: 2}}, - UpdateTime: time.Now(), - } - err = d.DeleteBidAskByID(updates, false) - assert.ErrorIs(t, err, errIDCannotBeMatched, "DeleteBidAskByID should error correctly") - - updates = &Update{ - Asks: Levels{{Price: 1337, Amount: 2, ID: 2}}, - UpdateTime: time.Now(), - } - err = d.DeleteBidAskByID(updates, true) - assert.NoError(t, err, "DeleteBidAskByID should not error") -} - -func TestUpdateBidAskByID(t *testing.T) { - t.Parallel() - d := NewDepth(id) - err := d.LoadSnapshot(Levels{{Price: 1337, Amount: 1, ID: 1}}, Levels{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Now(), time.Now(), false) - assert.NoError(t, err, "LoadSnapshot should not error") - - updates := &Update{ - Bids: Levels{{Price: 1337, Amount: 2, ID: 1}}, - Asks: Levels{{Price: 1337, Amount: 2, ID: 2}}, - } - - err = d.UpdateBidAskByID(updates) - assert.ErrorIs(t, err, errLastUpdatedNotSet, "UpdateBidAskByID should error correctly") - - updates.UpdateTime = time.Now() - err = d.UpdateBidAskByID(updates) - assert.NoError(t, err, "UpdateBidAskByID should not error") - - ob, err := d.Retrieve() - assert.NoError(t, err, "Retrieve should not error") - assert.Equal(t, 2.0, ob.Asks[0].Amount, "First ask amount should be correct") - assert.Equal(t, 2.0, ob.Bids[0].Amount, "First bid amount should be correct") - - updates = &Update{ - Bids: Levels{{Price: 1337, Amount: 2, ID: 666}}, - UpdateTime: time.Now(), - } - // random unmatching IDs - err = d.UpdateBidAskByID(updates) - assert.ErrorIs(t, err, errIDCannotBeMatched, "UpdateBidAskByID should error correctly") - - updates = &Update{ - Asks: Levels{{Price: 1337, Amount: 2, ID: 69}}, - UpdateTime: time.Now(), - } - err = d.UpdateBidAskByID(updates) - assert.ErrorIs(t, err, errIDCannotBeMatched, "UpdateBidAskByID should error correctly") -} - -func TestInsertBidAskByID(t *testing.T) { - t.Parallel() - d := NewDepth(id) - err := d.LoadSnapshot(Levels{{Price: 1337, Amount: 1, ID: 1}}, Levels{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Now(), time.Now(), false) - assert.NoError(t, err, "LoadSnapshot should not error") - - updates := &Update{ - Asks: Levels{{Price: 1337, Amount: 2, ID: 3}}, - } - err = d.InsertBidAskByID(updates) - assert.ErrorIs(t, err, errLastUpdatedNotSet, "InsertBidAskByID should error correctly") - - updates.UpdateTime = time.Now() - - err = d.InsertBidAskByID(updates) - assert.ErrorIs(t, err, errCollisionDetected, "InsertBidAskByID should error correctly on collision") - - err = d.LoadSnapshot(Levels{{Price: 1337, Amount: 1, ID: 1}}, Levels{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Now(), time.Now(), false) - assert.NoError(t, err, "LoadSnapshot should not error") - - updates = &Update{ - Bids: Levels{{Price: 1337, Amount: 2, ID: 3}}, - UpdateTime: time.Now(), - } - - err = d.InsertBidAskByID(updates) - assert.ErrorIs(t, err, errCollisionDetected, "InsertBidAskByID should error correctly on collision") - - err = d.LoadSnapshot(Levels{{Price: 1337, Amount: 1, ID: 1}}, Levels{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Now(), time.Now(), false) - assert.NoError(t, err, "LoadSnapshot should not error") - - updates = &Update{ - Bids: Levels{{Price: 1338, Amount: 2, ID: 3}}, - Asks: Levels{{Price: 1336, Amount: 2, ID: 4}}, - UpdateTime: time.Now(), - } - err = d.InsertBidAskByID(updates) - assert.NoError(t, err, "InsertBidAskByID should not error") - - ob, err := d.Retrieve() - assert.NoError(t, err, "Retrieve should not error") - assert.Len(t, ob.Asks, 2, "Should have correct Asks") - assert.Len(t, ob.Bids, 2, "Should have correct Bids") -} - -func TestUpdateInsertByID(t *testing.T) { - t.Parallel() - d := NewDepth(id) - err := d.LoadSnapshot(Levels{{Price: 1337, Amount: 1, ID: 1}}, Levels{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Now(), time.Now(), false) - assert.NoError(t, err, "LoadSnapshot should not error") - - updates := &Update{ - Bids: Levels{{Price: 1338, Amount: 0, ID: 3}}, - Asks: Levels{{Price: 1336, Amount: 2, ID: 4}}, - } - err = d.UpdateInsertByID(updates) - assert.ErrorIs(t, err, errLastUpdatedNotSet, "UpdateInsertByID should error correctly") - - updates.UpdateTime = time.Now() - err = d.UpdateInsertByID(updates) - assert.ErrorIs(t, err, errAmountCannotBeLessOrEqualToZero, "UpdateInsertByID should error correctly") - - // Above will invalidate the book - _, err = d.Retrieve() - assert.ErrorIs(t, err, ErrOrderbookInvalid, "Retrieve should error correctly") - - err = d.LoadSnapshot(Levels{{Price: 1337, Amount: 1, ID: 1}}, Levels{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Now(), time.Now(), false) - assert.NoError(t, err, "LoadSnapshot should not error") - - updates = &Update{ - Bids: Levels{{Price: 1338, Amount: 2, ID: 3}}, - Asks: Levels{{Price: 1336, Amount: 0, ID: 4}}, - UpdateTime: time.Now(), - } - err = d.UpdateInsertByID(updates) - assert.ErrorIs(t, err, errAmountCannotBeLessOrEqualToZero, "UpdateInsertByID should error correctly") - - // Above will invalidate the book - _, err = d.Retrieve() - assert.ErrorIs(t, err, ErrOrderbookInvalid, "Retrieve should error correctly") - - err = d.LoadSnapshot(Levels{{Price: 1337, Amount: 1, ID: 1}}, Levels{{Price: 1337, Amount: 10, ID: 2}}, 0, time.Now(), time.Now(), false) - assert.NoError(t, err, "LoadSnapshot should not error") - - updates = &Update{ - Bids: Levels{{Price: 1338, Amount: 2, ID: 3}}, - Asks: Levels{{Price: 1336, Amount: 2, ID: 4}}, - UpdateTime: time.Now(), - } - err = d.UpdateInsertByID(updates) - assert.NoError(t, err, "UpdateInsertByID should not error") - - ob, err := d.Retrieve() - assert.NoError(t, err, "Retrieve should not error") - assert.Len(t, ob.Asks, 2, "Should have correct Asks") - assert.Len(t, ob.Bids, 2, "Should have correct Bids") -} - func TestAssignOptions(t *testing.T) { t.Parallel() d := Depth{} cp := currency.NewPair(currency.LINK, currency.BTC) tn := time.Now() d.AssignOptions(&Book{ - Exchange: "test", - Pair: cp, - Asset: asset.Spot, - LastUpdated: tn, - LastUpdateID: 1337, - PriceDuplication: true, - IsFundingRate: true, - VerifyOrderbook: true, - RestSnapshot: true, - IDAlignment: true, + Exchange: "test", + Pair: cp, + Asset: asset.Spot, + LastUpdated: tn, + LastUpdateID: 1337, + PriceDuplication: true, + IsFundingRate: true, + ValidateOrderbook: true, + RestSnapshot: true, + IDAlignment: true, }) assert.Equal(t, "test", d.exchange, "exchange should be correct") @@ -457,7 +231,7 @@ func TestAssignOptions(t *testing.T) { assert.EqualValues(t, 1337, d.lastUpdateID, "lastUpdatedID should be correct") assert.True(t, d.priceDuplication, "priceDuplication should be correct") assert.True(t, d.IsFundingRate(), "IsFundingRate should be correct") - assert.True(t, d.VerifyOrderbook(), "VerifyOrderbook should be correct") + assert.True(t, d.ValidateOrderbook(), "ValidateOrderbook should be correct") assert.True(t, d.restSnapshot, "restSnapshot should be correct") assert.True(t, d.idAligned, "idAligned should be correct") } @@ -536,7 +310,7 @@ func TestGetMidPrice_Depth(t *testing.T) { _, err = depth.GetMidPrice() assert.ErrorIs(t, err, errNoLiquidity, "GetMidPrice should error correctly") - err = depth.LoadSnapshot(bid, ask, 0, time.Now(), time.Now(), true) + err = depth.LoadSnapshot(&Book{Bids: bid, Asks: ask, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") mid, err := depth.GetMidPrice() @@ -550,13 +324,13 @@ func TestGetMidPriceNoLock_Depth(t *testing.T) { _, err := depth.getMidPriceNoLock() assert.ErrorIs(t, err, errNoLiquidity, "getMidPriceNoLock should error correctly") - err = depth.LoadSnapshot(bid, nil, 0, time.Now(), time.Now(), true) + err = depth.LoadSnapshot(&Book{Bids: bid, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") _, err = depth.getMidPriceNoLock() assert.ErrorIs(t, err, errNoLiquidity, "getMidPriceNoLock should error correctly") - err = depth.LoadSnapshot(bid, ask, 0, time.Now(), time.Now(), true) + err = depth.LoadSnapshot(&Book{Bids: bid, Asks: ask, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") mid, err := depth.getMidPriceNoLock() @@ -579,7 +353,7 @@ func TestGetBestBidASk_Depth(t *testing.T) { _, err = depth.GetBestAsk() assert.ErrorIs(t, err, errNoLiquidity, "GetBestAsk should error correctly") - err = depth.LoadSnapshot(bid, ask, 0, time.Now(), time.Now(), true) + err = depth.LoadSnapshot(&Book{Bids: bid, Asks: ask, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") mid, err := depth.GetBestBid() @@ -601,13 +375,13 @@ func TestGetSpreadAmount(t *testing.T) { _, err = depth.GetSpreadAmount() assert.ErrorIs(t, err, errNoLiquidity, "GetSpreadAmount should error correctly") - err = depth.LoadSnapshot(nil, ask, 0, time.Now(), time.Now(), true) + err = depth.LoadSnapshot(&Book{Asks: ask, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") _, err = depth.GetSpreadAmount() assert.ErrorIs(t, err, errNoLiquidity, "GetSpreadAmount should error correctly") - err = depth.LoadSnapshot(bid, ask, 0, time.Now(), time.Now(), true) + err = depth.LoadSnapshot(&Book{Bids: bid, Asks: ask, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") spread, err := depth.GetSpreadAmount() @@ -625,13 +399,13 @@ func TestGetSpreadPercentage(t *testing.T) { _, err = depth.GetSpreadPercentage() assert.ErrorIs(t, err, errNoLiquidity, "GetSpreadPercentage should error correctly") - err = depth.LoadSnapshot(nil, ask, 0, time.Now(), time.Now(), true) + err = depth.LoadSnapshot(&Book{Asks: ask, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") _, err = depth.GetSpreadPercentage() assert.ErrorIs(t, err, errNoLiquidity, "GetSpreadPercentage should error correctly") - err = depth.LoadSnapshot(bid, ask, 0, time.Now(), time.Now(), true) + err = depth.LoadSnapshot(&Book{Bids: bid, Asks: ask, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") spread, err := depth.GetSpreadPercentage() @@ -649,13 +423,13 @@ func TestGetImbalance_Depth(t *testing.T) { _, err = depth.GetImbalance() assert.ErrorIs(t, err, errNoLiquidity, "GetImbalance should error correctly") - err = depth.LoadSnapshot(nil, ask, 0, time.Now(), time.Now(), true) + err = depth.LoadSnapshot(&Book{Asks: ask, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") _, err = depth.GetImbalance() assert.ErrorIs(t, err, errNoLiquidity, "GetImbalance should error correctly") - err = depth.LoadSnapshot(bid, ask, 0, time.Now(), time.Now(), true) + err = depth.LoadSnapshot(&Book{Bids: bid, Asks: ask, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") imbalance, err := depth.GetImbalance() @@ -678,7 +452,7 @@ func TestGetLevels(t *testing.T) { assert.Empty(t, askL, "Ask level should be empty") assert.Empty(t, bidL, "Bid level should be empty") - err = depth.LoadSnapshot(bid, ask, 0, time.Now(), time.Now(), true) + err = depth.LoadSnapshot(&Book{Bids: bid, Asks: ask, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") askL, bidL, err = depth.GetLevels(0) @@ -728,7 +502,7 @@ func TestMovementMethods(t *testing.T) { _, err = callMethod(depth, methodName, tt.tests[0].inputs) assert.ErrorIs(t, err, errNoLiquidity, "should error correctly with no liquidity") - err = depth.LoadSnapshot(bid, ask, 0, time.Now(), time.Now(), true) + err = depth.LoadSnapshot(&Book{Bids: bid, Asks: ask, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") for i, subT := range tt.tests { diff --git a/exchanges/orderbook/incremental_updates.go b/exchanges/orderbook/incremental_updates.go new file mode 100644 index 00000000..e10da9d5 --- /dev/null +++ b/exchanges/orderbook/incremental_updates.go @@ -0,0 +1,249 @@ +package orderbook + +import ( + "errors" + "fmt" + "time" + + "github.com/thrasher-corp/gocryptotrader/currency" + "github.com/thrasher-corp/gocryptotrader/exchanges/asset" +) + +// ActionType defines the behaviour of an orderbook update +type ActionType uint8 + +// ActionType constants for use with ProcessUpdate +const ( + UnknownAction ActionType = iota + InsertAction + UpdateOrInsertAction + UpdateAction + DeleteAction +) + +// Public error vars +var ( + ErrDepthNotFound = errors.New("orderbook depth not found") + ErrEmptyUpdate = errors.New("update contains no bids or asks") +) + +var ( + errInvalidAction = errors.New("invalid action") + errUpdateFailed = errors.New("orderbook update failed") + errDeleteFailed = errors.New("orderbook update delete failed") + errRESTSnapshot = errors.New("cannot update REST protocol loaded snapshot") + errChecksumMismatch = errors.New("checksum mismatch") + errChecksumGeneratorUnset = errors.New("checksum generator unset") +) + +// Update holds changes that are to be applied to a stored orderbook +type Update struct { + UpdateID int64 + UpdateTime time.Time + LastPushed time.Time + Asset asset.Item + Bids Levels + Asks Levels + Pair currency.Pair + + // ExpectedChecksum defines the expected value when the books have been verified + ExpectedChecksum uint32 + // GenerateChecksum is a function that will be called to generate a checksum from the stored orderbook post update + GenerateChecksum func(snapshot *Book) uint32 + // AllowEmpty, when true, permits loading an empty order book update to set an UpdateID without including actual data + AllowEmpty bool + // Action defines the action to be performed on the orderbook e.g. amend, delete, insert, update/insert + // Orderbook IDs are used to identify the orderbook level to be updated, deleted or inserted + Action ActionType + + SkipOutOfOrderLastUpdateID bool +} + +// ProcessUpdate applies updates to the orderbook depth, on error it will invalidate the orderbook and return the +// error, this is to ensure the orderbook is always in a valid state. +func (d *Depth) ProcessUpdate(u *Update) error { + if len(u.Bids) == 0 && len(u.Asks) == 0 && !u.AllowEmpty { + return d.Invalidate(ErrEmptyUpdate) + } + + // TODO: Enforce LastPushed set to determine server latency + + d.m.Lock() + defer d.m.Unlock() + + if d.validationError != nil { + return d.validationError + } + + // This will process out of order updates but will not error on them. + // TODO: Error on out of order updates; this is intentionally kept as is from the buffer package. + // Add update.UpdateTime time check to ensure that the update is newer than the last update, + // this should screen zero values as well. + if u.SkipOutOfOrderLastUpdateID && d.lastUpdateID >= u.UpdateID { + return nil + } + + if d.options.restSnapshot { + return d.invalidate(errRESTSnapshot) + } + + if u.Action != UnknownAction { + if err := d.update(u); err != nil { + return d.invalidate(err) + } + } else { + if err := d.updateBidAskByPrice(u); err != nil { + return d.invalidate(err) + } + } + + if !d.validateOrderbook { + return nil + } + + if u.ExpectedChecksum != 0 { + if u.GenerateChecksum == nil { + return d.invalidate(errChecksumGeneratorUnset) + } + if checksum := u.GenerateChecksum(d.snapshot()); checksum != u.ExpectedChecksum { + return d.invalidate(fmt.Errorf("%s %s %s %w: expected '%d', got '%d'", d.exchange, d.pair, d.asset, errChecksumMismatch, u.ExpectedChecksum, checksum)) + } + } + + if err := validate(d.snapshot()); err != nil { + return d.invalidate(err) + } + + return nil +} + +func (d *Depth) snapshot() *Book { + return &Book{ + Bids: d.bidLevels.Levels, + Asks: d.askLevels.Levels, + Exchange: d.options.exchange, + Pair: d.pair, + Asset: d.asset, + IsFundingRate: d.options.isFundingRate, + PriceDuplication: d.options.priceDuplication, + IDAlignment: d.options.idAligned, + ChecksumStringRequired: d.options.checksumStringRequired, + } +} + +// update will receive an action to execute against the orderbook it will then match by IDs instead of +// price to perform the action +func (d *Depth) update(u *Update) error { + switch u.Action { + case UpdateAction: + if err := d.updateBidAskByID(u); err != nil { + return fmt.Errorf("%w for %q: %w", errUpdateFailed, u.Action, err) + } + case DeleteAction: + // edge case for Bitfinex as their streaming endpoint duplicates deletes + bypassErr := d.options.exchange == "Bitfinex" && d.options.isFundingRate // TODO: Confirm this is still correct + if err := d.delete(u, bypassErr); err != nil { + return fmt.Errorf("%w for %q: %w", errDeleteFailed, u.Action, err) + } + case InsertAction: + if err := d.insert(u); err != nil { + return fmt.Errorf("%w for %q: %w", errUpdateFailed, u.Action, err) + } + case UpdateOrInsertAction: + if err := d.updateOrInsert(u); err != nil { + return fmt.Errorf("%w for %q: %w", errUpdateFailed, u.Action, err) + } + default: + return fmt.Errorf("%w [%s]", errInvalidAction, u.Action) + } + return nil +} + +// updateBidAskByPrice updates the bid and ask spread and enforces Depth.options.maxDepth +func (d *Depth) updateBidAskByPrice(update *Update) error { + if update.UpdateTime.IsZero() { + return fmt.Errorf("%s %s %s %w", d.exchange, d.pair, d.asset, ErrLastUpdatedNotSet) + } + d.bidLevels.updateInsertByPrice(update.Bids, d.options.maxDepth) + d.askLevels.updateInsertByPrice(update.Asks, d.options.maxDepth) + d.updateAndAlert(update) + return nil +} + +// updateBidAskByID amends details by ID +func (d *Depth) updateBidAskByID(update *Update) error { + if update.UpdateTime.IsZero() { + return fmt.Errorf("%s %s %s %w", d.exchange, d.pair, d.asset, ErrLastUpdatedNotSet) + } + if err := d.bidLevels.updateByID(update.Bids); err != nil { + return err + } + if err := d.askLevels.updateByID(update.Asks); err != nil { + return err + } + d.updateAndAlert(update) + return nil +} + +// delete deletes a price level by ID +func (d *Depth) delete(update *Update, bypassErr bool) error { + if update.UpdateTime.IsZero() { + return fmt.Errorf("%s %s %s %w", d.exchange, d.pair, d.asset, ErrLastUpdatedNotSet) + } + if err := d.bidLevels.deleteByID(update.Bids, bypassErr); err != nil { + return err + } + if err := d.askLevels.deleteByID(update.Asks, bypassErr); err != nil { + return err + } + d.updateAndAlert(update) + return nil +} + +// insert inserts new updates +func (d *Depth) insert(update *Update) error { + if update.UpdateTime.IsZero() { + return fmt.Errorf("%s %s %s %w", d.exchange, d.pair, d.asset, ErrLastUpdatedNotSet) + } + if err := d.bidLevels.insertUpdates(update.Bids); err != nil { + return err + } + if err := d.askLevels.insertUpdates(update.Asks); err != nil { + return err + } + d.updateAndAlert(update) + return nil +} + +// updateOrInsert updates or inserts by ID at current price level. +func (d *Depth) updateOrInsert(update *Update) error { + if update.UpdateTime.IsZero() { + return fmt.Errorf("%s %s %s %w", d.exchange, d.pair, d.asset, ErrLastUpdatedNotSet) + } + if err := d.bidLevels.updateInsertByID(update.Bids); err != nil { + return err + } + if err := d.askLevels.updateInsertByID(update.Asks); err != nil { + return err + } + d.updateAndAlert(update) + return nil +} + +// String returns a string representation of the ActionType +func (a ActionType) String() string { + switch a { + case UnknownAction: + return "Unknown" + case InsertAction: + return "Insert" + case UpdateOrInsertAction: + return "UpdateOrInsert" + case UpdateAction: + return "Update" + case DeleteAction: + return "Delete" + default: + return fmt.Sprintf("Unknown(%d)", a) + } +} diff --git a/exchanges/orderbook/incremental_updates_test.go b/exchanges/orderbook/incremental_updates_test.go new file mode 100644 index 00000000..f92a7103 --- /dev/null +++ b/exchanges/orderbook/incremental_updates_test.go @@ -0,0 +1,374 @@ +package orderbook + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newSnapshot(length int) *Book { + return &Book{ + Bids: newBids(length), + Asks: newAsks(length, length), + LastUpdated: time.Now(), + LastPushed: time.Now(), + LastUpdateID: 1, + } +} + +func newBids(length int) Levels { + bids := make(Levels, length) + for i := range length { + bids[i] = Level{Price: 1337 - float64(i), Amount: 1, ID: int64(i + 1)} + } + return bids +} + +func newAsks(idOffset, length int) Levels { + asks := make(Levels, length) + for i := range length { + asks[i] = Level{Price: 1338 + float64(i), Amount: 1, ID: int64(i + 1 + idOffset)} + } + return asks +} + +func TestProcessUpdate(t *testing.T) { + t.Parallel() + d := NewDepth(id) + require.NoError(t, d.LoadSnapshot(newSnapshot(69))) + assert.ErrorIs(t, d.ProcessUpdate(&Update{}), ErrEmptyUpdate) + assert.ErrorIs(t, d.ProcessUpdate(&Update{AllowEmpty: true}), ErrEmptyUpdate, "exercise validation error return from last ProcessUpdate call which invalidates the orderbook") + require.NoError(t, d.LoadSnapshot(newSnapshot(20))) + assert.NoError(t, d.ProcessUpdate(&Update{UpdateTime: time.Now(), Asks: Levels{{Price: 1337.5, Amount: 69420, ID: 69420}}, SkipOutOfOrderLastUpdateID: true})) + ob, err := d.Retrieve() + require.NoError(t, err) + assert.NotEqual(t, int64(69420), ob.Asks[0].ID, "Update above should skip insertion") + d.options.restSnapshot = true // Simulate the snapshot has been loaded from REST + err = d.ProcessUpdate(&Update{UpdateTime: time.Now(), Asks: Levels{{Price: 1337.5, Amount: 69420, ID: 69420}}}) + assert.ErrorIs(t, err, errRESTSnapshot) + + require.NoError(t, d.LoadSnapshot(newSnapshot(20))) + err = d.ProcessUpdate(&Update{Asks: Levels{{Price: 1337.5, Amount: 69420, ID: 69420}}}) + assert.ErrorIs(t, err, ErrLastUpdatedNotSet) + + require.NoError(t, d.LoadSnapshot(newSnapshot(20))) + err = d.ProcessUpdate(&Update{Action: InsertAction, Asks: Levels{{Price: 1337.5, Amount: 69420, ID: 69420}}}) + assert.ErrorIs(t, err, ErrLastUpdatedNotSet) + + require.NoError(t, d.LoadSnapshot(newSnapshot(20))) + d.validateOrderbook = true + d.askLevels.Levels[0].Amount = 0 + err = d.ProcessUpdate(&Update{UpdateTime: time.Now(), Asks: Levels{{Price: 1337.5, Amount: 69420, ID: 69420}}}) + assert.ErrorIs(t, err, errAmountInvalid) + + require.NoError(t, d.LoadSnapshot(newSnapshot(20))) + err = d.ProcessUpdate(&Update{UpdateTime: time.Now(), Asks: Levels{{Price: 1337.5, Amount: 69420, ID: 69420}}}) + require.NoError(t, err) + + require.NoError(t, d.LoadSnapshot(newSnapshot(20))) + err = d.ProcessUpdate(&Update{UpdateTime: time.Now(), Asks: Levels{{Price: 1337.5, Amount: 69420, ID: 69420}}, ExpectedChecksum: 1337}) + require.ErrorIs(t, err, errChecksumGeneratorUnset) + + require.NoError(t, d.LoadSnapshot(newSnapshot(20))) + err = d.ProcessUpdate(&Update{UpdateTime: time.Now(), Asks: Levels{{Price: 1337.5, Amount: 69420, ID: 69420}}, ExpectedChecksum: 1337, GenerateChecksum: func(*Book) uint32 { return 1336 }}) + require.ErrorIs(t, err, errChecksumMismatch) + + require.NoError(t, d.LoadSnapshot(newSnapshot(20))) + err = d.ProcessUpdate(&Update{UpdateTime: time.Now(), Asks: Levels{{Price: 1337.5, Amount: 69420, ID: 69420}}, ExpectedChecksum: 1337, GenerateChecksum: func(*Book) uint32 { return 1337 }}) + require.NoError(t, err) + + require.NoError(t, d.LoadSnapshot(newSnapshot(20))) + d.askLevels.Levels[0].Amount = 0 + d.validateOrderbook = false // Disable verification + err = d.ProcessUpdate(&Update{UpdateTime: time.Now(), Asks: Levels{{Price: 1337.5, Amount: 69420, ID: 69420}}, ExpectedChecksum: 1337, GenerateChecksum: func(*Book) uint32 { return 1337 }}) + require.NoError(t, err, "must not error when ValidateOrderbook is false") +} + +func TestUpdate(t *testing.T) { + t.Parallel() + d := NewDepth(id) + + require.NoError(t, d.LoadSnapshot(newSnapshot(20))) + err := d.update(&Update{}) + assert.ErrorIs(t, err, errInvalidAction, "update should error correctly") + + err = d.update(&Update{Action: UpdateAction, UpdateTime: time.Now(), Asks: Levels{{Price: 1338, Amount: 69420, ID: 69420}}}) + assert.ErrorIs(t, err, errUpdateFailed, "update should error correctly") + assert.ErrorContains(t, err, "Update") + err = d.update(&Update{Action: UpdateAction, UpdateTime: time.Now(), Asks: Levels{{Price: 1338, Amount: 69420, ID: 21}}}) + assert.NoError(t, err, "update should not error") + ob, err := d.Retrieve() + require.NoError(t, err) + assert.Equal(t, 69420.0, ob.Asks[0].Amount, "First ask amount should be correct") + + require.NoError(t, d.LoadSnapshot(newSnapshot(20))) + err = d.update(&Update{Action: DeleteAction, UpdateTime: time.Now(), Asks: Levels{{Price: 1338, Amount: 1, ID: 69420}}}) + assert.ErrorIs(t, err, errDeleteFailed, "update should error correctly") + assert.ErrorContains(t, err, "Delete") + err = d.update(&Update{Action: DeleteAction, UpdateTime: time.Now(), Asks: Levels{{ID: 21}}}) + assert.NoError(t, err, "update should not error") + ob, err = d.Retrieve() + require.NoError(t, err) + assert.NotEqual(t, 21, ob.Asks[0].ID, "Ask element should be deleted") + assert.Len(t, ob.Asks, 19, "Asks length should be correct") + + require.NoError(t, d.LoadSnapshot(newSnapshot(20))) + err = d.update(&Update{Action: InsertAction, UpdateTime: time.Now(), Asks: Levels{{Price: 1338, Amount: 1, ID: 21}}}) + assert.ErrorIs(t, err, errUpdateFailed, "update should error correctly") + assert.ErrorContains(t, err, "Insert") + err = d.update(&Update{Action: InsertAction, UpdateTime: time.Now(), Asks: Levels{{Price: 1337.5, Amount: 1, ID: 69420}}}) + assert.NoError(t, err, "update should not error") + ob, err = d.Retrieve() + require.NoError(t, err) + assert.Equal(t, int64(69420), ob.Asks[0].ID, "First ask ID should be correct") + + require.NoError(t, d.LoadSnapshot(newSnapshot(20))) + err = d.update(&Update{Action: UpdateOrInsertAction, UpdateTime: time.Now(), Asks: Levels{{Price: 1338, Amount: 0, ID: 21}}}) + assert.ErrorIs(t, err, errUpdateFailed, "update should error correctly") + assert.ErrorContains(t, err, "UpdateOrInsert") + err = d.update(&Update{Action: UpdateOrInsertAction, UpdateTime: time.Now(), Asks: Levels{{Price: 1337.5, Amount: 1, ID: 69420}}}) + assert.NoError(t, err, "update should not error") + ob, err = d.Retrieve() + require.NoError(t, err) + assert.Equal(t, int64(69420), ob.Asks[0].ID, "First ask ID should be correct") +} + +func TestUpdateBidAskByID(t *testing.T) { + t.Parallel() + d := NewDepth(id) + err := d.LoadSnapshot(&Book{Bids: Levels{{Price: 1337, Amount: 1, ID: 1}}, Asks: Levels{{Price: 1337, Amount: 10, ID: 2}}, LastUpdated: time.Now(), LastPushed: time.Now()}) + assert.NoError(t, err, "LoadSnapshot should not error") + + updates := &Update{ + Bids: Levels{{Price: 1337, Amount: 2, ID: 1}}, + Asks: Levels{{Price: 1337, Amount: 2, ID: 2}}, + } + + err = d.updateBidAskByID(updates) + assert.ErrorIs(t, err, ErrLastUpdatedNotSet, "UpdateBidAskByID should error correctly") + + updates.UpdateTime = time.Now() + err = d.updateBidAskByID(updates) + assert.NoError(t, err, "UpdateBidAskByID should not error") + + ob, err := d.Retrieve() + assert.NoError(t, err, "Retrieve should not error") + assert.Equal(t, 2.0, ob.Asks[0].Amount, "First ask amount should be correct") + assert.Equal(t, 2.0, ob.Bids[0].Amount, "First bid amount should be correct") + + updates = &Update{ + Bids: Levels{{Price: 1337, Amount: 2, ID: 666}}, + UpdateTime: time.Now(), + } + // random unmatching IDs + err = d.updateBidAskByID(updates) + assert.ErrorIs(t, err, errIDCannotBeMatched, "UpdateBidAskByID should error correctly") + + updates = &Update{ + Asks: Levels{{Price: 1337, Amount: 2, ID: 69}}, + UpdateTime: time.Now(), + } + err = d.updateBidAskByID(updates) + assert.ErrorIs(t, err, errIDCannotBeMatched, "UpdateBidAskByID should error correctly") +} + +func TestDelete(t *testing.T) { + t.Parallel() + d := NewDepth(id) + err := d.LoadSnapshot(&Book{Bids: Levels{{Price: 1337, Amount: 1, ID: 1}}, Asks: Levels{{Price: 1337, Amount: 10, ID: 2}}, LastUpdated: time.Now(), LastPushed: time.Now()}) + assert.NoError(t, err, "LoadSnapshot should not error") + + updates := &Update{ + Bids: Levels{{Price: 1337, Amount: 2, ID: 1}}, + Asks: Levels{{Price: 1337, Amount: 2, ID: 2}}, + } + + err = d.delete(updates, false) + assert.ErrorIs(t, err, ErrLastUpdatedNotSet, "delete should error correctly") + + updates.UpdateTime = time.Now() + err = d.delete(updates, false) + assert.NoError(t, err, "delete should not error") + + ob, err := d.Retrieve() + assert.NoError(t, err, "Retrieve should not error") + assert.Empty(t, ob.Asks, "Asks should be empty") + assert.Empty(t, ob.Bids, "Bids should be empty") + + updates = &Update{ + Bids: Levels{{Price: 1337, Amount: 2, ID: 1}}, + UpdateTime: time.Now(), + } + err = d.delete(updates, false) + assert.ErrorIs(t, err, errIDCannotBeMatched, "delete should error correctly") + + updates = &Update{ + Asks: Levels{{Price: 1337, Amount: 2, ID: 2}}, + UpdateTime: time.Now(), + } + err = d.delete(updates, false) + assert.ErrorIs(t, err, errIDCannotBeMatched, "delete should error correctly") + + updates = &Update{ + Asks: Levels{{Price: 1337, Amount: 2, ID: 2}}, + UpdateTime: time.Now(), + } + err = d.delete(updates, true) + assert.NoError(t, err, "delete should not error") +} + +func TestInsert(t *testing.T) { + t.Parallel() + d := NewDepth(id) + err := d.LoadSnapshot(&Book{Bids: Levels{{Price: 1337, Amount: 1, ID: 1}}, Asks: Levels{{Price: 1337, Amount: 10, ID: 2}}, LastUpdated: time.Now(), LastPushed: time.Now()}) + assert.NoError(t, err, "LoadSnapshot should not error") + + updates := &Update{ + Asks: Levels{{Price: 1337, Amount: 2, ID: 3}}, + } + err = d.insert(updates) + assert.ErrorIs(t, err, ErrLastUpdatedNotSet, "insert should error correctly") + + updates.UpdateTime = time.Now() + + err = d.insert(updates) + assert.ErrorIs(t, err, errCollisionDetected, "insert should error correctly on collision") + + err = d.LoadSnapshot(&Book{Bids: Levels{{Price: 1337, Amount: 1, ID: 1}}, Asks: Levels{{Price: 1337, Amount: 10, ID: 2}}, LastUpdated: time.Now(), LastPushed: time.Now()}) + assert.NoError(t, err, "LoadSnapshot should not error") + + updates = &Update{ + Bids: Levels{{Price: 1337, Amount: 2, ID: 3}}, + UpdateTime: time.Now(), + } + + err = d.insert(updates) + assert.ErrorIs(t, err, errCollisionDetected, "insert should error correctly on collision") + + err = d.LoadSnapshot(&Book{Bids: Levels{{Price: 1337, Amount: 1, ID: 1}}, Asks: Levels{{Price: 1337, Amount: 10, ID: 2}}, LastUpdated: time.Now(), LastPushed: time.Now()}) + assert.NoError(t, err, "LoadSnapshot should not error") + + updates = &Update{ + Bids: Levels{{Price: 1338, Amount: 2, ID: 3}}, + Asks: Levels{{Price: 1336, Amount: 2, ID: 4}}, + UpdateTime: time.Now(), + } + err = d.insert(updates) + assert.NoError(t, err, "InsertBidAskByID should not error") + + ob, err := d.Retrieve() + assert.NoError(t, err, "Retrieve should not error") + assert.Len(t, ob.Asks, 2, "Should have correct Asks") + assert.Len(t, ob.Bids, 2, "Should have correct Bids") +} + +func TestUpdateOrInsert(t *testing.T) { + t.Parallel() + d := NewDepth(id) + err := d.LoadSnapshot(&Book{Bids: Levels{{Price: 1337, Amount: 1, ID: 1}}, Asks: Levels{{Price: 1337, Amount: 10, ID: 2}}, LastUpdated: time.Now(), LastPushed: time.Now()}) + assert.NoError(t, err, "LoadSnapshot should not error") + + updates := &Update{ + Bids: Levels{{Price: 1338, Amount: 0, ID: 3}}, + Asks: Levels{{Price: 1336, Amount: 2, ID: 4}}, + } + err = d.updateOrInsert(updates) + assert.ErrorIs(t, err, ErrLastUpdatedNotSet, "updateOrInsert should error correctly") + + updates.UpdateTime = time.Now() + err = d.updateOrInsert(updates) + assert.ErrorIs(t, err, errAmountCannotBeLessOrEqualToZero, "updateOrInsert should error correctly") + + err = d.LoadSnapshot(&Book{Bids: Levels{{Price: 1337, Amount: 1, ID: 1}}, Asks: Levels{{Price: 1337, Amount: 10, ID: 2}}, LastUpdated: time.Now(), LastPushed: time.Now()}) + assert.NoError(t, err, "LoadSnapshot should not error") + + updates = &Update{ + Bids: Levels{{Price: 1338, Amount: 2, ID: 3}}, + Asks: Levels{{Price: 1336, Amount: 0, ID: 4}}, + UpdateTime: time.Now(), + } + err = d.updateOrInsert(updates) + assert.ErrorIs(t, err, errAmountCannotBeLessOrEqualToZero, "updateOrInsert should error correctly") + + err = d.LoadSnapshot(&Book{Bids: Levels{{Price: 1337, Amount: 1, ID: 1}}, Asks: Levels{{Price: 1337, Amount: 10, ID: 2}}, LastUpdated: time.Now(), LastPushed: time.Now()}) + assert.NoError(t, err, "LoadSnapshot should not error") + + updates = &Update{ + Bids: Levels{{Price: 1338, Amount: 2, ID: 3}}, + Asks: Levels{{Price: 1336, Amount: 2, ID: 4}}, + UpdateTime: time.Now(), + } + err = d.updateOrInsert(updates) + assert.NoError(t, err, "updateOrInsert should not error") + + ob, err := d.Retrieve() + assert.NoError(t, err, "Retrieve should not error") + assert.Len(t, ob.Asks, 2, "Should have correct Asks") + assert.Len(t, ob.Bids, 2, "Should have correct Bids") +} + +func TestUpdateBidAskByPrice(t *testing.T) { + t.Parallel() + d := NewDepth(id) + err := d.LoadSnapshot(&Book{Bids: Levels{{Price: 1337, Amount: 1, ID: 1}}, Asks: Levels{{Price: 1338, Amount: 10, ID: 2}}, LastUpdated: time.Now(), LastPushed: time.Now()}) + assert.NoError(t, err, "LoadSnapshot should not error") + + err = d.updateBidAskByPrice(&Update{}) + assert.ErrorIs(t, err, ErrLastUpdatedNotSet, "UpdateBidAskByPrice should error correctly") + + err = d.updateBidAskByPrice(&Update{UpdateTime: time.Now()}) + assert.NoError(t, err, "UpdateBidAskByPrice should not error") + + updates := &Update{ + Bids: Levels{{Price: 1337, Amount: 2, ID: 1}}, + Asks: Levels{{Price: 1338, Amount: 3, ID: 2}}, + UpdateID: 1, + UpdateTime: time.Now(), + } + err = d.updateBidAskByPrice(updates) + assert.NoError(t, err, "UpdateBidAskByPrice should not error") + + ob, err := d.Retrieve() + assert.NoError(t, err, "Retrieve should not error") + assert.Equal(t, 3.0, ob.Asks[0].Amount, "Asks amount should be correct") + assert.Equal(t, 2.0, ob.Bids[0].Amount, "Bids amount should be correct") + + updates = &Update{ + Bids: Levels{{Price: 1337, Amount: 0, ID: 1}}, + Asks: Levels{{Price: 1338, Amount: 0, ID: 2}}, + UpdateID: 2, + UpdateTime: time.Now(), + } + err = d.updateBidAskByPrice(updates) + assert.NoError(t, err, "UpdateBidAskByPrice should not error") + + askLen, err := d.GetAskLength() + assert.NoError(t, err, "GetAskLength should not error") + assert.Zero(t, askLen, "Ask Length should be correct") + + bidLen, err := d.GetBidLength() + assert.NoError(t, err, "GetBidLength should not error") + assert.Zero(t, bidLen, "Bid Length should be correct") +} + +func TestString(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + action ActionType + expected string + }{ + {action: UpdateAction, expected: "Update"}, + {action: InsertAction, expected: "Insert"}, + {action: UpdateOrInsertAction, expected: "UpdateOrInsert"}, + {action: DeleteAction, expected: "Delete"}, + {action: UnknownAction, expected: "Unknown"}, + {action: ActionType(69), expected: "Unknown(69)"}, + } { + t.Run(tc.expected, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tc.expected, tc.action.String(), "String representation should match") + }) + } +} diff --git a/exchanges/orderbook/levels.go b/exchanges/orderbook/levels.go index 87105c0f..4aff0629 100644 --- a/exchanges/orderbook/levels.go +++ b/exchanges/orderbook/levels.go @@ -68,7 +68,7 @@ updates: l[y].StrAmount = updts[x].StrAmount continue updates } - return fmt.Errorf("update error: %w ID: %d not found", errIDCannotBeMatched, updts[x].ID) + return fmt.Errorf("update error: %w; ID: %d not found", errIDCannotBeMatched, updts[x].ID) } return nil } diff --git a/exchanges/orderbook/levels_test.go b/exchanges/orderbook/levels_test.go index b06859f0..50ca57c7 100644 --- a/exchanges/orderbook/levels_test.go +++ b/exchanges/orderbook/levels_test.go @@ -1240,7 +1240,7 @@ func TestGetMovementByBaseAmount(t *testing.T) { t.Run(tt.Name, func(t *testing.T) { t.Parallel() depth := NewDepth(id) - err := depth.LoadSnapshot(tt.BidLiquidity, nil, 0, time.Now(), time.Now(), true) + err := depth.LoadSnapshot(&Book{Bids: tt.BidLiquidity, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) if err != nil { t.Fatal(err) } @@ -1372,7 +1372,7 @@ func TestGetBaseAmountFromNominalSlippage(t *testing.T) { t.Run(tt.Name, func(t *testing.T) { t.Parallel() depth := NewDepth(id) - err := depth.LoadSnapshot(tt.BidLiquidity, nil, 0, time.Now(), time.Now(), true) + err := depth.LoadSnapshot(&Book{Bids: tt.BidLiquidity, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") base, err := depth.bidLevels.hitBidsByNominalSlippage(tt.NominalSlippage, tt.ReferencePrice) @@ -1479,7 +1479,7 @@ func TestGetBaseAmountFromImpact(t *testing.T) { t.Run(tt.Name, func(t *testing.T) { t.Parallel() depth := NewDepth(id) - err := depth.LoadSnapshot(tt.BidLiquidity, nil, 0, time.Now(), time.Now(), true) + err := depth.LoadSnapshot(&Book{Bids: tt.BidLiquidity, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) if err != nil { t.Fatal(err) } @@ -1563,7 +1563,7 @@ func TestGetMovementByQuoteAmount(t *testing.T) { t.Run(tt.Name, func(t *testing.T) { t.Parallel() depth := NewDepth(id) - err := depth.LoadSnapshot(nil, tt.AskLiquidity, 0, time.Now(), time.Now(), true) + err := depth.LoadSnapshot(&Book{Asks: tt.AskLiquidity, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) if err != nil { t.Fatal(err) } @@ -1693,7 +1693,7 @@ func TestGetQuoteAmountFromNominalSlippage(t *testing.T) { t.Run(tt.Name, func(t *testing.T) { t.Parallel() depth := NewDepth(id) - err := depth.LoadSnapshot(nil, tt.AskLiquidity, 0, time.Now(), time.Now(), true) + err := depth.LoadSnapshot(&Book{Asks: tt.AskLiquidity, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") quote, err := depth.askLevels.liftAsksByNominalSlippage(tt.NominalSlippage, tt.ReferencePrice) @@ -1781,7 +1781,7 @@ func TestGetQuoteAmountFromImpact(t *testing.T) { t.Run(tt.Name, func(t *testing.T) { t.Parallel() depth := NewDepth(id) - err := depth.LoadSnapshot(nil, tt.AskLiquidity, 0, time.Now(), time.Now(), true) + err := depth.LoadSnapshot(&Book{Asks: tt.AskLiquidity, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) assert.NoError(t, err, "LoadSnapshot should not error") quote, err := depth.askLevels.liftAsksByImpactSlippage(tt.ImpactSlippage, tt.ReferencePrice) @@ -1802,7 +1802,7 @@ func TestGetHeadPrice(t *testing.T) { _, err = depth.askLevels.getHeadPriceNoLock() require.ErrorIs(t, err, errNoLiquidity) - err = depth.LoadSnapshot(bid, ask, 0, time.Now(), time.Now(), true) + err = depth.LoadSnapshot(&Book{Bids: bid, Asks: ask, LastUpdated: time.Now(), LastPushed: time.Now(), RestSnapshot: true}) require.NoError(t, err, "LoadSnapshot must not error") val, err := depth.bidLevels.getHeadPriceNoLock() diff --git a/exchanges/orderbook/orderbook.go b/exchanges/orderbook/orderbook.go index ae5b1289..8800df97 100644 --- a/exchanges/orderbook/orderbook.go +++ b/exchanges/orderbook/orderbook.go @@ -9,7 +9,6 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" "github.com/thrasher-corp/gocryptotrader/dispatch" "github.com/thrasher-corp/gocryptotrader/exchanges/asset" - "github.com/thrasher-corp/gocryptotrader/log" ) // Get checks and returns the orderbook given an exchange name and currency pair @@ -52,7 +51,8 @@ func (s *store) Update(b *Book) error { return err } } - if err := book.Depth.LoadSnapshot(b.Bids, b.Asks, b.LastUpdateID, b.LastUpdated, b.LastPushed, true); err != nil { + b.RestSnapshot = true + if err := book.Depth.LoadSnapshot(b); err != nil { return err } return s.signalMux.Publish(book.Depth, book.RouterID) @@ -80,7 +80,7 @@ func (s *store) track(b *Book) (book, error) { // DeployDepth used for subsystem deployment creates a depth item in the struct then returns a ptr to that Depth item func (s *store) DeployDepth(exchange string, p currency.Pair, a asset.Item) (*Depth, error) { if exchange == "" { - return nil, errExchangeNameUnset + return nil, ErrExchangeNameEmpty } if p.IsEmpty() { return nil, errPairNotSet @@ -153,105 +153,11 @@ func (b *Book) TotalAsksAmount() (amountCollated, total float64) { return amountCollated, total } -// Verify ensures that the orderbook items are correctly sorted prior to being -// set and will reject any book with incorrect values. -// Bids should always go from a high price to a low price and -// Asks should always go from a low price to a higher price -func (b *Book) Verify() error { - if !b.VerifyOrderbook { - return nil - } - - // Checking for both ask and bid lengths being zero has been removed and - // a warning has been put in place for some exchanges that return zero - // level books. In the event that there is a massive liquidity change where - // a book dries up, this will still update so we do not traverse potential - // incorrect old data. - if (len(b.Asks) == 0 || len(b.Bids) == 0) && !b.Asset.IsOptions() { - log.Warnf(log.OrderBook, - bookLengthIssue, - b.Exchange, - b.Pair, - b.Asset, - len(b.Bids), - len(b.Asks)) - } - err := checkAlignment(b.Bids, b.IsFundingRate, b.PriceDuplication, b.IDAlignment, b.ChecksumStringRequired, dsc, b.Exchange) - if err != nil { - return fmt.Errorf(bidLoadBookFailure, b.Exchange, b.Pair, b.Asset, err) - } - err = checkAlignment(b.Asks, b.IsFundingRate, b.PriceDuplication, b.IDAlignment, b.ChecksumStringRequired, asc, b.Exchange) - if err != nil { - return fmt.Errorf(askLoadBookFailure, b.Exchange, b.Pair, b.Asset, err) - } - return nil -} - -// checker defines specific functionality to determine ascending/descending -// validation -type checker func(current, previous Level) error - -// asc specifically defines ascending price check -var asc = func(current, previous Level) error { - if current.Price < previous.Price { - return errPriceOutOfOrder - } - return nil -} - -// dsc specifically defines descending price check -var dsc = func(current, previous Level) error { - if current.Price > previous.Price { - return errPriceOutOfOrder - } - return nil -} - -// checkAlignment validates full orderbook -func checkAlignment(depth Levels, fundingRate, priceDuplication, isIDAligned, requiresChecksumString bool, c checker, exch string) error { - for i := range depth { - if depth[i].Price == 0 { - switch { - case exch == "Bitfinex" && fundingRate: /* funding rate can be 0 it seems on Bitfinex */ - default: - return errPriceNotSet - } - } - - if depth[i].Amount <= 0 { - return errAmountInvalid - } - if fundingRate && depth[i].Period == 0 { - return errPeriodUnset - } - if requiresChecksumString && (depth[i].StrAmount == "" || depth[i].StrPrice == "") { - return errChecksumStringNotSet - } - - if i != 0 { - prev := i - 1 - if err := c(depth[i], depth[prev]); err != nil { - return err - } - if isIDAligned && depth[i].ID < depth[prev].ID { - return errIDOutOfOrder - } - if !priceDuplication && depth[i].Price == depth[prev].Price { - return errDuplication - } - if depth[i].ID != 0 && depth[i].ID == depth[prev].ID { - return errIDDuplication - } - } - } - return nil -} - // Process processes incoming orderbooks, creating or updating the orderbook // list func (b *Book) Process() error { if b.Exchange == "" { - return errExchangeNameUnset + return ErrExchangeNameEmpty } if b.Pair.IsEmpty() { @@ -266,7 +172,7 @@ func (b *Book) Process() error { b.LastUpdated = time.Now() } - if err := b.Verify(); err != nil { + if err := b.Validate(); err != nil { return err } return s.Update(b) diff --git a/exchanges/orderbook/orderbook_test.go b/exchanges/orderbook/orderbook_test.go index 20733527..0e4d3d0f 100644 --- a/exchanges/orderbook/orderbook_test.go +++ b/exchanges/orderbook/orderbook_test.go @@ -45,71 +45,68 @@ func TestSubscribeToExchangeOrderbooks(t *testing.T) { assert.NoError(t, err, "SubscribeToExchangeOrderbooks should not error") } -func TestVerify(t *testing.T) { +func TestValidate(t *testing.T) { t.Parallel() b := Book{ - Exchange: "TestExchange", - Asset: asset.Spot, - Pair: currency.NewBTCUSD(), - VerifyOrderbook: true, + Exchange: "TestExchange", + Asset: asset.Spot, + Pair: currency.NewBTCUSD(), + ValidateOrderbook: true, } - err := b.Verify() - if err != nil { - t.Fatalf("expecting %v error but received %v", nil, err) - } + require.NoError(t, b.Validate()) b.Asks = []Level{{ID: 1337, Price: 99, Amount: 1}, {ID: 1337, Price: 100, Amount: 1}} - err = b.Verify() + err := b.Validate() require.ErrorIs(t, err, errIDDuplication) b.Asks = []Level{{Price: 100, Amount: 1}, {Price: 100, Amount: 1}} - err = b.Verify() + err = b.Validate() require.ErrorIs(t, err, errDuplication) b.Asks = []Level{{Price: 100, Amount: 1}, {Price: 99, Amount: 1}} b.IsFundingRate = true - err = b.Verify() + err = b.Validate() require.ErrorIs(t, err, errPeriodUnset) b.IsFundingRate = false - err = b.Verify() + err = b.Validate() require.ErrorIs(t, err, errPriceOutOfOrder) b.Asks = []Level{{Price: 100, Amount: 1}, {Price: 100, Amount: 0}} - err = b.Verify() + err = b.Validate() require.ErrorIs(t, err, errAmountInvalid) b.Asks = []Level{{Price: 100, Amount: 1}, {Price: 0, Amount: 100}} - err = b.Verify() - require.ErrorIs(t, err, errPriceNotSet) + err = b.Validate() + require.ErrorIs(t, err, ErrPriceZero) b.Bids = []Level{{ID: 1337, Price: 100, Amount: 1}, {ID: 1337, Price: 99, Amount: 1}} - err = b.Verify() + err = b.Validate() require.ErrorIs(t, err, errIDDuplication) b.Bids = []Level{{Price: 100, Amount: 1}, {Price: 100, Amount: 1}} - err = b.Verify() + err = b.Validate() require.ErrorIs(t, err, errDuplication) b.Bids = []Level{{Price: 99, Amount: 1}, {Price: 100, Amount: 1}} b.IsFundingRate = true - err = b.Verify() + err = b.Validate() require.ErrorIs(t, err, errPeriodUnset) b.IsFundingRate = false - err = b.Verify() + err = b.Validate() require.ErrorIs(t, err, errPriceOutOfOrder) b.Bids = []Level{{Price: 100, Amount: 1}, {Price: 100, Amount: 0}} - err = b.Verify() + err = b.Validate() require.ErrorIs(t, err, errAmountInvalid) b.Bids = []Level{{Price: 100, Amount: 1}, {Price: 0, Amount: 100}} - err = b.Verify() - require.ErrorIs(t, err, errPriceNotSet) + err = b.Validate() + require.ErrorIs(t, err, ErrPriceZero) } func TestTotalBidsAmount(t *testing.T) { @@ -226,7 +223,7 @@ func TestBookGetDepth(t *testing.T) { func TestDeployDepth(t *testing.T) { pair := currency.NewBTCUSD() _, err := DeployDepth("", pair, asset.Spot) - require.ErrorIs(t, err, errExchangeNameUnset) + require.ErrorIs(t, err, ErrExchangeNameEmpty) _, err = DeployDepth("test", currency.EMPTYPAIR, asset.Spot) require.ErrorIs(t, err, errPairNotSet) _, err = DeployDepth("test", pair, asset.Empty) @@ -406,27 +403,23 @@ func levelsFixtureRandom() Levels { func TestSorting(t *testing.T) { var b Book - b.VerifyOrderbook = true + b.ValidateOrderbook = true b.Asks = levelsFixtureRandom() - err := b.Verify() + err := b.Validate() require.ErrorIs(t, err, errPriceOutOfOrder) b.Asks.SortAsks() - err = b.Verify() - if err != nil { - t.Fatal(err) - } + err = b.Validate() + require.NoError(t, err) b.Bids = levelsFixtureRandom() - err = b.Verify() + err = b.Validate() require.ErrorIs(t, err, errPriceOutOfOrder) b.Bids.SortBids() - err = b.Verify() - if err != nil { - t.Fatal(err) - } + err = b.Validate() + require.NoError(t, err) } func levelsFixture() Levels { @@ -438,17 +431,17 @@ func levelsFixture() Levels { } func TestReverse(t *testing.T) { - b := Book{VerifyOrderbook: true, Bids: levelsFixture()} - assert.ErrorIs(t, b.Verify(), errPriceOutOfOrder) + b := Book{ValidateOrderbook: true, Bids: levelsFixture()} + assert.ErrorIs(t, b.Validate(), errPriceOutOfOrder) b.Bids.Reverse() - assert.NoError(t, b.Verify()) + assert.NoError(t, b.Validate()) b.Asks = slices.Clone(b.Bids) - assert.ErrorIs(t, b.Verify(), errPriceOutOfOrder) + assert.ErrorIs(t, b.Validate(), errPriceOutOfOrder) b.Asks.Reverse() - assert.NoError(t, b.Verify()) + assert.NoError(t, b.Validate()) } // 705985 1856 ns/op 0 B/op 0 allocs/op @@ -534,23 +527,23 @@ func BenchmarkSortBidsDescending(b *testing.B) { func TestCheckAlignment(t *testing.T) { t.Parallel() itemWithFunding := Levels{{Amount: 1337, Price: 0, Period: 1337}} - err := checkAlignment(itemWithFunding, true, true, false, false, dsc, "Bitfinex") + err := checkAlignment(itemWithFunding, true, true, false, false, isDsc, "Bitfinex") if err != nil { t.Error(err) } - err = checkAlignment(itemWithFunding, false, true, false, false, dsc, "Bitfinex") - require.ErrorIs(t, err, errPriceNotSet) + err = checkAlignment(itemWithFunding, false, true, false, false, isDsc, "Bitfinex") + require.ErrorIs(t, err, ErrPriceZero) - err = checkAlignment(itemWithFunding, true, true, false, false, dsc, "Binance") - require.ErrorIs(t, err, errPriceNotSet) + err = checkAlignment(itemWithFunding, true, true, false, false, isDsc, "Binance") + require.ErrorIs(t, err, ErrPriceZero) itemWithFunding[0].Price = 1337 - err = checkAlignment(itemWithFunding, true, true, false, true, dsc, "Binance") + err = checkAlignment(itemWithFunding, true, true, false, true, isDsc, "Binance") require.ErrorIs(t, err, errChecksumStringNotSet) itemWithFunding[0].StrAmount = "1337.0000000" itemWithFunding[0].StrPrice = "1337.0000000" - err = checkAlignment(itemWithFunding, true, true, false, true, dsc, "Binance") + err = checkAlignment(itemWithFunding, true, true, false, true, isDsc, "Binance") require.NoError(t, err) } diff --git a/exchanges/orderbook/orderbook_types.go b/exchanges/orderbook/orderbook_types.go index 66856113..6acf0794 100644 --- a/exchanges/orderbook/orderbook_types.go +++ b/exchanges/orderbook/orderbook_types.go @@ -21,13 +21,13 @@ const ( // Public errors var ( ErrOrderbookNotFound = errors.New("cannot find orderbook(s)") + ErrPriceZero = errors.New("price cannot be zero") + ErrExchangeNameEmpty = errors.New("empty orderbook exchange name") ) var ( - errExchangeNameUnset = errors.New("orderbook exchange name not set") errPairNotSet = errors.New("orderbook currency pair not set") errAssetTypeNotSet = errors.New("orderbook asset type not set") - errPriceNotSet = errors.New("price cannot be zero") errAmountInvalid = errors.New("amount cannot be less or equal to zero") errPriceOutOfOrder = errors.New("pricing out of order") errIDOutOfOrder = errors.New("ID out of order") @@ -111,10 +111,10 @@ type Book struct { // prices in a payload PriceDuplication bool IsFundingRate bool - // VerifyOrderbook allows for a toggle between orderbook verification set by + // ValidateOrderbook allows for a toggle between orderbook verification set by // user configuration, this allows for a potential processing boost but // a potential for orderbook integrity being deminished. - VerifyOrderbook bool + ValidateOrderbook bool // RestSnapshot defines if the depth was applied via the REST protocol thus // an update cannot be applied via websocket mechanics and a resubscription // would need to take place to maintain book integrity @@ -141,45 +141,13 @@ type options struct { lastUpdateID int64 priceDuplication bool isFundingRate bool - verifyOrderbook bool + validateOrderbook bool restSnapshot bool idAligned bool checksumStringRequired bool maxDepth int } -// Action defines a set of differing states required to implement an incoming -// orderbook update used in conjunction with UpdateEntriesByID -type Action uint8 - -const ( - // Amend applies amount adjustment by ID - Amend Action = iota + 1 - // Delete removes price level from book by ID - Delete - // Insert adds price level to book - Insert - // UpdateInsert on conflict applies amount adjustment or appends new amount - // to book - UpdateInsert -) - -// Update and things and stuff -type Update struct { - UpdateID int64 - UpdateTime time.Time - LastPushed time.Time - Asset asset.Item - Action - Bids []Level - Asks []Level - Pair currency.Pair - // Checksum defines the expected value when the books have been verified - Checksum uint32 - // AllowEmpty, when true, permits loading an empty order book update to set an UpdateID without including actual data. - AllowEmpty bool -} - // Movement defines orderbook traversal details from either hitting the bids or // lifting the asks. type Movement struct { diff --git a/exchanges/orderbook/validate.go b/exchanges/orderbook/validate.go new file mode 100644 index 00000000..c665450d --- /dev/null +++ b/exchanges/orderbook/validate.go @@ -0,0 +1,96 @@ +package orderbook + +import ( + "fmt" + + "github.com/thrasher-corp/gocryptotrader/common" + "github.com/thrasher-corp/gocryptotrader/log" +) + +// checker defines specific functionality to determine ascending/descending validation +type checker func(current, previous Level) error + +// isAsc specifically defines ascending price check +var isAsc = func(current, previous Level) error { + if current.Price < previous.Price { + return errPriceOutOfOrder + } + return nil +} + +// isDsc specifically defines descending price check +var isDsc = func(current, previous Level) error { + if current.Price > previous.Price { + return errPriceOutOfOrder + } + return nil +} + +// Validate ensures that the orderbook items are correctly sorted and all fields are valid +// Bids should always go from a high price to a low price and Asks should always go from a low price to a higher price +func (b *Book) Validate() error { + if err := common.NilGuard(b); err != nil { + return err + } + if !b.ValidateOrderbook { + return nil + } + return validate(b) +} + +func validate(b *Book) error { + // Some exchanges may return empty sides, but it's not an error + // Options have empty sides too frequently for this warning to be useful + if (len(b.Asks) == 0 || len(b.Bids) == 0) && !b.Asset.IsOptions() { + log.Warnf(log.OrderBook, bookLengthIssue, b.Exchange, b.Pair, b.Asset, len(b.Bids), len(b.Asks)) + } + err := checkAlignment(b.Bids, b.IsFundingRate, b.PriceDuplication, b.IDAlignment, b.ChecksumStringRequired, isDsc, b.Exchange) + if err != nil { + return fmt.Errorf(bidLoadBookFailure, b.Exchange, b.Pair, b.Asset, err) + } + err = checkAlignment(b.Asks, b.IsFundingRate, b.PriceDuplication, b.IDAlignment, b.ChecksumStringRequired, isAsc, b.Exchange) + if err != nil { + return fmt.Errorf(askLoadBookFailure, b.Exchange, b.Pair, b.Asset, err) + } + return nil +} + +// checkAlignment validates an orderbook side is sequential and does not contain any invalid data +func checkAlignment(depth Levels, fundingRate, priceDuplication, isIDAligned, requiresChecksumString bool, c checker, exch string) error { + for i := range depth { + if depth[i].Price == 0 { + switch { + case exch == "Bitfinex" && fundingRate: /* funding rate can be 0 it seems on Bitfinex */ + default: + return ErrPriceZero + } + } + + if depth[i].Amount <= 0 { + return errAmountInvalid + } + if fundingRate && depth[i].Period == 0 { + return errPeriodUnset + } + if requiresChecksumString && (depth[i].StrAmount == "" || depth[i].StrPrice == "") { + return errChecksumStringNotSet + } + + if i != 0 { + prev := i - 1 + if err := c(depth[i], depth[prev]); err != nil { + return err + } + if isIDAligned && depth[i].ID < depth[prev].ID { + return errIDOutOfOrder + } + if !priceDuplication && depth[i].Price == depth[prev].Price { + return errDuplication + } + if depth[i].ID != 0 && depth[i].ID == depth[prev].ID { + return errIDDuplication + } + } + } + return nil +} diff --git a/exchanges/poloniex/poloniex_websocket.go b/exchanges/poloniex/poloniex_websocket.go index fc241ce8..5ebe1652 100644 --- a/exchanges/poloniex/poloniex_websocket.go +++ b/exchanges/poloniex/poloniex_websocket.go @@ -481,7 +481,7 @@ func (p *Poloniex) WsProcessOrderbookSnapshot(data []any) error { book.Asks.SortAsks() book.Bids.SortBids() book.Asset = asset.Spot - book.VerifyOrderbook = p.CanVerifyOrderbook + book.ValidateOrderbook = p.ValidateOrderbook book.LastUpdated = time.UnixMilli(tsMilli) book.Pair, err = currency.NewPairFromString(pair) if err != nil { diff --git a/exchanges/poloniex/poloniex_wrapper.go b/exchanges/poloniex/poloniex_wrapper.go index 7eef5f08..b094c07f 100644 --- a/exchanges/poloniex/poloniex_wrapper.go +++ b/exchanges/poloniex/poloniex_wrapper.go @@ -283,10 +283,10 @@ func (p *Poloniex) UpdateOrderbook(ctx context.Context, pair currency.Pair, asse return nil, err } callingBook := &orderbook.Book{ - Exchange: p.Name, - Pair: pair, - Asset: assetType, - VerifyOrderbook: p.CanVerifyOrderbook, + Exchange: p.Name, + Pair: pair, + Asset: assetType, + ValidateOrderbook: p.ValidateOrderbook, } orderbookNew, err := p.GetOrderbook(ctx, "", poloniexMaxOrderbookDepth) if err != nil { @@ -311,10 +311,10 @@ func (p *Poloniex) UpdateOrderbook(ctx context.Context, pair currency.Pair, asse } } book := &orderbook.Book{ - Exchange: p.Name, - Pair: enabledPairs[i], - Asset: assetType, - VerifyOrderbook: p.CanVerifyOrderbook, + Exchange: p.Name, + Pair: enabledPairs[i], + Asset: assetType, + ValidateOrderbook: p.ValidateOrderbook, } book.Bids = make(orderbook.Levels, len(data.Bids)) diff --git a/exchanges/yobit/yobit_wrapper.go b/exchanges/yobit/yobit_wrapper.go index c90ea9c9..45081560 100644 --- a/exchanges/yobit/yobit_wrapper.go +++ b/exchanges/yobit/yobit_wrapper.go @@ -200,10 +200,10 @@ func (y *Yobit) UpdateOrderbook(ctx context.Context, p currency.Pair, assetType return nil, err } book := &orderbook.Book{ - Exchange: y.Name, - Pair: p, - Asset: assetType, - VerifyOrderbook: y.CanVerifyOrderbook, + Exchange: y.Name, + Pair: p, + Asset: assetType, + ValidateOrderbook: y.ValidateOrderbook, } fPair, err := y.FormatExchangeCurrency(p, assetType) if err != nil {