From 7b718700f760423c11df86c7ca07b96520465a1d Mon Sep 17 00:00:00 2001 From: Ryan O'Hara-Reid Date: Fri, 23 Apr 2021 15:16:01 +1000 Subject: [PATCH] orderbook: Implement initial linked list (#643) * Exchanges: Initial implementation after rebase of depth (WIP) * orderbook/buffer: convert and couple orderbook interaction functionality from buffer to orderbook linked list - Use single point reference for orderbook depth * buffer/orderbook: conversion continued (WIP) * exchange: buffer/linkedlist handover (WIP) * Added some tests for yesterday * linkedList: added more testing and trying to figure out broken things * Started tying everything in * continuous integration and testing * orderbook: expanded tests * go mod tidy * Add in different synchornisation levels for protocols Add in timer for the streaming system to reduce updates to datahandler Add in more test code as I integrate more exchanges * Depth: Add tests, add length check to call linked list updating, add in constructor. Linked List: Improve tests, add in checks for zero liquidity on books. Node: Added in cleaner POC, add in contructor. Buffer: Fixed tests, checked benchmarks. * orderbook: reinstate dispatch calls * Addr glorious & madcozbad nits * fix functionality and add tests * Address linterinos * remove label * expanded comment * fix races and and bitmex test * reinstate go routine for alerting changes * rm line :D * fix more tests * Addr glorious nits * rm glorious field * depth: defer unlock to stop deadlock * orderbook: remove unused vars * buffer: fix test to what it should be * nits: madcosbad addr * nits: glorious nits * linkedlist: remove unused params * orderbook: shift time call to outside of push to inline, add in case for update inster price for zero liquidity, nits * orderbook: nits addressed * engine: change stream -> websocket convention and remove unused function * nits: glorious nits * Websocket Buffer: Add verbosity switch * linked list: Add comment * linked list: fix spelling * nits: glorious nits * orderbook: Adds in test and explicit time type with constructor, fix nits * linter * spelling: removed the dere fence * depth: Update alerting mechanism to a more battle tested state * depth: spelling * nits: glorious nits * linked list: match cases * buffer: fix linter issue * golangci: increase timeout by 30 seconds * nodes: update atomic checks * spelling: fix * node: add in commentary * exchanges/syncer: add function to switch over to REST when websocket functionality is not available for a specific asset type * linter: exchange linter issues * syncer: Add in warning * nits: glorious nits * AssetWebsocketSupport: unexport map * Nits: Adrr * rm letter * exchanges: Orderbook verification change for naming, deprecate checksum bypass as it has the potential to obfuscate errors that are at the tail end of the book, add in verification for websocket stream updates * general: fix spelling remove breakpoint * nits: fix more glorious nits until more are found * orderbook: fix tests * orderbook: fix wait tests and add in more checks * nits: addr * orderbook: remove dispatch reference * linkedlist: consolidate bid/ask functions * linked lisdt: remove words * fix spelling --- .golangci.yml | 2 +- cmd/exchange_template/wrapper_file.tmpl | 6 +- engine/engine.go | 21 +- engine/engine_types.go | 3 +- engine/events_test.go | 10 +- engine/helpers_test.go | 8 +- engine/routines.go | 14 +- engine/routines_test.go | 4 +- engine/rpcserver.go | 57 +- engine/syncer.go | 111 +- engine/syncer_types.go | 15 +- exchanges/alphapoint/alphapoint_wrapper.go | 4 +- exchanges/binance/binance_test.go | 12 +- exchanges/binance/binance_websocket.go | 10 +- exchanges/binance/binance_wrapper.go | 20 +- exchanges/bitfinex/bitfinex_websocket.go | 22 +- exchanges/bitfinex/bitfinex_wrapper.go | 10 +- exchanges/bitflyer/bitflyer_wrapper.go | 8 +- exchanges/bithumb/bithumb_wrapper.go | 8 +- exchanges/bitmex/bitmex_test.go | 2 +- exchanges/bitmex/bitmex_websocket.go | 8 +- exchanges/bitmex/bitmex_wrapper.go | 15 +- exchanges/bitstamp/bitstamp_websocket.go | 20 +- exchanges/bitstamp/bitstamp_wrapper.go | 8 +- exchanges/bittrex/bittrex_wrapper.go | 8 +- exchanges/btcmarkets/btcmarkets_websocket.go | 17 +- exchanges/btcmarkets/btcmarkets_wrapper.go | 10 +- exchanges/btse/btse_websocket.go | 8 +- exchanges/btse/btse_wrapper.go | 14 +- .../coinbasepro/coinbasepro_websocket.go | 6 +- exchanges/coinbasepro/coinbasepro_wrapper.go | 8 +- exchanges/coinbene/coinbene_websocket.go | 6 +- exchanges/coinbene/coinbene_wrapper.go | 8 +- exchanges/coinut/coinut_websocket.go | 6 +- exchanges/coinut/coinut_wrapper.go | 8 +- exchanges/exchange.go | 690 ++++---- exchanges/exchange_test.go | 46 + exchanges/exchange_types.go | 20 +- exchanges/exmo/exmo_wrapper.go | 16 +- exchanges/ftx/ftx_websocket.go | 20 +- exchanges/ftx/ftx_wrapper.go | 8 +- exchanges/gateio/gateio_websocket.go | 6 +- exchanges/gateio/gateio_wrapper.go | 8 +- exchanges/gemini/gemini_websocket.go | 6 +- exchanges/gemini/gemini_wrapper.go | 8 +- exchanges/hitbtc/hitbtc_websocket.go | 6 +- exchanges/hitbtc/hitbtc_wrapper.go | 8 +- exchanges/huobi/huobi_websocket.go | 6 +- exchanges/huobi/huobi_wrapper.go | 8 +- exchanges/interfaces.go | 1 + exchanges/itbit/itbit_wrapper.go | 10 +- exchanges/kraken/kraken_websocket.go | 27 +- exchanges/kraken/kraken_wrapper.go | 13 +- exchanges/lakebtc/lakebtc_websocket.go | 10 +- exchanges/lakebtc/lakebtc_wrapper.go | 8 +- exchanges/lbank/lbank_wrapper.go | 8 +- .../localbitcoins/localbitcoins_wrapper.go | 11 +- exchanges/okgroup/okgroup_websocket.go | 20 +- exchanges/okgroup/okgroup_wrapper.go | 8 +- exchanges/orderbook/calculator_test.go | 4 +- exchanges/orderbook/depth.go | 353 ++++ exchanges/orderbook/depth_test.go | 404 +++++ exchanges/orderbook/linked_list.go | 530 ++++++ exchanges/orderbook/linked_list_test.go | 1449 +++++++++++++++++ exchanges/orderbook/node.go | 135 ++ exchanges/orderbook/node_test.go | 101 ++ exchanges/orderbook/orderbook.go | 326 ++-- exchanges/orderbook/orderbook_test.go | 349 ++-- exchanges/orderbook/orderbook_types.go | 106 +- exchanges/poloniex/poloniex_websocket.go | 10 +- exchanges/poloniex/poloniex_wrapper.go | 16 +- exchanges/stream/buffer/buffer.go | 354 ++-- exchanges/stream/buffer/buffer_test.go | 477 +++--- exchanges/stream/buffer/buffer_types.go | 14 +- exchanges/stream/websocket.go | 1 + exchanges/yobit/yobit_wrapper.go | 8 +- exchanges/zb/zb_websocket.go | 8 +- exchanges/zb/zb_wrapper.go | 8 +- gctscript/modules/gct/exchange.go | 4 +- gctscript/wrappers/validator/validator.go | 6 +- main.go | 6 +- 81 files changed, 4546 insertions(+), 1592 deletions(-) create mode 100644 exchanges/orderbook/depth.go create mode 100644 exchanges/orderbook/depth_test.go create mode 100644 exchanges/orderbook/linked_list.go create mode 100644 exchanges/orderbook/linked_list_test.go create mode 100644 exchanges/orderbook/node.go create mode 100644 exchanges/orderbook/node_test.go diff --git a/.golangci.yml b/.golangci.yml index 6528b87f..c61cdb1a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,5 +1,5 @@ run: - timeout: 2m0s + timeout: 2m30s issues-exit-code: 1 tests: true skip-dirs: diff --git a/cmd/exchange_template/wrapper_file.tmpl b/cmd/exchange_template/wrapper_file.tmpl index 1fbbbd8c..af580c78 100644 --- a/cmd/exchange_template/wrapper_file.tmpl +++ b/cmd/exchange_template/wrapper_file.tmpl @@ -283,10 +283,10 @@ func ({{.Variable}} *{{.CapitalName}}) FetchOrderbook(currency currency.Pair, as // UpdateOrderbook updates and returns the orderbook for a currency pair func ({{.Variable}} *{{.CapitalName}}) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: {{.Variable}}.Name, + Exchange: {{.Variable}}.Name, Pair: p, - AssetType: assetType, - VerificationBypass: {{.Variable}}.OrderbookVerificationBypass, + Asset: assetType, + VerifyOrderbook: {{.Variable}}.CanVerifyOrderbook, } // NOTE: UPDATE ORDERBOOK EXAMPLE diff --git a/engine/engine.go b/engine/engine.go index ebcf7588..468c1d4a 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -208,7 +208,8 @@ func validateSettings(b *Engine, s *Settings, flagSet map[string]bool) { b.Settings.EnableOrderbookSyncing = s.EnableOrderbookSyncing b.Settings.EnableTradeSyncing = s.EnableTradeSyncing b.Settings.SyncWorkers = s.SyncWorkers - b.Settings.SyncTimeout = s.SyncTimeout + b.Settings.SyncTimeoutREST = s.SyncTimeoutREST + b.Settings.SyncTimeoutWebsocket = s.SyncTimeoutWebsocket b.Settings.SyncContinuously = s.SyncContinuously b.Settings.EnableDepositAddressManager = s.EnableDepositAddressManager b.Settings.EnableExchangeAutoPairUpdates = s.EnableExchangeAutoPairUpdates @@ -305,7 +306,8 @@ func PrintSettings(s *Settings) { gctlog.Debugf(gctlog.Global, "\t Enable ticker syncing: %v\n", s.EnableTickerSyncing) gctlog.Debugf(gctlog.Global, "\t Enable orderbook syncing: %v\n", s.EnableOrderbookSyncing) gctlog.Debugf(gctlog.Global, "\t Enable trade syncing: %v\n", s.EnableTradeSyncing) - gctlog.Debugf(gctlog.Global, "\t Exchange sync timeout: %v\n", s.SyncTimeout) + gctlog.Debugf(gctlog.Global, "\t Exchange REST sync timeout: %v\n", s.SyncTimeoutREST) + gctlog.Debugf(gctlog.Global, "\t Exchange Websocket sync timeout: %v\n", s.SyncTimeoutWebsocket) gctlog.Debugf(gctlog.Global, "- FOREX SETTINGS:") gctlog.Debugf(gctlog.Global, "\t Enable currency conveter: %v", s.EnableCurrencyConverter) gctlog.Debugf(gctlog.Global, "\t Enable currency layer: %v", s.EnableCurrencyLayer) @@ -464,13 +466,14 @@ func (bot *Engine) Start() error { if bot.Settings.EnableExchangeSyncManager { exchangeSyncCfg := CurrencyPairSyncerConfig{ - SyncTicker: bot.Settings.EnableTickerSyncing, - SyncOrderbook: bot.Settings.EnableOrderbookSyncing, - SyncTrades: bot.Settings.EnableTradeSyncing, - SyncContinuously: bot.Settings.SyncContinuously, - NumWorkers: bot.Settings.SyncWorkers, - Verbose: bot.Settings.Verbose, - SyncTimeout: bot.Settings.SyncTimeout, + SyncTicker: bot.Settings.EnableTickerSyncing, + SyncOrderbook: bot.Settings.EnableOrderbookSyncing, + SyncTrades: bot.Settings.EnableTradeSyncing, + SyncContinuously: bot.Settings.SyncContinuously, + NumWorkers: bot.Settings.SyncWorkers, + Verbose: bot.Settings.Verbose, + SyncTimeoutREST: bot.Settings.SyncTimeoutREST, + SyncTimeoutWebsocket: bot.Settings.SyncTimeoutWebsocket, } bot.ExchangeCurrencyPairManager, err = NewCurrencyPairSyncer(exchangeSyncCfg) diff --git a/engine/engine_types.go b/engine/engine_types.go index 1aaae2b7..d9bbf5d2 100644 --- a/engine/engine_types.go +++ b/engine/engine_types.go @@ -44,7 +44,8 @@ type Settings struct { EnableTradeSyncing bool SyncWorkers int SyncContinuously bool - SyncTimeout time.Duration + SyncTimeoutREST time.Duration + SyncTimeoutWebsocket time.Duration // Forex settings EnableCurrencyConverter bool diff --git a/engine/events_test.go b/engine/events_test.go index 0e88df76..b6016524 100644 --- a/engine/events_test.go +++ b/engine/events_test.go @@ -210,11 +210,11 @@ func TestProcessOrderbook(t *testing.T) { // now populate it with a 0 entry o := orderbook.Base{ - Pair: currency.NewPair(currency.BTC, currency.USD), - Bids: []orderbook.Item{{Amount: 24, Price: 23}}, - Asks: []orderbook.Item{{Amount: 24, Price: 23}}, - ExchangeName: e.Exchange, - AssetType: e.Asset, + Pair: currency.NewPair(currency.BTC, currency.USD), + Bids: []orderbook.Item{{Amount: 24, Price: 23}}, + Asks: []orderbook.Item{{Amount: 24, Price: 23}}, + Exchange: e.Exchange, + Asset: e.Asset, } if err := o.Process(); err != nil { t.Fatal("unexpected result:", err) diff --git a/engine/helpers_test.go b/engine/helpers_test.go index c8d554f0..07a2dca9 100644 --- a/engine/helpers_test.go +++ b/engine/helpers_test.go @@ -535,10 +535,10 @@ func TestGetSpecificOrderbook(t *testing.T) { bids = append(bids, orderbook.Item{Price: 1000, Amount: 1}) base := orderbook.Base{ - Pair: currency.NewPair(currency.BTC, currency.USD), - Bids: bids, - ExchangeName: "Bitstamp", - AssetType: asset.Spot, + Pair: currency.NewPair(currency.BTC, currency.USD), + Bids: bids, + Exchange: "Bitstamp", + Asset: asset.Spot, } err := base.Process() diff --git a/engine/routines.go b/engine/routines.go index 1789cec1..49deb313 100644 --- a/engine/routines.go +++ b/engine/routines.go @@ -132,17 +132,17 @@ func printOrderbookSummary(result *orderbook.Base, protocol string, bot *Engine, if err == common.ErrNotYetImplemented { log.Warnf(log.OrderBook, "Failed to get %s orderbook for %s %s %s. Error: %s\n", protocol, - result.ExchangeName, + result.Exchange, result.Pair, - result.AssetType, + result.Asset, err) return } log.Errorf(log.OrderBook, "Failed to get %s orderbook for %s %s %s. Error: %s\n", protocol, - result.ExchangeName, + result.Exchange, result.Pair, - result.AssetType, + result.Asset, err) return } @@ -165,10 +165,10 @@ func printOrderbookSummary(result *orderbook.Base, protocol string, bot *Engine, } log.Infof(log.OrderBook, book, - result.ExchangeName, + result.Exchange, protocol, bot.FormatCurrency(result.Pair), - strings.ToUpper(result.AssetType.String()), + strings.ToUpper(result.Asset.String()), len(result.Bids), bidsAmount, result.Pair.Base, @@ -317,7 +317,7 @@ func (bot *Engine) WebsocketDataHandler(exchName string, data interface{}) error if bot.Settings.EnableExchangeSyncManager && bot.ExchangeCurrencyPairManager != nil { bot.ExchangeCurrencyPairManager.update(exchName, d.Pair, - d.AssetType, + d.Asset, SyncItemOrderbook, nil) } diff --git a/engine/routines_test.go b/engine/routines_test.go index a691116d..af49f40a 100644 --- a/engine/routines_test.go +++ b/engine/routines_test.go @@ -118,8 +118,8 @@ func TestHandleData(t *testing.T) { } err = b.WebsocketDataHandler(exchName, &orderbook.Base{ - ExchangeName: fakePassExchange, - Pair: currency.NewPair(currency.BTC, currency.USD), + Exchange: fakePassExchange, + Pair: currency.NewPair(currency.BTC, currency.USD), }) if err != nil { t.Error(err) diff --git a/engine/rpcserver.go b/engine/rpcserver.go index 1548eb29..e550844d 100644 --- a/engine/rpcserver.go +++ b/engine/rpcserver.go @@ -1672,45 +1672,37 @@ func (s *RPCServer) GetOrderbookStream(r *gctrpc.GetOrderbookStreamRequest, stre return err } - pipe, err := orderbook.SubscribeOrderbook(r.Exchange, p, a) + depth, err := orderbook.GetDepth(r.Exchange, p, a) if err != nil { return err } - defer pipe.Release() - for { - data, ok := <-pipe.C - if !ok { - return errDispatchSystem + base := depth.Retrieve() + bids := make([]*gctrpc.OrderbookItem, len(base.Bids)) + for i := range base.Bids { + bids[i] = &gctrpc.OrderbookItem{ + Amount: base.Bids[i].Amount, + Price: base.Bids[i].Price, + Id: base.Bids[i].ID} } - - ob := (*data.(*interface{})).(orderbook.Base) - var bids, asks []*gctrpc.OrderbookItem - for i := range ob.Bids { - bids = append(bids, &gctrpc.OrderbookItem{ - Amount: ob.Bids[i].Amount, - Price: ob.Bids[i].Price, - Id: ob.Bids[i].ID, - }) - } - for i := range ob.Asks { - asks = append(asks, &gctrpc.OrderbookItem{ - Amount: ob.Asks[i].Amount, - Price: ob.Asks[i].Price, - Id: ob.Asks[i].ID, - }) + asks := make([]*gctrpc.OrderbookItem, len(base.Asks)) + for i := range base.Asks { + asks[i] = &gctrpc.OrderbookItem{ + Amount: base.Asks[i].Amount, + Price: base.Asks[i].Price, + Id: base.Asks[i].ID} } err := stream.Send(&gctrpc.OrderbookResponse{ - Pair: &gctrpc.CurrencyPair{Base: ob.Pair.Base.String(), - Quote: ob.Pair.Quote.String()}, + Pair: &gctrpc.CurrencyPair{Base: r.Pair.Base, Quote: r.Pair.Quote}, Bids: bids, Asks: asks, - AssetType: ob.AssetType.String(), + AssetType: r.AssetType, }) if err != nil { return err } + <-depth.Wait(nil) } } @@ -1734,27 +1726,26 @@ func (s *RPCServer) GetExchangeOrderbookStream(r *gctrpc.GetExchangeOrderbookStr } ob := (*data.(*interface{})).(orderbook.Base) - var bids, asks []*gctrpc.OrderbookItem + bids := make([]*gctrpc.OrderbookItem, len(ob.Bids)) for i := range ob.Bids { - bids = append(bids, &gctrpc.OrderbookItem{ + bids[i] = &gctrpc.OrderbookItem{ Amount: ob.Bids[i].Amount, Price: ob.Bids[i].Price, - Id: ob.Bids[i].ID, - }) + Id: ob.Bids[i].ID} } + asks := make([]*gctrpc.OrderbookItem, len(ob.Asks)) for i := range ob.Asks { - asks = append(asks, &gctrpc.OrderbookItem{ + asks[i] = &gctrpc.OrderbookItem{ Amount: ob.Asks[i].Amount, Price: ob.Asks[i].Price, - Id: ob.Asks[i].ID, - }) + Id: ob.Asks[i].ID} } err := stream.Send(&gctrpc.OrderbookResponse{ Pair: &gctrpc.CurrencyPair{Base: ob.Pair.Base.String(), Quote: ob.Pair.Quote.String()}, Bids: bids, Asks: asks, - AssetType: ob.AssetType.String(), + AssetType: ob.Asset.String(), }) if err != nil { return err diff --git a/engine/syncer.go b/engine/syncer.go index 87a1a65d..b14b68d2 100644 --- a/engine/syncer.go +++ b/engine/syncer.go @@ -18,8 +18,9 @@ const ( SyncItemOrderbook SyncItemTrade - DefaultSyncerWorkers = 15 - DefaultSyncerTimeout = time.Second * 15 + DefaultSyncerWorkers = 15 + DefaultSyncerTimeoutREST = time.Second * 15 + DefaultSyncerTimeoutWebsocket = time.Minute ) var ( @@ -37,28 +38,25 @@ func NewCurrencyPairSyncer(c CurrencyPairSyncerConfig) (*ExchangeCurrencyPairSyn c.NumWorkers = DefaultSyncerWorkers } - if c.SyncTimeout <= time.Duration(0) { - c.SyncTimeout = DefaultSyncerTimeout + if c.SyncTimeoutREST <= time.Duration(0) { + c.SyncTimeoutREST = DefaultSyncerTimeoutREST } - s := ExchangeCurrencyPairSyncer{ - Cfg: CurrencyPairSyncerConfig{ - SyncTicker: c.SyncTicker, - SyncOrderbook: c.SyncOrderbook, - SyncTrades: c.SyncTrades, - SyncContinuously: c.SyncContinuously, - SyncTimeout: c.SyncTimeout, - NumWorkers: c.NumWorkers, - }, + if c.SyncTimeoutWebsocket <= time.Duration(0) { + c.SyncTimeoutWebsocket = DefaultSyncerTimeoutWebsocket } + s := ExchangeCurrencyPairSyncer{Cfg: c} + s.tickerBatchLastRequested = make(map[string]time.Time) log.Debugf(log.SyncMgr, "Exchange currency pair syncer config: continuous: %v ticker: %v"+ - " orderbook: %v trades: %v workers: %v verbose: %v timeout: %v\n", + " orderbook: %v trades: %v workers: %v verbose: %v timeout REST: %v"+ + " timeout Websocket: %v\n", s.Cfg.SyncContinuously, s.Cfg.SyncTicker, s.Cfg.SyncOrderbook, - s.Cfg.SyncTrades, s.Cfg.NumWorkers, s.Cfg.Verbose, s.Cfg.SyncTimeout) + s.Cfg.SyncTrades, s.Cfg.NumWorkers, s.Cfg.Verbose, s.Cfg.SyncTimeoutREST, + s.Cfg.SyncTimeoutWebsocket) return &s, nil } @@ -138,20 +136,6 @@ func (e *ExchangeCurrencyPairSyncer) add(c *CurrencyPairSyncAgent) { e.CurrencyPairs = append(e.CurrencyPairs, *c) } -func (e *ExchangeCurrencyPairSyncer) remove(c *CurrencyPairSyncAgent) { - e.mux.Lock() - defer e.mux.Unlock() - - for x := range e.CurrencyPairs { - if e.CurrencyPairs[x].Exchange == c.Exchange && - e.CurrencyPairs[x].Pair.Equal(c.Pair) && - e.CurrencyPairs[x].AssetType == c.AssetType { - e.CurrencyPairs = append(e.CurrencyPairs[:x], e.CurrencyPairs[x+1:]...) - return - } - } -} - func (e *ExchangeCurrencyPairSyncer) isProcessing(exchangeName string, p currency.Pair, a asset.Item, syncType int) bool { e.mux.Lock() defer e.mux.Unlock() @@ -325,6 +309,8 @@ func (e *ExchangeCurrencyPairSyncer) worker() { if exchanges[x].GetBase().CurrencyPairs.IsAssetEnabled(assetTypes[y]) != nil { continue } + + wsAssetSupported := exchanges[x].IsAssetWebsocketSupported(assetTypes[y]) enabledPairs, err := exchanges[x].GetEnabledPairs(assetTypes[y]) if err != nil { log.Errorf(log.SyncMgr, @@ -345,25 +331,21 @@ func (e *ExchangeCurrencyPairSyncer) worker() { Pair: enabledPairs[i], } + sBase := SyncBase{ + IsUsingREST: usingREST || !wsAssetSupported, + IsUsingWebsocket: usingWebsocket && wsAssetSupported, + } + if e.Cfg.SyncTicker { - c.Ticker = SyncBase{ - IsUsingREST: usingREST, - IsUsingWebsocket: usingWebsocket, - } + c.Ticker = sBase } if e.Cfg.SyncOrderbook { - c.Orderbook = SyncBase{ - IsUsingREST: usingREST, - IsUsingWebsocket: usingWebsocket, - } + c.Orderbook = sBase } if e.Cfg.SyncTrades { - c.Trade = SyncBase{ - IsUsingREST: usingREST, - IsUsingWebsocket: usingWebsocket, - } + c.Trade = sBase } e.add(&c) @@ -382,9 +364,11 @@ func (e *ExchangeCurrencyPairSyncer) worker() { } if e.Cfg.SyncTicker { if !e.isProcessing(exchangeName, c.Pair, c.AssetType, SyncItemTicker) { - if c.Ticker.LastUpdated.IsZero() || time.Since(c.Ticker.LastUpdated) > e.Cfg.SyncTimeout { + if c.Ticker.LastUpdated.IsZero() || + (time.Since(c.Ticker.LastUpdated) > e.Cfg.SyncTimeoutREST && c.Ticker.IsUsingREST) || + (time.Since(c.Ticker.LastUpdated) > e.Cfg.SyncTimeoutWebsocket && c.Ticker.IsUsingWebsocket) { if c.Ticker.IsUsingWebsocket { - if time.Since(c.Created) < e.Cfg.SyncTimeout { + if time.Since(c.Created) < e.Cfg.SyncTimeoutWebsocket { continue } @@ -397,7 +381,7 @@ func (e *ExchangeCurrencyPairSyncer) worker() { c.Exchange, Bot.FormatCurrency(enabledPairs[i]).String(), strings.ToUpper(c.AssetType.String()), - e.Cfg.SyncTimeout, + e.Cfg.SyncTimeoutWebsocket, ) switchedToRest = true e.setProcessing(c.Exchange, c.Pair, c.AssetType, SyncItemTicker, false) @@ -417,7 +401,7 @@ func (e *ExchangeCurrencyPairSyncer) worker() { } e.mux.Unlock() - if batchLastDone.IsZero() || time.Since(batchLastDone) > e.Cfg.SyncTimeout { + if batchLastDone.IsZero() || time.Since(batchLastDone) > e.Cfg.SyncTimeoutREST { e.mux.Lock() if e.Cfg.Verbose { log.Debugf(log.SyncMgr, "%s Init'ing REST ticker batching\n", exchangeName) @@ -450,9 +434,11 @@ func (e *ExchangeCurrencyPairSyncer) worker() { if e.Cfg.SyncOrderbook { if !e.isProcessing(exchangeName, c.Pair, c.AssetType, SyncItemOrderbook) { - if c.Orderbook.LastUpdated.IsZero() || time.Since(c.Orderbook.LastUpdated) > e.Cfg.SyncTimeout { + if c.Orderbook.LastUpdated.IsZero() || + (time.Since(c.Orderbook.LastUpdated) > e.Cfg.SyncTimeoutREST && c.Orderbook.IsUsingREST) || + (time.Since(c.Orderbook.LastUpdated) > e.Cfg.SyncTimeoutWebsocket && c.Orderbook.IsUsingWebsocket) { if c.Orderbook.IsUsingWebsocket { - if time.Since(c.Created) < e.Cfg.SyncTimeout { + if time.Since(c.Created) < e.Cfg.SyncTimeoutWebsocket { continue } if supportsREST { @@ -464,7 +450,7 @@ func (e *ExchangeCurrencyPairSyncer) worker() { c.Exchange, Bot.FormatCurrency(c.Pair).String(), strings.ToUpper(c.AssetType.String()), - e.Cfg.SyncTimeout, + e.Cfg.SyncTimeoutWebsocket, ) switchedToRest = true e.setProcessing(c.Exchange, c.Pair, c.AssetType, SyncItemOrderbook, false) @@ -486,7 +472,7 @@ func (e *ExchangeCurrencyPairSyncer) worker() { } if e.Cfg.SyncTrades { if !e.isProcessing(exchangeName, c.Pair, c.AssetType, SyncItemTrade) { - if c.Trade.LastUpdated.IsZero() || time.Since(c.Trade.LastUpdated) > e.Cfg.SyncTimeout { + if c.Trade.LastUpdated.IsZero() || time.Since(c.Trade.LastUpdated) > e.Cfg.SyncTimeoutREST { e.setProcessing(c.Exchange, c.Pair, c.AssetType, SyncItemTrade, true) e.update(c.Exchange, c.Pair, c.AssetType, SyncItemTrade, nil) } @@ -560,6 +546,13 @@ func (e *ExchangeCurrencyPairSyncer) Start() { continue } + wsAssetSupported := exchanges[x].IsAssetWebsocketSupported(assetTypes[y]) + if !wsAssetSupported { + log.Warnf(log.SyncMgr, + "%s asset type %s websocket functionality is unsupported, REST fetching only.", + exchangeName, + assetTypes[y]) + } enabledPairs, err := exchanges[x].GetEnabledPairs(assetTypes[y]) if err != nil { log.Errorf(log.SyncMgr, @@ -579,25 +572,21 @@ func (e *ExchangeCurrencyPairSyncer) Start() { Pair: enabledPairs[i], } + sBase := SyncBase{ + IsUsingREST: usingREST || !wsAssetSupported, + IsUsingWebsocket: usingWebsocket && wsAssetSupported, + } + if e.Cfg.SyncTicker { - c.Ticker = SyncBase{ - IsUsingREST: usingREST, - IsUsingWebsocket: usingWebsocket, - } + c.Ticker = sBase } if e.Cfg.SyncOrderbook { - c.Orderbook = SyncBase{ - IsUsingREST: usingREST, - IsUsingWebsocket: usingWebsocket, - } + c.Orderbook = sBase } if e.Cfg.SyncTrades { - c.Trade = SyncBase{ - IsUsingREST: usingREST, - IsUsingWebsocket: usingWebsocket, - } + c.Trade = sBase } e.add(&c) diff --git a/engine/syncer_types.go b/engine/syncer_types.go index 4701b503..3a3c0f5e 100644 --- a/engine/syncer_types.go +++ b/engine/syncer_types.go @@ -10,13 +10,14 @@ import ( // CurrencyPairSyncerConfig stores the currency pair config type CurrencyPairSyncerConfig struct { - SyncTicker bool - SyncOrderbook bool - SyncTrades bool - SyncContinuously bool - SyncTimeout time.Duration - NumWorkers int - Verbose bool + SyncTicker bool + SyncOrderbook bool + SyncTrades bool + SyncContinuously bool + SyncTimeoutREST time.Duration + SyncTimeoutWebsocket time.Duration + NumWorkers int + Verbose bool } // ExchangeSyncerConfig stores the exchange syncer config diff --git a/exchanges/alphapoint/alphapoint_wrapper.go b/exchanges/alphapoint/alphapoint_wrapper.go index acd650d3..5e4f23ac 100644 --- a/exchanges/alphapoint/alphapoint_wrapper.go +++ b/exchanges/alphapoint/alphapoint_wrapper.go @@ -187,8 +187,8 @@ func (a *Alphapoint) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*or } orderBook.Pair = p - orderBook.ExchangeName = a.Name - orderBook.AssetType = assetType + orderBook.Exchange = a.Name + orderBook.Asset = assetType err = orderBook.Process() if err != nil { diff --git a/exchanges/binance/binance_test.go b/exchanges/binance/binance_test.go index 0ed59862..2e3bce4b 100644 --- a/exchanges/binance/binance_test.go +++ b/exchanges/binance/binance_test.go @@ -2128,7 +2128,10 @@ func TestWsDepthUpdate(t *testing.T) { t.Error(err) } - ob := b.Websocket.Orderbook.GetOrderbook(p, asset.Spot) + ob, err := b.Websocket.Orderbook.GetOrderbook(p, asset.Spot) + if err != nil { + t.Fatal(err) + } if exp, got := seedLastUpdateID, ob.LastUpdateID; got != exp { t.Fatalf("Unexpected Last update id of orderbook for old update. Exp: %d, got: %d", exp, got) } @@ -2154,11 +2157,14 @@ func TestWsDepthUpdate(t *testing.T) { ] }}`) - if err := b.wsHandleData(update2); err != nil { + if err = b.wsHandleData(update2); err != nil { t.Error(err) } - ob = b.Websocket.Orderbook.GetOrderbook(p, asset.Spot) + ob, err = b.Websocket.Orderbook.GetOrderbook(p, asset.Spot) + if err != nil { + t.Fatal(err) + } if exp, got := int64(165), ob.LastUpdateID; got != exp { t.Fatalf("Unexpected Last update id of orderbook for new update. Exp: %d, got: %d", exp, got) } diff --git a/exchanges/binance/binance_websocket.go b/exchanges/binance/binance_websocket.go index 10bd891c..7f6b4f5c 100644 --- a/exchanges/binance/binance_websocket.go +++ b/exchanges/binance/binance_websocket.go @@ -463,10 +463,10 @@ func (b *Binance) SeedLocalCacheWithBook(p currency.Pair, orderbookNew *OrderBoo } newOrderBook.Pair = p - newOrderBook.AssetType = asset.Spot - newOrderBook.ExchangeName = b.Name + newOrderBook.Asset = asset.Spot + newOrderBook.Exchange = b.Name newOrderBook.LastUpdateID = orderbookNew.LastUpdateID - newOrderBook.VerificationBypass = b.OrderbookVerificationBypass + newOrderBook.VerifyOrderbook = b.CanVerifyOrderbook return b.Websocket.Orderbook.LoadSnapshot(&newOrderBook) } @@ -632,8 +632,8 @@ func (b *Binance) applyBufferUpdate(pair currency.Pair) error { return nil } - recent := b.Websocket.Orderbook.GetOrderbook(pair, asset.Spot) - if recent == nil || (recent.Asks == nil && recent.Bids == nil) { + recent, err := b.Websocket.Orderbook.GetOrderbook(pair, asset.Spot) + if err != nil || (recent.Asks == nil && recent.Bids == nil) { return b.obm.fetchBookViaREST(pair) } diff --git a/exchanges/binance/binance_wrapper.go b/exchanges/binance/binance_wrapper.go index 6523d06c..301f8163 100644 --- a/exchanges/binance/binance_wrapper.go +++ b/exchanges/binance/binance_wrapper.go @@ -92,14 +92,26 @@ func (b *Binance) SetDefaults() { if err != nil { log.Errorln(log.ExchangeSys, err) } + err = b.DisableAssetWebsocketSupport(asset.Margin) + if err != nil { + log.Errorln(log.ExchangeSys, err) + } err = b.StoreAssetPairFormat(asset.CoinMarginedFutures, coinFutures) if err != nil { log.Errorln(log.ExchangeSys, err) } + err = b.DisableAssetWebsocketSupport(asset.CoinMarginedFutures) + if err != nil { + log.Errorln(log.ExchangeSys, err) + } err = b.StoreAssetPairFormat(asset.USDTMarginedFutures, usdtFutures) if err != nil { log.Errorln(log.ExchangeSys, err) } + err = b.DisableAssetWebsocketSupport(asset.USDTMarginedFutures) + if err != nil { + log.Errorln(log.ExchangeSys, err) + } b.Features = exchange.Features{ Supports: exchange.FeaturesSupported{ REST: true, @@ -520,10 +532,10 @@ func (b *Binance) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderb // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *Binance) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: b.CanVerifyOrderbook, } var orderbookNew OrderBook var err error diff --git a/exchanges/bitfinex/bitfinex_websocket.go b/exchanges/bitfinex/bitfinex_websocket.go index 1eac2b8d..a8cb90cb 100644 --- a/exchanges/bitfinex/bitfinex_websocket.go +++ b/exchanges/bitfinex/bitfinex_websocket.go @@ -962,13 +962,12 @@ func (b *Bitfinex) WsInsertSnapshot(p currency.Pair, assetType asset.Item, books } } - book.AssetType = assetType + book.Asset = assetType book.Pair = p - book.ExchangeName = b.Name - book.NotAggregated = true - book.HasChecksumValidation = true + book.Exchange = b.Name + book.PriceDuplication = true book.IsFundingRate = fundingRate - book.VerificationBypass = b.OrderbookVerificationBypass + book.VerifyOrderbook = b.CanVerifyOrderbook return b.Websocket.Orderbook.LoadSnapshot(&book) } @@ -1036,14 +1035,15 @@ func (b *Bitfinex) WsUpdateOrderbook(p currency.Pair, assetType asset.Item, book if checkme.Sequence+1 == sequenceNo { // Sequence numbers get dropped, if checksum is not in line with // sequence, do not check. - ob := b.Websocket.Orderbook.GetOrderbook(p, assetType) - if ob == nil { - return fmt.Errorf("cannot calculate websocket checksum: book not found for %s %s", + ob, err := b.Websocket.Orderbook.GetOrderbook(p, assetType) + if err != nil { + return fmt.Errorf("cannot calculate websocket checksum: book not found for %s %s %w", p, - assetType) + assetType, + err) } - err := validateCRC32(ob, checkme.Token) + err = validateCRC32(ob, checkme.Token) if err != nil { return err } @@ -1398,7 +1398,7 @@ func validateCRC32(book *orderbook.Base, token int) error { return nil } return fmt.Errorf("invalid checksum for %s %s: calculated [%d] does not match [%d]", - book.AssetType, + book.Asset, book.Pair, checksum, uint32(token)) diff --git a/exchanges/bitfinex/bitfinex_wrapper.go b/exchanges/bitfinex/bitfinex_wrapper.go index ab540aa4..050f2949 100644 --- a/exchanges/bitfinex/bitfinex_wrapper.go +++ b/exchanges/bitfinex/bitfinex_wrapper.go @@ -395,11 +395,11 @@ func (b *Bitfinex) FetchOrderbook(p currency.Pair, assetType asset.Item) (*order // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *Bitfinex) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { o := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - NotAggregated: true, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + PriceDuplication: true, + VerifyOrderbook: b.CanVerifyOrderbook, } fPair, err := b.FormatExchangeCurrency(p, assetType) diff --git a/exchanges/bitflyer/bitflyer_wrapper.go b/exchanges/bitflyer/bitflyer_wrapper.go index adf382d5..8a375aaa 100644 --- a/exchanges/bitflyer/bitflyer_wrapper.go +++ b/exchanges/bitflyer/bitflyer_wrapper.go @@ -254,10 +254,10 @@ func (b *Bitflyer) FetchOrderbook(p currency.Pair, assetType asset.Item) (*order // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *Bitflyer) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: b.CanVerifyOrderbook, } fPair, err := b.FormatExchangeCurrency(p, assetType) diff --git a/exchanges/bithumb/bithumb_wrapper.go b/exchanges/bithumb/bithumb_wrapper.go index ae685905..8aa2944b 100644 --- a/exchanges/bithumb/bithumb_wrapper.go +++ b/exchanges/bithumb/bithumb_wrapper.go @@ -246,10 +246,10 @@ func (b *Bithumb) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderb // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *Bithumb) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: b.CanVerifyOrderbook, } curr := p.Base.String() diff --git a/exchanges/bitmex/bitmex_test.go b/exchanges/bitmex/bitmex_test.go index a2de8cdf..d4868d8c 100644 --- a/exchanges/bitmex/bitmex_test.go +++ b/exchanges/bitmex/bitmex_test.go @@ -955,7 +955,7 @@ func TestWSOrderbookHandling(t *testing.T) { ] }`) err = b.wsHandleData(pressXToJSON) - if err != nil && err.Error() != "perpetualcontract ETHUSD update cannot be deleted id: 17999995000 not found" { + if err != nil && err.Error() != "delete error: cannot match ID on linked list 17999995000 not found" { t.Error(err) } if err == nil { diff --git a/exchanges/bitmex/bitmex_websocket.go b/exchanges/bitmex/bitmex_websocket.go index 8f4de3fe..e4f73618 100644 --- a/exchanges/bitmex/bitmex_websocket.go +++ b/exchanges/bitmex/bitmex_websocket.go @@ -508,11 +508,11 @@ func (b *Bitmex) processOrderbook(data []OrderBookL2, action string, p currency. data[i].Side) } } - orderbook.Reverse(book.Asks) // Reverse asks for correct alignment - book.AssetType = a + book.Asks.Reverse() // Reverse asks for correct alignment + book.Asset = a book.Pair = p - book.ExchangeName = b.Name - book.VerificationBypass = b.OrderbookVerificationBypass + book.Exchange = b.Name + book.VerifyOrderbook = b.CanVerifyOrderbook err := b.Websocket.Orderbook.LoadSnapshot(&book) if err != nil { diff --git a/exchanges/bitmex/bitmex_wrapper.go b/exchanges/bitmex/bitmex_wrapper.go index 40654faf..efa2a58f 100644 --- a/exchanges/bitmex/bitmex_wrapper.go +++ b/exchanges/bitmex/bitmex_wrapper.go @@ -69,6 +69,11 @@ func (b *Bitmex) SetDefaults() { log.Errorln(log.ExchangeSys, err) } + err = b.DisableAssetWebsocketSupport(asset.Index) + if err != nil { + log.Errorln(log.ExchangeSys, err) + } + b.Features = exchange.Features{ Supports: exchange.FeaturesSupported{ REST: true, @@ -346,10 +351,10 @@ func (b *Bitmex) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbo // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *Bitmex) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: b.CanVerifyOrderbook, } if assetType == asset.Index { @@ -384,7 +389,7 @@ func (b *Bitmex) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderb orderbookNew[i].Side) } } - orderbook.Reverse(book.Asks) + book.Asks.Reverse() // Reverse order of asks to ascending err = book.Process() if err != nil { diff --git a/exchanges/bitstamp/bitstamp_websocket.go b/exchanges/bitstamp/bitstamp_websocket.go index 58c56268..6f7032a2 100644 --- a/exchanges/bitstamp/bitstamp_websocket.go +++ b/exchanges/bitstamp/bitstamp_websocket.go @@ -239,13 +239,13 @@ func (b *Bitstamp) wsUpdateOrderbook(update websocketOrderBook, p currency.Pair, bids = append(bids, orderbook.Item{Price: target, Amount: amount}) } return b.Websocket.Orderbook.LoadSnapshot(&orderbook.Base{ - Bids: bids, - Asks: asks, - Pair: p, - LastUpdated: time.Unix(update.Timestamp, 0), - AssetType: assetType, - ExchangeName: b.Name, - VerificationBypass: b.OrderbookVerificationBypass, + Bids: bids, + Asks: asks, + Pair: p, + LastUpdated: time.Unix(update.Timestamp, 0), + Asset: assetType, + Exchange: b.Name, + VerifyOrderbook: b.CanVerifyOrderbook, }) } @@ -275,9 +275,9 @@ func (b *Bitstamp) seedOrderBook() error { }) } newOrderBook.Pair = p[x] - newOrderBook.AssetType = asset.Spot - newOrderBook.ExchangeName = b.Name - newOrderBook.VerificationBypass = b.OrderbookVerificationBypass + newOrderBook.Asset = asset.Spot + newOrderBook.Exchange = b.Name + newOrderBook.VerifyOrderbook = b.CanVerifyOrderbook err = b.Websocket.Orderbook.LoadSnapshot(&newOrderBook) if err != nil { diff --git a/exchanges/bitstamp/bitstamp_wrapper.go b/exchanges/bitstamp/bitstamp_wrapper.go index 54f3fd91..bfc6f1f7 100644 --- a/exchanges/bitstamp/bitstamp_wrapper.go +++ b/exchanges/bitstamp/bitstamp_wrapper.go @@ -324,10 +324,10 @@ func (b *Bitstamp) FetchOrderbook(p currency.Pair, assetType asset.Item) (*order // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *Bitstamp) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: b.CanVerifyOrderbook, } fPair, err := b.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/bittrex/bittrex_wrapper.go b/exchanges/bittrex/bittrex_wrapper.go index 87b402cf..a4cd72ca 100644 --- a/exchanges/bittrex/bittrex_wrapper.go +++ b/exchanges/bittrex/bittrex_wrapper.go @@ -333,10 +333,10 @@ func (b *Bittrex) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderb // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *Bittrex) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: b.CanVerifyOrderbook, } fpair, err := b.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/btcmarkets/btcmarkets_websocket.go b/exchanges/btcmarkets/btcmarkets_websocket.go index 5df16d99..76fc2d92 100644 --- a/exchanges/btcmarkets/btcmarkets_websocket.go +++ b/exchanges/btcmarkets/btcmarkets_websocket.go @@ -83,7 +83,7 @@ func (b *BTCMarkets) wsHandleData(respRaw []byte) error { return err } - var bids, asks []orderbook.Item + var bids, asks orderbook.Items for x := range ob.Bids { var price, amount float64 price, err = strconv.ParseFloat(ob.Bids[x][0].(string), 64) @@ -117,14 +117,15 @@ func (b *BTCMarkets) wsHandleData(respRaw []byte) error { }) } if ob.Snapshot { + bids.SortBids() // Alignment completely out, sort is needed. err = b.Websocket.Orderbook.LoadSnapshot(&orderbook.Base{ - Pair: p, - Bids: orderbook.SortBids(bids), // Alignment completely out sort is needed - Asks: asks, - LastUpdated: ob.Timestamp, - AssetType: asset.Spot, - ExchangeName: b.Name, - VerificationBypass: b.OrderbookVerificationBypass, + Pair: p, + Bids: bids, + Asks: asks, + LastUpdated: ob.Timestamp, + Asset: asset.Spot, + Exchange: b.Name, + VerifyOrderbook: b.CanVerifyOrderbook, }) } else { err = b.Websocket.Orderbook.Update(&buffer.Update{ diff --git a/exchanges/btcmarkets/btcmarkets_wrapper.go b/exchanges/btcmarkets/btcmarkets_wrapper.go index c598ff5a..bcd04d46 100644 --- a/exchanges/btcmarkets/btcmarkets_wrapper.go +++ b/exchanges/btcmarkets/btcmarkets_wrapper.go @@ -360,11 +360,11 @@ func (b *BTCMarkets) FetchOrderbook(p currency.Pair, assetType asset.Item) (*ord // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *BTCMarkets) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - NotAggregated: true, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + PriceDuplication: true, + VerifyOrderbook: b.CanVerifyOrderbook, } fpair, err := b.FormatExchangeCurrency(p, assetType) diff --git a/exchanges/btse/btse_websocket.go b/exchanges/btse/btse_websocket.go index 221cd23f..148afc8e 100644 --- a/exchanges/btse/btse_websocket.go +++ b/exchanges/btse/btse_websocket.go @@ -311,10 +311,10 @@ func (b *BTSE) wsHandleData(respRaw []byte) error { return err } newOB.Pair = p - newOB.AssetType = a - newOB.ExchangeName = b.Name - orderbook.Reverse(newOB.Asks) // Reverse asks for correct alignment - newOB.VerificationBypass = b.OrderbookVerificationBypass + newOB.Asset = a + newOB.Exchange = b.Name + newOB.Asks.Reverse() // Reverse asks for correct alignment + newOB.VerifyOrderbook = b.CanVerifyOrderbook err = b.Websocket.Orderbook.LoadSnapshot(&newOB) if err != nil { return err diff --git a/exchanges/btse/btse_wrapper.go b/exchanges/btse/btse_wrapper.go index 47215adb..dd8f7945 100644 --- a/exchanges/btse/btse_wrapper.go +++ b/exchanges/btse/btse_wrapper.go @@ -336,10 +336,10 @@ func (b *BTSE) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbook // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *BTSE) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: b.CanVerifyOrderbook, } fPair, err := b.FormatExchangeCurrency(p, assetType) if err != nil { @@ -366,10 +366,10 @@ func (b *BTSE) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderboo Price: a.SellQuote[x].Price, Amount: a.SellQuote[x].Size}) } - orderbook.Reverse(book.Asks) // Reverse asks for correct alignment + book.Asks.Reverse() // Reverse asks for correct alignment book.Pair = p - book.ExchangeName = b.Name - book.AssetType = assetType + book.Exchange = b.Name + book.Asset = assetType err = book.Process() if err != nil { return book, err diff --git a/exchanges/coinbasepro/coinbasepro_websocket.go b/exchanges/coinbasepro/coinbasepro_websocket.go index 621ab0bc..3b34ce76 100644 --- a/exchanges/coinbasepro/coinbasepro_websocket.go +++ b/exchanges/coinbasepro/coinbasepro_websocket.go @@ -309,10 +309,10 @@ func (c *CoinbasePro) ProcessSnapshot(snapshot *WebsocketOrderbookSnapshot) erro return err } - base.AssetType = asset.Spot + base.Asset = asset.Spot base.Pair = pair - base.ExchangeName = c.Name - base.VerificationBypass = c.OrderbookVerificationBypass + base.Exchange = c.Name + base.VerifyOrderbook = c.CanVerifyOrderbook return c.Websocket.Orderbook.LoadSnapshot(&base) } diff --git a/exchanges/coinbasepro/coinbasepro_wrapper.go b/exchanges/coinbasepro/coinbasepro_wrapper.go index 26cbb10d..4d4f57fc 100644 --- a/exchanges/coinbasepro/coinbasepro_wrapper.go +++ b/exchanges/coinbasepro/coinbasepro_wrapper.go @@ -411,10 +411,10 @@ func (c *CoinbasePro) FetchOrderbook(p currency.Pair, assetType asset.Item) (*or // UpdateOrderbook updates and returns the orderbook for a currency pair func (c *CoinbasePro) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: c.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: c.OrderbookVerificationBypass, + Exchange: c.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: c.CanVerifyOrderbook, } fpair, err := c.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/coinbene/coinbene_websocket.go b/exchanges/coinbene/coinbene_websocket.go index e769d598..3d5c9447 100644 --- a/exchanges/coinbene/coinbene_websocket.go +++ b/exchanges/coinbene/coinbene_websocket.go @@ -285,11 +285,11 @@ func (c *Coinbene) wsHandleData(respRaw []byte) error { var newOB orderbook.Base newOB.Asks = asks newOB.Bids = bids - newOB.AssetType = assetType + newOB.Asset = assetType newOB.Pair = newPair - newOB.ExchangeName = c.Name + newOB.Exchange = c.Name newOB.LastUpdated = time.Unix(orderBook.Data[0].Timestamp, 0) - newOB.VerificationBypass = c.OrderbookVerificationBypass + newOB.VerifyOrderbook = c.CanVerifyOrderbook err = c.Websocket.Orderbook.LoadSnapshot(&newOB) if err != nil { return err diff --git a/exchanges/coinbene/coinbene_wrapper.go b/exchanges/coinbene/coinbene_wrapper.go index 87262d68..2685ec63 100644 --- a/exchanges/coinbene/coinbene_wrapper.go +++ b/exchanges/coinbene/coinbene_wrapper.go @@ -411,10 +411,10 @@ func (c *Coinbene) FetchOrderbook(p currency.Pair, assetType asset.Item) (*order // UpdateOrderbook updates and returns the orderbook for a currency pair func (c *Coinbene) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: c.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: c.OrderbookVerificationBypass, + Exchange: c.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: c.CanVerifyOrderbook, } if !c.SupportsAsset(assetType) { return book, diff --git a/exchanges/coinut/coinut_websocket.go b/exchanges/coinut/coinut_websocket.go index 77ac3cdf..1143c8f7 100644 --- a/exchanges/coinut/coinut_websocket.go +++ b/exchanges/coinut/coinut_websocket.go @@ -530,7 +530,7 @@ func (c *COINUT) WsProcessOrderbookSnapshot(ob *WsOrderbookSnapshot) error { var newOrderBook orderbook.Base newOrderBook.Asks = asks newOrderBook.Bids = bids - newOrderBook.VerificationBypass = c.OrderbookVerificationBypass + newOrderBook.VerifyOrderbook = c.CanVerifyOrderbook pairs, err := c.GetEnabledPairs(asset.Spot) if err != nil { @@ -550,8 +550,8 @@ func (c *COINUT) WsProcessOrderbookSnapshot(ob *WsOrderbookSnapshot) error { return err } - newOrderBook.AssetType = asset.Spot - newOrderBook.ExchangeName = c.Name + newOrderBook.Asset = asset.Spot + newOrderBook.Exchange = c.Name return c.Websocket.Orderbook.LoadSnapshot(&newOrderBook) } diff --git a/exchanges/coinut/coinut_wrapper.go b/exchanges/coinut/coinut_wrapper.go index fefd2ef5..d310a7ab 100644 --- a/exchanges/coinut/coinut_wrapper.go +++ b/exchanges/coinut/coinut_wrapper.go @@ -463,10 +463,10 @@ func (c *COINUT) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbo // UpdateOrderbook updates and returns the orderbook for a currency pair func (c *COINUT) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: c.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: c.OrderbookVerificationBypass, + Exchange: c.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: c.CanVerifyOrderbook, } err := c.loadInstrumentsIfNotLoaded() if err != nil { diff --git a/exchanges/exchange.go b/exchanges/exchange.go index 2eb78946..969dd516 100644 --- a/exchanges/exchange.go +++ b/exchanges/exchange.go @@ -39,19 +39,19 @@ const ( DefaultWebsocketOrderbookBufferLimit = 5 ) -func (e *Base) checkAndInitRequester() { - if e.Requester == nil { - e.Requester = request.New(e.Name, +func (b *Base) checkAndInitRequester() { + if b.Requester == nil { + b.Requester = request.New(b.Name, &http.Client{Transport: new(http.Transport)}) } } // SetHTTPClientTimeout sets the timeout value for the exchanges HTTP Client and // also the underlying transports idle connection timeout -func (e *Base) SetHTTPClientTimeout(t time.Duration) error { - e.checkAndInitRequester() - e.Requester.HTTPClient.Timeout = t - tr, ok := e.Requester.HTTPClient.Transport.(*http.Transport) +func (b *Base) SetHTTPClientTimeout(t time.Duration) error { + b.checkAndInitRequester() + b.Requester.HTTPClient.Timeout = t + tr, ok := b.Requester.HTTPClient.Transport.(*http.Transport) if !ok { return errors.New("transport not set, cannot set timeout") } @@ -60,31 +60,31 @@ func (e *Base) SetHTTPClientTimeout(t time.Duration) error { } // SetHTTPClient sets exchanges HTTP client -func (e *Base) SetHTTPClient(h *http.Client) { - e.checkAndInitRequester() - e.Requester.HTTPClient = h +func (b *Base) SetHTTPClient(h *http.Client) { + b.checkAndInitRequester() + b.Requester.HTTPClient = h } // GetHTTPClient gets the exchanges HTTP client -func (e *Base) GetHTTPClient() *http.Client { - e.checkAndInitRequester() - return e.Requester.HTTPClient +func (b *Base) GetHTTPClient() *http.Client { + b.checkAndInitRequester() + return b.Requester.HTTPClient } // SetHTTPClientUserAgent sets the exchanges HTTP user agent -func (e *Base) SetHTTPClientUserAgent(ua string) { - e.checkAndInitRequester() - e.Requester.UserAgent = ua - e.HTTPUserAgent = ua +func (b *Base) SetHTTPClientUserAgent(ua string) { + b.checkAndInitRequester() + b.Requester.UserAgent = ua + b.HTTPUserAgent = ua } // GetHTTPClientUserAgent gets the exchanges HTTP user agent -func (e *Base) GetHTTPClientUserAgent() string { - return e.HTTPUserAgent +func (b *Base) GetHTTPClientUserAgent() string { + return b.HTTPUserAgent } // SetClientProxyAddress sets a proxy address for REST and websocket requests -func (e *Base) SetClientProxyAddress(addr string) error { +func (b *Base) SetClientProxyAddress(addr string) error { if addr == "" { return nil } @@ -94,13 +94,13 @@ func (e *Base) SetClientProxyAddress(addr string) error { err) } - err = e.Requester.SetProxy(proxy) + err = b.Requester.SetProxy(proxy) if err != nil { return err } - if e.Websocket != nil { - err = e.Websocket.SetProxyAddress(addr) + if b.Websocket != nil { + err = b.Websocket.SetProxyAddress(addr) if err != nil { return err } @@ -110,98 +110,98 @@ func (e *Base) SetClientProxyAddress(addr string) error { // SetFeatureDefaults sets the exchanges default feature // support set -func (e *Base) SetFeatureDefaults() { - if e.Config.Features == nil { +func (b *Base) SetFeatureDefaults() { + if b.Config.Features == nil { s := &config.FeaturesConfig{ Supports: config.FeaturesSupportedConfig{ - Websocket: e.Features.Supports.Websocket, - REST: e.Features.Supports.REST, + Websocket: b.Features.Supports.Websocket, + REST: b.Features.Supports.REST, RESTCapabilities: protocol.Features{ - AutoPairUpdates: e.Features.Supports.RESTCapabilities.AutoPairUpdates, + AutoPairUpdates: b.Features.Supports.RESTCapabilities.AutoPairUpdates, }, }, } - if e.Config.SupportsAutoPairUpdates != nil { - s.Supports.RESTCapabilities.AutoPairUpdates = *e.Config.SupportsAutoPairUpdates - s.Enabled.AutoPairUpdates = *e.Config.SupportsAutoPairUpdates + if b.Config.SupportsAutoPairUpdates != nil { + s.Supports.RESTCapabilities.AutoPairUpdates = *b.Config.SupportsAutoPairUpdates + s.Enabled.AutoPairUpdates = *b.Config.SupportsAutoPairUpdates } else { - s.Supports.RESTCapabilities.AutoPairUpdates = e.Features.Supports.RESTCapabilities.AutoPairUpdates - s.Enabled.AutoPairUpdates = e.Features.Supports.RESTCapabilities.AutoPairUpdates + s.Supports.RESTCapabilities.AutoPairUpdates = b.Features.Supports.RESTCapabilities.AutoPairUpdates + s.Enabled.AutoPairUpdates = b.Features.Supports.RESTCapabilities.AutoPairUpdates if !s.Supports.RESTCapabilities.AutoPairUpdates { - e.Config.CurrencyPairs.LastUpdated = time.Now().Unix() - e.CurrencyPairs.LastUpdated = e.Config.CurrencyPairs.LastUpdated + b.Config.CurrencyPairs.LastUpdated = time.Now().Unix() + b.CurrencyPairs.LastUpdated = b.Config.CurrencyPairs.LastUpdated } } - e.Config.Features = s - e.Config.SupportsAutoPairUpdates = nil + b.Config.Features = s + b.Config.SupportsAutoPairUpdates = nil } else { - if e.Features.Supports.RESTCapabilities.AutoPairUpdates != e.Config.Features.Supports.RESTCapabilities.AutoPairUpdates { - e.Config.Features.Supports.RESTCapabilities.AutoPairUpdates = e.Features.Supports.RESTCapabilities.AutoPairUpdates + if b.Features.Supports.RESTCapabilities.AutoPairUpdates != b.Config.Features.Supports.RESTCapabilities.AutoPairUpdates { + b.Config.Features.Supports.RESTCapabilities.AutoPairUpdates = b.Features.Supports.RESTCapabilities.AutoPairUpdates - if !e.Config.Features.Supports.RESTCapabilities.AutoPairUpdates { - e.Config.CurrencyPairs.LastUpdated = time.Now().Unix() + if !b.Config.Features.Supports.RESTCapabilities.AutoPairUpdates { + b.Config.CurrencyPairs.LastUpdated = time.Now().Unix() } } - if e.Features.Supports.REST != e.Config.Features.Supports.REST { - e.Config.Features.Supports.REST = e.Features.Supports.REST + if b.Features.Supports.REST != b.Config.Features.Supports.REST { + b.Config.Features.Supports.REST = b.Features.Supports.REST } - if e.Features.Supports.RESTCapabilities.TickerBatching != e.Config.Features.Supports.RESTCapabilities.TickerBatching { - e.Config.Features.Supports.RESTCapabilities.TickerBatching = e.Features.Supports.RESTCapabilities.TickerBatching + if b.Features.Supports.RESTCapabilities.TickerBatching != b.Config.Features.Supports.RESTCapabilities.TickerBatching { + b.Config.Features.Supports.RESTCapabilities.TickerBatching = b.Features.Supports.RESTCapabilities.TickerBatching } - if e.Features.Supports.Websocket != e.Config.Features.Supports.Websocket { - e.Config.Features.Supports.Websocket = e.Features.Supports.Websocket + if b.Features.Supports.Websocket != b.Config.Features.Supports.Websocket { + b.Config.Features.Supports.Websocket = b.Features.Supports.Websocket } - if e.IsSaveTradeDataEnabled() != e.Config.Features.Enabled.SaveTradeData { - e.SetSaveTradeDataStatus(e.Config.Features.Enabled.SaveTradeData) + if b.IsSaveTradeDataEnabled() != b.Config.Features.Enabled.SaveTradeData { + b.SetSaveTradeDataStatus(b.Config.Features.Enabled.SaveTradeData) } - e.Features.Enabled.AutoPairUpdates = e.Config.Features.Enabled.AutoPairUpdates + b.Features.Enabled.AutoPairUpdates = b.Config.Features.Enabled.AutoPairUpdates } } // SetAPICredentialDefaults sets the API Credential validator defaults -func (e *Base) SetAPICredentialDefaults() { +func (b *Base) SetAPICredentialDefaults() { // Exchange hardcoded settings take precedence and overwrite the config settings - if e.Config.API.CredentialsValidator == nil { - e.Config.API.CredentialsValidator = new(config.APICredentialsValidatorConfig) + if b.Config.API.CredentialsValidator == nil { + b.Config.API.CredentialsValidator = new(config.APICredentialsValidatorConfig) } - if e.Config.API.CredentialsValidator.RequiresKey != e.API.CredentialsValidator.RequiresKey { - e.Config.API.CredentialsValidator.RequiresKey = e.API.CredentialsValidator.RequiresKey + if b.Config.API.CredentialsValidator.RequiresKey != b.API.CredentialsValidator.RequiresKey { + b.Config.API.CredentialsValidator.RequiresKey = b.API.CredentialsValidator.RequiresKey } - if e.Config.API.CredentialsValidator.RequiresSecret != e.API.CredentialsValidator.RequiresSecret { - e.Config.API.CredentialsValidator.RequiresSecret = e.API.CredentialsValidator.RequiresSecret + if b.Config.API.CredentialsValidator.RequiresSecret != b.API.CredentialsValidator.RequiresSecret { + b.Config.API.CredentialsValidator.RequiresSecret = b.API.CredentialsValidator.RequiresSecret } - if e.Config.API.CredentialsValidator.RequiresBase64DecodeSecret != e.API.CredentialsValidator.RequiresBase64DecodeSecret { - e.Config.API.CredentialsValidator.RequiresBase64DecodeSecret = e.API.CredentialsValidator.RequiresBase64DecodeSecret + if b.Config.API.CredentialsValidator.RequiresBase64DecodeSecret != b.API.CredentialsValidator.RequiresBase64DecodeSecret { + b.Config.API.CredentialsValidator.RequiresBase64DecodeSecret = b.API.CredentialsValidator.RequiresBase64DecodeSecret } - if e.Config.API.CredentialsValidator.RequiresClientID != e.API.CredentialsValidator.RequiresClientID { - e.Config.API.CredentialsValidator.RequiresClientID = e.API.CredentialsValidator.RequiresClientID + if b.Config.API.CredentialsValidator.RequiresClientID != b.API.CredentialsValidator.RequiresClientID { + b.Config.API.CredentialsValidator.RequiresClientID = b.API.CredentialsValidator.RequiresClientID } - if e.Config.API.CredentialsValidator.RequiresPEM != e.API.CredentialsValidator.RequiresPEM { - e.Config.API.CredentialsValidator.RequiresPEM = e.API.CredentialsValidator.RequiresPEM + if b.Config.API.CredentialsValidator.RequiresPEM != b.API.CredentialsValidator.RequiresPEM { + b.Config.API.CredentialsValidator.RequiresPEM = b.API.CredentialsValidator.RequiresPEM } } // SupportsRESTTickerBatchUpdates returns whether or not the // exhange supports REST batch ticker fetching -func (e *Base) SupportsRESTTickerBatchUpdates() bool { - return e.Features.Supports.RESTCapabilities.TickerBatching +func (b *Base) SupportsRESTTickerBatchUpdates() bool { + return b.Features.Supports.RESTCapabilities.TickerBatching } // SupportsAutoPairUpdates returns whether or not the exchange supports // auto currency pair updating -func (e *Base) SupportsAutoPairUpdates() bool { - if e.Features.Supports.RESTCapabilities.AutoPairUpdates || - e.Features.Supports.WebsocketCapabilities.AutoPairUpdates { +func (b *Base) SupportsAutoPairUpdates() bool { + if b.Features.Supports.RESTCapabilities.AutoPairUpdates || + b.Features.Supports.WebsocketCapabilities.AutoPairUpdates { return true } return false @@ -209,22 +209,22 @@ func (e *Base) SupportsAutoPairUpdates() bool { // GetLastPairsUpdateTime returns the unix timestamp of when the exchanges // currency pairs were last updated -func (e *Base) GetLastPairsUpdateTime() int64 { - return e.CurrencyPairs.LastUpdated +func (b *Base) GetLastPairsUpdateTime() int64 { + return b.CurrencyPairs.LastUpdated } // GetAssetTypes returns the available asset types for an individual exchange -func (e *Base) GetAssetTypes() asset.Items { - return e.CurrencyPairs.GetAssetTypes() +func (b *Base) GetAssetTypes() asset.Items { + return b.CurrencyPairs.GetAssetTypes() } // GetPairAssetType returns the associated asset type for the currency pair // This method is only useful for exchanges that have pair names with multiple delimiters (BTC-USD-0626) // Helpful if the exchange has only a single asset type but in that case the asset type can be hard coded -func (e *Base) GetPairAssetType(c currency.Pair) (asset.Item, error) { - assetTypes := e.GetAssetTypes() +func (b *Base) GetPairAssetType(c currency.Pair) (asset.Item, error) { + assetTypes := b.GetAssetTypes() for i := range assetTypes { - avail, err := e.GetAvailablePairs(assetTypes[i]) + avail, err := b.GetAvailablePairs(assetTypes[i]) if err != nil { return "", err } @@ -237,137 +237,137 @@ func (e *Base) GetPairAssetType(c currency.Pair) (asset.Item, error) { // GetClientBankAccounts returns banking details associated with // a client for withdrawal purposes -func (e *Base) GetClientBankAccounts(exchangeName, withdrawalCurrency string) (*banking.Account, error) { +func (b *Base) GetClientBankAccounts(exchangeName, withdrawalCurrency string) (*banking.Account, error) { cfg := config.GetConfig() return cfg.GetClientBankAccounts(exchangeName, withdrawalCurrency) } // GetExchangeBankAccounts returns banking details associated with an // exchange for funding purposes -func (e *Base) GetExchangeBankAccounts(id, depositCurrency string) (*banking.Account, error) { +func (b *Base) GetExchangeBankAccounts(id, depositCurrency string) (*banking.Account, error) { cfg := config.GetConfig() - return cfg.GetExchangeBankAccounts(e.Name, id, depositCurrency) + return cfg.GetExchangeBankAccounts(b.Name, id, depositCurrency) } // SetCurrencyPairFormat checks the exchange request and config currency pair // formats and syncs it with the exchanges SetDefault settings -func (e *Base) SetCurrencyPairFormat() { - if e.Config.CurrencyPairs == nil { - e.Config.CurrencyPairs = new(currency.PairsManager) +func (b *Base) SetCurrencyPairFormat() { + if b.Config.CurrencyPairs == nil { + b.Config.CurrencyPairs = new(currency.PairsManager) } - e.Config.CurrencyPairs.UseGlobalFormat = e.CurrencyPairs.UseGlobalFormat - if e.Config.CurrencyPairs.UseGlobalFormat { - e.Config.CurrencyPairs.RequestFormat = e.CurrencyPairs.RequestFormat - e.Config.CurrencyPairs.ConfigFormat = e.CurrencyPairs.ConfigFormat + b.Config.CurrencyPairs.UseGlobalFormat = b.CurrencyPairs.UseGlobalFormat + if b.Config.CurrencyPairs.UseGlobalFormat { + b.Config.CurrencyPairs.RequestFormat = b.CurrencyPairs.RequestFormat + b.Config.CurrencyPairs.ConfigFormat = b.CurrencyPairs.ConfigFormat return } - if e.Config.CurrencyPairs.ConfigFormat != nil { - e.Config.CurrencyPairs.ConfigFormat = nil + if b.Config.CurrencyPairs.ConfigFormat != nil { + b.Config.CurrencyPairs.ConfigFormat = nil } - if e.Config.CurrencyPairs.RequestFormat != nil { - e.Config.CurrencyPairs.RequestFormat = nil + if b.Config.CurrencyPairs.RequestFormat != nil { + b.Config.CurrencyPairs.RequestFormat = nil } - assetTypes := e.GetAssetTypes() + assetTypes := b.GetAssetTypes() for x := range assetTypes { - if _, err := e.Config.CurrencyPairs.Get(assetTypes[x]); err != nil { - ps, err := e.CurrencyPairs.Get(assetTypes[x]) + if _, err := b.Config.CurrencyPairs.Get(assetTypes[x]); err != nil { + ps, err := b.CurrencyPairs.Get(assetTypes[x]) if err != nil { continue } - e.Config.CurrencyPairs.Store(assetTypes[x], *ps) + b.Config.CurrencyPairs.Store(assetTypes[x], *ps) } } } // SetConfigPairs sets the exchanges currency pairs to the pairs set in the config -func (e *Base) SetConfigPairs() error { - assetTypes := e.Config.CurrencyPairs.GetAssetTypes() - exchangeAssets := e.CurrencyPairs.GetAssetTypes() +func (b *Base) SetConfigPairs() error { + assetTypes := b.Config.CurrencyPairs.GetAssetTypes() + exchangeAssets := b.CurrencyPairs.GetAssetTypes() for x := range assetTypes { if !exchangeAssets.Contains(assetTypes[x]) { log.Warnf(log.ExchangeSys, "%s exchange asset type %s unsupported, please manually remove from configuration", - e.Name, + b.Name, assetTypes[x]) } - cfgPS, err := e.Config.CurrencyPairs.Get(assetTypes[x]) + cfgPS, err := b.Config.CurrencyPairs.Get(assetTypes[x]) if err != nil { return err } var enabledAsset bool - if e.Config.CurrencyPairs.IsAssetEnabled(assetTypes[x]) == nil { + if b.Config.CurrencyPairs.IsAssetEnabled(assetTypes[x]) == nil { enabledAsset = true } - e.CurrencyPairs.SetAssetEnabled(assetTypes[x], enabledAsset) + b.CurrencyPairs.SetAssetEnabled(assetTypes[x], enabledAsset) - if e.Config.CurrencyPairs.UseGlobalFormat { - e.CurrencyPairs.StorePairs(assetTypes[x], cfgPS.Available, false) - e.CurrencyPairs.StorePairs(assetTypes[x], cfgPS.Enabled, true) + if b.Config.CurrencyPairs.UseGlobalFormat { + b.CurrencyPairs.StorePairs(assetTypes[x], cfgPS.Available, false) + b.CurrencyPairs.StorePairs(assetTypes[x], cfgPS.Enabled, true) continue } - exchPS, err := e.CurrencyPairs.Get(assetTypes[x]) + exchPS, err := b.CurrencyPairs.Get(assetTypes[x]) if err != nil { return err } cfgPS.ConfigFormat = exchPS.ConfigFormat cfgPS.RequestFormat = exchPS.RequestFormat - e.CurrencyPairs.StorePairs(assetTypes[x], cfgPS.Available, false) - e.CurrencyPairs.StorePairs(assetTypes[x], cfgPS.Enabled, true) + b.CurrencyPairs.StorePairs(assetTypes[x], cfgPS.Available, false) + b.CurrencyPairs.StorePairs(assetTypes[x], cfgPS.Enabled, true) } return nil } // GetAuthenticatedAPISupport returns whether the exchange supports // authenticated API requests -func (e *Base) GetAuthenticatedAPISupport(endpoint uint8) bool { +func (b *Base) GetAuthenticatedAPISupport(endpoint uint8) bool { switch endpoint { case RestAuthentication: - return e.API.AuthenticatedSupport + return b.API.AuthenticatedSupport case WebsocketAuthentication: - return e.API.AuthenticatedWebsocketSupport + return b.API.AuthenticatedWebsocketSupport } return false } // GetName is a method that returns the name of the exchange base -func (e *Base) GetName() string { - return e.Name +func (b *Base) GetName() string { + return b.Name } // GetEnabledFeatures returns the exchanges enabled features -func (e *Base) GetEnabledFeatures() FeaturesEnabled { - return e.Features.Enabled +func (b *Base) GetEnabledFeatures() FeaturesEnabled { + return b.Features.Enabled } // GetSupportedFeatures returns the exchanges supported features -func (e *Base) GetSupportedFeatures() FeaturesSupported { - return e.Features.Supports +func (b *Base) GetSupportedFeatures() FeaturesSupported { + return b.Features.Supports } // GetPairFormat returns the pair format based on the exchange and // asset type -func (e *Base) GetPairFormat(assetType asset.Item, requestFormat bool) (currency.PairFormat, error) { - if e.CurrencyPairs.UseGlobalFormat { +func (b *Base) GetPairFormat(assetType asset.Item, requestFormat bool) (currency.PairFormat, error) { + if b.CurrencyPairs.UseGlobalFormat { if requestFormat { - if e.CurrencyPairs.RequestFormat == nil { + if b.CurrencyPairs.RequestFormat == nil { return currency.PairFormat{}, errors.New("global request format is nil") } - return *e.CurrencyPairs.RequestFormat, nil + return *b.CurrencyPairs.RequestFormat, nil } - if e.CurrencyPairs.ConfigFormat == nil { + if b.CurrencyPairs.ConfigFormat == nil { return currency.PairFormat{}, errors.New("global config format is nil") } - return *e.CurrencyPairs.ConfigFormat, nil + return *b.CurrencyPairs.ConfigFormat, nil } - ps, err := e.CurrencyPairs.Get(assetType) + ps, err := b.CurrencyPairs.Get(assetType) if err != nil { return currency.PairFormat{}, err } @@ -390,16 +390,16 @@ func (e *Base) GetPairFormat(assetType asset.Item, requestFormat bool) (currency // GetEnabledPairs is a method that returns the enabled currency pairs of // the exchange by asset type, if the asset type is disabled this will return no // enabled pairs -func (e *Base) GetEnabledPairs(a asset.Item) (currency.Pairs, error) { - err := e.CurrencyPairs.IsAssetEnabled(a) +func (b *Base) GetEnabledPairs(a asset.Item) (currency.Pairs, error) { + err := b.CurrencyPairs.IsAssetEnabled(a) if err != nil { return nil, nil } - format, err := e.GetPairFormat(a, false) + format, err := b.GetPairFormat(a, false) if err != nil { return nil, err } - enabledpairs, err := e.CurrencyPairs.GetPairs(a, true) + enabledpairs, err := b.CurrencyPairs.GetPairs(a, true) if err != nil { return nil, err } @@ -411,16 +411,16 @@ func (e *Base) GetEnabledPairs(a asset.Item) (currency.Pairs, error) { // GetRequestFormattedPairAndAssetType is a method that returns the enabled currency pair of // along with its asset type. Only use when there is no chance of the same name crossing over -func (e *Base) GetRequestFormattedPairAndAssetType(p string) (currency.Pair, asset.Item, error) { - assetTypes := e.GetAssetTypes() +func (b *Base) GetRequestFormattedPairAndAssetType(p string) (currency.Pair, asset.Item, error) { + assetTypes := b.GetAssetTypes() var response currency.Pair for i := range assetTypes { - format, err := e.GetPairFormat(assetTypes[i], true) + format, err := b.GetPairFormat(assetTypes[i], true) if err != nil { return response, assetTypes[i], err } - pairs, err := e.CurrencyPairs.GetPairs(assetTypes[i], true) + pairs, err := b.CurrencyPairs.GetPairs(assetTypes[i], true) if err != nil { return response, assetTypes[i], err } @@ -437,12 +437,12 @@ func (e *Base) GetRequestFormattedPairAndAssetType(p string) (currency.Pair, ass // GetAvailablePairs is a method that returns the available currency pairs // of the exchange by asset type -func (e *Base) GetAvailablePairs(assetType asset.Item) (currency.Pairs, error) { - format, err := e.GetPairFormat(assetType, false) +func (b *Base) GetAvailablePairs(assetType asset.Item) (currency.Pairs, error) { + format, err := b.GetPairFormat(assetType, false) if err != nil { return nil, err } - pairs, err := e.CurrencyPairs.GetPairs(assetType, false) + pairs, err := b.CurrencyPairs.GetPairs(assetType, false) if err != nil { return nil, err } @@ -451,9 +451,9 @@ func (e *Base) GetAvailablePairs(assetType asset.Item) (currency.Pairs, error) { // SupportsPair returns true or not whether a currency pair exists in the // exchange available currencies or not -func (e *Base) SupportsPair(p currency.Pair, enabledPairs bool, assetType asset.Item) error { +func (b *Base) SupportsPair(p currency.Pair, enabledPairs bool, assetType asset.Item) error { if enabledPairs { - pairs, err := e.GetEnabledPairs(assetType) + pairs, err := b.GetEnabledPairs(assetType) if err != nil { return err } @@ -463,7 +463,7 @@ func (e *Base) SupportsPair(p currency.Pair, enabledPairs bool, assetType asset. return errors.New("pair not supported") } - avail, err := e.GetAvailablePairs(assetType) + avail, err := b.GetAvailablePairs(assetType) if err != nil { return err } @@ -475,15 +475,15 @@ func (e *Base) SupportsPair(p currency.Pair, enabledPairs bool, assetType asset. // FormatExchangeCurrencies returns a string containing // the exchanges formatted currency pairs -func (e *Base) FormatExchangeCurrencies(pairs []currency.Pair, assetType asset.Item) (string, error) { +func (b *Base) FormatExchangeCurrencies(pairs []currency.Pair, assetType asset.Item) (string, error) { var currencyItems strings.Builder - pairFmt, err := e.GetPairFormat(assetType, true) + pairFmt, err := b.GetPairFormat(assetType, true) if err != nil { return "", err } for x := range pairs { - format, err := e.FormatExchangeCurrency(pairs[x], assetType) + format, err := b.FormatExchangeCurrency(pairs[x], assetType) if err != nil { return "", err } @@ -502,8 +502,8 @@ func (e *Base) FormatExchangeCurrencies(pairs []currency.Pair, assetType asset.I // FormatExchangeCurrency is a method that formats and returns a currency pair // based on the user currency display preferences -func (e *Base) FormatExchangeCurrency(p currency.Pair, assetType asset.Item) (currency.Pair, error) { - pairFmt, err := e.GetPairFormat(assetType, true) +func (b *Base) FormatExchangeCurrency(p currency.Pair, assetType asset.Item) (currency.Pair, error) { + pairFmt, err := b.GetPairFormat(assetType, true) if err != nil { return currency.Pair{}, err } @@ -511,47 +511,47 @@ func (e *Base) FormatExchangeCurrency(p currency.Pair, assetType asset.Item) (cu } // SetEnabled is a method that sets if the exchange is enabled -func (e *Base) SetEnabled(enabled bool) { - e.Enabled = enabled +func (b *Base) SetEnabled(enabled bool) { + b.Enabled = enabled } // IsEnabled is a method that returns if the current exchange is enabled -func (e *Base) IsEnabled() bool { - return e.Enabled +func (b *Base) IsEnabled() bool { + return b.Enabled } // SetAPIKeys is a method that sets the current API keys for the exchange -func (e *Base) SetAPIKeys(apiKey, apiSecret, clientID string) { - e.API.Credentials.Key = apiKey - e.API.Credentials.ClientID = clientID +func (b *Base) SetAPIKeys(apiKey, apiSecret, clientID string) { + b.API.Credentials.Key = apiKey + b.API.Credentials.ClientID = clientID - if e.API.CredentialsValidator.RequiresBase64DecodeSecret { + if b.API.CredentialsValidator.RequiresBase64DecodeSecret { result, err := crypto.Base64Decode(apiSecret) if err != nil { - e.API.AuthenticatedSupport = false - e.API.AuthenticatedWebsocketSupport = false + b.API.AuthenticatedSupport = false + b.API.AuthenticatedWebsocketSupport = false log.Warnf(log.ExchangeSys, warningBase64DecryptSecretKeyFailed, - e.Name) + b.Name) return } - e.API.Credentials.Secret = string(result) + b.API.Credentials.Secret = string(result) } else { - e.API.Credentials.Secret = apiSecret + b.API.Credentials.Secret = apiSecret } } // SetupDefaults sets the exchange settings based on the supplied config -func (e *Base) SetupDefaults(exch *config.ExchangeConfig) error { - e.Enabled = true - e.LoadedByConfig = true - e.Config = exch - e.Verbose = exch.Verbose +func (b *Base) SetupDefaults(exch *config.ExchangeConfig) error { + b.Enabled = true + b.LoadedByConfig = true + b.Config = exch + b.Verbose = exch.Verbose - e.API.AuthenticatedSupport = exch.API.AuthenticatedSupport - e.API.AuthenticatedWebsocketSupport = exch.API.AuthenticatedWebsocketSupport - if e.API.AuthenticatedSupport || e.API.AuthenticatedWebsocketSupport { - e.SetAPIKeys(exch.API.Credentials.Key, + b.API.AuthenticatedSupport = exch.API.AuthenticatedSupport + b.API.AuthenticatedWebsocketSupport = exch.API.AuthenticatedWebsocketSupport + if b.API.AuthenticatedSupport || b.API.AuthenticatedWebsocketSupport { + b.SetAPIKeys(exch.API.Credentials.Key, exch.API.Credentials.Secret, exch.API.Credentials.ClientID) } @@ -560,7 +560,7 @@ func (e *Base) SetupDefaults(exch *config.ExchangeConfig) error { exch.HTTPTimeout = DefaultHTTPTimeout } - err := e.SetHTTPClientTimeout(exch.HTTPTimeout) + err := b.SetHTTPClientTimeout(exch.HTTPTimeout) if err != nil { return err } @@ -569,109 +569,115 @@ func (e *Base) SetupDefaults(exch *config.ExchangeConfig) error { exch.CurrencyPairs = new(currency.PairsManager) } - e.HTTPDebugging = exch.HTTPDebugging - e.SetHTTPClientUserAgent(exch.HTTPUserAgent) - e.SetCurrencyPairFormat() + b.HTTPDebugging = exch.HTTPDebugging + b.SetHTTPClientUserAgent(exch.HTTPUserAgent) + b.SetCurrencyPairFormat() - err = e.SetConfigPairs() + err = b.SetConfigPairs() if err != nil { return err } - e.SetFeatureDefaults() + b.SetFeatureDefaults() - if e.API.Endpoints == nil { - e.API.Endpoints = e.NewEndpoints() + if b.API.Endpoints == nil { + b.API.Endpoints = b.NewEndpoints() } - err = e.SetAPIURL() + err = b.SetAPIURL() if err != nil { return err } - e.SetAPICredentialDefaults() + b.SetAPICredentialDefaults() - err = e.SetClientProxyAddress(exch.ProxyAddress) + err = b.SetClientProxyAddress(exch.ProxyAddress) if err != nil { return err } - e.BaseCurrencies = exch.BaseCurrencies - e.OrderbookVerificationBypass = exch.OrderbookConfig.VerificationBypass + b.BaseCurrencies = exch.BaseCurrencies + + if exch.OrderbookConfig.VerificationBypass { + log.Warnf(log.ExchangeSys, + "%s orderbook verification has been bypassed via config.", + b.Name) + } + b.CanVerifyOrderbook = !exch.OrderbookConfig.VerificationBypass return nil } // AllowAuthenticatedRequest checks to see if the required fields have been set // before sending an authenticated API request -func (e *Base) AllowAuthenticatedRequest() bool { - if e.SkipAuthCheck { +func (b *Base) AllowAuthenticatedRequest() bool { + if b.SkipAuthCheck { return true } // Individual package usage, allow request if API credentials are valid a // and without needing to set AuthenticatedSupport to true - if !e.LoadedByConfig { - return e.ValidateAPICredentials() + if !b.LoadedByConfig { + return b.ValidateAPICredentials() } // Bot usage, AuthenticatedSupport can be disabled by user if desired, so // don't allow authenticated requests. - if !e.API.AuthenticatedSupport && !e.API.AuthenticatedWebsocketSupport { + if !b.API.AuthenticatedSupport && !b.API.AuthenticatedWebsocketSupport { return false } // Check to see if the user has enabled AuthenticatedSupport, but has // invalid API credentials set and loaded by config - return e.ValidateAPICredentials() + return b.ValidateAPICredentials() } // ValidateAPICredentials validates the exchanges API credentials -func (e *Base) ValidateAPICredentials() bool { - if e.API.CredentialsValidator.RequiresKey { - if e.API.Credentials.Key == "" || - e.API.Credentials.Key == config.DefaultAPIKey { +func (b *Base) ValidateAPICredentials() bool { + if b.API.CredentialsValidator.RequiresKey { + if b.API.Credentials.Key == "" || + b.API.Credentials.Key == config.DefaultAPIKey { log.Warnf(log.ExchangeSys, "exchange %s requires API key but default/empty one set", - e.Name) + b.Name) return false } } - if e.API.CredentialsValidator.RequiresSecret { - if e.API.Credentials.Secret == "" || - e.API.Credentials.Secret == config.DefaultAPISecret { + if b.API.CredentialsValidator.RequiresSecret { + if b.API.Credentials.Secret == "" || + b.API.Credentials.Secret == config.DefaultAPISecret { log.Warnf(log.ExchangeSys, "exchange %s requires API secret but default/empty one set", - e.Name) + b.Name) return false } } - if e.API.CredentialsValidator.RequiresPEM { - if e.API.Credentials.PEMKey == "" || - strings.Contains(e.API.Credentials.PEMKey, "JUSTADUMMY") { + if b.API.CredentialsValidator.RequiresPEM { + if b.API.Credentials.PEMKey == "" || + strings.Contains(b.API.Credentials.PEMKey, "JUSTADUMMY") { log.Warnf(log.ExchangeSys, "exchange %s requires API PEM key but default/empty one set", - e.Name) + b.Name) return false } } - if e.API.CredentialsValidator.RequiresClientID { - if e.API.Credentials.ClientID == "" || - e.API.Credentials.ClientID == config.DefaultAPIClientID { + if b.API.CredentialsValidator.RequiresClientID { + if b.API.Credentials.ClientID == "" || + b.API.Credentials.ClientID == config.DefaultAPIClientID { log.Warnf(log.ExchangeSys, "exchange %s requires API ClientID but default/empty one set", - e.Name) + b.Name) return false } } - if e.API.CredentialsValidator.RequiresBase64DecodeSecret && !e.LoadedByConfig { - _, err := crypto.Base64Decode(e.API.Credentials.Secret) + if b.API.CredentialsValidator.RequiresBase64DecodeSecret && !b.LoadedByConfig { + _, err := crypto.Base64Decode(b.API.Credentials.Secret) if err != nil { log.Warnf(log.ExchangeSys, "exchange %s API secret base64 decode failed: %s", - e.Name, err) + b.Name, err) return false } } @@ -680,12 +686,12 @@ func (e *Base) ValidateAPICredentials() bool { // SetPairs sets the exchange currency pairs for either enabledPairs or // availablePairs -func (e *Base) SetPairs(pairs currency.Pairs, assetType asset.Item, enabled bool) error { +func (b *Base) SetPairs(pairs currency.Pairs, assetType asset.Item, enabled bool) error { if len(pairs) == 0 { - return fmt.Errorf("%s SetPairs error - pairs is empty", e.Name) + return fmt.Errorf("%s SetPairs error - pairs is empty", b.Name) } - pairFmt, err := e.GetPairFormat(assetType, false) + pairFmt, err := b.GetPairFormat(assetType, false) if err != nil { return err } @@ -696,14 +702,14 @@ func (e *Base) SetPairs(pairs currency.Pairs, assetType asset.Item, enabled bool pairFmt.Uppercase)) } - e.CurrencyPairs.StorePairs(assetType, newPairs, enabled) - e.Config.CurrencyPairs.StorePairs(assetType, newPairs, enabled) + b.CurrencyPairs.StorePairs(assetType, newPairs, enabled) + b.Config.CurrencyPairs.StorePairs(assetType, newPairs, enabled) return nil } // UpdatePairs updates the exchange currency pairs for either enabledPairs or // availablePairs -func (e *Base) UpdatePairs(exchangeProducts currency.Pairs, assetType asset.Item, enabled, force bool) error { +func (b *Base) UpdatePairs(exchangeProducts currency.Pairs, assetType asset.Item, enabled, force bool) error { exchangeProducts = exchangeProducts.Upper() var products currency.Pairs for x := range exchangeProducts { @@ -714,7 +720,7 @@ func (e *Base) UpdatePairs(exchangeProducts currency.Pairs, assetType asset.Item } var updateType string - targetPairs, err := e.CurrencyPairs.GetPairs(assetType, enabled) + targetPairs, err := b.CurrencyPairs.GetPairs(assetType, enabled) if err != nil { return err } @@ -730,14 +736,14 @@ func (e *Base) UpdatePairs(exchangeProducts currency.Pairs, assetType asset.Item if force { log.Debugf(log.ExchangeSys, "%s forced update of %s [%v] pairs.", - e.Name, + b.Name, updateType, strings.ToUpper(assetType.String())) } else { if len(newPairs) > 0 { log.Debugf(log.ExchangeSys, "%s Updating %s pairs [%v] - Added: %s.\n", - e.Name, + b.Name, updateType, strings.ToUpper(assetType.String()), newPairs) @@ -745,20 +751,20 @@ func (e *Base) UpdatePairs(exchangeProducts currency.Pairs, assetType asset.Item if len(removedPairs) > 0 { log.Debugf(log.ExchangeSys, "%s Updating %s pairs [%v] - Removed: %s.\n", - e.Name, + b.Name, updateType, strings.ToUpper(assetType.String()), removedPairs) } } - e.Config.CurrencyPairs.StorePairs(assetType, products, enabled) - e.CurrencyPairs.StorePairs(assetType, products, enabled) + b.Config.CurrencyPairs.StorePairs(assetType, products, enabled) + b.CurrencyPairs.StorePairs(assetType, products, enabled) if !enabled { // If available pairs are changed we will remove currency pair items // that are still included in the enabled pairs list. - enabledPairs, err := e.CurrencyPairs.GetPairs(assetType, true) + enabledPairs, err := b.CurrencyPairs.GetPairs(assetType, true) if err == nil { return nil } @@ -770,11 +776,11 @@ func (e *Base) UpdatePairs(exchangeProducts currency.Pairs, assetType asset.Item if len(remove) > 0 { log.Debugf(log.ExchangeSys, "%s Checked and updated enabled pairs [%v] - Removed: %s.\n", - e.Name, + b.Name, strings.ToUpper(assetType.String()), remove) - e.Config.CurrencyPairs.StorePairs(assetType, enabledPairs, true) - e.CurrencyPairs.StorePairs(assetType, enabledPairs, true) + b.Config.CurrencyPairs.StorePairs(assetType, enabledPairs, true) + b.CurrencyPairs.StorePairs(assetType, enabledPairs, true) } } } @@ -782,7 +788,7 @@ func (e *Base) UpdatePairs(exchangeProducts currency.Pairs, assetType asset.Item } // SetAPIURL sets configuration API URL for an exchange -func (e *Base) SetAPIURL() error { +func (b *Base) SetAPIURL() error { checkInsecureEndpoint := func(endpoint string) { if strings.Contains(endpoint, "https") || strings.Contains(endpoint, "wss") { return @@ -791,74 +797,74 @@ func (e *Base) SetAPIURL() error { "%s is using HTTP instead of HTTPS or WS instead of WSS [%s] for API functionality, an"+ " attacker could eavesdrop on this connection. Use at your"+ " own risk.", - e.Name, endpoint) + b.Name, endpoint) } var err error - if e.Config.API.OldEndPoints != nil { - if e.Config.API.OldEndPoints.URL != "" && e.Config.API.OldEndPoints.URL != config.APIURLNonDefaultMessage { - err = e.API.Endpoints.SetRunning(RestSpot.String(), e.Config.API.OldEndPoints.URL) + if b.Config.API.OldEndPoints != nil { + if b.Config.API.OldEndPoints.URL != "" && b.Config.API.OldEndPoints.URL != config.APIURLNonDefaultMessage { + err = b.API.Endpoints.SetRunning(RestSpot.String(), b.Config.API.OldEndPoints.URL) if err != nil { return err } - checkInsecureEndpoint(e.Config.API.OldEndPoints.URL) + checkInsecureEndpoint(b.Config.API.OldEndPoints.URL) } - if e.Config.API.OldEndPoints.URLSecondary != "" && e.Config.API.OldEndPoints.URLSecondary != config.APIURLNonDefaultMessage { - err = e.API.Endpoints.SetRunning(RestSpotSupplementary.String(), e.Config.API.OldEndPoints.URLSecondary) + if b.Config.API.OldEndPoints.URLSecondary != "" && b.Config.API.OldEndPoints.URLSecondary != config.APIURLNonDefaultMessage { + err = b.API.Endpoints.SetRunning(RestSpotSupplementary.String(), b.Config.API.OldEndPoints.URLSecondary) if err != nil { return err } - checkInsecureEndpoint(e.Config.API.OldEndPoints.URLSecondary) + checkInsecureEndpoint(b.Config.API.OldEndPoints.URLSecondary) } - if e.Config.API.OldEndPoints.WebsocketURL != "" && e.Config.API.OldEndPoints.WebsocketURL != config.WebsocketURLNonDefaultMessage { - err = e.API.Endpoints.SetRunning(WebsocketSpot.String(), e.Config.API.OldEndPoints.WebsocketURL) + if b.Config.API.OldEndPoints.WebsocketURL != "" && b.Config.API.OldEndPoints.WebsocketURL != config.WebsocketURLNonDefaultMessage { + err = b.API.Endpoints.SetRunning(WebsocketSpot.String(), b.Config.API.OldEndPoints.WebsocketURL) if err != nil { return err } - checkInsecureEndpoint(e.Config.API.OldEndPoints.WebsocketURL) + checkInsecureEndpoint(b.Config.API.OldEndPoints.WebsocketURL) } - e.Config.API.OldEndPoints = nil - } else if e.Config.API.Endpoints != nil { - for key, val := range e.Config.API.Endpoints { + b.Config.API.OldEndPoints = nil + } else if b.Config.API.Endpoints != nil { + for key, val := range b.Config.API.Endpoints { if val == "" || val == config.APIURLNonDefaultMessage || val == config.WebsocketURLNonDefaultMessage { continue } checkInsecureEndpoint(val) - err = e.API.Endpoints.SetRunning(key, val) + err = b.API.Endpoints.SetRunning(key, val) if err != nil { return err } } } - runningMap := e.API.Endpoints.GetURLMap() - e.Config.API.Endpoints = runningMap + runningMap := b.API.Endpoints.GetURLMap() + b.Config.API.Endpoints = runningMap return nil } // SupportsREST returns whether or not the exchange supports // REST -func (e *Base) SupportsREST() bool { - return e.Features.Supports.REST +func (b *Base) SupportsREST() bool { + return b.Features.Supports.REST } // GetWithdrawPermissions passes through the exchange's withdraw permissions -func (e *Base) GetWithdrawPermissions() uint32 { - return e.Features.Supports.WithdrawPermissions +func (b *Base) GetWithdrawPermissions() uint32 { + return b.Features.Supports.WithdrawPermissions } // SupportsWithdrawPermissions compares the supplied permissions with the exchange's to verify they're supported -func (e *Base) SupportsWithdrawPermissions(permissions uint32) bool { - exchangePermissions := e.GetWithdrawPermissions() +func (b *Base) SupportsWithdrawPermissions(permissions uint32) bool { + exchangePermissions := b.GetWithdrawPermissions() return permissions&exchangePermissions == permissions } // FormatWithdrawPermissions will return each of the exchange's compatible withdrawal methods in readable form -func (e *Base) FormatWithdrawPermissions() string { +func (b *Base) FormatWithdrawPermissions() string { var services []string for i := 0; i < 32; i++ { var check uint32 = 1 << uint32(i) - if e.GetWithdrawPermissions()&check != 0 { + if b.GetWithdrawPermissions()&check != 0 { switch check { case AutoWithdrawCrypto: services = append(services, AutoWithdrawCryptoText) @@ -912,29 +918,29 @@ func (e *Base) FormatWithdrawPermissions() string { // SupportsAsset whether or not the supplied asset is supported // by the exchange -func (e *Base) SupportsAsset(a asset.Item) bool { - _, ok := e.CurrencyPairs.Pairs[a] +func (b *Base) SupportsAsset(a asset.Item) bool { + _, ok := b.CurrencyPairs.Pairs[a] return ok } // PrintEnabledPairs prints the exchanges enabled asset pairs -func (e *Base) PrintEnabledPairs() { - for k, v := range e.CurrencyPairs.Pairs { +func (b *Base) PrintEnabledPairs() { + for k, v := range b.CurrencyPairs.Pairs { log.Infof(log.ExchangeSys, "%s Asset type %v:\n\t Enabled pairs: %v", - e.Name, strings.ToUpper(k.String()), v.Enabled) + b.Name, strings.ToUpper(k.String()), v.Enabled) } } // GetBase returns the exchange base -func (e *Base) GetBase() *Base { return e } +func (b *Base) GetBase() *Base { return b } // CheckTransientError catches transient errors and returns nil if found, used // for validation of API credentials -func (e *Base) CheckTransientError(err error) error { +func (b *Base) CheckTransientError(err error) error { if _, ok := err.(net.Error); ok { log.Warnf(log.ExchangeSys, "%s net error captured, will not disable authentication %s", - e.Name, + b.Name, err) return nil } @@ -942,20 +948,20 @@ func (e *Base) CheckTransientError(err error) error { } // DisableRateLimiter disables the rate limiting system for the exchange -func (e *Base) DisableRateLimiter() error { - return e.Requester.DisableRateLimiter() +func (b *Base) DisableRateLimiter() error { + return b.Requester.DisableRateLimiter() } // EnableRateLimiter enables the rate limiting system for the exchange -func (e *Base) EnableRateLimiter() error { - return e.Requester.EnableRateLimiter() +func (b *Base) EnableRateLimiter() error { + return b.Requester.EnableRateLimiter() } // StoreAssetPairFormat initialises and stores a defined asset format -func (e *Base) StoreAssetPairFormat(a asset.Item, f currency.PairStore) error { +func (b *Base) StoreAssetPairFormat(a asset.Item, f currency.PairStore) error { if a.String() == "" { return fmt.Errorf("%s cannot add to pairs manager, no asset provided", - e.Name) + b.Name) } if f.AssetEnabled == nil { @@ -964,152 +970,152 @@ func (e *Base) StoreAssetPairFormat(a asset.Item, f currency.PairStore) error { if f.RequestFormat == nil { return fmt.Errorf("%s cannot add to pairs manager, request pair format not provided", - e.Name) + b.Name) } if f.ConfigFormat == nil { return fmt.Errorf("%s cannot add to pairs manager, config pair format not provided", - e.Name) + b.Name) } - if e.CurrencyPairs.Pairs == nil { - e.CurrencyPairs.Pairs = make(map[asset.Item]*currency.PairStore) + if b.CurrencyPairs.Pairs == nil { + b.CurrencyPairs.Pairs = make(map[asset.Item]*currency.PairStore) } - e.CurrencyPairs.Pairs[a] = &f + b.CurrencyPairs.Pairs[a] = &f return nil } // SetGlobalPairsManager sets defined asset and pairs management system with // with global formatting -func (e *Base) SetGlobalPairsManager(request, config *currency.PairFormat, assets ...asset.Item) error { +func (b *Base) SetGlobalPairsManager(request, config *currency.PairFormat, assets ...asset.Item) error { if request == nil { return fmt.Errorf("%s cannot set pairs manager, request pair format not provided", - e.Name) + b.Name) } if config == nil { return fmt.Errorf("%s cannot set pairs manager, config pair format not provided", - e.Name) + b.Name) } if len(assets) == 0 { return fmt.Errorf("%s cannot set pairs manager, no assets provided", - e.Name) + b.Name) } - e.CurrencyPairs.UseGlobalFormat = true - e.CurrencyPairs.RequestFormat = request - e.CurrencyPairs.ConfigFormat = config + b.CurrencyPairs.UseGlobalFormat = true + b.CurrencyPairs.RequestFormat = request + b.CurrencyPairs.ConfigFormat = config - if e.CurrencyPairs.Pairs != nil { + if b.CurrencyPairs.Pairs != nil { return fmt.Errorf("%s cannot set pairs manager, pairs already set", - e.Name) + b.Name) } - e.CurrencyPairs.Pairs = make(map[asset.Item]*currency.PairStore) + b.CurrencyPairs.Pairs = make(map[asset.Item]*currency.PairStore) for i := range assets { if assets[i].String() == "" { - e.CurrencyPairs.Pairs = nil + b.CurrencyPairs.Pairs = nil return fmt.Errorf("%s cannot set pairs manager, asset is empty string", - e.Name) + b.Name) } - e.CurrencyPairs.Pairs[assets[i]] = new(currency.PairStore) - e.CurrencyPairs.Pairs[assets[i]].ConfigFormat = config - e.CurrencyPairs.Pairs[assets[i]].RequestFormat = request + b.CurrencyPairs.Pairs[assets[i]] = new(currency.PairStore) + b.CurrencyPairs.Pairs[assets[i]].ConfigFormat = config + b.CurrencyPairs.Pairs[assets[i]].RequestFormat = request } return nil } // GetWebsocket returns a pointer to the exchange websocket -func (e *Base) GetWebsocket() (*stream.Websocket, error) { - if e.Websocket == nil { +func (b *Base) GetWebsocket() (*stream.Websocket, error) { + if b.Websocket == nil { return nil, common.ErrFunctionNotSupported } - return e.Websocket, nil + return b.Websocket, nil } // SupportsWebsocket returns whether or not the exchange supports // websocket -func (e *Base) SupportsWebsocket() bool { - return e.Features.Supports.Websocket +func (b *Base) SupportsWebsocket() bool { + return b.Features.Supports.Websocket } // IsWebsocketEnabled returns whether or not the exchange has its // websocket client enabled -func (e *Base) IsWebsocketEnabled() bool { - if e.Websocket == nil { +func (b *Base) IsWebsocketEnabled() bool { + if b.Websocket == nil { return false } - return e.Websocket.IsEnabled() + return b.Websocket.IsEnabled() } // FlushWebsocketChannels refreshes websocket channel subscriptions based on // websocket features. Used in the event of a pair/asset or subscription change. -func (e *Base) FlushWebsocketChannels() error { - if e.Websocket == nil { +func (b *Base) FlushWebsocketChannels() error { + if b.Websocket == nil { return nil } - return e.Websocket.FlushChannels() + return b.Websocket.FlushChannels() } // SubscribeToWebsocketChannels appends to ChannelsToSubscribe // which lets websocket.manageSubscriptions handle subscribing -func (e *Base) SubscribeToWebsocketChannels(channels []stream.ChannelSubscription) error { - if e.Websocket == nil { +func (b *Base) SubscribeToWebsocketChannels(channels []stream.ChannelSubscription) error { + if b.Websocket == nil { return common.ErrFunctionNotSupported } - return e.Websocket.SubscribeToChannels(channels) + return b.Websocket.SubscribeToChannels(channels) } // UnsubscribeToWebsocketChannels removes from ChannelsToSubscribe // which lets websocket.manageSubscriptions handle unsubscribing -func (e *Base) UnsubscribeToWebsocketChannels(channels []stream.ChannelSubscription) error { - if e.Websocket == nil { +func (b *Base) UnsubscribeToWebsocketChannels(channels []stream.ChannelSubscription) error { + if b.Websocket == nil { return common.ErrFunctionNotSupported } - return e.Websocket.UnsubscribeChannels(channels) + return b.Websocket.UnsubscribeChannels(channels) } // GetSubscriptions returns a copied list of subscriptions -func (e *Base) GetSubscriptions() ([]stream.ChannelSubscription, error) { - if e.Websocket == nil { +func (b *Base) GetSubscriptions() ([]stream.ChannelSubscription, error) { + if b.Websocket == nil { return nil, common.ErrFunctionNotSupported } - return e.Websocket.GetSubscriptions(), nil + return b.Websocket.GetSubscriptions(), nil } // AuthenticateWebsocket sends an authentication message to the websocket -func (e *Base) AuthenticateWebsocket() error { +func (b *Base) AuthenticateWebsocket() error { return common.ErrFunctionNotSupported } // KlineIntervalEnabled returns if requested interval is enabled on exchange -func (e *Base) klineIntervalEnabled(in kline.Interval) bool { - return e.Features.Enabled.Kline.Intervals[in.Word()] +func (b *Base) klineIntervalEnabled(in kline.Interval) bool { + return b.Features.Enabled.Kline.Intervals[in.Word()] } // FormatExchangeKlineInterval returns Interval to string // Exchanges can override this if they require custom formatting -func (e *Base) FormatExchangeKlineInterval(in kline.Interval) string { +func (b *Base) FormatExchangeKlineInterval(in kline.Interval) string { return strconv.FormatFloat(in.Duration().Seconds(), 'f', 0, 64) } // ValidateKline confirms that the requested pair, asset & interval are supported and/or enabled by the requested exchange -func (e *Base) ValidateKline(pair currency.Pair, a asset.Item, interval kline.Interval) error { +func (b *Base) ValidateKline(pair currency.Pair, a asset.Item, interval kline.Interval) error { var errorList []string var err kline.ErrorKline - if e.CurrencyPairs.IsAssetEnabled(a) != nil { + if b.CurrencyPairs.IsAssetEnabled(a) != nil { err.Asset = a errorList = append(errorList, "asset not enabled") - } else if !e.CurrencyPairs.Pairs[a].Enabled.Contains(pair, true) { + } else if !b.CurrencyPairs.Pairs[a].Enabled.Contains(pair, true) { err.Pair = pair errorList = append(errorList, "pair not enabled") } - if !e.klineIntervalEnabled(interval) { + if !b.klineIntervalEnabled(interval) { err.Interval = interval errorList = append(errorList, "interval not supported") } @@ -1124,39 +1130,38 @@ func (e *Base) ValidateKline(pair currency.Pair, a asset.Item, interval kline.In // AddTradesToBuffer is a helper function that will only // add trades to the buffer if it is allowed -func (e *Base) AddTradesToBuffer(trades ...trade.Data) error { - if !e.IsSaveTradeDataEnabled() { +func (b *Base) AddTradesToBuffer(trades ...trade.Data) error { + if !b.IsSaveTradeDataEnabled() { return nil } - - return trade.AddTradesToBuffer(e.Name, trades...) + return trade.AddTradesToBuffer(b.Name, trades...) } // IsSaveTradeDataEnabled checks the state of // SaveTradeData in a concurrent-friendly manner -func (e *Base) IsSaveTradeDataEnabled() bool { - e.settingsMutex.RLock() - isEnabled := e.Features.Enabled.SaveTradeData - e.settingsMutex.RUnlock() +func (b *Base) IsSaveTradeDataEnabled() bool { + b.settingsMutex.RLock() + isEnabled := b.Features.Enabled.SaveTradeData + b.settingsMutex.RUnlock() return isEnabled } // SetSaveTradeDataStatus locks and sets the status of // the config and the exchange's setting for SaveTradeData -func (e *Base) SetSaveTradeDataStatus(enabled bool) { - e.settingsMutex.Lock() - defer e.settingsMutex.Unlock() - e.Features.Enabled.SaveTradeData = enabled - e.Config.Features.Enabled.SaveTradeData = enabled - if e.Verbose { - log.Debugf(log.Trade, "Set %v 'SaveTradeData' to %v", e.Name, enabled) +func (b *Base) SetSaveTradeDataStatus(enabled bool) { + b.settingsMutex.Lock() + defer b.settingsMutex.Unlock() + b.Features.Enabled.SaveTradeData = enabled + b.Config.Features.Enabled.SaveTradeData = enabled + if b.Verbose { + log.Debugf(log.Trade, "Set %v 'SaveTradeData' to %v", b.Name, enabled) } } // NewEndpoints declares default and running URLs maps -func (e *Base) NewEndpoints() *Endpoints { +func (b *Base) NewEndpoints() *Endpoints { return &Endpoints{ - Exchange: e.Name, + Exchange: b.Name, defaults: make(map[string]string), } } @@ -1182,7 +1187,11 @@ func (e *Endpoints) SetRunning(key, val string) error { } _, err = url.ParseRequestURI(val) if err != nil { - log.Warnf(log.ExchangeSys, "Could not set custom URL for %s to %s for exchange %s. invalid URI for request.", key, val, e.Exchange) + log.Warnf(log.ExchangeSys, + "Could not set custom URL for %s to %s for exchange %s. invalid URI for request.", + key, + val, + e.Exchange) return nil } e.defaults[key] = val @@ -1221,8 +1230,8 @@ func (e *Endpoints) GetURLMap() map[string]string { } // FormatSymbol formats the given pair to a string suitable for exchange API requests -func (e *Base) FormatSymbol(pair currency.Pair, assetType asset.Item) (string, error) { - pairFmt, err := e.GetPairFormat(assetType, true) +func (b *Base) FormatSymbol(pair currency.Pair, assetType asset.Item) (string, error) { + pairFmt, err := b.GetPairFormat(assetType, true) if err != nil { return pair.String(), err } @@ -1263,6 +1272,33 @@ func (u URL) String() string { } // UpdateOrderExecutionLimits updates order execution limits this is overridable -func (e *Base) UpdateOrderExecutionLimits(a asset.Item) error { +func (b *Base) UpdateOrderExecutionLimits(a asset.Item) error { return common.ErrNotYetImplemented } + +// DisableAssetWebsocketSupport disables websocket functionality for the +// supplied asset item. In the case that websocket functionality has not yet +// been implemented for that specific asset type. This is a base method to +// check availability of asset type. +func (b *Base) DisableAssetWebsocketSupport(aType asset.Item) error { + if !b.SupportsAsset(aType) { + return fmt.Errorf("%s %w", + aType, + asset.ErrNotSupported) + } + b.AssetWebsocketSupport.m.Lock() + if b.AssetWebsocketSupport.unsupported == nil { + b.AssetWebsocketSupport.unsupported = make(map[asset.Item]bool) + } + b.AssetWebsocketSupport.unsupported[aType] = true + b.AssetWebsocketSupport.m.Unlock() + return nil +} + +// IsAssetWebsocketSupported checks to see if the supplied asset type is +// supported by websocket. +func (a *AssetWebsocketSupport) IsAssetWebsocketSupported(aType asset.Item) bool { + a.m.RLock() + defer a.m.RUnlock() + return a.unsupported == nil || !a.unsupported[aType] +} diff --git a/exchanges/exchange_test.go b/exchanges/exchange_test.go index 2c76503d..71ac4f91 100644 --- a/exchanges/exchange_test.go +++ b/exchanges/exchange_test.go @@ -2323,3 +2323,49 @@ func TestSetRunning(t *testing.T) { t.Error(err) } } + +func TestAssetWebsocketFunctionality(t *testing.T) { + b := Base{} + if !b.IsAssetWebsocketSupported(asset.Spot) { + t.Fatal("error asset is not turned off, unexpected response") + } + + err := b.DisableAssetWebsocketSupport(asset.Spot) + if !errors.Is(err, asset.ErrNotSupported) { + t.Fatalf("expected error: %v but received: %v", asset.ErrNotSupported, err) + } + + err = b.StoreAssetPairFormat(asset.Spot, currency.PairStore{ + RequestFormat: ¤cy.PairFormat{ + Uppercase: true, + }, + ConfigFormat: ¤cy.PairFormat{ + Uppercase: true, + }, + }) + if err != nil { + log.Errorln(log.ExchangeSys, err) + } + + err = b.DisableAssetWebsocketSupport(asset.Spot) + if !errors.Is(err, nil) { + t.Fatalf("expected error: %v but received: %v", nil, err) + } + + if b.IsAssetWebsocketSupported(asset.Spot) { + t.Fatal("error asset is not turned off, unexpected response") + } + + // Edge case + b.AssetWebsocketSupport.unsupported = make(map[asset.Item]bool) + b.AssetWebsocketSupport.unsupported[asset.Spot] = true + b.AssetWebsocketSupport.unsupported[asset.Futures] = false + + if b.IsAssetWebsocketSupported(asset.Spot) { + t.Fatal("error asset is turned off, unexpected response") + } + + if !b.IsAssetWebsocketSupported(asset.Futures) { + t.Fatal("error asset is not turned off, unexpected response") + } +} diff --git a/exchanges/exchange_types.go b/exchanges/exchange_types.go index 153b6a36..dda436da 100644 --- a/exchanges/exchange_types.go +++ b/exchanges/exchange_types.go @@ -6,6 +6,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/config" "github.com/thrasher-corp/gocryptotrader/currency" + "github.com/thrasher-corp/gocryptotrader/exchanges/asset" "github.com/thrasher-corp/gocryptotrader/exchanges/kline" "github.com/thrasher-corp/gocryptotrader/exchanges/order" "github.com/thrasher-corp/gocryptotrader/exchanges/protocol" @@ -220,10 +221,15 @@ type Base struct { WebsocketOrderbookBufferLimit int64 Websocket *stream.Websocket *request.Requester - Config *config.ExchangeConfig - settingsMutex sync.RWMutex - OrderbookVerificationBypass bool + Config *config.ExchangeConfig + settingsMutex sync.RWMutex + // CanVerifyOrderbook determines if the orderbook verification can be bypassed, + // increasing potential update speed but decreasing confidence in orderbook + // integrity. + CanVerifyOrderbook bool order.ExecutionLimits + + AssetWebsocketSupport } // url lookup consts @@ -259,3 +265,11 @@ var keyURLs = []URL{RestSpot, // URL stores uint conversions type URL uint16 + +// AssetWebsocketSupport defines the availability of websocket functionality to +// the specific asset type. TODO: Deprecate as this is a temp item to address +// certain limitations quickly. +type AssetWebsocketSupport struct { + unsupported map[asset.Item]bool + m sync.RWMutex +} diff --git a/exchanges/exmo/exmo_wrapper.go b/exchanges/exmo/exmo_wrapper.go index 5379c604..df3088f4 100644 --- a/exchanges/exmo/exmo_wrapper.go +++ b/exchanges/exmo/exmo_wrapper.go @@ -240,10 +240,10 @@ func (e *EXMO) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbook // UpdateOrderbook updates and returns the orderbook for a currency pair func (e *EXMO) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { callingBook := &orderbook.Base{ - ExchangeName: e.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: e.OrderbookVerificationBypass, + Exchange: e.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: e.CanVerifyOrderbook, } enabledPairs, err := e.GetEnabledPairs(assetType) if err != nil { @@ -262,10 +262,10 @@ func (e *EXMO) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderboo for i := range enabledPairs { book := &orderbook.Base{ - ExchangeName: e.Name, - Pair: enabledPairs[i], - AssetType: assetType, - VerificationBypass: e.OrderbookVerificationBypass, + Exchange: e.Name, + Pair: enabledPairs[i], + Asset: assetType, + VerifyOrderbook: e.CanVerifyOrderbook, } curr, err := e.FormatExchangeCurrency(enabledPairs[i], assetType) diff --git a/exchanges/ftx/ftx_websocket.go b/exchanges/ftx/ftx_websocket.go index 13ee4aae..4a10e541 100644 --- a/exchanges/ftx/ftx_websocket.go +++ b/exchanges/ftx/ftx_websocket.go @@ -452,7 +452,10 @@ func (f *FTX) WsProcessUpdateOB(data *WsOrderbookData, p currency.Pair, a asset. return err } - updatedOb := f.Websocket.Orderbook.GetOrderbook(p, a) + updatedOb, err := f.Websocket.Orderbook.GetOrderbook(p, a) + if err != nil { + return err + } checksum := f.CalcUpdateOBChecksum(updatedOb) if checksum != data.Checksum { @@ -506,14 +509,13 @@ func (f *FTX) WsProcessPartialOB(data *WsOrderbookData, p currency.Pair, a asset } newOrderBook := orderbook.Base{ - Asks: asks, - Bids: bids, - AssetType: a, - LastUpdated: timestampFromFloat64(data.Time), - Pair: p, - ExchangeName: f.Name, - HasChecksumValidation: true, - VerificationBypass: f.OrderbookVerificationBypass, + Asks: asks, + Bids: bids, + Asset: a, + LastUpdated: timestampFromFloat64(data.Time), + Pair: p, + Exchange: f.Name, + VerifyOrderbook: f.CanVerifyOrderbook, } return f.Websocket.Orderbook.LoadSnapshot(&newOrderBook) } diff --git a/exchanges/ftx/ftx_wrapper.go b/exchanges/ftx/ftx_wrapper.go index 520d78ae..18c2e1f7 100644 --- a/exchanges/ftx/ftx_wrapper.go +++ b/exchanges/ftx/ftx_wrapper.go @@ -344,10 +344,10 @@ func (f *FTX) FetchOrderbook(currency currency.Pair, assetType asset.Item) (*ord // UpdateOrderbook updates and returns the orderbook for a currency pair func (f *FTX) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: f.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: f.OrderbookVerificationBypass, + Exchange: f.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: f.CanVerifyOrderbook, } formattedPair, err := f.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/gateio/gateio_websocket.go b/exchanges/gateio/gateio_websocket.go index d7c70cc4..0687d93e 100644 --- a/exchanges/gateio/gateio_websocket.go +++ b/exchanges/gateio/gateio_websocket.go @@ -350,10 +350,10 @@ func (g *Gateio) wsHandleData(respRaw []byte) error { var newOrderBook orderbook.Base newOrderBook.Asks = asks newOrderBook.Bids = bids - newOrderBook.AssetType = asset.Spot + newOrderBook.Asset = asset.Spot newOrderBook.Pair = p - newOrderBook.ExchangeName = g.Name - newOrderBook.VerificationBypass = g.OrderbookVerificationBypass + newOrderBook.Exchange = g.Name + newOrderBook.VerifyOrderbook = g.CanVerifyOrderbook err = g.Websocket.Orderbook.LoadSnapshot(&newOrderBook) if err != nil { diff --git a/exchanges/gateio/gateio_wrapper.go b/exchanges/gateio/gateio_wrapper.go index f55922a7..3cfe0aaa 100644 --- a/exchanges/gateio/gateio_wrapper.go +++ b/exchanges/gateio/gateio_wrapper.go @@ -287,10 +287,10 @@ func (g *Gateio) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbo // UpdateOrderbook updates and returns the orderbook for a currency pair func (g *Gateio) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: g.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: g.OrderbookVerificationBypass, + Exchange: g.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: g.CanVerifyOrderbook, } curr, err := g.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/gemini/gemini_websocket.go b/exchanges/gemini/gemini_websocket.go index 446e697c..be5047de 100644 --- a/exchanges/gemini/gemini_websocket.go +++ b/exchanges/gemini/gemini_websocket.go @@ -526,10 +526,10 @@ func (g *Gemini) wsProcessUpdate(result *wsL2MarketData) error { var newOrderBook orderbook.Base newOrderBook.Asks = asks newOrderBook.Bids = bids - newOrderBook.AssetType = asset.Spot + newOrderBook.Asset = asset.Spot newOrderBook.Pair = pair - newOrderBook.ExchangeName = g.Name - newOrderBook.VerificationBypass = g.OrderbookVerificationBypass + newOrderBook.Exchange = g.Name + newOrderBook.VerifyOrderbook = g.CanVerifyOrderbook err := g.Websocket.Orderbook.LoadSnapshot(&newOrderBook) if err != nil { return err diff --git a/exchanges/gemini/gemini_wrapper.go b/exchanges/gemini/gemini_wrapper.go index 22c9aa6a..4139a4fd 100644 --- a/exchanges/gemini/gemini_wrapper.go +++ b/exchanges/gemini/gemini_wrapper.go @@ -403,10 +403,10 @@ func (g *Gemini) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbo // UpdateOrderbook updates and returns the orderbook for a currency pair func (g *Gemini) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: g.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: g.OrderbookVerificationBypass, + Exchange: g.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: g.CanVerifyOrderbook, } fPair, err := g.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/hitbtc/hitbtc_websocket.go b/exchanges/hitbtc/hitbtc_websocket.go index 0a05e317..37f57eba 100644 --- a/exchanges/hitbtc/hitbtc_websocket.go +++ b/exchanges/hitbtc/hitbtc_websocket.go @@ -327,10 +327,10 @@ func (h *HitBTC) WsProcessOrderbookSnapshot(ob WsOrderbook) error { h.Websocket.DataHandler <- err return err } - newOrderBook.AssetType = asset.Spot + newOrderBook.Asset = asset.Spot newOrderBook.Pair = p - newOrderBook.ExchangeName = h.Name - newOrderBook.VerificationBypass = h.OrderbookVerificationBypass + newOrderBook.Exchange = h.Name + newOrderBook.VerifyOrderbook = h.CanVerifyOrderbook return h.Websocket.Orderbook.LoadSnapshot(&newOrderBook) } diff --git a/exchanges/hitbtc/hitbtc_wrapper.go b/exchanges/hitbtc/hitbtc_wrapper.go index b530cc27..b1d79531 100644 --- a/exchanges/hitbtc/hitbtc_wrapper.go +++ b/exchanges/hitbtc/hitbtc_wrapper.go @@ -376,10 +376,10 @@ func (h *HitBTC) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbo // UpdateOrderbook updates and returns the orderbook for a currency pair func (h *HitBTC) UpdateOrderbook(c currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: h.Name, - Pair: c, - AssetType: assetType, - VerificationBypass: h.OrderbookVerificationBypass, + Exchange: h.Name, + Pair: c, + Asset: assetType, + VerifyOrderbook: h.CanVerifyOrderbook, } fpair, err := h.FormatExchangeCurrency(c, assetType) if err != nil { diff --git a/exchanges/huobi/huobi_websocket.go b/exchanges/huobi/huobi_websocket.go index a77db319..5369f120 100644 --- a/exchanges/huobi/huobi_websocket.go +++ b/exchanges/huobi/huobi_websocket.go @@ -464,9 +464,9 @@ func (h *HUOBI) WsProcessOrderbook(update *WsDepth, symbol string) error { newOrderBook.Asks = asks newOrderBook.Bids = bids newOrderBook.Pair = p - newOrderBook.AssetType = asset.Spot - newOrderBook.ExchangeName = h.Name - newOrderBook.VerificationBypass = h.OrderbookVerificationBypass + newOrderBook.Asset = asset.Spot + newOrderBook.Exchange = h.Name + newOrderBook.VerifyOrderbook = h.CanVerifyOrderbook return h.Websocket.Orderbook.LoadSnapshot(&newOrderBook) } diff --git a/exchanges/huobi/huobi_wrapper.go b/exchanges/huobi/huobi_wrapper.go index 9c3161d0..5dc9c78d 100644 --- a/exchanges/huobi/huobi_wrapper.go +++ b/exchanges/huobi/huobi_wrapper.go @@ -522,10 +522,10 @@ func (h *HUOBI) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderboo // UpdateOrderbook updates and returns the orderbook for a currency pair func (h *HUOBI) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: h.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: h.OrderbookVerificationBypass, + Exchange: h.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: h.CanVerifyOrderbook, } var err error switch assetType { diff --git a/exchanges/interfaces.go b/exchanges/interfaces.go index 39b196d2..a956ba00 100644 --- a/exchanges/interfaces.go +++ b/exchanges/interfaces.go @@ -82,6 +82,7 @@ type IBotExchange interface { SupportsWebsocket() bool SubscribeToWebsocketChannels(channels []stream.ChannelSubscription) error UnsubscribeToWebsocketChannels(channels []stream.ChannelSubscription) error + IsAssetWebsocketSupported(aType asset.Item) bool // FlushWebsocketChannels checks and flushes subscriptions if there is a // pair,asset, url/proxy or subscription change FlushWebsocketChannels() error diff --git a/exchanges/itbit/itbit_wrapper.go b/exchanges/itbit/itbit_wrapper.go index 4269fce2..4c7aa5df 100644 --- a/exchanges/itbit/itbit_wrapper.go +++ b/exchanges/itbit/itbit_wrapper.go @@ -191,11 +191,11 @@ func (i *ItBit) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderboo // UpdateOrderbook updates and returns the orderbook for a currency pair func (i *ItBit) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: i.Name, - Pair: p, - AssetType: assetType, - NotAggregated: true, - VerificationBypass: i.OrderbookVerificationBypass, + Exchange: i.Name, + Pair: p, + Asset: assetType, + PriceDuplication: true, + VerifyOrderbook: i.CanVerifyOrderbook, } fpair, err := i.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/kraken/kraken_websocket.go b/exchanges/kraken/kraken_websocket.go index a3af907d..d768e7a4 100644 --- a/exchanges/kraken/kraken_websocket.go +++ b/exchanges/kraken/kraken_websocket.go @@ -795,9 +795,9 @@ func (k *Kraken) wsProcessOrderBook(channelData *WebsocketChannelData, data map[ // wsProcessOrderBookPartial creates a new orderbook entry for a given currency pair func (k *Kraken) wsProcessOrderBookPartial(channelData *WebsocketChannelData, askData, bidData []interface{}) error { base := orderbook.Base{ - Pair: channelData.Pair, - AssetType: asset.Spot, - VerificationBypass: k.OrderbookVerificationBypass, + Pair: channelData.Pair, + Asset: asset.Spot, + VerifyOrderbook: k.CanVerifyOrderbook, } // Kraken ob data is timestamped per price, GCT orderbook data is // timestamped per entry using the highest last update time, we can attempt @@ -851,8 +851,7 @@ func (k *Kraken) wsProcessOrderBookPartial(channelData *WebsocketChannelData, as } } base.LastUpdated = highestLastUpdate - base.ExchangeName = k.Name - base.HasChecksumValidation = true + base.Exchange = k.Name return k.Websocket.Orderbook.LoadSnapshot(&base) } @@ -995,11 +994,12 @@ func (k *Kraken) wsProcessOrderBookUpdate(channelData *WebsocketChannelData, ask return err } - book := k.Websocket.Orderbook.GetOrderbook(channelData.Pair, asset.Spot) - if book == nil { - return fmt.Errorf("cannot calculate websocket checksum: book not found for %s %s", + book, err := k.Websocket.Orderbook.GetOrderbook(channelData.Pair, asset.Spot) + if err != nil { + return fmt.Errorf("cannot calculate websocket checksum: book not found for %s %s %w", channelData.Pair, - asset.Spot) + asset.Spot, + err) } token, err := strconv.ParseInt(checksum, 10, 64) @@ -1014,12 +1014,13 @@ func validateCRC32(b *orderbook.Base, token uint32, decPrice, decAmount int) err if len(b.Asks) < 10 || len(b.Bids) < 10 { return fmt.Errorf("%s %s insufficient bid and asks to calculate checksum", b.Pair, - b.AssetType) + b.Asset) } if decPrice == 0 || decAmount == 0 { - return fmt.Errorf("%s %s trailing decimal count not calculated", b.Pair, - b.AssetType) + return fmt.Errorf("%s %s trailing decimal count not calculated", + b.Pair, + b.Asset) } var checkStr strings.Builder @@ -1040,7 +1041,7 @@ func validateCRC32(b *orderbook.Base, token uint32, decPrice, decAmount int) err if check := crc32.ChecksumIEEE([]byte(checkStr.String())); check != token { return fmt.Errorf("%s %s invalid checksum %d, expected %d", b.Pair, - b.AssetType, + b.Asset, check, token) } diff --git a/exchanges/kraken/kraken_wrapper.go b/exchanges/kraken/kraken_wrapper.go index 9fb0e8da..96db312c 100644 --- a/exchanges/kraken/kraken_wrapper.go +++ b/exchanges/kraken/kraken_wrapper.go @@ -93,6 +93,11 @@ func (k *Kraken) SetDefaults() { log.Errorln(log.ExchangeSys, err) } + err = k.DisableAssetWebsocketSupport(asset.Futures) + if err != nil { + log.Errorln(log.ExchangeSys, err) + } + k.Features = exchange.Features{ Supports: exchange.FeaturesSupported{ REST: true, @@ -497,10 +502,10 @@ func (k *Kraken) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbo // UpdateOrderbook updates and returns the orderbook for a currency pair func (k *Kraken) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: k.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: k.OrderbookVerificationBypass, + Exchange: k.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: k.CanVerifyOrderbook, } var err error switch assetType { diff --git a/exchanges/lakebtc/lakebtc_websocket.go b/exchanges/lakebtc/lakebtc_websocket.go index aadb8cb8..b4624eab 100644 --- a/exchanges/lakebtc/lakebtc_websocket.go +++ b/exchanges/lakebtc/lakebtc_websocket.go @@ -174,11 +174,11 @@ func (l *LakeBTC) processOrderbook(obUpdate, channel string) error { } book := orderbook.Base{ - Pair: p, - LastUpdated: time.Now(), - AssetType: asset.Spot, - ExchangeName: l.Name, - VerificationBypass: l.OrderbookVerificationBypass, + Pair: p, + LastUpdated: time.Now(), + Asset: asset.Spot, + Exchange: l.Name, + VerifyOrderbook: l.CanVerifyOrderbook, } for i := range update.Asks { diff --git a/exchanges/lakebtc/lakebtc_wrapper.go b/exchanges/lakebtc/lakebtc_wrapper.go index a8a12d8f..09c48329 100644 --- a/exchanges/lakebtc/lakebtc_wrapper.go +++ b/exchanges/lakebtc/lakebtc_wrapper.go @@ -279,10 +279,10 @@ func (l *LakeBTC) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderb // UpdateOrderbook updates and returns the orderbook for a currency pair func (l *LakeBTC) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: l.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: l.OrderbookVerificationBypass, + Exchange: l.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: l.CanVerifyOrderbook, } fPair, err := l.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/lbank/lbank_wrapper.go b/exchanges/lbank/lbank_wrapper.go index 1eb8ad96..0a1f366f 100644 --- a/exchanges/lbank/lbank_wrapper.go +++ b/exchanges/lbank/lbank_wrapper.go @@ -247,10 +247,10 @@ func (l *Lbank) FetchOrderbook(currency currency.Pair, assetType asset.Item) (*o // UpdateOrderbook updates and returns the orderbook for a currency pair func (l *Lbank) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: l.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: l.OrderbookVerificationBypass, + Exchange: l.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: l.CanVerifyOrderbook, } fpair, err := l.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/localbitcoins/localbitcoins_wrapper.go b/exchanges/localbitcoins/localbitcoins_wrapper.go index f0b7397a..ed162069 100644 --- a/exchanges/localbitcoins/localbitcoins_wrapper.go +++ b/exchanges/localbitcoins/localbitcoins_wrapper.go @@ -218,11 +218,12 @@ func (l *LocalBitcoins) FetchOrderbook(p currency.Pair, assetType asset.Item) (* // UpdateOrderbook updates and returns the orderbook for a currency pair func (l *LocalBitcoins) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: l.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: l.OrderbookVerificationBypass, + Exchange: l.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: l.CanVerifyOrderbook, } + orderbookNew, err := l.GetOrderbook(p.Quote.String()) if err != nil { return book, err @@ -242,7 +243,7 @@ func (l *LocalBitcoins) UpdateOrderbook(p currency.Pair, assetType asset.Item) ( }) } - book.NotAggregated = true + book.PriceDuplication = true err = book.Process() if err != nil { return book, err diff --git a/exchanges/okgroup/okgroup_websocket.go b/exchanges/okgroup/okgroup_websocket.go index bb3168cf..8e1a161a 100644 --- a/exchanges/okgroup/okgroup_websocket.go +++ b/exchanges/okgroup/okgroup_websocket.go @@ -678,14 +678,13 @@ func (o *OKGroup) WsProcessPartialOrderBook(wsEventData *WebsocketOrderBook, ins } newOrderBook := orderbook.Base{ - Asks: asks, - Bids: bids, - AssetType: a, - LastUpdated: wsEventData.Timestamp, - Pair: instrument, - ExchangeName: o.Name, - HasChecksumValidation: true, - VerificationBypass: o.OrderbookVerificationBypass, + Asks: asks, + Bids: bids, + Asset: a, + LastUpdated: wsEventData.Timestamp, + Pair: instrument, + Exchange: o.Name, + VerifyOrderbook: o.CanVerifyOrderbook, } return o.Websocket.Orderbook.LoadSnapshot(&newOrderBook) } @@ -715,7 +714,10 @@ func (o *OKGroup) WsProcessUpdateOrderbook(wsEventData *WebsocketOrderBook, inst return err } - updatedOb := o.Websocket.Orderbook.GetOrderbook(instrument, a) + updatedOb, err := o.Websocket.Orderbook.GetOrderbook(instrument, a) + if err != nil { + return err + } checksum := o.CalculateUpdateOrderbookChecksum(updatedOb) if checksum != wsEventData.Checksum { diff --git a/exchanges/okgroup/okgroup_wrapper.go b/exchanges/okgroup/okgroup_wrapper.go index 30bee556..4e3f234d 100644 --- a/exchanges/okgroup/okgroup_wrapper.go +++ b/exchanges/okgroup/okgroup_wrapper.go @@ -86,10 +86,10 @@ func (o *OKGroup) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderb // UpdateOrderbook updates and returns the orderbook for a currency pair func (o *OKGroup) UpdateOrderbook(p currency.Pair, a asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: o.Name, - Pair: p, - AssetType: a, - VerificationBypass: o.OrderbookVerificationBypass, + Exchange: o.Name, + Pair: p, + Asset: a, + VerifyOrderbook: o.CanVerifyOrderbook, } if a == asset.Index { diff --git a/exchanges/orderbook/calculator_test.go b/exchanges/orderbook/calculator_test.go index 410fa9f2..69fea175 100644 --- a/exchanges/orderbook/calculator_test.go +++ b/exchanges/orderbook/calculator_test.go @@ -8,8 +8,8 @@ import ( func testSetup() Base { return Base{ - ExchangeName: "a", - Pair: currency.NewPair(currency.BTC, currency.USD), + Exchange: "a", + Pair: currency.NewPair(currency.BTC, currency.USD), Asks: []Item{ {Price: 7000, Amount: 1}, {Price: 7001, Amount: 2}, diff --git a/exchanges/orderbook/depth.go b/exchanges/orderbook/depth.go new file mode 100644 index 00000000..c87fdab5 --- /dev/null +++ b/exchanges/orderbook/depth.go @@ -0,0 +1,353 @@ +package orderbook + +import ( + "sync" + "sync/atomic" + "time" + + "github.com/gofrs/uuid" + "github.com/thrasher-corp/gocryptotrader/dispatch" + "github.com/thrasher-corp/gocryptotrader/log" +) + +// Depth defines a linked list of orderbook items +type Depth struct { + asks + bids + + // unexported stack of nodes + stack *stack + + Alert + + mux *dispatch.Mux + id uuid.UUID + + options + m sync.Mutex +} + +// NewDepth returns a new depth item +func newDepth(id uuid.UUID) *Depth { + return &Depth{ + stack: newStack(), + id: id, + mux: service.Mux, + } +} + +// Publish alerts any subscribed routines using a dispatch mux +func (d *Depth) Publish() { + err := d.mux.Publish([]uuid.UUID{d.id}, d.Retrieve()) + if err != nil { + log.Errorf(log.ExchangeSys, "Cannot publish orderbook update to mux %v", err) + } +} + +// GetAskLength returns length of asks +func (d *Depth) GetAskLength() int { + d.m.Lock() + defer d.m.Unlock() + return d.asks.length +} + +// GetBidLength returns length of bids +func (d *Depth) GetBidLength() int { + d.m.Lock() + defer d.m.Unlock() + return d.bids.length +} + +// Retrieve returns the orderbook base a copy of the underlying linked list +// spread +func (d *Depth) Retrieve() *Base { + d.m.Lock() + defer d.m.Unlock() + return &Base{ + Bids: d.bids.retrieve(), + Asks: d.asks.retrieve(), + Exchange: d.exchange, + Asset: d.asset, + Pair: d.pair, + LastUpdated: d.lastUpdated, + LastUpdateID: d.lastUpdateID, + PriceDuplication: d.priceDuplication, + IsFundingRate: d.isFundingRate, + VerifyOrderbook: d.VerifyOrderbook, + } +} + +// TotalBidAmounts returns the total amount of bids and the total orderbook +// bids value +func (d *Depth) TotalBidAmounts() (liquidity, value float64) { + d.m.Lock() + defer d.m.Unlock() + return d.bids.amount() +} + +// TotalAskAmounts returns the total amount of asks and the total orderbook +// asks value +func (d *Depth) TotalAskAmounts() (liquidity, value float64) { + d.m.Lock() + defer d.m.Unlock() + return d.asks.amount() +} + +// LoadSnapshot flushes the bids and asks with a snapshot +func (d *Depth) LoadSnapshot(bids, asks []Item) { + d.m.Lock() + d.bids.load(bids, d.stack) + d.asks.load(asks, d.stack) + d.alert() + d.m.Unlock() +} + +// Flush flushes the bid and ask depths +func (d *Depth) Flush() { + d.m.Lock() + d.bids.load(nil, d.stack) + d.asks.load(nil, d.stack) + d.alert() + d.m.Unlock() +} + +// UpdateBidAskByPrice updates the bid and ask spread by supplied updates, this +// will trim total length of depth level to a specified supplied number +func (d *Depth) UpdateBidAskByPrice(bidUpdts, askUpdts Items, maxDepth int) { + if len(bidUpdts) == 0 && len(askUpdts) == 0 { + return + } + d.m.Lock() + tn := getNow() + if len(bidUpdts) != 0 { + d.bids.updateInsertByPrice(bidUpdts, d.stack, maxDepth, tn) + } + if len(askUpdts) != 0 { + d.asks.updateInsertByPrice(askUpdts, d.stack, maxDepth, tn) + } + d.alert() + d.m.Unlock() +} + +// UpdateBidAskByID amends details by ID +func (d *Depth) UpdateBidAskByID(bidUpdts, askUpdts Items) error { + if len(bidUpdts) == 0 && len(askUpdts) == 0 { + return nil + } + d.m.Lock() + defer d.m.Unlock() + if len(bidUpdts) != 0 { + err := d.bids.updateByID(bidUpdts) + if err != nil { + return err + } + } + if len(askUpdts) != 0 { + err := d.asks.updateByID(askUpdts) + if err != nil { + return err + } + } + d.alert() + return nil +} + +// DeleteBidAskByID deletes a price level by ID +func (d *Depth) DeleteBidAskByID(bidUpdts, askUpdts Items, bypassErr bool) error { + if len(bidUpdts) == 0 && len(askUpdts) == 0 { + return nil + } + d.m.Lock() + defer d.m.Unlock() + if len(bidUpdts) != 0 { + err := d.bids.deleteByID(bidUpdts, d.stack, bypassErr) + if err != nil { + return err + } + } + if len(askUpdts) != 0 { + err := d.asks.deleteByID(askUpdts, d.stack, bypassErr) + if err != nil { + return err + } + } + d.alert() + return nil +} + +// InsertBidAskByID inserts new updates +func (d *Depth) InsertBidAskByID(bidUpdts, askUpdts Items) error { + if len(bidUpdts) == 0 && len(askUpdts) == 0 { + return nil + } + d.m.Lock() + defer d.m.Unlock() + if len(bidUpdts) != 0 { + err := d.bids.insertUpdates(bidUpdts, d.stack) + if err != nil { + return err + } + } + if len(askUpdts) != 0 { + err := d.asks.insertUpdates(askUpdts, d.stack) + if err != nil { + return err + } + } + d.alert() + return nil +} + +// UpdateInsertByID updates or inserts by ID at current price level. +func (d *Depth) UpdateInsertByID(bidUpdts, askUpdts Items) error { + if len(bidUpdts) == 0 && len(askUpdts) == 0 { + return nil + } + d.m.Lock() + defer d.m.Unlock() + if len(bidUpdts) != 0 { + err := d.bids.updateInsertByID(bidUpdts, d.stack) + if err != nil { + return err + } + } + if len(askUpdts) != 0 { + err := d.asks.updateInsertByID(askUpdts, d.stack) + if err != nil { + return err + } + } + d.alert() + return nil +} + +// AssignOptions assigns the initial options for the depth instance +func (d *Depth) AssignOptions(b *Base) { + d.m.Lock() + d.options = options{ + exchange: b.Exchange, + pair: b.Pair, + asset: b.Asset, + lastUpdated: b.LastUpdated, + lastUpdateID: b.LastUpdateID, + priceDuplication: b.PriceDuplication, + isFundingRate: b.IsFundingRate, + VerifyOrderbook: b.VerifyOrderbook, + restSnapshot: b.RestSnapshot, + idAligned: b.IDAlignment, + } + d.m.Unlock() +} + +// SetLastUpdate sets details of last update information +func (d *Depth) SetLastUpdate(lastUpdate time.Time, lastUpdateID int64, updateByREST bool) { + d.m.Lock() + d.lastUpdated = lastUpdate + d.lastUpdateID = lastUpdateID + d.restSnapshot = updateByREST + d.m.Unlock() +} + +// GetName returns name of exchange +func (d *Depth) GetName() string { + d.m.Lock() + defer d.m.Unlock() + return d.exchange +} + +// IsRestSnapshot returns if the depth item was updated via REST +func (d *Depth) IsRestSnapshot() bool { + d.m.Lock() + defer d.m.Unlock() + return d.restSnapshot +} + +// LastUpdateID returns the last Update ID +func (d *Depth) LastUpdateID() int64 { + d.m.Lock() + defer d.m.Unlock() + return d.lastUpdateID +} + +// IsFundingRate returns if the depth is a funding rate +func (d *Depth) IsFundingRate() bool { + d.m.Lock() + defer d.m.Unlock() + return d.isFundingRate +} + +// Alert defines fields required to alert sub-systems of a change of state to +// re-check depth list +type Alert struct { + // Channel to wait for an alert on. + forAlert chan struct{} + // Lets the updater functions know if there are any routines waiting for an + // alert. + sema uint32 + // After closing the forAlert channel this will notify when all the routines + // that have waited, have either checked the orderbook depth or finished. + wg sync.WaitGroup + // Segregated lock only for waiting routines, so as this does not interfere + // with the main depth lock, acts as a rolling gate. + m sync.Mutex +} + +// alert establishes a state change on the orderbook depth. +func (a *Alert) alert() { + // CompareAndSwap is used to swap from 1 -> 2 so we don't keep actuating + // the opposing compare and swap in method wait. This function can return + // freely when an alert operation is in process. + if !atomic.CompareAndSwapUint32(&a.sema, 1, 2) { + // Return if no waiting routines or currently alerting. + return + } + + go func() { + // Actuate lock in a different routine, as alerting is a second order + // priority compared to updating and releasing calling routine. + a.m.Lock() + // Closing; alerts many waiting routines. + close(a.forAlert) + // Wait for waiting routines to receive alert and return. + a.wg.Wait() + atomic.SwapUint32(&a.sema, 0) // Swap back to neutral state. + a.m.Unlock() + }() +} + +// Wait pauses calling routine until depth change has been established via depth +// method alert. Kick allows for cancellation of waiting or when the caller has +// has been shut down, if this is not needed it can be set to nil. This +// returns a channel so strategies can cleanly wait on a select statement case. +func (a *Alert) Wait(kick <-chan struct{}) <-chan bool { + reply := make(chan bool) + a.m.Lock() + a.wg.Add(1) + if atomic.CompareAndSwapUint32(&a.sema, 0, 1) { + a.forAlert = make(chan struct{}) + } + go a.hold(reply, kick) + a.m.Unlock() + return reply +} + +// hold waits on either channel in the event that the routine has finished or an +// alert from a depth update has occurred. +func (a *Alert) hold(ch chan<- bool, kick <-chan struct{}) { + select { + // In a select statement, if by chance there is no receiver or its late, + // we can still close and return, limiting dead-lock potential. + case <-a.forAlert: // Main waiting channel from alert + select { + case ch <- false: + default: + } + case <-kick: // This can be nil. + select { + case ch <- true: + default: + } + } + a.wg.Done() + close(ch) +} diff --git a/exchanges/orderbook/depth_test.go b/exchanges/orderbook/depth_test.go new file mode 100644 index 00000000..ed49c23c --- /dev/null +++ b/exchanges/orderbook/depth_test.go @@ -0,0 +1,404 @@ +package orderbook + +import ( + "errors" + "log" + "reflect" + "sync" + "testing" + "time" + + "github.com/gofrs/uuid" + "github.com/thrasher-corp/gocryptotrader/currency" + "github.com/thrasher-corp/gocryptotrader/exchanges/asset" +) + +var id, _ = uuid.NewV4() + +func TestGetLength(t *testing.T) { + d := newDepth(id) + if d.GetAskLength() != 0 { + t.Errorf("expected len %v, but received %v", 0, d.GetAskLength()) + } + + d.asks.load([]Item{{Price: 1337}}, d.stack) + + if d.GetAskLength() != 1 { + t.Errorf("expected len %v, but received %v", 1, d.GetAskLength()) + } + + d = newDepth(id) + if d.GetBidLength() != 0 { + t.Errorf("expected len %v, but received %v", 0, d.GetBidLength()) + } + + d.bids.load([]Item{{Price: 1337}}, d.stack) + + if d.GetBidLength() != 1 { + t.Errorf("expected len %v, but received %v", 1, d.GetBidLength()) + } +} + +func TestRetrieve(t *testing.T) { + d := newDepth(id) + d.asks.load([]Item{{Price: 1337}}, d.stack) + d.bids.load([]Item{{Price: 1337}}, d.stack) + d.options = options{ + exchange: "THE BIG ONE!!!!!!", + pair: currency.NewPair(currency.THETA, currency.USD), + asset: "Silly asset", + lastUpdated: time.Now(), + lastUpdateID: 007, + priceDuplication: true, + isFundingRate: true, + VerifyOrderbook: true, + restSnapshot: true, + idAligned: true, + } + + // If we add anymore options to the options struct later this will complain + // generally want to return a full carbon copy + mirrored := reflect.Indirect(reflect.ValueOf(d.options)) + for n := 0; n < mirrored.NumField(); n++ { + structVal := mirrored.Field(n) + if structVal.IsZero() { + t.Fatalf("struct value options not set for field %v", + mirrored.Type().Field(n).Name) + } + } + theBigD := d.Retrieve() + if len(theBigD.Asks) != 1 { + t.Errorf("expected len %v, but received %v", 1, len(theBigD.Bids)) + } + + if len(theBigD.Bids) != 1 { + t.Errorf("expected len %v, but received %v", 1, len(theBigD.Bids)) + } +} + +func TestTotalAmounts(t *testing.T) { + d := newDepth(id) + + liquidity, value := d.TotalBidAmounts() + if liquidity != 0 || value != 0 { + t.Fatalf("liquidity expected %f received %f value expected %f received %f", + 0., + liquidity, + 0., + value) + } + + liquidity, value = d.TotalAskAmounts() + if liquidity != 0 || value != 0 { + t.Fatalf("liquidity expected %f received %f value expected %f received %f", + 0., + liquidity, + 0., + value) + } + + d.asks.load([]Item{{Price: 1337, Amount: 1}}, d.stack) + d.bids.load([]Item{{Price: 1337, Amount: 10}}, d.stack) + + liquidity, value = d.TotalBidAmounts() + if liquidity != 10 || value != 13370 { + t.Fatalf("liquidity expected %f received %f value expected %f received %f", + 10., + liquidity, + 13370., + value) + } + + liquidity, value = d.TotalAskAmounts() + if liquidity != 1 || value != 1337 { + t.Fatalf("liquidity expected %f received %f value expected %f received %f", + 1., + liquidity, + 1337., + value) + } +} + +func TestLoadSnapshot(t *testing.T) { + d := newDepth(id) + d.LoadSnapshot(Items{{Price: 1337, Amount: 1}}, Items{{Price: 1337, Amount: 10}}) + if d.Retrieve().Asks[0].Price != 1337 || d.Retrieve().Bids[0].Price != 1337 { + t.Fatal("not set") + } +} + +func TestFlush(t *testing.T) { + d := newDepth(id) + d.LoadSnapshot(Items{{Price: 1337, Amount: 1}}, Items{{Price: 1337, Amount: 10}}) + d.Flush() + if len(d.Retrieve().Asks) != 0 || len(d.Retrieve().Bids) != 0 { + t.Fatal("not flushed") + } + d.LoadSnapshot(Items{{Price: 1337, Amount: 1}}, Items{{Price: 1337, Amount: 10}}) + d.Flush() + if len(d.Retrieve().Asks) != 0 || len(d.Retrieve().Bids) != 0 { + t.Fatal("not flushed") + } +} + +func TestUpdateBidAskByPrice(t *testing.T) { + d := newDepth(id) + d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}) + d.UpdateBidAskByPrice(Items{{Price: 1337, Amount: 2, ID: 1}}, Items{{Price: 1337, Amount: 2, ID: 2}}, 0) + if d.Retrieve().Asks[0].Amount != 2 || d.Retrieve().Bids[0].Amount != 2 { + t.Fatal("orderbook amounts not updated correctly") + } + d.UpdateBidAskByPrice(Items{{Price: 1337, Amount: 0, ID: 1}}, Items{{Price: 1337, Amount: 0, ID: 2}}, 0) + if d.GetAskLength() != 0 || d.GetBidLength() != 0 { + t.Fatal("orderbook amounts not updated correctly") + } +} + +func TestDeleteBidAskByID(t *testing.T) { + d := newDepth(id) + d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}) + err := d.DeleteBidAskByID(Items{{Price: 1337, Amount: 2, ID: 1}}, Items{{Price: 1337, Amount: 2, ID: 2}}, false) + if err != nil { + t.Fatal(err) + } + if len(d.Retrieve().Asks) != 0 || len(d.Retrieve().Bids) != 0 { + t.Fatal("items not deleted") + } + + err = d.DeleteBidAskByID(Items{{Price: 1337, Amount: 2, ID: 1}}, nil, false) + if !errors.Is(err, errIDCannotBeMatched) { + t.Fatalf("error expected %v received %v", errIDCannotBeMatched, err) + } + + err = d.DeleteBidAskByID(nil, Items{{Price: 1337, Amount: 2, ID: 2}}, false) + if !errors.Is(err, errIDCannotBeMatched) { + t.Fatalf("error expected %v received %v", errIDCannotBeMatched, err) + } + + err = d.DeleteBidAskByID(nil, Items{{Price: 1337, Amount: 2, ID: 2}}, true) + if !errors.Is(err, nil) { + t.Fatalf("error expected %v received %v", nil, err) + } +} + +func TestUpdateBidAskByID(t *testing.T) { + d := newDepth(id) + d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}) + err := d.UpdateBidAskByID(Items{{Price: 1337, Amount: 2, ID: 1}}, Items{{Price: 1337, Amount: 2, ID: 2}}) + if err != nil { + t.Fatal(err) + } + if d.Retrieve().Asks[0].Amount != 2 || d.Retrieve().Bids[0].Amount != 2 { + t.Fatal("orderbook amounts not updated correctly") + } + + // random unmatching IDs + err = d.UpdateBidAskByID(Items{{Price: 1337, Amount: 2, ID: 666}}, nil) + if !errors.Is(err, errIDCannotBeMatched) { + t.Fatalf("error expected %v received %v", errIDCannotBeMatched, err) + } + + err = d.UpdateBidAskByID(nil, Items{{Price: 1337, Amount: 2, ID: 69}}) + if !errors.Is(err, errIDCannotBeMatched) { + t.Fatalf("error expected %v received %v", errIDCannotBeMatched, err) + } +} + +func TestInsertBidAskByID(t *testing.T) { + d := newDepth(id) + d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}) + err := d.InsertBidAskByID(Items{{Price: 1338, Amount: 2, ID: 3}}, Items{{Price: 1336, Amount: 2, ID: 4}}) + if err != nil { + t.Fatal(err) + } + if len(d.Retrieve().Asks) != 2 || len(d.Retrieve().Bids) != 2 { + t.Fatal("items not added correctly") + } +} + +func TestUpdateInsertByID(t *testing.T) { + d := newDepth(id) + d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}) + + err := d.UpdateInsertByID(Items{{Price: 1338, Amount: 0, ID: 3}}, Items{{Price: 1336, Amount: 2, ID: 4}}) + if !errors.Is(err, errAmountCannotBeLessOrEqualToZero) { + t.Fatalf("expected: %v but received: %v", errAmountCannotBeLessOrEqualToZero, err) + } + + err = d.UpdateInsertByID(Items{{Price: 1338, Amount: 2, ID: 3}}, Items{{Price: 1336, Amount: 0, ID: 4}}) + if !errors.Is(err, errAmountCannotBeLessOrEqualToZero) { + t.Fatalf("expected: %v but received: %v", errAmountCannotBeLessOrEqualToZero, err) + } + + err = d.UpdateInsertByID(Items{{Price: 1338, Amount: 2, ID: 3}}, Items{{Price: 1336, Amount: 2, ID: 4}}) + if err != nil { + t.Fatal(err) + } + + if len(d.Retrieve().Asks) != 2 || len(d.Retrieve().Bids) != 2 { + t.Fatal("items not added correctly") + } +} + +func TestAssignOptions(t *testing.T) { + d := Depth{} + cp := currency.NewPair(currency.LINK, currency.BTC) + tn := time.Now() + d.AssignOptions(&Base{ + Exchange: "test", + Pair: cp, + Asset: asset.Spot, + LastUpdated: tn, + LastUpdateID: 1337, + PriceDuplication: true, + IsFundingRate: true, + VerifyOrderbook: true, + RestSnapshot: true, + IDAlignment: true, + }) + + if d.exchange != "test" || + d.pair != cp || + d.asset != asset.Spot || + d.lastUpdated != tn || + d.lastUpdateID != 1337 || + !d.priceDuplication || + !d.isFundingRate || + !d.VerifyOrderbook || + !d.restSnapshot || + !d.idAligned { + t.Fatal("failed to set correctly") + } +} + +func TestSetLastUpdate(t *testing.T) { + d := Depth{} + tn := time.Now() + d.SetLastUpdate(tn, 1337, true) + if d.lastUpdated != tn || + d.lastUpdateID != 1337 || + !d.restSnapshot { + t.Fatal("failed to set correctly") + } +} + +func TestGetName(t *testing.T) { + d := Depth{} + d.exchange = "test" + if d.GetName() != "test" { + t.Fatal("failed to get correct value") + } +} + +func TestIsRestSnapshot(t *testing.T) { + d := Depth{} + d.restSnapshot = true + if !d.IsRestSnapshot() { + t.Fatal("failed to set correctly") + } +} + +func TestLastUpdateID(t *testing.T) { + d := Depth{} + d.lastUpdateID = 1337 + if d.LastUpdateID() != 1337 { + t.Fatal("failed to get correct value") + } +} + +func TestIsFundingRate(t *testing.T) { + d := Depth{} + d.isFundingRate = true + if !d.IsFundingRate() { + t.Fatal("failed to get correct value") + } +} + +func TestPublish(t *testing.T) { + d := Depth{} + d.Publish() +} + +func TestWait(t *testing.T) { + wait := Alert{} + var wg sync.WaitGroup + + // standard alert + wg.Add(100) + for x := 0; x < 100; x++ { + go func() { + w := wait.Wait(nil) + wg.Done() + if <-w { + log.Fatal("incorrect routine wait response for alert expecting false") + } + wg.Done() + }() + } + + wg.Wait() + wg.Add(100) + isLeaky(&wait, nil, t) + wait.alert() + wg.Wait() + isLeaky(&wait, nil, t) + + // use kick + ch := make(chan struct{}) + wg.Add(100) + for x := 0; x < 100; x++ { + go func() { + w := wait.Wait(ch) + wg.Done() + if !<-w { + log.Fatal("incorrect routine wait response for kick expecting true") + } + wg.Done() + }() + } + wg.Wait() + wg.Add(100) + isLeaky(&wait, ch, t) + close(ch) + wg.Wait() + ch = make(chan struct{}) + isLeaky(&wait, ch, t) + + // late receivers + wg.Add(100) + for x := 0; x < 100; x++ { + go func(x int) { + bb := wait.Wait(ch) + wg.Done() + if x%2 == 0 { + time.Sleep(time.Millisecond * 5) + } + b := <-bb + if b { + log.Fatal("incorrect routine wait response since we call alert below; expecting false") + } + wg.Done() + }(x) + } + wg.Wait() + wg.Add(100) + isLeaky(&wait, ch, t) + wait.alert() + wg.Wait() + isLeaky(&wait, ch, t) +} + +// isLeaky tests to see if the wait functionality is returning an abnormal +// channel that is operational when it shouldn't be. +func isLeaky(a *Alert, ch chan struct{}, t *testing.T) { + t.Helper() + check := a.Wait(ch) + time.Sleep(time.Millisecond * 5) // When we call wait a routine for hold is + // spawned, so for a test we need to add in a time for goschedular to allow + // routine to actually wait on the forAlert and kick channels + select { + case <-check: + t.Fatal("leaky waiter") + default: + } +} diff --git a/exchanges/orderbook/linked_list.go b/exchanges/orderbook/linked_list.go new file mode 100644 index 00000000..d0050311 --- /dev/null +++ b/exchanges/orderbook/linked_list.go @@ -0,0 +1,530 @@ +package orderbook + +import ( + "errors" + "fmt" +) + +var errIDCannotBeMatched = errors.New("cannot match ID on linked list") +var errCollisionDetected = errors.New("cannot insert update collision detected") +var errAmountCannotBeLessOrEqualToZero = errors.New("amount cannot be less or equal to zero") + +// linkedList defines a linked list for a depth level, reutilisation of nodes +// to and from a stack. +type linkedList struct { + length int + head *node +} + +// comparison defines expected functionality to compare between two reference +// price levels +type comparison func(float64, float64) bool + +// load iterates across new items and refreshes linked list. It creates a linked +// list exactly the same as the item slice that is supplied, if items is of nil +// value it will flush entire list. +func (ll *linkedList) load(items Items, stack *stack) { + // Tip sets up a pointer to a struct field variable pointer. This is used + // so when a node is popped from the stack we can reference that current + // nodes' struct 'next' field and set on next iteration without utilising + // assignment for example `prev.next = *node`. + var tip = &ll.head + // Prev denotes a place holder to node and all of its next references need + // to be pushed back onto stack. + var prev *node + for i := range items { + if *tip == nil { + // Extend node chain + *tip = stack.Pop() + // Set current node prev to last node + (*tip).prev = prev + ll.length++ + } + // Set item value + (*tip).value = items[i] + // Set previous to current node + prev = *tip + // Set tip to next node + tip = &(*tip).next + } + + // Push has references to dangling nodes that need to be removed and pushed + // back onto stack for re-use + var push *node + // Cleave unused reference chain from main chain + if prev == nil { + // The entire chain will need to be pushed back on to stack + push = *tip + ll.head = nil + } else { + push = prev.next + prev.next = nil + } + + // Push unused pointers back on stack + for push != nil { + pending := push.next + stack.Push(push, getNow()) + ll.length-- + push = pending + } +} + +// updateByID amends price by corresponding ID and returns an error if not found +func (ll *linkedList) updateByID(updts []Item) error { +updates: + for x := range updts { + for tip := ll.head; tip != nil; tip = tip.next { + if updts[x].ID != tip.value.ID { // Filter IDs that don't match + continue + } + if updts[x].Price > 0 { + // Only apply changes when zero values are not present, Bitmex + // for example sends 0 price values. + tip.value.Price = updts[x].Price + } + tip.value.Amount = updts[x].Amount + continue updates + } + return fmt.Errorf("update error: %w %d not found", + errIDCannotBeMatched, + updts[x].ID) + } + return nil +} + +// deleteByID deletes reference by ID +func (ll *linkedList) deleteByID(updts Items, stack *stack, bypassErr bool) error { +updates: + for x := range updts { + for tip := &ll.head; *tip != nil; tip = &(*tip).next { + if updts[x].ID != (*tip).value.ID { + continue + } + stack.Push(deleteAtTip(ll, tip), getNow()) + continue updates + } + if !bypassErr { + return fmt.Errorf("delete error: %w %d not found", + errIDCannotBeMatched, + updts[x].ID) + } + } + return nil +} + +// cleanup reduces the max size of the depth length if exceeded. Is used after +// updates have been applied instead of adhoc, reason being its easier to prune +// at the end. (cant inline) +func (ll *linkedList) cleanup(maxChainLength int, stack *stack) { + // Reduces the max length of total linked list chain, occurs after updates + // have been implemented as updates can push length out of bounds, if + // cleaved after that update, new update might not applied correctly. + n := ll.head + for i := 0; i < maxChainLength; i++ { + if n.next == nil { + return + } + n = n.next + } + + // cleave reference to current node + if n.prev != nil { + n.prev.next = nil + } else { + ll.head = nil + } + + var pruned int + for n != nil { + pruned++ + pending := n.next + stack.Push(n, getNow()) + n = pending + } + ll.length -= pruned +} + +// amount returns total depth liquidity and value +func (ll *linkedList) amount() (liquidity, value float64) { + for tip := ll.head; tip != nil; tip = tip.next { + liquidity += tip.value.Amount + value += tip.value.Amount * tip.value.Price + } + return +} + +// retrieve returns a full slice of contents from the linked list +func (ll *linkedList) retrieve() Items { + depth := make(Items, ll.length) + iterator := 0 + for tip := ll.head; tip != nil; tip = tip.next { + depth[iterator] = tip.value + iterator++ + } + return depth +} + +// updateInsertByPrice amends, inserts, moves and cleaves length of depth by +// updates +func (ll *linkedList) updateInsertByPrice(updts Items, stack *stack, maxChainLength int, compare func(float64, float64) bool, tn now) { + for x := range updts { + for tip := &ll.head; ; tip = &(*tip).next { + if *tip == nil { + insertHeadSpecific(ll, updts[x], stack) + break + } + if (*tip).value.Price == updts[x].Price { // Match check + if updts[x].Amount <= 0 { // Capture delete update + stack.Push(deleteAtTip(ll, tip), tn) + } else { // Amend current amount value + (*tip).value.Amount = updts[x].Amount + } + break // Continue updates + } + + if compare((*tip).value.Price, updts[x].Price) { // Insert + // This check below filters zero values and provides an + // optimisation for when select exchanges send a delete update + // to a non-existent price level (OTC/Hidden order) so we can + // break instantly and reduce the traversal of the entire chain. + if updts[x].Amount > 0 { + insertAtTip(ll, tip, updts[x], stack) + } + break // Continue updates + } + + if (*tip).next == nil { // Tip is at tail + // This check below is just a catch all in the event the above + // zero value check fails + if updts[x].Amount > 0 { + insertAtTail(ll, tip, updts[x], stack) + } + break + } + } + } + // Reduces length of total linked list chain to a maxChainLength value + if maxChainLength != 0 && ll.length > maxChainLength { + ll.cleanup(maxChainLength, stack) + } +} + +// updateInsertByID updates or inserts if not found for a bid or ask depth +// 1) node ID found amount amended (best case) +// 2) node ID found amount and price amended and node moved to correct position +// (medium case) +// 3) Update price exceeds traversal node price before ID found, save node +// address for either; node ID matches then re-address node or end of depth pop +// a node from the stack (worst case) +func (ll *linkedList) updateInsertByID(updts Items, stack *stack, compare comparison) error { +updates: + for x := range updts { + if updts[x].Amount <= 0 { + return errAmountCannotBeLessOrEqualToZero + } + // bookmark allows for saving of a position of a node in the event that + // an update price exceeds the current node price. We can then match an + // ID and re-assign that ID's node to that positioning without popping + // from the stack and then pushing to the stack later for cleanup. + // If the ID is not found we can pop from stack then insert into that + // price level + var bookmark *node + for tip := ll.head; tip != nil; tip = tip.next { + if tip.value.ID == updts[x].ID { + if tip.value.Price != updts[x].Price { // Price level change + if tip.next == nil { + // no movement needed just a re-adjustment + tip.value.Price = updts[x].Price + tip.value.Amount = updts[x].Amount + continue updates + } + // bookmark tip to move this node to correct price level + bookmark = tip + continue // continue through node depth + } + // no price change, amend amount and continue update + tip.value.Amount = updts[x].Amount + continue updates // continue to next update + } + + if compare(tip.value.Price, updts[x].Price) { + if bookmark != nil { // shift bookmarked node to current tip + bookmark.value = updts[x] + move(&ll.head, bookmark, tip) + continue updates + } + + // search for ID + for n := tip.next; n != nil; n = n.next { + if n.value.ID == updts[x].ID { + n.value = updts[x] + // inserting before the tip + move(&ll.head, n, tip) + continue updates + } + } + // ID not matched in depth so add correct level for insert + if tip.next == nil { + n := stack.Pop() + n.value = updts[x] + ll.length++ + if tip.prev == nil { + tip.prev = n + n.next = tip + ll.head = n + continue updates + } + tip.prev.next = n + n.prev = tip.prev + tip.prev = n + n.next = tip + continue updates + } + bookmark = tip + break + } + + if tip.next == nil { + if shiftBookmark(tip, &bookmark, &ll.head, updts[x]) { + continue updates + } + } + } + n := stack.Pop() + n.value = updts[x] + insertNodeAtBookmark(ll, bookmark, n) // Won't inline with stack + } + return nil +} + +// insertUpdates inserts new updates for bids or asks based on price level +func (ll *linkedList) insertUpdates(updts Items, stack *stack, comp comparison) error { + for x := range updts { + var prev *node + for tip := &ll.head; ; tip = &(*tip).next { + if *tip == nil { // Head + n := stack.Pop() + n.value = updts[x] + n.prev = prev + ll.length++ + *tip = n + break // Continue updates + } + + if (*tip).value.Price == updts[x].Price { // Price already found + return fmt.Errorf("%w for price %f", + errCollisionDetected, + updts[x].Price) + } + + if comp((*tip).value.Price, updts[x].Price) { // Alignment + n := stack.Pop() + n.value = updts[x] + n.prev = prev + ll.length++ + // Reference current with new node + (*tip).prev = n + // Push tip to the right + n.next = *tip + // This is the same as prev.next = n + *tip = n + break // Continue updates + } + + if (*tip).next == nil { // Tail + insertAtTail(ll, tip, updts[x], stack) + break // Continue updates + } + prev = *tip + } + } + return nil +} + +// bids embed a linked list to attach methods for bid depth specific +// functionality +type bids struct { + linkedList +} + +// bidCompare ensures price is in correct descending alignment (can inline) +func bidCompare(left, right float64) bool { + return left < right +} + +// updateInsertByPrice amends, inserts, moves and cleaves length of depth by +// updates +func (ll *bids) updateInsertByPrice(updts Items, stack *stack, maxChainLength int, tn now) { + ll.linkedList.updateInsertByPrice(updts, stack, maxChainLength, bidCompare, tn) +} + +// updateInsertByID updates or inserts if not found +func (ll *bids) updateInsertByID(updts Items, stack *stack) error { + return ll.linkedList.updateInsertByID(updts, stack, bidCompare) +} + +// insertUpdates inserts new updates for bids based on price level +func (ll *bids) insertUpdates(updts Items, stack *stack) error { + return ll.linkedList.insertUpdates(updts, stack, bidCompare) +} + +// asks embed a linked list to attach methods for ask depth specific +// functionality +type asks struct { + linkedList +} + +// askCompare ensures price is in correct ascending alignment (can inline) +func askCompare(left, right float64) bool { + return left > right +} + +// updateInsertByPrice amends, inserts, moves and cleaves length of depth by +// updates +func (ll *asks) updateInsertByPrice(updts Items, stack *stack, maxChainLength int, tn now) { + ll.linkedList.updateInsertByPrice(updts, stack, maxChainLength, askCompare, tn) +} + +// updateInsertByID updates or inserts if not found +func (ll *asks) updateInsertByID(updts Items, stack *stack) error { + return ll.linkedList.updateInsertByID(updts, stack, askCompare) +} + +// insertUpdates inserts new updates for asks based on price level +func (ll *asks) insertUpdates(updts Items, stack *stack) error { + return ll.linkedList.insertUpdates(updts, stack, askCompare) +} + +// move moves a node from a point in a node chain to another node position, +// this left justified towards head as element zero is the top of the depth +// side. (can inline) +func move(head **node, from, to *node) { + if from.next != nil { // From is at tail + from.next.prev = from.prev + } + if from.prev == nil { // From is at head + (*head).next.prev = nil + *head = (*head).next + } else { + from.prev.next = from.next + } + // insert from node next to 'to' node + if to.prev == nil { // Destination is at head position + *head = from + } else { + to.prev.next = from + } + from.prev = to.prev + to.prev = from + from.next = to +} + +// deleteAtTip removes a node from tip target returns old node (can inline) +func deleteAtTip(ll *linkedList, tip **node) *node { + // Old is a placeholder for current tips node value to push + // back on to the stack. + old := *tip + switch { + case old.prev == nil: // At head position + // shift current tip head to the right + *tip = old.next + // Remove reference to node from chain + if old.next != nil { // This is when liquidity hits zero + old.next.prev = nil + } + case old.next == nil: // At tail position + // Remove reference to node from chain + old.prev.next = nil + default: + // Reference prior node in chain to next node in chain + // bypassing current node + old.prev.next = old.next + old.next.prev = old.prev + } + ll.length-- + return old +} + +// insertAtTip inserts at a tip target (can inline) +func insertAtTip(ll *linkedList, tip **node, updt Item, stack *stack) { + n := stack.Pop() + n.value = updt + n.next = *tip + n.prev = (*tip).prev + if (*tip).prev == nil { // Tip is at head + // Replace head which will push everything to the right + // when this node will reference new node below + *tip = n + } else { + // Reference new node to previous node + (*tip).prev.next = n + } + // Reference next node to new node + n.next.prev = n + ll.length++ +} + +// insertAtTail inserts at tail end of node chain (can inline) +func insertAtTail(ll *linkedList, tip **node, updt Item, stack *stack) { + n := stack.Pop() + n.value = updt + // Reference tip to new node + (*tip).next = n + // Reference new node with current tip + n.prev = *tip + ll.length++ +} + +// insertHeadSpecific inserts at head specifically there might be an instance +// where the liquidity on an exchange does fall to zero through a streaming +// endpoint then it comes back online. (can inline) +func insertHeadSpecific(ll *linkedList, updt Item, stack *stack) { + n := stack.Pop() + n.value = updt + ll.head = n + ll.length++ +} + +// insertNodeAtBookmark inserts a new node at a bookmarked node position +// returns if a node needs to replace head (can inline) +func insertNodeAtBookmark(ll *linkedList, bookmark, n *node) { + switch { + case bookmark == nil: // Zero liquidity and we are rebuilding from scratch + ll.head = n + case bookmark.prev == nil: + n.prev = bookmark.prev + bookmark.prev = n + n.next = bookmark + ll.head = n + case bookmark.next == nil: + n.prev = bookmark + bookmark.next = n + default: + bookmark.prev.next = n + n.prev = bookmark.prev + bookmark.prev = n + n.next = bookmark + } + ll.length++ +} + +// shiftBookmark moves a bookmarked node to the tip's next position or if nil, +// sets tip as bookmark (can inline) +func shiftBookmark(tip *node, bookmark, head **node, updt Item) bool { + if *bookmark == nil { // End of the chain and no bookmark set + *bookmark = tip // Set tip to bookmark so we can set a new node there + return false + } + (*bookmark).value = updt + (*bookmark).next.prev = (*bookmark).prev + if (*bookmark).prev == nil { // Bookmark is at head + *head = (*bookmark).next + } else { + (*bookmark).prev.next = (*bookmark).next + } + tip.next = *bookmark + (*bookmark).prev = tip + (*bookmark).next = nil + return true +} diff --git a/exchanges/orderbook/linked_list_test.go b/exchanges/orderbook/linked_list_test.go new file mode 100644 index 00000000..21d7df69 --- /dev/null +++ b/exchanges/orderbook/linked_list_test.go @@ -0,0 +1,1449 @@ +package orderbook + +import ( + "errors" + "fmt" + "testing" + "time" +) + +var ask = Items{ + Item{Price: 1337, Amount: 1}, + Item{Price: 1338, Amount: 1}, + Item{Price: 1339, Amount: 1}, + Item{Price: 1340, Amount: 1}, + Item{Price: 1341, Amount: 1}, + Item{Price: 1342, Amount: 1}, + Item{Price: 1343, Amount: 1}, + Item{Price: 1344, Amount: 1}, + Item{Price: 1345, Amount: 1}, + Item{Price: 1346, Amount: 1}, + Item{Price: 1347, Amount: 1}, + Item{Price: 1348, Amount: 1}, + Item{Price: 1349, Amount: 1}, + Item{Price: 1350, Amount: 1}, + Item{Price: 1351, Amount: 1}, + Item{Price: 1352, Amount: 1}, + Item{Price: 1353, Amount: 1}, + Item{Price: 1354, Amount: 1}, + Item{Price: 1355, Amount: 1}, + Item{Price: 1356, Amount: 1}, +} + +// Display displays depth content for tests +func (ll *linkedList) display() { + for tip := ll.head; tip != nil; tip = tip.next { + fmt.Printf("NODE: %+v %p \n", tip, tip) + } + fmt.Println() +} + +func TestLoad(t *testing.T) { + list := asks{} + Check(list, 0, 0, 0, t) + + stack := newStack() + list.load(Items{ + {Price: 1, Amount: 1}, + {Price: 3, Amount: 1}, + {Price: 5, Amount: 1}, + {Price: 7, Amount: 1}, + {Price: 9, Amount: 1}, + {Price: 11, Amount: 1}, + }, stack) + + if stack.getCount() != 0 { + t.Fatalf("incorrect stack count expected: %v received: %v", 0, stack.getCount()) + } + + Check(list, 6, 36, 6, t) + + list.load(Items{ + {Price: 1, Amount: 1}, + {Price: 3, Amount: 1}, + {Price: 5, Amount: 1}, + }, stack) + + if stack.getCount() != 3 { + t.Fatalf("incorrect stack count expected: %v received: %v", 3, stack.getCount()) + } + + Check(list, 3, 9, 3, t) + + list.load(Items{ + {Price: 1, Amount: 1}, + {Price: 3, Amount: 1}, + {Price: 5, Amount: 1}, + {Price: 7, Amount: 1}, + }, stack) + + if stack.getCount() != 2 { + t.Fatalf("incorrect stack count expected: %v received: %v", 2, stack.getCount()) + } + + Check(list, 4, 16, 4, t) + + // purge entire list + list.load(nil, stack) + + if stack.getCount() != 6 { + t.Fatalf("incorrect stack count expected: %v received: %v", 6, stack.getCount()) + } + + Check(list, 0, 0, 0, t) +} + +// 22222386 57.3 ns/op 0 B/op 0 allocs/op (old) +// 27906781 42.4 ns/op 0 B/op 0 allocs/op (new) +func BenchmarkLoad(b *testing.B) { + ll := linkedList{} + s := newStack() + for i := 0; i < b.N; i++ { + ll.load(ask, s) + } +} + +func TestUpdateInsertByPrice(t *testing.T) { + a := asks{} + stack := newStack() + asksSnapshot := Items{ + {Price: 1, Amount: 1}, + {Price: 3, Amount: 1}, + {Price: 5, Amount: 1}, + {Price: 7, Amount: 1}, + {Price: 9, Amount: 1}, + {Price: 11, Amount: 1}, + } + a.load(asksSnapshot, stack) + + // Update one instance with matching price + a.updateInsertByPrice(Items{ + {Price: 1, Amount: 2}, + }, stack, 0, getNow()) + + Check(a, 7, 37, 6, t) + + if stack.getCount() != 0 { + t.Fatalf("incorrect stack count expected: %v received: %v", 0, stack.getCount()) + } + + // Insert at head + a.updateInsertByPrice(Items{ + {Price: 0.5, Amount: 2}, + }, stack, 0, getNow()) + + Check(a, 9, 38, 7, t) + + if stack.getCount() != 0 { + t.Fatalf("incorrect stack count expected: %v received: %v", 0, stack.getCount()) + } + + // Insert at tail + a.updateInsertByPrice(Items{ + {Price: 12, Amount: 2}, + }, stack, 0, getNow()) + + Check(a, 11, 62, 8, t) + + if stack.getCount() != 0 { + t.Fatalf("incorrect stack count expected: %v received: %v", 0, stack.getCount()) + } + + // Insert between price and up to and beyond max allowable depth level + a.updateInsertByPrice(Items{ + {Price: 11.5, Amount: 2}, + {Price: 10.5, Amount: 2}, + {Price: 13, Amount: 2}, + }, stack, 10, getNow()) + + Check(a, 15, 106, 10, t) + + if stack.getCount() != 1 { + t.Fatalf("incorrect stack count expected: %v received: %v", 1, stack.getCount()) + } + + // delete at tail + a.updateInsertByPrice(Items{ + {Price: 12, Amount: 0}, + }, stack, 0, getNow()) + + Check(a, 13, 82, 9, t) + + if stack.getCount() != 2 { + t.Fatalf("incorrect stack count expected: %v received: %v", 2, stack.getCount()) + } + + // delete at mid + a.updateInsertByPrice(Items{ + {Price: 7, Amount: 0}, + }, stack, 0, getNow()) + + Check(a, 12, 75, 8, t) + + if stack.getCount() != 3 { + t.Fatalf("incorrect stack count expected: %v received: %v", 3, stack.getCount()) + } + + // delete at head + a.updateInsertByPrice(Items{ + {Price: 0.5, Amount: 0}, + }, stack, 0, getNow()) + + Check(a, 10, 74, 7, t) + + if stack.getCount() != 4 { + t.Fatalf("incorrect stack count expected: %v received: %v", 4, stack.getCount()) + } + + // purge if liquidity plunges to zero + a.load(nil, stack) + + // rebuild everything again + a.updateInsertByPrice(Items{ + {Price: 1, Amount: 1}, + {Price: 3, Amount: 1}, + {Price: 5, Amount: 1}, + {Price: 7, Amount: 1}, + {Price: 9, Amount: 1}, + {Price: 11, Amount: 1}, + }, stack, 0, getNow()) + + Check(a, 6, 36, 6, t) + + if stack.getCount() != 5 { + t.Fatalf("incorrect stack count expected: %v received: %v", 4, stack.getCount()) + } + + b := bids{} + bidsSnapshot := Items{ + {Price: 11, Amount: 1}, + {Price: 9, Amount: 1}, + {Price: 7, Amount: 1}, + {Price: 5, Amount: 1}, + {Price: 3, Amount: 1}, + {Price: 1, Amount: 1}, + } + b.load(bidsSnapshot, stack) + + // Update one instance with matching price + b.updateInsertByPrice(Items{ + {Price: 11, Amount: 2}, + }, stack, 0, getNow()) + + Check(b, 7, 47, 6, t) + + if stack.getCount() != 0 { + t.Fatalf("incorrect stack count expected: %v received: %v", 0, stack.getCount()) + } + + // Insert at head + b.updateInsertByPrice(Items{ + {Price: 12, Amount: 2}, + }, stack, 0, getNow()) + + Check(b, 9, 71, 7, t) + + if stack.getCount() != 0 { + t.Fatalf("incorrect stack count expected: %v received: %v", 0, stack.getCount()) + } + + // Insert at tail + b.updateInsertByPrice(Items{ + {Price: 0.5, Amount: 2}, + }, stack, 0, getNow()) + + Check(b, 11, 72, 8, t) + + if stack.getCount() != 0 { + t.Fatalf("incorrect stack count expected: %v received: %v", 0, stack.getCount()) + } + + // Insert between price and up to and beyond max allowable depth level + b.updateInsertByPrice(Items{ + {Price: 11.5, Amount: 2}, + {Price: 10.5, Amount: 2}, + {Price: 13, Amount: 2}, + }, stack, 10, getNow()) + + Check(b, 15, 141, 10, t) + + if stack.getCount() != 1 { + t.Fatalf("incorrect stack count expected: %v received: %v", 0, stack.getCount()) + } + + // Insert between price and up to and beyond max allowable depth level + b.updateInsertByPrice(Items{ + {Price: 1, Amount: 0}, + }, stack, 0, getNow()) + + Check(b, 14, 140, 9, t) + + if stack.getCount() != 2 { + t.Fatalf("incorrect stack count expected: %v received: %v", 2, stack.getCount()) + } + + // delete at mid + b.updateInsertByPrice(Items{ + {Price: 10.5, Amount: 0}, + }, stack, 0, getNow()) + + Check(b, 12, 119, 8, t) + + if stack.getCount() != 3 { + t.Fatalf("incorrect stack count expected: %v received: %v", 3, stack.getCount()) + } + + // delete at head + b.updateInsertByPrice(Items{ + {Price: 13, Amount: 0}, + }, stack, 0, getNow()) + + Check(b, 10, 93, 7, t) + + if stack.getCount() != 4 { + t.Fatalf("incorrect stack count expected: %v received: %v", 4, stack.getCount()) + } + + // purge if liquidity plunges to zero + b.load(nil, stack) + + // rebuild everything again + b.updateInsertByPrice(Items{ + {Price: 1, Amount: 1}, + {Price: 3, Amount: 1}, + {Price: 5, Amount: 1}, + {Price: 7, Amount: 1}, + {Price: 9, Amount: 1}, + {Price: 11, Amount: 1}, + }, stack, 0, getNow()) + + Check(b, 6, 36, 6, t) + + if stack.getCount() != 5 { + t.Fatalf("incorrect stack count expected: %v received: %v", 4, stack.getCount()) + } +} + +func TestCleanup(t *testing.T) { + a := asks{} + stack := newStack() + asksSnapshot := Items{ + {Price: 1, Amount: 1}, + {Price: 3, Amount: 1}, + {Price: 5, Amount: 1}, + {Price: 7, Amount: 1}, + {Price: 9, Amount: 1}, + {Price: 11, Amount: 1}, + } + a.load(asksSnapshot, stack) + + a.cleanup(6, stack) + Check(a, 6, 36, 6, t) + a.cleanup(5, stack) + Check(a, 5, 25, 5, t) + a.cleanup(1, stack) + Check(a, 1, 1, 1, t) + a.cleanup(10, stack) + Check(a, 1, 1, 1, t) + a.cleanup(0, stack) // will purge, underlying checks are done elseware to prevent this + Check(a, 0, 0, 0, t) +} + +// 46154023 24.0 ns/op 0 B/op 0 allocs/op (old) +// 134830672 9.83 ns/op 0 B/op 0 allocs/op (new) +func BenchmarkUpdateInsertByPrice_Amend(b *testing.B) { + a := asks{} + stack := newStack() + + a.load(ask, stack) + + updates := Items{ + { + Price: 1337, // Amend + Amount: 2, + }, + { + Price: 1337, // Amend + Amount: 1, + }, + } + + for i := 0; i < b.N; i++ { + a.updateInsertByPrice(updates, stack, 0, getNow()) + } +} + +// 49763002 24.9 ns/op 0 B/op 0 allocs/op +func BenchmarkUpdateInsertByPrice_Insert_Delete(b *testing.B) { + a := asks{} + stack := newStack() + + a.load(ask, stack) + + updates := Items{ + { + Price: 1337.5, // Insert + Amount: 2, + }, + { + Price: 1337.5, // Delete + Amount: 0, + }, + } + + for i := 0; i < b.N; i++ { + a.updateInsertByPrice(updates, stack, 0, getNow()) + } +} + +func TestUpdateByID(t *testing.T) { + a := asks{} + s := newStack() + asksSnapshot := Items{ + {Price: 1, Amount: 1, ID: 1}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 11, Amount: 1, ID: 11}, + } + a.load(asksSnapshot, s) + + err := a.updateByID(Items{ + {Price: 1, Amount: 1, ID: 1}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 11, Amount: 1, ID: 11}, + }) + if err != nil { + t.Fatal(err) + } + + Check(a, 6, 36, 6, t) + + err = a.updateByID(Items{ + {Price: 11, Amount: 1, ID: 1337}, + }) + if !errors.Is(err, errIDCannotBeMatched) { + t.Fatalf("expecting %s but received %v", errIDCannotBeMatched, err) + } + + err = a.updateByID(Items{ // Simulate Bitmex updating + {Price: 0, Amount: 1337, ID: 3}, + }) + if !errors.Is(err, nil) { + t.Fatalf("expecting %v but received %v", nil, err) + } + + if a.retrieve()[1].Price == 0 { + t.Fatal("price should not be replaced with zero") + } + + if a.retrieve()[1].Amount != 1337 { + t.Fatal("unexpected value for update") + } +} + +// 46043871 25.9 ns/op 0 B/op 0 allocs/op +func BenchmarkUpdateByID(b *testing.B) { + asks := linkedList{} + s := newStack() + asksSnapshot := Items{ + {Price: 1, Amount: 1, ID: 1}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 11, Amount: 1, ID: 11}, + } + asks.load(asksSnapshot, s) + + for i := 0; i < b.N; i++ { + err := asks.updateByID(Items{ + {Price: 1, Amount: 1, ID: 1}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 11, Amount: 1, ID: 11}, + }) + if err != nil { + b.Fatal(err) + } + } +} + +func TestDeleteByID(t *testing.T) { + a := asks{} + s := newStack() + asksSnapshot := Items{ + {Price: 1, Amount: 1, ID: 1}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 11, Amount: 1, ID: 11}, + } + a.load(asksSnapshot, s) + + // Delete at head + err := a.deleteByID(Items{ + {Price: 1, Amount: 1, ID: 1}, + }, s, false) + if err != nil { + t.Fatal(err) + } + + Check(a, 5, 35, 5, t) + + // Delete at tail + err = a.deleteByID(Items{ + {Price: 1, Amount: 1, ID: 11}, + }, s, false) + if err != nil { + t.Fatal(err) + } + + Check(a, 4, 24, 4, t) + + // Delete in middle + err = a.deleteByID(Items{ + {Price: 1, Amount: 1, ID: 5}, + }, s, false) + if err != nil { + t.Fatal(err) + } + + Check(a, 3, 19, 3, t) + + // Intentional error + err = a.deleteByID(Items{ + {Price: 11, Amount: 1, ID: 1337}, + }, s, false) + if !errors.Is(err, errIDCannotBeMatched) { + t.Fatalf("expecting %s but received %v", errIDCannotBeMatched, err) + } + + // Error bypass + err = a.deleteByID(Items{ + {Price: 11, Amount: 1, ID: 1337}, + }, s, true) + if err != nil { + t.Fatal(err) + } +} + +func TestUpdateInsertByIDAsk(t *testing.T) { + a := asks{} + s := newStack() + asksSnapshot := Items{ + {Price: 1, Amount: 1, ID: 1}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 11, Amount: 1, ID: 11}, + } + a.load(asksSnapshot, s) + + // Update one instance with matching ID + err := a.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 7, 37, 6, t) + + // Reset + a.load(asksSnapshot, s) + + // Update all instances with matching ID in order + err = a.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 5, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 12, 72, 6, t) + + // Update all instances with matching ID in backwards + err = a.updateInsertByID(Items{ + {Price: 11, Amount: 2, ID: 11}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 5, Amount: 2, ID: 5}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 1, Amount: 2, ID: 1}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 12, 72, 6, t) + + // Update all instances with matching ID all over the ship + err = a.updateInsertByID(Items{ + {Price: 11, Amount: 2, ID: 11}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 1, Amount: 2, ID: 1}, + {Price: 5, Amount: 2, ID: 5}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 12, 72, 6, t) + + // Update all instances move one before ID in middle + err = a.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 2, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 12, 66, 6, t) + + // Update all instances move one before ID at head + err = a.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: .5, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 12, 63, 6, t) + + // Reset + a.load(asksSnapshot, s) + + // Update all instances move one after ID + err = a.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 8, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 12, 78, 6, t) + + // Reset + a.load(asksSnapshot, s) + + // Update all instances move one after ID to tail + err = a.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 12, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 12, 86, 6, t) + + // Update all instances then pop new instance + err = a.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 12, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + {Price: 10, Amount: 2, ID: 10}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 14, 106, 7, t) + + // Reset + a.load(asksSnapshot, s) + + // Update all instances pop at head + err = a.updateInsertByID(Items{ + {Price: 0.5, Amount: 2, ID: 0}, + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 12, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 14, 87, 7, t) + + // bookmark head and move to mid + err = a.updateInsertByID(Items{ + {Price: 7.5, Amount: 2, ID: 0}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 14, 101, 7, t) + + // bookmark head and move to tail + err = a.updateInsertByID(Items{ + {Price: 12.5, Amount: 2, ID: 1}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 14, 124, 7, t) + + // move tail location to head + err = a.updateInsertByID(Items{ + {Price: 2.5, Amount: 2, ID: 1}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 14, 104, 7, t) + + // move tail location to mid + err = a.updateInsertByID(Items{ + {Price: 8, Amount: 2, ID: 5}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 14, 96, 7, t) + + // insert at tail dont match + err = a.updateInsertByID(Items{ + {Price: 30, Amount: 2, ID: 1234}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 16, 156, 8, t) + + // insert between last and 2nd last + err = a.updateInsertByID(Items{ + {Price: 12, Amount: 2, ID: 12345}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 18, 180, 9, t) + + // readjust at end + err = a.updateInsertByID(Items{ + {Price: 29, Amount: 3, ID: 1234}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 19, 207, 9, t) + + // readjust further and decrease price past tail + err = a.updateInsertByID(Items{ + {Price: 31, Amount: 3, ID: 1234}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 19, 213, 9, t) + + // purge + a.load(nil, s) + + // insert with no liquidity and jumbled + err = a.updateInsertByID(Items{ + {Price: 11, Amount: 2, ID: 11}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 0.5, Amount: 2, ID: 0}, + {Price: 12, Amount: 2, ID: 5}, + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 14, 87, 7, t) +} + +func TestUpdateInsertByIDBids(t *testing.T) { + b := bids{} + s := newStack() + bidsSnapshot := Items{ + {Price: 11, Amount: 1, ID: 11}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 1, Amount: 1, ID: 1}, + } + b.load(bidsSnapshot, s) + + // Update one instance with matching ID + err := b.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 7, 37, 6, t) + + // Reset + b.load(bidsSnapshot, s) + + // Update all instances with matching ID in order + err = b.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 5, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 12, 72, 6, t) + + // Update all instances with matching ID in backwards + err = b.updateInsertByID(Items{ + {Price: 11, Amount: 2, ID: 11}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 5, Amount: 2, ID: 5}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 1, Amount: 2, ID: 1}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 12, 72, 6, t) + + // Update all instances with matching ID all over the ship + err = b.updateInsertByID(Items{ + {Price: 11, Amount: 2, ID: 11}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 1, Amount: 2, ID: 1}, + {Price: 5, Amount: 2, ID: 5}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 12, 72, 6, t) + + // Update all instances move one before ID in middle + err = b.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 2, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 12, 66, 6, t) + + // Update all instances move one before ID at head + err = b.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: .5, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 12, 63, 6, t) + + // Reset + b.load(bidsSnapshot, s) + + // Update all instances move one after ID + err = b.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 8, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 12, 78, 6, t) + + // Reset + b.load(bidsSnapshot, s) + + // Update all instances move one after ID to tail + err = b.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 12, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 12, 86, 6, t) + + // Update all instances then pop new instance + err = b.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 12, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + {Price: 10, Amount: 2, ID: 10}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 14, 106, 7, t) + + // Reset + b.load(bidsSnapshot, s) + + // Update all instances pop at tail + err = b.updateInsertByID(Items{ + {Price: 0.5, Amount: 2, ID: 0}, + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 12, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 14, 87, 7, t) + + // bookmark head and move to mid + err = b.updateInsertByID(Items{ + {Price: 9.5, Amount: 2, ID: 5}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 14, 82, 7, t) + + // bookmark head and move to tail + err = b.updateInsertByID(Items{ + {Price: 0.25, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 14, 60.5, 7, t) + + // move tail location to head + err = b.updateInsertByID(Items{ + {Price: 10, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 14, 80, 7, t) + + // move tail location to mid + err = b.updateInsertByID(Items{ + {Price: 7.5, Amount: 2, ID: 0}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 14, 94, 7, t) + + // insert at head dont match + err = b.updateInsertByID(Items{ + {Price: 30, Amount: 2, ID: 1234}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 16, 154, 8, t) + + // insert between last and 2nd last + err = b.updateInsertByID(Items{ + {Price: 1.5, Amount: 2, ID: 12345}, + }, s) + if err != nil { + t.Fatal(err) + } + Check(b, 18, 157, 9, t) + + // readjust at end + err = b.updateInsertByID(Items{ + {Price: 1, Amount: 3, ID: 1}, + }, s) + if err != nil { + t.Fatal(err) + } + Check(b, 19, 158, 9, t) + + // readjust further and decrease price past tail + err = b.updateInsertByID(Items{ + {Price: .9, Amount: 3, ID: 1}, + }, s) + if err != nil { + t.Fatal(err) + } + Check(b, 19, 157.7, 9, t) + + // purge + b.load(nil, s) + + // insert with no liquidity and jumbled + err = b.updateInsertByID(Items{ + {Price: 0.5, Amount: 2, ID: 0}, + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 12, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 14, 87, 7, t) +} + +func TestInsertUpdatesBid(t *testing.T) { + b := bids{} + s := newStack() + bidsSnapshot := Items{ + {Price: 11, Amount: 1, ID: 11}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 1, Amount: 1, ID: 1}, + } + b.load(bidsSnapshot, s) + + err := b.insertUpdates(Items{ + {Price: 11, Amount: 1, ID: 11}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 1, Amount: 1, ID: 1}, + }, s) + if !errors.Is(err, errCollisionDetected) { + t.Fatalf("expected error %s but received %v", errCollisionDetected, err) + } + + Check(b, 6, 36, 6, t) + + // Insert at head + err = b.insertUpdates(Items{ + {Price: 12, Amount: 1, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 7, 48, 7, t) + + // Insert at tail + err = b.insertUpdates(Items{ + {Price: 0.5, Amount: 1, ID: 12}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 8, 48.5, 8, t) + + // Insert at mid + err = b.insertUpdates(Items{ + {Price: 5.5, Amount: 1, ID: 13}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 9, 54, 9, t) + + // purge + b.load(nil, s) + + // Add one at head + err = b.insertUpdates(Items{ + {Price: 5.5, Amount: 1, ID: 13}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 1, 5.5, 1, t) +} + +func TestInsertUpdatesAsk(t *testing.T) { + a := asks{} + s := newStack() + askSnapshot := Items{ + {Price: 1, Amount: 1, ID: 1}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 11, Amount: 1, ID: 11}, + } + a.load(askSnapshot, s) + + err := a.insertUpdates(Items{ + {Price: 11, Amount: 1, ID: 11}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 1, Amount: 1, ID: 1}, + }, s) + if !errors.Is(err, errCollisionDetected) { + t.Fatalf("expected error %s but received %v", errCollisionDetected, err) + } + + Check(a, 6, 36, 6, t) + + // Insert at tail + err = a.insertUpdates(Items{ + {Price: 12, Amount: 1, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 7, 48, 7, t) + + // Insert at head + err = a.insertUpdates(Items{ + {Price: 0.5, Amount: 1, ID: 12}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 8, 48.5, 8, t) + + // Insert at mid + err = a.insertUpdates(Items{ + {Price: 5.5, Amount: 1, ID: 13}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 9, 54, 9, t) + + // purge + a.load(nil, s) + + // Add one at head + err = a.insertUpdates(Items{ + {Price: 5.5, Amount: 1, ID: 13}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 1, 5.5, 1, t) +} + +// check checks depth values after an update has taken place +func Check(depth interface{}, liquidity, value float64, nodeCount int, t *testing.T) { + t.Helper() + b, isBid := depth.(bids) + a, isAsk := depth.(asks) + + var ll linkedList + switch { + case isBid: + ll = b.linkedList + case isAsk: + ll = a.linkedList + default: + t.Fatal("value passed in is not of type bids or asks") + } + + liquidityTotal, valueTotal := ll.amount() + + if liquidityTotal != liquidity { + ll.display() + t.Fatalf("mismatched liquidity expecting %v but received %v", + liquidity, + liquidityTotal) + } + + if valueTotal != value { + ll.display() + t.Fatalf("mismatched total value expecting %v but received %v", + value, + valueTotal) + } + + if ll.length != nodeCount { + ll.display() + t.Fatalf("mismatched node count expecting %v but received %v", + nodeCount, + ll.length) + } + + if ll.head == nil { + return + } + + var tail *node + var price float64 + for tip := ll.head; ; tip = tip.next { + switch { + case price == 0: + price = tip.value.Price + case isBid && price < tip.value.Price: + ll.display() + t.Fatal("Bid pricing out of order should be descending") + case isAsk && price > tip.value.Price: + ll.display() + t.Fatal("Ask pricing out of order should be ascending") + default: + price = tip.value.Price + } + + if tip.next == nil { + tail = tip + break + } + } + + var liqReversed, valReversed float64 + var nodeReversed int + for tip := tail; tip != nil; tip = tip.prev { + liqReversed += tip.value.Amount + valReversed += tip.value.Amount * tip.value.Price + nodeReversed++ + } + + if liquidity-liqReversed != 0 { + ll.display() + fmt.Println(liquidity, liqReversed) + t.Fatalf("mismatched liquidity when reversing direction expecting %v but received %v", + 0, + liquidity-liqReversed) + } + + if nodeCount-nodeReversed != 0 { + ll.display() + t.Fatalf("mismatched node count when reversing direction expecting %v but received %v", + 0, + nodeCount-nodeReversed) + } + + if value-valReversed != 0 { + ll.display() + fmt.Println(valReversed, value) + t.Fatalf("mismatched total book value when reversing direction expecting %v but received %v", + 0, + value-valReversed) + } +} + +func TestAmount(t *testing.T) { + a := asks{} + s := newStack() + askSnapshot := Items{ + {Price: 1, Amount: 1, ID: 1}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 11, Amount: 1, ID: 11}, + } + a.load(askSnapshot, s) + + liquidity, value := a.amount() + if liquidity != 6 { + t.Fatalf("incorrect liquidity calculation expected 6 but received %f", liquidity) + } + + if value != 36 { + t.Fatalf("incorrect value calculation expected 36 but received %f", value) + } +} + +func TestShiftBookmark(t *testing.T) { + bookmarkedNode := &node{ + value: Item{ + ID: 1337, + Amount: 1, + Price: 2, + }, + next: nil, + prev: nil, + shelved: time.Time{}, + } + + originalBookmarkPrev := &node{ + value: Item{ + ID: 1336, + }, + next: bookmarkedNode, + prev: nil, // At head + shelved: time.Time{}, + } + originalBookmarkNext := &node{ + value: Item{ + ID: 1338, + }, + next: nil, // This can be left nil in actuality this will be + // populated + prev: bookmarkedNode, + shelved: time.Time{}, + } + + // associate previous and next nodes to bookmarked node + bookmarkedNode.prev = originalBookmarkPrev + bookmarkedNode.next = originalBookmarkNext + + tip := &node{ + value: Item{ + ID: 69420, + }, + next: nil, // In this case tip will be at tail + prev: nil, + shelved: time.Time{}, + } + + tipprev := &node{ + value: Item{ + ID: 69419, + }, + next: tip, + prev: nil, // This can be left nil in actuality this will be + // populated + shelved: time.Time{}, + } + + // associate tips prev field with the correct prev node + tip.prev = tipprev + + if !shiftBookmark(tip, &bookmarkedNode, nil, Item{Amount: 1336, ID: 1337, Price: 9999}) { + t.Fatal("There should be liquidity so we don't need to set tip to bookmark") + } + + if bookmarkedNode.value.Price != 9999 || + bookmarkedNode.value.Amount != 1336 || + bookmarkedNode.value.ID != 1337 { + t.Fatal("bookmarked details are not set correctly with shift") + } + + if bookmarkedNode.prev != tip { + t.Fatal("bookmarked prev memory address does not point to tip") + } + + if bookmarkedNode.next != nil { + t.Fatal("bookmarked next is at tail and should be nil") + } + + if bookmarkedNode.next != nil { + t.Fatal("bookmarked next is at tail and should be nil") + } + + if originalBookmarkPrev.next != originalBookmarkNext { + t.Fatal("original bookmarked prev node should be associated with original bookmarked next node") + } + + if originalBookmarkNext.prev != originalBookmarkPrev { + t.Fatal("original bookmarked next node should be associated with original bookmarked prev node") + } + + var nilBookmark *node + + if shiftBookmark(tip, &nilBookmark, nil, Item{Amount: 1336, ID: 1337, Price: 9999}) { + t.Fatal("there should not be a bookmarked node") + } + + if tip != nilBookmark { + t.Fatal("nilBookmark not reassigned") + } + + head := bookmarkedNode + bookmarkedNode.prev = nil + bookmarkedNode.next = originalBookmarkNext + tip.next = nil + + if !shiftBookmark(tip, &bookmarkedNode, &head, Item{Amount: 1336, ID: 1337, Price: 9999}) { + t.Fatal("There should be liquidity so we don't need to set tip to bookmark") + } + + if head != originalBookmarkNext { + t.Fatal("unexpected pointer variable") + } +} diff --git a/exchanges/orderbook/node.go b/exchanges/orderbook/node.go new file mode 100644 index 00000000..c35b72bc --- /dev/null +++ b/exchanges/orderbook/node.go @@ -0,0 +1,135 @@ +package orderbook + +import ( + "sync/atomic" + "time" +) + +const ( + neutral uint32 = iota + active +) + +var ( + defaultInterval = time.Minute + defaultAllowance = time.Second * 30 +) + +// node defines a linked list node for an orderbook item +type node struct { + value Item + next *node + prev *node + // Denotes time pushed to stack, this will influence cleanup routine when + // there is a pause or minimal actions during period + shelved time.Time +} + +// stack defines a FILO list of reusable nodes +type stack struct { + nodes []*node + sema uint32 + count int32 +} + +// newStack returns a ptr to a new stack instance, also starts the cleaning +// service +func newStack() *stack { + s := &stack{} + go s.cleaner() + return s +} + +// now defines a time which is now to ensure no other values get passed in +type now time.Time + +// getNow returns the time at which it is called +func getNow() now { + return now(time.Now()) +} + +// Push pushes a node pointer into the stack to be reused the time is passed in +// to allow for inlining which sets the time at which the node is theoretically +// pushed to a stack. +func (s *stack) Push(n *node, tn now) { + if !atomic.CompareAndSwapUint32(&s.sema, neutral, active) { + // Stack is in use, for now we can dereference pointer + n = nil + return + } + // Adds a time when its placed back on to stack. + n.shelved = time.Time(tn) + n.next = nil + n.prev = nil + n.value = Item{} + + // Allows for resize when overflow TODO: rethink this + s.nodes = append(s.nodes[:s.count], n) + s.count++ + atomic.StoreUint32(&s.sema, neutral) +} + +// Pop returns the last pointer off the stack and reduces the count and if empty +// will produce a lovely fresh node +func (s *stack) Pop() *node { + if !atomic.CompareAndSwapUint32(&s.sema, neutral, active) { + // Stack is in use, for now we can allocate a new node pointer + return &node{} + } + + if s.count == 0 { + // Create an empty node when no nodes are in slice or when cleaning + // service is running + atomic.StoreUint32(&s.sema, neutral) + return &node{} + } + s.count-- + n := s.nodes[s.count] + atomic.StoreUint32(&s.sema, neutral) + return n +} + +// cleaner (POC) runs to the defaultTimer to clean excess nodes (nodes not being +// utilised) TODO: Couple time parameters to check for a reduction in activity. +// Add in counter per second function (?) so if there is a lot of activity don't +// inhibit stack performance. +func (s *stack) cleaner() { + tt := time.NewTimer(defaultInterval) +sleeperino: + for range tt.C { + if !atomic.CompareAndSwapUint32(&s.sema, neutral, active) { + // Stack is in use, reset timer to zero to recheck for neutral state. + tt.Reset(0) + continue + } + // As the old nodes are going to be left justified on this slice we + // should just be able to shift the nodes that are still within time + // allowance all the way to the left. Not going to resize capacity + // because if it can get this big, it might as well stay this big. + // TODO: Test and rethink if sizing is an issue + for x := int32(0); x < s.count; x++ { + if time.Since(s.nodes[x].shelved) > defaultAllowance { + // Old node found continue + continue + } + // First good node found, everything to the left of this on the + // slice can be reassigned + var counter int32 + for y := int32(0); y+x < s.count; y++ { // Go through good nodes + // Reassign + s.nodes[y] = s.nodes[y+x] + // Add to the changed counter to remove from main + // counter + counter++ + } + s.count -= counter + atomic.StoreUint32(&s.sema, neutral) + tt.Reset(defaultInterval) + continue sleeperino + } + // Nodes are old, flush entirety. + s.count = 0 + atomic.StoreUint32(&s.sema, neutral) + tt.Reset(defaultInterval) + } +} diff --git a/exchanges/orderbook/node_test.go b/exchanges/orderbook/node_test.go new file mode 100644 index 00000000..553a675c --- /dev/null +++ b/exchanges/orderbook/node_test.go @@ -0,0 +1,101 @@ +package orderbook + +import ( + "fmt" + "sync/atomic" + "testing" + "time" +) + +func TestPushPop(t *testing.T) { + s := newStack() + var nSlice []*node + for i := 0; i < 100; i++ { + nSlice = append(nSlice, s.Pop()) + } + + if s.getCount() != 0 { + t.Fatalf("incorrect stack count expected %v but received %v", 0, s.getCount()) + } + + for i := 0; i < 100; i++ { + s.Push(nSlice[i], getNow()) + } + + if s.getCount() != 100 { + t.Fatalf("incorrect stack count expected %v but received %v", 100, s.getCount()) + } +} + +func TestCleaner(t *testing.T) { + s := newStack() + var nSlice []*node + for i := 0; i < 100; i++ { + nSlice = append(nSlice, s.Pop()) + } + + tn := getNow() + for i := 0; i < 50; i++ { + s.Push(nSlice[i], tn) + } + // Makes all the 50 pushed nodes invalid + time.Sleep(time.Millisecond * 550) + tn = getNow() + for i := 50; i < 100; i++ { + s.Push(nSlice[i], tn) + } + time.Sleep(time.Millisecond * 550) + if s.getCount() != 50 { + t.Fatalf("incorrect stack count expected %v but received %v", 50, s.getCount()) + } + time.Sleep(time.Second) + if s.getCount() != 0 { + t.Fatalf("incorrect stack count expected %v but received %v", 0, s.getCount()) + } +} + +// Display nodes for testing purposes +func (s *stack) Display() { + for i := int32(0); i < s.getCount(); i++ { + fmt.Printf("NODE IN STACK: %+v %p \n", s.nodes[i], s.nodes[i]) + } + fmt.Println("TOTAL COUNT:", s.getCount()) +} + +// 158 9,521,717 ns/op 9600104 B/op 100001 allocs/op +func BenchmarkWithoutStack(b *testing.B) { + var n *node + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := 0; j < 100000; j++ { + n = new(node) + n.value.Price = 1337 + } + } +} + +// 316 3,485,211 ns/op 1 B/op 0 allocs/op +func BenchmarkWithStack(b *testing.B) { + var n *node + stack := newStack() + b.ReportAllocs() + b.ResetTimer() + tn := getNow() + for i := 0; i < b.N; i++ { + for j := 0; j < 100000; j++ { + n = stack.Pop() + n.value.Price = 1337 + stack.Push(n, tn) + } + } +} + +// getCount is a test helper function to derive the count that does not race. +func (s *stack) getCount() int32 { + if !atomic.CompareAndSwapUint32(&s.sema, neutral, active) { + return -1 + } + defer atomic.StoreUint32(&s.sema, neutral) + return s.count +} diff --git a/exchanges/orderbook/orderbook.go b/exchanges/orderbook/orderbook.go index f5e664be..4c242ac3 100644 --- a/exchanges/orderbook/orderbook.go +++ b/exchanges/orderbook/orderbook.go @@ -18,144 +18,181 @@ func Get(exchange string, p currency.Pair, a asset.Item) (*Base, error) { return service.Retrieve(exchange, p, a) } -// SubscribeOrderbook subcribes to an orderbook and returns a communication -// channel to stream orderbook data updates -func SubscribeOrderbook(exchange string, p currency.Pair, a asset.Item) (dispatch.Pipe, error) { - exchange = strings.ToLower(exchange) - service.Lock() - defer service.Unlock() - book, ok := service.Books[exchange][a][p.Base.Item][p.Quote.Item] - if !ok { - return dispatch.Pipe{}, - fmt.Errorf("orderbook item not found for %s %s %s", - exchange, - p, - a) - } - return service.mux.Subscribe(book.Main) +// GetDepth returns a Depth pointer allowing the caller to stream orderbook +// changes +func GetDepth(exchange string, p currency.Pair, a asset.Item) (*Depth, error) { + return service.GetDepth(exchange, p, a) } -// SubscribeToExchangeOrderbooks subcribes to all orderbooks on an exchange +// DeployDepth sets a depth struct and returns a depth pointer. This allows for +// the loading of a new orderbook snapshot and incremental updates via the +// streaming package. +func DeployDepth(exchange string, p currency.Pair, a asset.Item) (*Depth, error) { + return service.DeployDepth(exchange, p, a) +} + +// SubscribeToExchangeOrderbooks returns a pipe to an exchange feed func SubscribeToExchangeOrderbooks(exchange string) (dispatch.Pipe, error) { service.Lock() defer service.Unlock() - id, ok := service.Exchange[strings.ToLower(exchange)] + exch, ok := service.books[strings.ToLower(exchange)] if !ok { - return dispatch.Pipe{}, fmt.Errorf("%s exchange orderbooks not found", - exchange) + return dispatch.Pipe{}, fmt.Errorf("%w for %s exchange", + errCannotFindOrderbook, exchange) } - return service.mux.Subscribe(id) + return service.Mux.Subscribe(exch.ID) } // Update stores orderbook data func (s *Service) Update(b *Base) error { - name := strings.ToLower(b.ExchangeName) + name := strings.ToLower(b.Exchange) s.Lock() - m1, ok := s.Books[name] + m1, ok := s.books[name] if !ok { - m1 = make(map[asset.Item]map[*currency.Item]map[*currency.Item]*Book) - s.Books[name] = m1 + id, err := s.Mux.GetID() + if err != nil { + s.Unlock() + return err + } + m1 = Exchange{ + m: make(map[asset.Item]map[*currency.Item]map[*currency.Item]*Depth), + ID: id, + } + s.books[name] = m1 } - m2, ok := m1[b.AssetType] + m2, ok := m1.m[b.Asset] if !ok { - m2 = make(map[*currency.Item]map[*currency.Item]*Book) - m1[b.AssetType] = m2 + m2 = make(map[*currency.Item]map[*currency.Item]*Depth) + m1.m[b.Asset] = m2 } m3, ok := m2[b.Pair.Base.Item] if !ok { - m3 = make(map[*currency.Item]*Book) + m3 = make(map[*currency.Item]*Depth) m2[b.Pair.Base.Item] = m3 } book, ok := m3[b.Pair.Quote.Item] if !ok { - book = new(Book) + book = newDepth(m1.ID) + book.AssignOptions(b) m3[b.Pair.Quote.Item] = book - err := s.SetNewData(b, book, name) - s.Unlock() - return err } - - book.b.Bids = append(b.Bids[:0:0], b.Bids...) // nolint:gocritic // Short hand to not use make and copy - book.b.Asks = append(b.Asks[:0:0], b.Asks...) // nolint:gocritic // Short hand to not use make and copy - book.b.LastUpdated = b.LastUpdated - ids := append(book.Assoc, book.Main) + book.SetLastUpdate(b.LastUpdated, b.LastUpdateID, true) + book.LoadSnapshot(b.Bids, b.Asks) s.Unlock() - return s.mux.Publish(ids, b) + return s.Mux.Publish([]uuid.UUID{m1.ID}, book.Retrieve()) } -// SetNewData sets new data -func (s *Service) SetNewData(ob *Base, book *Book, exch string) error { - var err error - book.Assoc, err = s.getAssociations(strings.ToLower(exch)) - if err != nil { - return err +// DeployDepth used for subsystem deployment creates a depth item in the struct +// then returns a ptr to that Depth item +func (s *Service) DeployDepth(exchange string, p currency.Pair, a asset.Item) (*Depth, error) { + if exchange == "" { + return nil, errExchangeNameUnset } - book.Main, err = s.mux.GetID() - if err != nil { - return err + if p.IsEmpty() { + return nil, errPairNotSet } - - // Below instigates orderbook item separation so we can ensure, in the event - // of a simultaneous update via websocket/rest/fix, we don't affect package - // scoped orderbook data which could result in a potential panic - cpy := *ob - cpy.Bids = append(cpy.Bids[:0:0], cpy.Bids...) - cpy.Asks = append(cpy.Asks[:0:0], cpy.Asks...) - book.b = &cpy - return nil -} - -// GetAssociations links a singular book with it's dispatch associations -func (s *Service) getAssociations(exch string) ([]uuid.UUID, error) { - var ids []uuid.UUID - exchangeID, ok := s.Exchange[exch] + if !a.IsValid() { + return nil, errAssetTypeNotSet + } + s.Lock() + m1, ok := s.books[strings.ToLower(exchange)] if !ok { - var err error - exchangeID, err = s.mux.GetID() + id, err := s.Mux.GetID() if err != nil { return nil, err } - s.Exchange[exch] = exchangeID + m1 = Exchange{ + m: make(map[asset.Item]map[*currency.Item]map[*currency.Item]*Depth), + ID: id, + } + s.books[strings.ToLower(exchange)] = m1 } - - ids = append(ids, exchangeID) - return ids, nil + m2, ok := m1.m[a] + if !ok { + m2 = make(map[*currency.Item]map[*currency.Item]*Depth) + m1.m[a] = m2 + } + m3, ok := m2[p.Base.Item] + if !ok { + m3 = make(map[*currency.Item]*Depth) + m2[p.Base.Item] = m3 + } + book, ok := m3[p.Quote.Item] + if !ok { + book = newDepth(m1.ID) + m3[p.Quote.Item] = book + } + s.Unlock() + return book, nil } -// Retrieve gets orderbook data from the slice -func (s *Service) Retrieve(exchange string, p currency.Pair, a asset.Item) (*Base, error) { +// GetDepth returns the actual depth struct for potential subsystems and +// strategies to interact with +func (s *Service) GetDepth(exchange string, p currency.Pair, a asset.Item) (*Depth, error) { s.Lock() defer s.Unlock() - m1, ok := s.Books[strings.ToLower(exchange)] + m1, ok := s.books[strings.ToLower(exchange)] if !ok { - return nil, fmt.Errorf("no orderbooks for %s exchange", exchange) + return nil, fmt.Errorf("%w for %s exchange", + errCannotFindOrderbook, exchange) } - m2, ok := m1[a] + m2, ok := m1.m[a] if !ok { - return nil, fmt.Errorf("no orderbooks associated with asset type %s", + return nil, fmt.Errorf("%w associated with asset type %s", + errCannotFindOrderbook, a) } m3, ok := m2[p.Base.Item] if !ok { - return nil, fmt.Errorf("no orderbooks associated with base currency %s", + return nil, fmt.Errorf("%w associated with base currency %s", + errCannotFindOrderbook, p.Base) } book, ok := m3[p.Quote.Item] if !ok { - return nil, fmt.Errorf("no orderbooks associated with base currency %s", + return nil, fmt.Errorf("%w associated with base currency %s", + errCannotFindOrderbook, p.Quote) } + return book, nil +} - ob := *book.b - ob.Bids = append(ob.Bids[:0:0], ob.Bids...) - ob.Asks = append(ob.Asks[:0:0], ob.Asks...) - return &ob, nil +// Retrieve gets orderbook depth data from the associated linked list and +// returns the base equivalent copy +func (s *Service) Retrieve(exchange string, p currency.Pair, a asset.Item) (*Base, error) { + s.Lock() + defer s.Unlock() + m1, ok := s.books[strings.ToLower(exchange)] + if !ok { + return nil, fmt.Errorf("%w for %s exchange", + errCannotFindOrderbook, + exchange) + } + m2, ok := m1.m[a] + if !ok { + return nil, fmt.Errorf("%w associated with asset type %s", + errCannotFindOrderbook, + a) + } + m3, ok := m2[p.Base.Item] + if !ok { + return nil, fmt.Errorf("%w associated with base currency %s", + errCannotFindOrderbook, + p.Base) + } + book, ok := m3[p.Quote.Item] + if !ok { + return nil, fmt.Errorf("%w associated with base currency %s", + errCannotFindOrderbook, + p.Quote) + } + return book.Retrieve(), nil } // TotalBidsAmount returns the total amount of bids and the total orderbook @@ -178,18 +215,15 @@ func (b *Base) TotalAsksAmount() (amountCollated, total float64) { return amountCollated, total } -// Update updates the bids and asks -func (b *Base) Update(bids, asks []Item) { - b.Bids = bids - b.Asks = asks - b.LastUpdated = time.Now() -} - // Verify ensures that the orderbook items are correctly sorted prior to being // set and will reject any book with incorrect values. // Bids should always go from a high price to a low price and // Asks should always go from a low price to a higher price func (b *Base) Verify() error { + if !b.VerifyOrderbook { + return nil + } + // Checking for both ask and bid lengths being zero has been removed and // a warning has been put in place some exchanges e.g. LakeBTC return zero // level books. In the event that there is a massive liquidity change where @@ -198,58 +232,68 @@ func (b *Base) Verify() error { if len(b.Asks) == 0 || len(b.Bids) == 0 { log.Warnf(log.OrderBook, bookLengthIssue, - b.ExchangeName, + b.Exchange, b.Pair, - b.AssetType, + b.Asset, len(b.Bids), len(b.Asks)) } - for i := range b.Bids { - if b.Bids[i].Price == 0 { - return fmt.Errorf(bidLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errPriceNotSet) - } - if b.Bids[i].Amount <= 0 { - return fmt.Errorf(bidLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errAmountInvalid) - } - if b.IsFundingRate && b.Bids[i].Period == 0 { - return fmt.Errorf(bidLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errPeriodUnset) - } - if i != 0 { - if b.Bids[i].Price > b.Bids[i-1].Price { - return fmt.Errorf(bidLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errOutOfOrder) - } - - if !b.NotAggregated && b.Bids[i].Price == b.Bids[i-1].Price { - return fmt.Errorf(bidLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errDuplication) - } - - if b.Bids[i].ID != 0 && b.Bids[i].ID == b.Bids[i-1].ID { - return fmt.Errorf(bidLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errIDDuplication) - } - } + err := checkAlignment(b.Bids, b.IsFundingRate, b.PriceDuplication, b.IDAlignment, dsc) + if err != nil { + return fmt.Errorf(bidLoadBookFailure, b.Exchange, b.Pair, b.Asset, err) } + err = checkAlignment(b.Asks, b.IsFundingRate, b.PriceDuplication, b.IDAlignment, asc) + if err != nil { + return fmt.Errorf(askLoadBookFailure, b.Exchange, b.Pair, b.Asset, err) + } + return nil +} - for i := range b.Asks { - if b.Asks[i].Price == 0 { - return fmt.Errorf(askLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errPriceNotSet) +// checker defines specific functionality to determine ascending/descending +// validation +type checker func(current Item, previous Item) error + +// asc specifically defines ascending price check +var asc = func(current Item, previous Item) error { + if current.Price < previous.Price { + return errPriceOutOfOrder + } + return nil +} + +// dsc specifically defines descending price check +var dsc = func(current Item, previous Item) error { + if current.Price > previous.Price { + return errPriceOutOfOrder + } + return nil +} + +// checkAlignment validates full orderbook +func checkAlignment(depth Items, fundingRate, priceDuplication, isIDAligned bool, c checker) error { + for i := range depth { + if depth[i].Price == 0 { + return errPriceNotSet } - if b.Asks[i].Amount <= 0 { - return fmt.Errorf(askLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errAmountInvalid) + if depth[i].Amount <= 0 { + return errAmountInvalid } - if b.IsFundingRate && b.Asks[i].Period == 0 { - return fmt.Errorf(askLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errPeriodUnset) + if fundingRate && depth[i].Period == 0 { + return errPeriodUnset } if i != 0 { - if b.Asks[i].Price < b.Asks[i-1].Price { - return fmt.Errorf(askLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errOutOfOrder) + prev := i - 1 + if err := c(depth[i], depth[prev]); err != nil { + return err } - - if !b.NotAggregated && b.Asks[i].Price == b.Asks[i-1].Price { - return fmt.Errorf(askLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errDuplication) + if isIDAligned && depth[i].ID < depth[prev].ID { + return errIDOutOfOrder } - - if b.Asks[i].ID != 0 && b.Asks[i].ID == b.Asks[i-1].ID { - return fmt.Errorf(askLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errIDDuplication) + if !priceDuplication && depth[i].Price == depth[prev].Price { + return errDuplication + } + if depth[i].ID != 0 && depth[i].ID == depth[prev].ID { + return errIDDuplication } } } @@ -259,7 +303,7 @@ func (b *Base) Verify() error { // Process processes incoming orderbooks, creating or updating the orderbook // list func (b *Base) Process() error { - if b.ExchangeName == "" { + if b.Exchange == "" { return errExchangeNameUnset } @@ -267,7 +311,7 @@ func (b *Base) Process() error { return errPairNotSet } - if b.AssetType.String() == "" { + if b.Asset.String() == "" { return errAssetTypeNotSet } @@ -275,11 +319,9 @@ func (b *Base) Process() error { b.LastUpdated = time.Now() } - if !b.VerificationBypass && !b.HasChecksumValidation { - err := b.Verify() - if err != nil { - return err - } + err := b.Verify() + if err != nil { + return err } return service.Update(b) } @@ -290,27 +332,25 @@ func (b *Base) Process() error { // using a sort algorithm as the algorithm could be impeded by a worst case time // complexity when elements are shifted as opposed to just swapping element // values. -func Reverse(elem []Item) { - eLen := len(elem) +func (elem *Items) Reverse() { + eLen := len(*elem) var target int for i := eLen/2 - 1; i >= 0; i-- { target = eLen - 1 - i - elem[i], elem[target] = elem[target], elem[i] + (*elem)[i], (*elem)[target] = (*elem)[target], (*elem)[i] } } // SortAsks sorts ask items to the correct ascending order if pricing values are // scattered. If order from exchange is descending consider using the Reverse // function. -func SortAsks(d []Item) []Item { - sort.Sort(byOBPrice(d)) - return d +func (elem *Items) SortAsks() { + sort.Sort(byOBPrice(*elem)) } // SortBids sorts bid items to the correct descending order if pricing values // are scattered. If order from exchange is ascending consider using the Reverse // function. -func SortBids(d []Item) []Item { - sort.Sort(sort.Reverse(byOBPrice(d))) - return d +func (elem *Items) SortBids() { + sort.Sort(sort.Reverse(byOBPrice(*elem))) } diff --git a/exchanges/orderbook/orderbook_test.go b/exchanges/orderbook/orderbook_test.go index 06eb5af5..760f3d04 100644 --- a/exchanges/orderbook/orderbook_test.go +++ b/exchanges/orderbook/orderbook_test.go @@ -16,105 +16,29 @@ import ( ) func TestMain(m *testing.M) { + // Sets up lower values for test environment + defaultInterval = time.Second + defaultAllowance = time.Millisecond * 500 err := dispatch.Start(1, dispatch.DefaultJobsLimit) if err != nil { log.Fatal(err) } - - cpyMux = service.mux - os.Exit(m.Run()) } -var cpyMux *dispatch.Mux - -func TestSubscribeOrderbook(t *testing.T) { - _, err := SubscribeOrderbook("", currency.Pair{}, asset.Item("")) - if err == nil { - t.Error("error cannot be nil") - } - - p := currency.NewPair(currency.BTC, currency.USD) - - b := Base{ - Pair: p, - AssetType: asset.Spot, - } - - err = b.Process() - if err == nil { - t.Error("error cannot be nil") - } - - b.ExchangeName = "SubscribeOBTest" - b.Bids = []Item{{Price: 100, Amount: 1}, {Price: 99, Amount: 1}} - err = b.Process() - if err != nil { - t.Error("process error", err) - } - - _, err = SubscribeOrderbook("SubscribeOBTest", p, asset.Spot) - if err != nil { - t.Error(err) - } - - // process redundant update - err = b.Process() - if err != nil { - t.Error("process error", err) - } -} - -func TestUpdateBooks(t *testing.T) { - p := currency.NewPair(currency.BTC, currency.USD) - - b := Base{ - Pair: p, - AssetType: asset.Spot, - ExchangeName: "UpdateTest", - } - - service.mux = nil - - err := service.Update(&b) - if err == nil { - t.Error("error cannot be nil") - } - - b.Pair.Base = currency.CYC - err = service.Update(&b) - if err == nil { - t.Error("error cannot be nil") - } - - b.Pair.Quote = currency.ENAU - err = service.Update(&b) - if err == nil { - t.Error("error cannot be nil") - } - - b.AssetType = "unicorns" - err = service.Update(&b) - if err == nil { - t.Error("error cannot be nil") - } - - service.mux = cpyMux -} - func TestSubscribeToExchangeOrderbooks(t *testing.T) { _, err := SubscribeToExchangeOrderbooks("") - if err == nil { - t.Error("error cannot be nil") + if !errors.Is(err, errCannotFindOrderbook) { + t.Fatalf("expected: %v but received: %v", errCannotFindOrderbook, err) } p := currency.NewPair(currency.BTC, currency.USD) b := Base{ - Pair: p, - AssetType: asset.Spot, - ExchangeName: "SubscribeToExchangeOrderbooks", - Bids: []Item{{Price: 100, Amount: 1}, {Price: 99, Amount: 1}}, + Pair: p, + Asset: asset.Spot, + Exchange: "SubscribeToExchangeOrderbooks", + Bids: []Item{{Price: 100, Amount: 1}, {Price: 99, Amount: 1}}, } err = b.Process() @@ -131,9 +55,10 @@ func TestSubscribeToExchangeOrderbooks(t *testing.T) { func TestVerify(t *testing.T) { t.Parallel() b := Base{ - ExchangeName: "TestExchange", - AssetType: asset.Spot, - Pair: currency.NewPair(currency.BTC, currency.USD), + Exchange: "TestExchange", + Asset: asset.Spot, + Pair: currency.NewPair(currency.BTC, currency.USD), + VerifyOrderbook: true, } err := b.Verify() @@ -143,75 +68,75 @@ func TestVerify(t *testing.T) { b.Asks = []Item{{ID: 1337, Price: 99, Amount: 1}, {ID: 1337, Price: 100, Amount: 1}} err = b.Verify() - if err == nil || !errors.Is(err, errIDDuplication) { + if !errors.Is(err, errIDDuplication) { t.Fatalf("expecting %s error but received %v", errIDDuplication, err) } b.Asks = []Item{{Price: 100, Amount: 1}, {Price: 100, Amount: 1}} err = b.Verify() - if err == nil || !errors.Is(err, errDuplication) { + if !errors.Is(err, errDuplication) { t.Fatalf("expecting %s error but received %v", errDuplication, err) } b.Asks = []Item{{Price: 100, Amount: 1}, {Price: 99, Amount: 1}} b.IsFundingRate = true err = b.Verify() - if err == nil || !errors.Is(err, errPeriodUnset) { + if !errors.Is(err, errPeriodUnset) { t.Fatalf("expecting %s error but received %v", errPeriodUnset, err) } b.IsFundingRate = false err = b.Verify() - if err == nil || !errors.Is(err, errOutOfOrder) { - t.Fatalf("expecting %s error but received %v", errOutOfOrder, err) + if !errors.Is(err, errPriceOutOfOrder) { + t.Fatalf("expecting %s error but received %v", errPriceOutOfOrder, err) } b.Asks = []Item{{Price: 100, Amount: 1}, {Price: 100, Amount: 0}} err = b.Verify() - if err == nil || !errors.Is(err, errAmountInvalid) { + if !errors.Is(err, errAmountInvalid) { t.Fatalf("expecting %s error but received %v", errAmountInvalid, err) } b.Asks = []Item{{Price: 100, Amount: 1}, {Price: 0, Amount: 100}} err = b.Verify() - if err == nil || !errors.Is(err, errPriceNotSet) { + if !errors.Is(err, errPriceNotSet) { t.Fatalf("expecting %s error but received %v", errPriceNotSet, err) } b.Bids = []Item{{ID: 1337, Price: 100, Amount: 1}, {ID: 1337, Price: 99, Amount: 1}} err = b.Verify() - if err == nil || !errors.Is(err, errIDDuplication) { + if !errors.Is(err, errIDDuplication) { t.Fatalf("expecting %s error but received %v", errIDDuplication, err) } b.Bids = []Item{{Price: 100, Amount: 1}, {Price: 100, Amount: 1}} err = b.Verify() - if err == nil || !errors.Is(err, errDuplication) { + if !errors.Is(err, errDuplication) { t.Fatalf("expecting %s error but received %v", errDuplication, err) } b.Bids = []Item{{Price: 99, Amount: 1}, {Price: 100, Amount: 1}} b.IsFundingRate = true err = b.Verify() - if err == nil || !errors.Is(err, errPeriodUnset) { + if !errors.Is(err, errPeriodUnset) { t.Fatalf("expecting %s error but received %v", errPeriodUnset, err) } b.IsFundingRate = false err = b.Verify() - if err == nil || !errors.Is(err, errOutOfOrder) { - t.Fatalf("expecting %s error but received %v", errOutOfOrder, err) + if !errors.Is(err, errPriceOutOfOrder) { + t.Fatalf("expecting %s error but received %v", errPriceOutOfOrder, err) } b.Bids = []Item{{Price: 100, Amount: 1}, {Price: 100, Amount: 0}} err = b.Verify() - if err == nil || !errors.Is(err, errAmountInvalid) { + if !errors.Is(err, errAmountInvalid) { t.Fatalf("expecting %s error but received %v", errAmountInvalid, err) } b.Bids = []Item{{Price: 100, Amount: 1}, {Price: 0, Amount: 100}} err = b.Verify() - if err == nil || !errors.Is(err, errPriceNotSet) { + if !errors.Is(err, errPriceNotSet) { t.Fatalf("expecting %s error but received %v", errPriceNotSet, err) } } @@ -251,51 +176,17 @@ func TestCalculateTotaAsks(t *testing.T) { } } -func TestUpdate(t *testing.T) { - t.Parallel() - curr, err := currency.NewPairFromStrings("BTC", "USD") - if err != nil { - t.Fatal(err) - } - timeNow := time.Now() - base := Base{ - Pair: curr, - Asks: []Item{{Price: 100, Amount: 10}}, - Bids: []Item{{Price: 200, Amount: 10}}, - LastUpdated: timeNow, - } - - asks := []Item{{Price: 200, Amount: 101}} - bids := []Item{{Price: 201, Amount: 100}} - time.Sleep(time.Millisecond * 50) - base.Update(bids, asks) - - if !base.LastUpdated.After(timeNow) { - t.Fatal("TestUpdate expected LastUpdated to be greater then original time") - } - - a, b := base.TotalAsksAmount() - if a != 100 && b != 20200 { - t.Fatal("TestUpdate expected a = 100 and b = 20100") - } - - a, b = base.TotalBidsAmount() - if a != 100 && b != 20100 { - t.Fatal("TestUpdate expected a = 100 and b = 20100") - } -} - func TestGetOrderbook(t *testing.T) { c, err := currency.NewPairFromStrings("BTC", "USD") if err != nil { t.Fatal(err) } base := &Base{ - Pair: c, - Asks: []Item{{Price: 100, Amount: 10}}, - Bids: []Item{{Price: 200, Amount: 10}}, - ExchangeName: "Exchange", - AssetType: asset.Spot, + Pair: c, + Asks: []Item{{Price: 100, Amount: 10}}, + Bids: []Item{{Price: 200, Amount: 10}}, + Exchange: "Exchange", + Asset: asset.Spot, } err = base.Process() @@ -344,17 +235,102 @@ func TestGetOrderbook(t *testing.T) { } } +func TestGetDepth(t *testing.T) { + c, err := currency.NewPairFromStrings("BTC", "USD") + if err != nil { + t.Fatal(err) + } + base := &Base{ + Pair: c, + Asks: []Item{{Price: 100, Amount: 10}}, + Bids: []Item{{Price: 200, Amount: 10}}, + Exchange: "Exchange", + Asset: asset.Spot, + } + + err = base.Process() + if err != nil { + t.Fatal(err) + } + + result, err := GetDepth("Exchange", c, asset.Spot) + if err != nil { + t.Fatalf("TestGetOrderbook failed to get orderbook. Error %s", + err) + } + if !result.pair.Equal(c) { + t.Fatal("TestGetOrderbook failed. Mismatched pairs") + } + + _, err = GetDepth("nonexistent", c, asset.Spot) + if !errors.Is(err, errCannotFindOrderbook) { + t.Fatalf("expecting %s error but received %v", errCannotFindOrderbook, err) + } + + c.Base = currency.NewCode("blah") + _, err = GetDepth("Exchange", c, asset.Spot) + if !errors.Is(err, errCannotFindOrderbook) { + t.Fatalf("expecting %s error but received %v", errCannotFindOrderbook, err) + } + + newCurrency, err := currency.NewPairFromStrings("BTC", "AUD") + if err != nil { + t.Fatal(err) + } + _, err = GetDepth("Exchange", newCurrency, asset.Futures) + if !errors.Is(err, errCannotFindOrderbook) { + t.Fatalf("expecting %s error but received %v", errCannotFindOrderbook, err) + } + + base.Pair = newCurrency + err = base.Process() + if err != nil { + t.Error(err) + } + + _, err = GetDepth("Exchange", newCurrency, "meowCats") + if !errors.Is(err, errCannotFindOrderbook) { + t.Fatalf("expecting %s error but received %v", errCannotFindOrderbook, err) + } +} + +func TestDeployDepth(t *testing.T) { + c, err := currency.NewPairFromStrings("BTC", "USD") + if err != nil { + t.Fatal(err) + } + _, err = DeployDepth("", c, asset.Spot) + if !errors.Is(err, errExchangeNameUnset) { + t.Fatalf("expecting %s error but received %v", errExchangeNameUnset, err) + } + _, err = DeployDepth("test", currency.Pair{}, asset.Spot) + if !errors.Is(err, errPairNotSet) { + t.Fatalf("expecting %s error but received %v", errPairNotSet, err) + } + _, err = DeployDepth("test", c, "") + if !errors.Is(err, errAssetTypeNotSet) { + t.Fatalf("expecting %s error but received %v", errAssetTypeNotSet, err) + } + d, err := DeployDepth("test", c, asset.Spot) + if err != nil { + t.Fatal(err) + } + if d == nil { + t.Fatal("depth ptr shall not be nill") + } +} + func TestCreateNewOrderbook(t *testing.T) { c, err := currency.NewPairFromStrings("BTC", "USD") if err != nil { t.Fatal(err) } base := &Base{ - Pair: c, - Asks: []Item{{Price: 100, Amount: 10}}, - Bids: []Item{{Price: 200, Amount: 10}}, - ExchangeName: "testCreateNewOrderbook", - AssetType: asset.Spot, + Pair: c, + Asks: []Item{{Price: 100, Amount: 10}}, + Bids: []Item{{Price: 200, Amount: 10}}, + Exchange: "testCreateNewOrderbook", + Asset: asset.Spot, } err = base.Process() @@ -388,9 +364,9 @@ func TestProcessOrderbook(t *testing.T) { t.Fatal(err) } base := Base{ - Asks: []Item{{Price: 100, Amount: 10}}, - Bids: []Item{{Price: 200, Amount: 10}}, - ExchangeName: "ProcessOrderbook", + Asks: []Item{{Price: 100, Amount: 10}}, + Bids: []Item{{Price: 200, Amount: 10}}, + Exchange: "ProcessOrderbook", } // test for empty pair @@ -408,7 +384,7 @@ func TestProcessOrderbook(t *testing.T) { } // now process a valid orderbook - base.AssetType = asset.Spot + base.Asset = asset.Spot err = base.Process() if err != nil { t.Error("unexpcted result: ", err) @@ -458,7 +434,7 @@ func TestProcessOrderbook(t *testing.T) { } base.Asks = []Item{{Price: 200, Amount: 200}} - base.AssetType = "monthly" + base.Asset = "monthly" err = base.Process() if err != nil { t.Error("Process() error", err) @@ -475,8 +451,8 @@ func TestProcessOrderbook(t *testing.T) { } base.Bids = []Item{{Price: 420, Amount: 200}} - base.ExchangeName = "Blah" - base.AssetType = "quarterly" + base.Exchange = "Blah" + base.Asset = "quarterly" err = base.Process() if err != nil { t.Error("Process() error", err) @@ -521,11 +497,11 @@ func TestProcessOrderbook(t *testing.T) { asks := []Item{{Price: rand.Float64(), Amount: rand.Float64()}} // nolint:gosec // no need to import crypo/rand for testing bids := []Item{{Price: rand.Float64(), Amount: rand.Float64()}} // nolint:gosec // no need to import crypo/rand for testing base := &Base{ - Pair: newPairs, - Asks: asks, - Bids: bids, - ExchangeName: newName, - AssetType: asset.Spot, + Pair: newPairs, + Asks: asks, + Bids: bids, + Exchange: newName, + Asset: asset.Spot, } m.Lock() @@ -577,7 +553,7 @@ func TestProcessOrderbook(t *testing.T) { wg.Wait() } -func deployUnorderedSlice() []Item { +func deployUnorderedSlice() Items { var items []Item rand.Seed(time.Now().UnixNano()) for i := 0; i < 1000; i++ { @@ -588,16 +564,15 @@ func deployUnorderedSlice() []Item { func TestSorting(t *testing.T) { var b Base + b.VerifyOrderbook = true b.Asks = deployUnorderedSlice() err := b.Verify() - if err == nil { - t.Fatal("error cannot be nil") + if !errors.Is(err, errPriceOutOfOrder) { + t.Fatalf("error expected %v received %v", errPriceOutOfOrder, err) } - SortAsks(nil) - - SortAsks(b.Asks) + b.Asks.SortAsks() err = b.Verify() if err != nil { t.Fatal(err) @@ -605,20 +580,18 @@ func TestSorting(t *testing.T) { b.Bids = deployUnorderedSlice() err = b.Verify() - if err == nil { - t.Fatal("error cannot be nil") + if !errors.Is(err, errPriceOutOfOrder) { + t.Fatalf("error expected %v received %v", errPriceOutOfOrder, err) } - SortBids(nil) - - SortBids(b.Bids) + b.Bids.SortBids() err = b.Verify() if err != nil { t.Fatal(err) } } -func deploySliceOrdered() []Item { +func deploySliceOrdered() Items { rand.Seed(time.Now().UnixNano()) var items []Item for i := 0; i < 1000; i++ { @@ -629,6 +602,7 @@ func deploySliceOrdered() []Item { func TestReverse(t *testing.T) { var b Base + b.VerifyOrderbook = true length := 1000 b.Bids = deploySliceOrdered() @@ -637,12 +611,11 @@ func TestReverse(t *testing.T) { } err := b.Verify() - if err == nil { - t.Fatal("error cannot be nil") + if !errors.Is(err, errPriceOutOfOrder) { + t.Fatalf("error expected %v received %v", errPriceOutOfOrder, err) } - Reverse(nil) - Reverse(b.Bids) + b.Bids.Reverse() err = b.Verify() if err != nil { t.Fatal(err) @@ -650,11 +623,11 @@ func TestReverse(t *testing.T) { b.Asks = append(b.Bids[:0:0], b.Bids...) // nolint:gocritic // Short hand err = b.Verify() - if err == nil { - t.Fatal("error cannot be nil") + if !errors.Is(err, errPriceOutOfOrder) { + t.Fatalf("error expected %v received %v", errPriceOutOfOrder, err) } - Reverse(b.Asks) + b.Asks.Reverse() err = b.Verify() if err != nil { t.Fatal(err) @@ -670,7 +643,7 @@ func BenchmarkReverse(b *testing.B) { } for i := 0; i < b.N; i++ { - Reverse(s) + s.Reverse() } } @@ -679,17 +652,17 @@ func BenchmarkSortAsksDecending(b *testing.B) { s := deploySliceOrdered() for i := 0; i < b.N; i++ { ts := append(s[:0:0], s...) - SortAsks(ts) + ts.SortAsks() } } // 14924 79199 ns/op 49206 B/op 3 allocs/op func BenchmarkSortBidsAscending(b *testing.B) { s := deploySliceOrdered() - Reverse(s) + s.Reverse() for i := 0; i < b.N; i++ { ts := append(s[:0:0], s...) - SortBids(ts) + ts.SortBids() } } @@ -698,7 +671,7 @@ func BenchmarkSortAsksStandard(b *testing.B) { s := deployUnorderedSlice() for i := 0; i < b.N; i++ { ts := append(s[:0:0], s...) - SortAsks(ts) + ts.SortAsks() } } @@ -707,7 +680,7 @@ func BenchmarkSortBidsStandard(b *testing.B) { s := deployUnorderedSlice() for i := 0; i < b.N; i++ { ts := append(s[:0:0], s...) - SortBids(ts) + ts.SortBids() } } @@ -716,21 +689,21 @@ func BenchmarkSortAsksAscending(b *testing.B) { s := deploySliceOrdered() for i := 0; i < b.N; i++ { ts := append(s[:0:0], s...) - SortAsks(ts) + ts.SortAsks() } } // 12565 97257 ns/op 49208 B/op 3 allocs/op func BenchmarkSortBidsDescending(b *testing.B) { s := deploySliceOrdered() - Reverse(s) + s.Reverse() for i := 0; i < b.N; i++ { ts := append(s[:0:0], s...) - SortBids(ts) + ts.SortBids() } } -// 923154 1169 ns/op 4096 B/op 1 allocs/op +// 124867 8480 ns/op 49152 B/op 1 allocs/op func BenchmarkDuplicatingSlice(b *testing.B) { s := deploySliceOrdered() for i := 0; i < b.N; i++ { @@ -738,7 +711,7 @@ func BenchmarkDuplicatingSlice(b *testing.B) { } } -// 705922 1546 ns/op 4096 B/op 1 allocs/op +// 122998 8441 ns/op 49152 B/op 1 allocs/op func BenchmarkCopySlice(b *testing.B) { s := deploySliceOrdered() for i := 0; i < b.N; i++ { diff --git a/exchanges/orderbook/orderbook_types.go b/exchanges/orderbook/orderbook_types.go index 48f53342..2e21a6b6 100644 --- a/exchanges/orderbook/orderbook_types.go +++ b/exchanges/orderbook/orderbook_types.go @@ -20,42 +20,38 @@ const ( // Vars for the orderbook package var ( - service *Service - - errExchangeNameUnset = errors.New("orderbook exchange name not set") - errPairNotSet = errors.New("orderbook currency pair not set") - errAssetTypeNotSet = errors.New("orderbook asset type not set") - errNoOrderbook = errors.New("orderbook bids and asks are empty") - errPriceNotSet = errors.New("price cannot be zero") - errAmountInvalid = errors.New("amount cannot be less or equal to zero") - errOutOfOrder = errors.New("pricing out of order") - errDuplication = errors.New("price duplication") - errIDDuplication = errors.New("id duplication") - errPeriodUnset = errors.New("funding rate period is unset") + errExchangeNameUnset = errors.New("orderbook exchange name not set") + errPairNotSet = errors.New("orderbook currency pair not set") + errAssetTypeNotSet = errors.New("orderbook asset type not set") + errCannotFindOrderbook = errors.New("cannot find orderbook(s)") + errPriceNotSet = errors.New("price cannot be zero") + errAmountInvalid = errors.New("amount cannot be less or equal to zero") + errPriceOutOfOrder = errors.New("pricing out of order") + errIDOutOfOrder = errors.New("ID out of order") + errDuplication = errors.New("price duplication") + errIDDuplication = errors.New("id duplication") + errPeriodUnset = errors.New("funding rate period is unset") ) -func init() { - service = new(Service) - service.mux = dispatch.GetNewMux() - service.Books = make(map[string]map[asset.Item]map[*currency.Item]map[*currency.Item]*Book) - service.Exchange = make(map[string]uuid.UUID) +var service = Service{ + books: make(map[string]Exchange), + Mux: dispatch.GetNewMux(), } -// Book defines an orderbook with its links to different dispatch outputs -type Book struct { - b *Base - Main uuid.UUID - Assoc []uuid.UUID -} - -// Service holds orderbook information for each individual exchange +// Service provides a store for difference exchange orderbooks type Service struct { - Books map[string]map[asset.Item]map[*currency.Item]map[*currency.Item]*Book - Exchange map[string]uuid.UUID - mux *dispatch.Mux + books map[string]Exchange + *dispatch.Mux sync.Mutex } +// Exchange defines a holder for the exchange specific depth items with a +// specific ID associated with that exchange +type Exchange struct { + m map[asset.Item]map[*currency.Item]map[*currency.Item]*Depth + ID uuid.UUID +} + // Item stores the amount and price values type Item struct { Amount float64 @@ -70,27 +66,34 @@ type Item struct { OrderCount int64 } +// Items defines a slice of orderbook items +type Items []Item + // Base holds the fields for the orderbook base type Base struct { - Pair currency.Pair `json:"pair"` - Bids []Item `json:"bids"` - Asks []Item `json:"asks"` - LastUpdated time.Time `json:"lastUpdated"` - LastUpdateID int64 `json:"lastUpdateId"` - AssetType asset.Item `json:"assetType"` - ExchangeName string `json:"exchangeName"` + Bids Items + Asks Items - // NotAggregated defines whether an orderbook can contain duplicate prices - // in a payload - NotAggregated bool `json:"-"` - IsFundingRate bool `json:"fundingRate"` + Exchange string + Pair currency.Pair + Asset asset.Item - // VerificationBypass is a complete orderbook verification bypass set by - // user configuration - VerificationBypass bool `json:"-"` - // HasChecksumValidation defines an allowance to bypass internal - // verification if the book has been verified by checksum. - HasChecksumValidation bool `json:"-"` + LastUpdated time.Time + LastUpdateID int64 + // PriceDuplication defines whether an orderbook can contain duplicate + // prices in a payload + PriceDuplication bool + IsFundingRate bool + // VerifyOrderbook allows for a toggle between orderbook verification set by + // user configuration, this allows for a potential processing boost but + // a potential for orderbook integrity being deminished. + VerifyOrderbook bool `json:"-"` + // RestSnapshot defines if the depth was applied via the REST protocol thus + // an update cannot be applied via websocket mechanics and a resubscription + // would need to take place to maintain book integrity + RestSnapshot bool + // Checks if the orderbook needs ID alignment as well as price alignment + IDAlignment bool } type byOBPrice []Item @@ -98,3 +101,16 @@ type byOBPrice []Item func (a byOBPrice) Len() int { return len(a) } func (a byOBPrice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a byOBPrice) Less(i, j int) bool { return a[i].Price < a[j].Price } + +type options struct { + exchange string + pair currency.Pair + asset asset.Item + lastUpdated time.Time + lastUpdateID int64 + priceDuplication bool + isFundingRate bool + VerifyOrderbook bool + restSnapshot bool + idAligned bool +} diff --git a/exchanges/poloniex/poloniex_websocket.go b/exchanges/poloniex/poloniex_websocket.go index 7ff3f25c..af26205e 100644 --- a/exchanges/poloniex/poloniex_websocket.go +++ b/exchanges/poloniex/poloniex_websocket.go @@ -503,17 +503,17 @@ func (p *Poloniex) WsProcessOrderbookSnapshot(ob []interface{}, symbol string) e } // Both sides are completely out of order - sort needs to be used - book.Asks = orderbook.SortAsks(book.Asks) - book.Bids = orderbook.SortBids(book.Bids) - book.AssetType = asset.Spot - book.VerificationBypass = p.OrderbookVerificationBypass + book.Asks.SortAsks() + book.Bids.SortBids() + book.Asset = asset.Spot + book.VerifyOrderbook = p.CanVerifyOrderbook var err error book.Pair, err = currency.NewPairFromString(symbol) if err != nil { return err } - book.ExchangeName = p.Name + book.Exchange = p.Name return p.Websocket.Orderbook.LoadSnapshot(&book) } diff --git a/exchanges/poloniex/poloniex_wrapper.go b/exchanges/poloniex/poloniex_wrapper.go index ff7dee1b..c6974774 100644 --- a/exchanges/poloniex/poloniex_wrapper.go +++ b/exchanges/poloniex/poloniex_wrapper.go @@ -330,10 +330,10 @@ func (p *Poloniex) FetchOrderbook(currencyPair currency.Pair, assetType asset.It // UpdateOrderbook updates and returns the orderbook for a currency pair func (p *Poloniex) UpdateOrderbook(c currency.Pair, assetType asset.Item) (*orderbook.Base, error) { callingBook := &orderbook.Base{ - ExchangeName: p.Name, - Pair: c, - AssetType: assetType, - VerificationBypass: p.OrderbookVerificationBypass, + Exchange: p.Name, + Pair: c, + Asset: assetType, + VerifyOrderbook: p.CanVerifyOrderbook, } orderbookNew, err := p.GetOrderbook("", poloniexMaxOrderbookDepth) if err != nil { @@ -346,10 +346,10 @@ func (p *Poloniex) UpdateOrderbook(c currency.Pair, assetType asset.Item) (*orde } for i := range enabledPairs { book := &orderbook.Base{ - ExchangeName: p.Name, - Pair: enabledPairs[i], - AssetType: assetType, - VerificationBypass: p.OrderbookVerificationBypass, + Exchange: p.Name, + Pair: enabledPairs[i], + Asset: assetType, + VerifyOrderbook: p.CanVerifyOrderbook, } fpair, err := p.FormatExchangeCurrency(enabledPairs[i], assetType) diff --git a/exchanges/stream/buffer/buffer.go b/exchanges/stream/buffer/buffer.go index d275ee6c..a85bfb5c 100644 --- a/exchanges/stream/buffer/buffer.go +++ b/exchanges/stream/buffer/buffer.go @@ -4,10 +4,12 @@ import ( "errors" "fmt" "sort" + "time" "github.com/thrasher-corp/gocryptotrader/currency" "github.com/thrasher-corp/gocryptotrader/exchanges/asset" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" + "github.com/thrasher-corp/gocryptotrader/log" ) const packageError = "websocket orderbook buffer error: %w" @@ -18,6 +20,8 @@ var ( errIssueBufferEnabledButNoLimit = errors.New("buffer enabled but no limit set") errUpdateIsNil = errors.New("update is nil") errUpdateNoTargets = errors.New("update bid/ask targets cannot be nil") + errDepthNotFound = errors.New("orderbook depth not found") + errRESTOverwrite = errors.New("orderbook has been overwritten by REST protocol") ) // Setup sets private variables @@ -25,7 +29,10 @@ func (w *Orderbook) Setup(obBufferLimit int, bufferEnabled, sortBuffer, sortBufferByUpdateIDs, - updateEntriesByID bool, exchangeName string, dataHandler chan interface{}) error { + updateEntriesByID, + verbose bool, + exchangeName string, + dataHandler chan interface{}) error { if exchangeName == "" { return fmt.Errorf(packageError, errUnsetExchangeName) } @@ -43,6 +50,7 @@ func (w *Orderbook) Setup(obBufferLimit int, w.exchangeName = exchangeName w.dataHandler = dataHandler w.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder) + w.verbose = verbose return nil } @@ -57,27 +65,48 @@ func (w *Orderbook) validate(u *Update) error { return nil } -// Update updates a local buffer using bid targets and ask targets then updates -// main orderbook -// Volume == 0; deletion at price target -// Price target not found; append of price target -// Price target found; amend volume of price target +// Update updates a stored pointer to an orderbook.Depth struct containing a +// linked list, this switches between the usage of a buffered update func (w *Orderbook) Update(u *Update) error { if err := w.validate(u); err != nil { return err } w.m.Lock() defer w.m.Unlock() - obLookup, ok := w.ob[u.Pair.Base][u.Pair.Quote][u.Asset] + book, ok := w.ob[u.Pair.Base][u.Pair.Quote][u.Asset] if !ok { - return fmt.Errorf("ob.Base could not be found for Exchange %s CurrencyPair: %s AssetType: %s", + return fmt.Errorf("%w for Exchange %s CurrencyPair: %s AssetType: %s", + errDepthNotFound, w.exchangeName, u.Pair, u.Asset) } + // Checks for when the rest protocol overwrites a streaming dominated book + // will stop updating book via incremental updates. This occurs because our + // sync manager (engine/sync.go) timer has elapsed for streaming. Usually + // because the book is highly illiquid. TODO: Book resubscribe on websocket. + if book.ob.IsRestSnapshot() { + if w.verbose { + log.Warnf(log.WebsocketMgr, + "%s for Exchange %s CurrencyPair: %s AssetType: %s consider extending synctimeoutwebsocket", + errRESTOverwrite, + w.exchangeName, + u.Pair, + u.Asset) + } + return fmt.Errorf("%w for Exchange %s CurrencyPair: %s AssetType: %s", + errRESTOverwrite, + w.exchangeName, + u.Pair, + u.Asset) + } + + // Apply new update information + book.ob.SetLastUpdate(u.UpdateTime, u.UpdateID, false) + if w.bufferEnabled { - processed, err := w.processBufferUpdate(obLookup, u) + processed, err := w.processBufferUpdate(book, u) if err != nil { return err } @@ -86,21 +115,36 @@ func (w *Orderbook) Update(u *Update) error { return nil } } else { - err := w.processObUpdate(obLookup, u) + err := w.processObUpdate(book, u) if err != nil { return err } } - err := obLookup.ob.Process() - if err != nil { - return err + if book.ob.VerifyOrderbook { // This is used here so as to not retrieve + // book if verification is off. + // On every update, this will retrieve and verify orderbook depths + err := book.ob.Retrieve().Verify() + if err != nil { + return err + } } - // Process in data handler select { - case w.dataHandler <- obLookup.ob: + case <-book.ticker.C: + // Opted to wait for receiver because we are limiting here and the sync + // manager requires update + go func() { + w.dataHandler <- book.ob.Retrieve() + book.ob.Publish() + }() default: + // We do not need to send an update to the sync manager within this time + // window unless verbose is turned on + if w.verbose { + w.dataHandler <- book.ob.Retrieve() + book.ob.Publish() + } } return nil } @@ -139,230 +183,42 @@ func (w *Orderbook) processBufferUpdate(o *orderbookHolder, u *Update) (bool, er // processObUpdate processes updates either by its corresponding id or by // price level func (w *Orderbook) processObUpdate(o *orderbookHolder, u *Update) error { - o.ob.LastUpdateID = u.UpdateID if w.updateEntriesByID { return o.updateByIDAndAction(u) } - return o.updateByPrice(u) + o.updateByPrice(u) + return nil } // updateByPrice ammends amount if match occurs by price, deletes if amount is // zero or less and inserts if not found. -func (o *orderbookHolder) updateByPrice(updts *Update) error { -askUpdates: - for j := range updts.Asks { - for target := range o.ob.Asks { - if o.ob.Asks[target].Price == updts.Asks[j].Price { - if updts.Asks[j].Amount == 0 { - o.ob.Asks = append(o.ob.Asks[:target], o.ob.Asks[target+1:]...) - continue askUpdates - } - o.ob.Asks[target].Amount = updts.Asks[j].Amount - continue askUpdates - } - } - if updts.Asks[j].Amount <= 0 { - continue - } - insertAsk(updts.Asks[j], &o.ob.Asks) - if updts.MaxDepth != 0 && len(o.ob.Asks) > updts.MaxDepth { - o.ob.Asks = o.ob.Asks[:updts.MaxDepth] - } - } -bidUpdates: - for j := range updts.Bids { - for target := range o.ob.Bids { - if o.ob.Bids[target].Price == updts.Bids[j].Price { - if updts.Bids[j].Amount == 0 { - o.ob.Bids = append(o.ob.Bids[:target], o.ob.Bids[target+1:]...) - continue bidUpdates - } - o.ob.Bids[target].Amount = updts.Bids[j].Amount - continue bidUpdates - } - } - if updts.Bids[j].Amount <= 0 { - continue - } - insertBid(updts.Bids[j], &o.ob.Bids) - if updts.MaxDepth != 0 && len(o.ob.Bids) > updts.MaxDepth { - o.ob.Bids = o.ob.Bids[:updts.MaxDepth] - } - } - return nil +func (o *orderbookHolder) updateByPrice(updts *Update) { + o.ob.UpdateBidAskByPrice(updts.Bids, updts.Asks, updts.MaxDepth) } // updateByIDAndAction will receive an action to execute against the orderbook // it will then match by IDs instead of price to perform the action -func (o *orderbookHolder) updateByIDAndAction(updts *Update) (err error) { +func (o *orderbookHolder) updateByIDAndAction(updts *Update) error { switch updts.Action { case Amend: - err = applyUpdates(updts.Bids, o.ob.Bids) - if err != nil { - return err - } - err = applyUpdates(updts.Asks, o.ob.Asks) - if err != nil { - return err - } + return o.ob.UpdateBidAskByID(updts.Bids, updts.Asks) case Delete: // edge case for Bitfinex as their streaming endpoint duplicates deletes - bypassErr := o.ob.ExchangeName == "Bitfinex" && o.ob.IsFundingRate - err = deleteUpdates(updts.Bids, &o.ob.Bids, bypassErr) - if err != nil { - return fmt.Errorf("%s %s %v", o.ob.AssetType, o.ob.Pair, err) - } - err = deleteUpdates(updts.Asks, &o.ob.Asks, bypassErr) - if err != nil { - return fmt.Errorf("%s %s %v", o.ob.AssetType, o.ob.Pair, err) - } + bypassErr := o.ob.GetName() == "Bitfinex" && o.ob.IsFundingRate() + return o.ob.DeleteBidAskByID(updts.Bids, updts.Asks, bypassErr) case Insert: - insertUpdatesBid(updts.Bids, &o.ob.Bids) - insertUpdatesAsk(updts.Asks, &o.ob.Asks) + return o.ob.InsertBidAskByID(updts.Bids, updts.Asks) case UpdateInsert: - updateBids: - for x := range updts.Bids { - for target := range o.ob.Bids { // First iteration finds ID matches - if o.ob.Bids[target].ID == updts.Bids[x].ID { - if o.ob.Bids[target].Price != updts.Bids[x].Price { - // Price change occurred so correct bid alignment is - // needed - delete instance and insert into correct - // price level - o.ob.Bids = append(o.ob.Bids[:target], o.ob.Bids[target+1:]...) - break - } - o.ob.Bids[target].Amount = updts.Bids[x].Amount - continue updateBids - } - } - insertBid(updts.Bids[x], &o.ob.Bids) - } - updateAsks: - for x := range updts.Asks { - for target := range o.ob.Asks { - if o.ob.Asks[target].ID == updts.Asks[x].ID { - if o.ob.Asks[target].Price != updts.Asks[x].Price { - // Price change occurred so correct ask alignment is - // needed - delete instance and insert into correct - // price level - o.ob.Asks = append(o.ob.Asks[:target], o.ob.Asks[target+1:]...) - break - } - o.ob.Asks[target].Amount = updts.Asks[x].Amount - continue updateAsks - } - } - insertAsk(updts.Asks[x], &o.ob.Asks) - } + return o.ob.UpdateInsertByID(updts.Bids, updts.Asks) default: return fmt.Errorf("invalid action [%s]", updts.Action) } - return nil } -// applyUpdates amends amount by ID and returns an error if not found -func applyUpdates(updts, book []orderbook.Item) error { -updates: - for x := range updts { - for y := range book { - if book[y].ID == updts[x].ID { - book[y].Amount = updts[x].Amount - continue updates - } - } - return fmt.Errorf("update cannot be applied id: %d not found", - updts[x].ID) - } - return nil -} - -// deleteUpdates removes updates from orderbook and returns an error if not -// found -func deleteUpdates(updt []orderbook.Item, book *[]orderbook.Item, bypassErr bool) error { -updates: - for x := range updt { - for y := range *book { - if (*book)[y].ID == updt[x].ID { - *book = append((*book)[:y], (*book)[y+1:]...) // nolint:gocritic - continue updates - } - } - // bypassErr is for expected duplication from endpoint. - if !bypassErr { - return fmt.Errorf("update cannot be deleted id: %d not found", - updt[x].ID) - } - } - return nil -} - -func insertAsk(updt orderbook.Item, book *[]orderbook.Item) { - for target := range *book { - if updt.Price < (*book)[target].Price { - insertItem(updt, book, target) - return - } - } - *book = append(*book, updt) -} - -func insertBid(updt orderbook.Item, book *[]orderbook.Item) { - for target := range *book { - if updt.Price > (*book)[target].Price { - insertItem(updt, book, target) - return - } - } - *book = append(*book, updt) -} - -// insertUpdatesBid inserts on **correctly aligned** book at price level -func insertUpdatesBid(updt []orderbook.Item, book *[]orderbook.Item) { -updates: - for x := range updt { - for target := range *book { - if updt[x].Price > (*book)[target].Price { - insertItem(updt[x], book, target) - continue updates - } - } - *book = append(*book, updt[x]) - } -} - -// insertUpdatesBid inserts on **correctly aligned** book at price level -func insertUpdatesAsk(updt []orderbook.Item, book *[]orderbook.Item) { -updates: - for x := range updt { - for target := range *book { - if updt[x].Price < (*book)[target].Price { - insertItem(updt[x], book, target) - continue updates - } - } - *book = append(*book, updt[x]) - } -} - -// insertItem inserts item in slice by target element this is an optimization -// to reduce the need for sorting algorithms -func insertItem(update orderbook.Item, book *[]orderbook.Item, target int) { - // TODO: extend slice by incoming update length before this gets hit - *book = append(*book, orderbook.Item{}) - copy((*book)[target+1:], (*book)[target:]) - (*book)[target] = update -} - -// LoadSnapshot loads initial snapshot of ob data from websocket +// LoadSnapshot loads initial snapshot of orderbook data from websocket func (w *Orderbook) LoadSnapshot(book *orderbook.Base) error { w.m.Lock() defer w.m.Unlock() - - err := book.Process() - if err != nil { - return err - } - m1, ok := w.ob[book.Pair.Base] if !ok { m1 = make(map[currency.Code]map[asset.Item]*orderbookHolder) @@ -373,31 +229,60 @@ func (w *Orderbook) LoadSnapshot(book *orderbook.Base) error { m2 = make(map[asset.Item]*orderbookHolder) m1[book.Pair.Quote] = m2 } - m3, ok := m2[book.AssetType] + holder, ok := m2[book.Asset] if !ok { - m3 = &orderbookHolder{ob: book, buffer: &[]Update{}} - m2[book.AssetType] = m3 - } else { - m3.ob.LastUpdateID = book.LastUpdateID - m3.ob.Bids = book.Bids - m3.ob.Asks = book.Asks + // Associate orderbook pointer with local exchange depth map + depth, err := orderbook.DeployDepth(book.Exchange, book.Pair, book.Asset) + if err != nil { + return err + } + depth.AssignOptions(book) + buffer := make([]Update, w.obBufferLimit) + ticker := time.NewTicker(timerDefault) + holder = &orderbookHolder{ + ob: depth, + buffer: &buffer, + ticker: ticker, + } + m2[book.Asset] = holder } - w.dataHandler <- book + + // Checks if book can deploy to linked list + err := book.Verify() + if err != nil { + return err + } + + holder.ob.LoadSnapshot(book.Bids, book.Asks) + + if holder.ob.VerifyOrderbook { // This is used here so as to not retrieve + // book if verification is off. + // Checks to see if orderbook snapshot that was deployed has not been + // altered in any way + err = holder.ob.Retrieve().Verify() + if err != nil { + return err + } + } + + w.dataHandler <- holder.ob.Retrieve() + holder.ob.Publish() return nil } -// GetOrderbook returns orderbook stored in current buffer -func (w *Orderbook) GetOrderbook(p currency.Pair, a asset.Item) *orderbook.Base { +// GetOrderbook returns an orderbook copy as orderbook.Base +func (w *Orderbook) GetOrderbook(p currency.Pair, a asset.Item) (*orderbook.Base, error) { w.m.Lock() defer w.m.Unlock() - ptr, ok := w.ob[p.Base][p.Quote][a] + book, ok := w.ob[p.Base][p.Quote][a] if !ok { - return nil + return nil, fmt.Errorf("%s %s %s %w", + w.exchangeName, + p, + a, + errDepthNotFound) } - cpy := *ptr.ob - cpy.Asks = append(cpy.Asks[:0:0], cpy.Asks...) - cpy.Bids = append(cpy.Bids[:0:0], cpy.Bids...) - return &cpy + return book.ob.Retrieve(), nil } // FlushBuffer flushes w.ob data to be garbage collected and refreshed when a @@ -414,9 +299,12 @@ func (w *Orderbook) FlushOrderbook(p currency.Pair, a asset.Item) error { defer w.m.Unlock() book, ok := w.ob[p.Base][p.Quote][a] if !ok { - return fmt.Errorf("orderbook not associated with pair: [%s] and asset [%s]", p, a) + return fmt.Errorf("cannot flush orderbook %s %s %s %w", + w.exchangeName, + p, + a, + errDepthNotFound) } - book.ob.Bids = nil - book.ob.Asks = nil + book.ob.Flush() return nil } diff --git a/exchanges/stream/buffer/buffer_test.go b/exchanges/stream/buffer/buffer_test.go index e7b10aeb..e5a96d72 100644 --- a/exchanges/stream/buffer/buffer_test.go +++ b/exchanges/stream/buffer/buffer_test.go @@ -26,26 +26,27 @@ const ( exchangeName = "exchangeTest" ) -func createSnapshot() (obl *Orderbook, asks, bids []orderbook.Item, err error) { - var snapShot1 orderbook.Base - snapShot1.ExchangeName = exchangeName - asks = []orderbook.Item{ - {Price: 4000, Amount: 1, ID: 6}, +func createSnapshot() (holder *Orderbook, asks, bids orderbook.Items, err error) { + asks = orderbook.Items{{Price: 4000, Amount: 1, ID: 6}} + bids = orderbook.Items{{Price: 4000, Amount: 1, ID: 6}} + + book := &orderbook.Base{ + Exchange: exchangeName, + Asks: asks, + Bids: bids, + Asset: asset.Spot, + Pair: cp, + PriceDuplication: true, } - bids = []orderbook.Item{ - {Price: 4000, Amount: 1, ID: 6}, - } - snapShot1.Asks = asks - snapShot1.Bids = bids - snapShot1.AssetType = asset.Spot - snapShot1.Pair = cp - snapShot1.NotAggregated = true - obl = &Orderbook{ + + newBook := make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder) + + holder = &Orderbook{ exchangeName: exchangeName, dataHandler: make(chan interface{}, 100), - ob: make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder), + ob: newBook, } - err = obl.LoadSnapshot(&snapShot1) + err = holder.LoadSnapshot(book) return } @@ -108,20 +109,13 @@ func BenchmarkUpdateAsksByPrice(b *testing.B) { // BenchmarkBufferPerformance demonstrates buffer more performant than multi // process calls +// 4219518 287 ns/op 176 B/op 1 allocs/op func BenchmarkBufferPerformance(b *testing.B) { - obl, asks, bids, err := createSnapshot() + holder, asks, bids, err := createSnapshot() if err != nil { b.Fatal(err) } - obl.bufferEnabled = true - // This is to ensure we do not send in zero orderbook info to our main book - // in orderbook.go, orderbooks should not be zero even after an update. - dummyItem := orderbook.Item{ - Amount: 1333337, - Price: 1337.1337, - ID: 1337, - } - obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids = append(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids, dummyItem) + holder.bufferEnabled = true update := &Update{ Bids: bids, Asks: asks, @@ -134,7 +128,7 @@ func BenchmarkBufferPerformance(b *testing.B) { randomIndex := rand.Intn(4) // nolint:gosec // no need to import crypo/rand for testing update.Asks = itemArray[randomIndex] update.Bids = itemArray[randomIndex] - err = obl.Update(update) + err = holder.Update(update) if err != nil { b.Fatal(err) } @@ -142,21 +136,14 @@ func BenchmarkBufferPerformance(b *testing.B) { } // BenchmarkBufferSortingPerformance benchmark +// 2693391 467 ns/op 208 B/op 2 allocs/op func BenchmarkBufferSortingPerformance(b *testing.B) { - obl, asks, bids, err := createSnapshot() + holder, asks, bids, err := createSnapshot() if err != nil { b.Fatal(err) } - obl.bufferEnabled = true - obl.sortBuffer = true - // This is to ensure we do not send in zero orderbook info to our main book - // in orderbook.go, orderbooks should not be zero even after an update. - dummyItem := orderbook.Item{ - Amount: 1333337, - Price: 1337.1337, - ID: 1337, - } - obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids = append(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids, dummyItem) + holder.bufferEnabled = true + holder.sortBuffer = true update := &Update{ Bids: bids, Asks: asks, @@ -169,7 +156,7 @@ func BenchmarkBufferSortingPerformance(b *testing.B) { randomIndex := rand.Intn(4) // nolint:gosec // no need to import crypo/rand for testing update.Asks = itemArray[randomIndex] update.Bids = itemArray[randomIndex] - err = obl.Update(update) + err = holder.Update(update) if err != nil { b.Fatal(err) } @@ -177,22 +164,15 @@ func BenchmarkBufferSortingPerformance(b *testing.B) { } // BenchmarkBufferSortingPerformance benchmark +// 1000000 1019 ns/op 208 B/op 2 allocs/op func BenchmarkBufferSortingByIDPerformance(b *testing.B) { - obl, asks, bids, err := createSnapshot() + holder, asks, bids, err := createSnapshot() if err != nil { b.Fatal(err) } - obl.bufferEnabled = true - obl.sortBuffer = true - obl.sortBufferByUpdateIDs = true - // This is to ensure we do not send in zero orderbook info to our main book - // in orderbook.go, orderbooks should not be zero even after an update. - dummyItem := orderbook.Item{ - Amount: 1333337, - Price: 1337.1337, - ID: 1337, - } - obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids = append(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids, dummyItem) + holder.bufferEnabled = true + holder.sortBuffer = true + holder.sortBufferByUpdateIDs = true update := &Update{ Bids: bids, Asks: asks, @@ -205,28 +185,21 @@ func BenchmarkBufferSortingByIDPerformance(b *testing.B) { randomIndex := rand.Intn(4) // nolint:gosec // no need to import crypo/rand for testing update.Asks = itemArray[randomIndex] update.Bids = itemArray[randomIndex] - err = obl.Update(update) + err = holder.Update(update) if err != nil { b.Fatal(err) } } } -// BenchmarkNoBufferPerformance demonstrates orderbook process less performant +// BenchmarkNoBufferPerformance demonstrates orderbook process more performant // than buffer +// 9516966 141 ns/op 0 B/op 0 allocs/op func BenchmarkNoBufferPerformance(b *testing.B) { obl, asks, bids, err := createSnapshot() if err != nil { b.Fatal(err) } - // This is to ensure we do not send in zero orderbook info to our main book - // in orderbook.go, orderbooks should not be zero even after an update. - dummyItem := orderbook.Item{ - Amount: 1333337, - Price: 1337.1337, - ID: 1337, - } - obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids = append(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids, dummyItem) update := &Update{ Bids: bids, Asks: asks, @@ -247,13 +220,13 @@ func BenchmarkNoBufferPerformance(b *testing.B) { } func TestUpdates(t *testing.T) { - obl, _, _, err := createSnapshot() + holder, _, _, err := createSnapshot() if err != nil { t.Error(err) } - holder := obl.ob[cp.Base][cp.Quote][asset.Spot] - holder.updateByPrice(&Update{ + book := holder.ob[cp.Base][cp.Quote][asset.Spot] + book.updateByPrice(&Update{ Bids: itemArray[5], Asks: itemArray[5], Pair: cp, @@ -264,7 +237,7 @@ func TestUpdates(t *testing.T) { t.Error(err) } - holder.updateByPrice(&Update{ + book.updateByPrice(&Update{ Bids: itemArray[0], Asks: itemArray[0], Pair: cp, @@ -275,23 +248,23 @@ func TestUpdates(t *testing.T) { t.Error(err) } - if len(holder.ob.Asks) != 3 { + if book.ob.GetAskLength() != 3 { t.Error("Did not update") } } // TestHittingTheBuffer logic test func TestHittingTheBuffer(t *testing.T) { - obl, _, _, err := createSnapshot() + holder, _, _, err := createSnapshot() if err != nil { t.Fatal(err) } - obl.bufferEnabled = true - obl.obBufferLimit = 5 + holder.bufferEnabled = true + holder.obBufferLimit = 5 for i := range itemArray { asks := itemArray[i] bids := itemArray[i] - err = obl.Update(&Update{ + err = holder.Update(&Update{ Bids: bids, Asks: asks, Pair: cp, @@ -302,67 +275,67 @@ func TestHittingTheBuffer(t *testing.T) { t.Fatal(err) } } - if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks) != 3 { - t.Errorf("expected 3 entries, received: %v", - len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks)) + + book := holder.ob[cp.Base][cp.Quote][asset.Spot] + if book.ob.GetAskLength() != 3 { + t.Errorf("expected 3 entries, received: %v", book.ob.GetAskLength()) } - if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids) != 3 { - t.Errorf("expected 3 entries, received: %v", - len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids)) + if book.ob.GetBidLength() != 3 { + t.Errorf("expected 3 entries, received: %v", book.ob.GetBidLength()) } } // TestInsertWithIDs logic test func TestInsertWithIDs(t *testing.T) { - obl, _, _, err := createSnapshot() + holder, _, _, err := createSnapshot() if err != nil { t.Fatal(err) } - obl.bufferEnabled = true - obl.updateEntriesByID = true - obl.obBufferLimit = 5 + holder.bufferEnabled = true + holder.updateEntriesByID = true + holder.obBufferLimit = 5 for i := range itemArray { asks := itemArray[i] if asks[0].Amount <= 0 { continue } bids := itemArray[i] - err = obl.Update(&Update{ + err = holder.Update(&Update{ Bids: bids, Asks: asks, Pair: cp, UpdateTime: time.Now(), Asset: asset.Spot, - Action: "insert", + Action: UpdateInsert, }) if err != nil { t.Fatal(err) } } - if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks) != 6 { - t.Errorf("expected 6 entries, received: %v", - len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks)) + + book := holder.ob[cp.Base][cp.Quote][asset.Spot] + if book.ob.GetAskLength() != 6 { + t.Errorf("expected 5 entries, received: %v", book.ob.GetAskLength()) } - if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids) != 6 { - t.Errorf("expected 6 entries, received: %v", - len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids)) + if book.ob.GetBidLength() != 6 { + t.Errorf("expected 5 entries, received: %v", book.ob.GetBidLength()) } } // TestSortIDs logic test func TestSortIDs(t *testing.T) { - obl, _, _, err := createSnapshot() + holder, _, _, err := createSnapshot() if err != nil { t.Fatal(err) } - obl.bufferEnabled = true - obl.sortBufferByUpdateIDs = true - obl.sortBuffer = true - obl.obBufferLimit = 5 + holder.bufferEnabled = true + holder.sortBufferByUpdateIDs = true + holder.sortBuffer = true + holder.obBufferLimit = 5 for i := range itemArray { asks := itemArray[i] bids := itemArray[i] - err = obl.Update(&Update{ + err = holder.Update(&Update{ Bids: bids, Asks: asks, Pair: cp, @@ -373,19 +346,18 @@ func TestSortIDs(t *testing.T) { t.Fatal(err) } } - if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks) != 3 { - t.Errorf("expected 3 entries, received: %v", - len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks)) + book := holder.ob[cp.Base][cp.Quote][asset.Spot] + if book.ob.GetAskLength() != 3 { + t.Errorf("expected 3 entries, received: %v", book.ob.GetAskLength()) } - if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids) != 3 { - t.Errorf("expected 3 entries, received: %v", - len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids)) + if book.ob.GetAskLength() != 3 { + t.Errorf("expected 3 entries, received: %v", book.ob.GetAskLength()) } } // TestOutOfOrderIDs logic test func TestOutOfOrderIDs(t *testing.T) { - obl, _, _, err := createSnapshot() + holder, _, _, err := createSnapshot() if err != nil { t.Fatal(err) } @@ -394,12 +366,12 @@ func TestOutOfOrderIDs(t *testing.T) { t.Errorf("expected sorted price to be 3000, received: %v", itemArray[1][0].Price) } - obl.bufferEnabled = true - obl.sortBuffer = true - obl.obBufferLimit = 5 + holder.bufferEnabled = true + holder.sortBuffer = true + holder.obBufferLimit = 5 for i := range itemArray { asks := itemArray[i] - err = obl.Update(&Update{ + err = holder.Update(&Update{ Asks: asks, Pair: cp, UpdateID: outOFOrderIDs[i], @@ -409,15 +381,16 @@ func TestOutOfOrderIDs(t *testing.T) { t.Fatal(err) } } + book := holder.ob[cp.Base][cp.Quote][asset.Spot] + cpy := book.ob.Retrieve() // Index 1 since index 0 is price 7000 - if obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks[1].Price != 2000 { - t.Errorf("expected sorted price to be 3000, received: %v", - obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks[1].Price) + if cpy.Asks[1].Price != 2000 { + t.Errorf("expected sorted price to be 2000, received: %v", cpy.Asks[1].Price) } } func TestOrderbookLastUpdateID(t *testing.T) { - obl, _, _, err := createSnapshot() + holder, _, _, err := createSnapshot() if err != nil { t.Fatal(err) } @@ -428,7 +401,7 @@ func TestOrderbookLastUpdateID(t *testing.T) { for i := range itemArray { asks := itemArray[i] - err = obl.Update(&Update{ + err = holder.Update(&Update{ Asks: asks, Pair: cp, UpdateID: int64(i) + 1, @@ -439,7 +412,10 @@ func TestOrderbookLastUpdateID(t *testing.T) { } } - ob := obl.GetOrderbook(cp, asset.Spot) + ob, err := holder.GetOrderbook(cp, asset.Spot) + if err != nil { + t.Fatal(err) + } if exp := len(itemArray); ob.LastUpdateID != int64(exp) { t.Errorf("expected last update id to be %d, received: %v", exp, ob.LastUpdateID) } @@ -447,7 +423,7 @@ func TestOrderbookLastUpdateID(t *testing.T) { // TestRunUpdateWithoutSnapshot logic test func TestRunUpdateWithoutSnapshot(t *testing.T) { - var obl Orderbook + var holder Orderbook var snapShot1 orderbook.Base asks := []orderbook.Item{ {Price: 4000, Amount: 1, ID: 8}, @@ -458,21 +434,18 @@ func TestRunUpdateWithoutSnapshot(t *testing.T) { } snapShot1.Asks = asks snapShot1.Bids = bids - snapShot1.AssetType = asset.Spot + snapShot1.Asset = asset.Spot snapShot1.Pair = cp - obl.exchangeName = exchangeName - err := obl.Update(&Update{ + holder.exchangeName = exchangeName + err := holder.Update(&Update{ Bids: bids, Asks: asks, Pair: cp, UpdateTime: time.Now(), Asset: asset.Spot, }) - if err == nil { - t.Fatal("expected an error running update with no snapshot loaded") - } - if err.Error() != "ob.Base could not be found for Exchange exchangeTest CurrencyPair: BTCUSD AssetType: spot" { - t.Fatal(err) + if !errors.Is(err, errDepthNotFound) { + t.Fatalf("expected %v but received %v", errDepthNotFound, err) } } @@ -482,7 +455,7 @@ func TestRunUpdateWithoutAnyUpdates(t *testing.T) { var snapShot1 orderbook.Base snapShot1.Asks = []orderbook.Item{} snapShot1.Bids = []orderbook.Item{} - snapShot1.AssetType = asset.Spot + snapShot1.Asset = asset.Spot snapShot1.Pair = cp obl.exchangeName = exchangeName err := obl.Update(&Update{ @@ -492,11 +465,8 @@ func TestRunUpdateWithoutAnyUpdates(t *testing.T) { UpdateTime: time.Now(), Asset: asset.Spot, }) - if err == nil { - t.Fatal("expected an error running update with no snapshot loaded") - } - if err.Error() != "websocket orderbook buffer error: update bid/ask targets cannot be nil" { - t.Fatal("expected nil asks and bids error") + if !errors.Is(err, errUpdateNoTargets) { + t.Fatalf("expected %v but received %v", errUpdateNoTargets, err) } } @@ -506,9 +476,9 @@ func TestRunSnapshotWithNoData(t *testing.T) { obl.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder) obl.dataHandler = make(chan interface{}, 1) var snapShot1 orderbook.Base - snapShot1.AssetType = asset.Spot + snapShot1.Asset = asset.Spot snapShot1.Pair = cp - snapShot1.ExchangeName = "test" + snapShot1.Exchange = "test" obl.exchangeName = "test" err := obl.LoadSnapshot(&snapShot1) if err != nil { @@ -522,7 +492,7 @@ func TestLoadSnapshot(t *testing.T) { obl.dataHandler = make(chan interface{}, 100) obl.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder) var snapShot1 orderbook.Base - snapShot1.ExchangeName = "SnapshotWithOverride" + snapShot1.Exchange = "SnapshotWithOverride" asks := []orderbook.Item{ {Price: 4000, Amount: 1, ID: 8}, } @@ -531,7 +501,7 @@ func TestLoadSnapshot(t *testing.T) { } snapShot1.Asks = asks snapShot1.Bids = bids - snapShot1.AssetType = asset.Spot + snapShot1.Asset = asset.Spot snapShot1.Pair = cp err := obl.LoadSnapshot(&snapShot1) if err != nil { @@ -556,11 +526,11 @@ func TestFlushbuffer(t *testing.T) { // TestInsertingSnapShots logic test func TestInsertingSnapShots(t *testing.T) { - var obl Orderbook - obl.dataHandler = make(chan interface{}, 100) - obl.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder) + var holder Orderbook + holder.dataHandler = make(chan interface{}, 100) + holder.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder) var snapShot1 orderbook.Base - snapShot1.ExchangeName = "WSORDERBOOKTEST1" + snapShot1.Exchange = "WSORDERBOOKTEST1" asks := []orderbook.Item{ {Price: 6000, Amount: 1, ID: 1}, {Price: 6001, Amount: 0.5, ID: 2}, @@ -591,14 +561,14 @@ func TestInsertingSnapShots(t *testing.T) { snapShot1.Asks = asks snapShot1.Bids = bids - snapShot1.AssetType = asset.Spot + snapShot1.Asset = asset.Spot snapShot1.Pair = cp - err := obl.LoadSnapshot(&snapShot1) + err := holder.LoadSnapshot(&snapShot1) if err != nil { t.Fatal(err) } var snapShot2 orderbook.Base - snapShot2.ExchangeName = "WSORDERBOOKTEST2" + snapShot2.Exchange = "WSORDERBOOKTEST2" asks = []orderbook.Item{ {Price: 51, Amount: 1, ID: 1}, {Price: 52, Amount: 0.5, ID: 2}, @@ -627,19 +597,21 @@ func TestInsertingSnapShots(t *testing.T) { {Price: 39, Amount: 7, ID: 22}, } - snapShot2.Asks = orderbook.SortAsks(asks) - snapShot2.Bids = orderbook.SortBids(bids) - snapShot2.AssetType = asset.Spot + snapShot2.Asks = asks + snapShot2.Asks.SortAsks() + snapShot2.Bids = bids + snapShot2.Bids.SortBids() + snapShot2.Asset = asset.Spot snapShot2.Pair, err = currency.NewPairFromString("LTCUSD") if err != nil { t.Fatal(err) } - err = obl.LoadSnapshot(&snapShot2) + err = holder.LoadSnapshot(&snapShot2) if err != nil { t.Fatal(err) } var snapShot3 orderbook.Base - snapShot3.ExchangeName = "WSORDERBOOKTEST3" + snapShot3.Exchange = "WSORDERBOOKTEST3" asks = []orderbook.Item{ {Price: 511, Amount: 1, ID: 1}, {Price: 52, Amount: 0.5, ID: 2}, @@ -668,74 +640,88 @@ func TestInsertingSnapShots(t *testing.T) { {Price: 39, Amount: 7, ID: 22}, } - snapShot3.Asks = orderbook.SortAsks(asks) - snapShot3.Bids = orderbook.SortBids(bids) - snapShot3.AssetType = "FUTURES" + snapShot3.Asks = asks + snapShot3.Asks.SortAsks() + snapShot3.Bids = bids + snapShot3.Bids.SortBids() + snapShot3.Asset = asset.Futures snapShot3.Pair, err = currency.NewPairFromString("LTCUSD") if err != nil { t.Fatal(err) } - err = obl.LoadSnapshot(&snapShot3) + err = holder.LoadSnapshot(&snapShot3) if err != nil { t.Fatal(err) } - if obl.ob[snapShot1.Pair.Base][snapShot1.Pair.Quote][snapShot1.AssetType].ob.Asks[0] != snapShot1.Asks[0] { + ob, err := holder.GetOrderbook(snapShot1.Pair, snapShot1.Asset) + if err != nil { + t.Fatal(err) + } + if ob.Asks[0] != snapShot1.Asks[0] { t.Errorf("loaded data mismatch. Expected %v, received %v", snapShot1.Asks[0], - obl.ob[snapShot1.Pair.Base][snapShot1.Pair.Quote][snapShot1.AssetType].ob.Asks[0]) + ob.Asks[0]) } - if obl.ob[snapShot2.Pair.Base][snapShot1.Pair.Quote][snapShot2.AssetType].ob.Asks[0] != snapShot2.Asks[0] { + ob, err = holder.GetOrderbook(snapShot2.Pair, snapShot2.Asset) + if err != nil { + t.Fatal(err) + } + if ob.Asks[0] != snapShot2.Asks[0] { t.Errorf("loaded data mismatch. Expected %v, received %v", snapShot2.Asks[0], - obl.ob[snapShot2.Pair.Base][snapShot1.Pair.Quote][snapShot2.AssetType].ob.Asks[0]) + ob.Asks[0]) } - if obl.ob[snapShot3.Pair.Base][snapShot1.Pair.Quote][snapShot3.AssetType].ob.Asks[0] != snapShot3.Asks[0] { + ob, err = holder.GetOrderbook(snapShot3.Pair, snapShot3.Asset) + if err != nil { + t.Fatal(err) + } + if ob.Asks[0] != snapShot3.Asks[0] { t.Errorf("loaded data mismatch. Expected %v, received %v", snapShot3.Asks[0], - obl.ob[snapShot3.Pair.Base][snapShot1.Pair.Quote][snapShot3.AssetType].ob.Asks[0]) + ob.Asks[0]) } } func TestGetOrderbook(t *testing.T) { - obl, _, _, err := createSnapshot() + holder, _, _, err := createSnapshot() if err != nil { t.Fatal(err) } - ob := obl.GetOrderbook(cp, asset.Spot) - bufferOb := obl.ob[cp.Base][cp.Quote][asset.Spot] - if bufferOb.ob == ob { - t.Error("orderbooks should be separate in pointer value and not linked to orderbook package") + ob, err := holder.GetOrderbook(cp, asset.Spot) + if err != nil { + t.Fatal(err) } - - if len(bufferOb.ob.Asks) != len(ob.Asks) || - len(bufferOb.ob.Bids) != len(ob.Bids) || - bufferOb.ob.AssetType != ob.AssetType || - bufferOb.ob.ExchangeName != ob.ExchangeName || - bufferOb.ob.LastUpdateID != ob.LastUpdateID || - bufferOb.ob.NotAggregated != ob.NotAggregated || - bufferOb.ob.Pair != ob.Pair { + bufferOb := holder.ob[cp.Base][cp.Quote][asset.Spot] + b := bufferOb.ob.Retrieve() + if bufferOb.ob.GetAskLength() != len(ob.Asks) || + bufferOb.ob.GetBidLength() != len(ob.Bids) || + b.Asset != ob.Asset || + b.Exchange != ob.Exchange || + b.LastUpdateID != ob.LastUpdateID || + b.PriceDuplication != ob.PriceDuplication || + b.Pair != ob.Pair { t.Fatal("data on both books should be the same") } } func TestSetup(t *testing.T) { w := Orderbook{} - err := w.Setup(0, false, false, false, false, "", nil) - if err == nil || !errors.Is(err, errUnsetExchangeName) { + err := w.Setup(0, false, false, false, false, true, "", nil) + if !errors.Is(err, errUnsetExchangeName) { t.Fatalf("expected error %v but received %v", errUnsetExchangeName, err) } - err = w.Setup(0, false, false, false, false, "test", nil) - if err == nil || !errors.Is(err, errUnsetDataHandler) { + err = w.Setup(0, false, false, false, false, false, "test", nil) + if !errors.Is(err, errUnsetDataHandler) { t.Fatalf("expected error %v but received %v", errUnsetDataHandler, err) } - err = w.Setup(0, true, false, false, false, "test", make(chan interface{})) - if err == nil || !errors.Is(err, errIssueBufferEnabledButNoLimit) { + err = w.Setup(0, true, false, false, false, true, "test", make(chan interface{})) + if !errors.Is(err, errIssueBufferEnabledButNoLimit) { t.Fatalf("expected error %v but received %v", errIssueBufferEnabledButNoLimit, err) } - err = w.Setup(1337, true, true, true, true, "test", make(chan interface{})) + err = w.Setup(1337, true, true, true, true, false, "test", make(chan interface{})) if err != nil { t.Fatal(err) } @@ -752,25 +738,25 @@ func TestSetup(t *testing.T) { func TestValidate(t *testing.T) { w := Orderbook{} err := w.validate(nil) - if err == nil || !errors.Is(err, errUpdateIsNil) { + if !errors.Is(err, errUpdateIsNil) { t.Fatalf("expected error %v but received %v", errUpdateIsNil, err) } err = w.validate(&Update{}) - if err == nil || !errors.Is(err, errUpdateNoTargets) { + if !errors.Is(err, errUpdateNoTargets) { t.Fatalf("expected error %v but received %v", errUpdateNoTargets, err) } } func TestEnsureMultipleUpdatesViaPrice(t *testing.T) { - obl, _, _, err := createSnapshot() + holder, _, _, err := createSnapshot() if err != nil { t.Error(err) } asks := bidAskGenerator() - holder := obl.ob[cp.Base][cp.Quote][asset.Spot] - holder.updateByPrice(&Update{ + book := holder.ob[cp.Base][cp.Quote][asset.Spot] + book.updateByPrice(&Update{ Bids: asks, Asks: asks, Pair: cp, @@ -781,69 +767,12 @@ func TestEnsureMultipleUpdatesViaPrice(t *testing.T) { t.Error(err) } - if len(holder.ob.Asks) <= 3 { + if book.ob.GetAskLength() <= 3 { t.Errorf("Insufficient updates") } } -func TestInsertItem(t *testing.T) { - update := []orderbook.Item{{Price: 4}} - - // Correctly aligned - asks := []orderbook.Item{ - { - Price: 1, - }, - { - Price: 2, - }, - { - Price: 3, - }, - { - Price: 5, - }, - { - Price: 6, - }, - { - Price: 7, - }, - } - - insertUpdatesAsk(update, &asks) - if asks[3].Price != 4 { - t.Fatal("incorrect insertion") - } - - bids := []orderbook.Item{ - { - Price: 7, - }, - { - Price: 6, - }, - { - Price: 5, - }, - { - Price: 3, - }, - { - Price: 2, - }, - { - Price: 1, - }, - } - - insertUpdatesBid(update, &bids) - if asks[3].Price != 4 { - t.Fatal("incorrect insertion") - } -} - -func deploySliceOrdered(size int) []orderbook.Item { +func deploySliceOrdered(size int) orderbook.Items { rand.Seed(time.Now().UnixNano()) var items []orderbook.Item for i := 0; i < size; i++ { @@ -857,14 +786,16 @@ func TestUpdateByIDAndAction(t *testing.T) { asks := deploySliceOrdered(100) bids := append(asks[:0:0], asks...) - orderbook.Reverse(bids) + bids.Reverse() - book := &orderbook.Base{ - Bids: append(bids[:0:0], bids...), - Asks: append(asks[:0:0], asks...), + book, err := orderbook.DeployDepth("test", cp, asset.Spot) + if err != nil { + t.Fatal(err) } - err := book.Verify() + book.LoadSnapshot(append(bids[:0:0], bids...), append(asks[:0:0], asks...)) + + err = book.Retrieve().Verify() if err != nil { t.Fatal(err) } @@ -894,14 +825,16 @@ func TestUpdateByIDAndAction(t *testing.T) { Action: UpdateInsert, Bids: []orderbook.Item{ { - Price: 0, - ID: 1337, + Price: 0, + ID: 1337, + Amount: 1, }, }, Asks: []orderbook.Item{ { - Price: 100, - ID: 1337, + Price: 100, + ID: 1337, + Amount: 1, }, }, }) @@ -909,10 +842,12 @@ func TestUpdateByIDAndAction(t *testing.T) { t.Fatal(err) } - if book.Bids[len(book.Bids)-1].Price != 0 { + cpy := book.Retrieve() + + if cpy.Bids[len(cpy.Bids)-1].Price != 0 { t.Fatal("did not append bid item") } - if book.Asks[len(book.Asks)-1].Price != 100 { + if cpy.Asks[len(cpy.Asks)-1].Price != 100 { t.Fatal("did not append ask item") } @@ -938,11 +873,13 @@ func TestUpdateByIDAndAction(t *testing.T) { t.Fatal(err) } - if book.Bids[len(book.Bids)-1].Amount != 100 { - t.Fatal("did not update bid amount") + cpy = book.Retrieve() + + if cpy.Bids[len(cpy.Bids)-1].Amount != 100 { + t.Fatal("did not update bid amount", cpy.Bids[len(cpy.Bids)-1].Amount) } - if book.Asks[len(book.Asks)-1].Amount != 100 { + if cpy.Asks[len(cpy.Asks)-1].Amount != 100 { t.Fatal("did not update ask amount") } @@ -968,16 +905,17 @@ func TestUpdateByIDAndAction(t *testing.T) { t.Fatal(err) } - if book.Bids[0].Amount != 99 && book.Bids[0].Price != 100 { + cpy = book.Retrieve() + + if cpy.Bids[0].Amount != 99 && cpy.Bids[0].Price != 100 { t.Fatal("did not adjust bid item placement and details") } - if book.Asks[0].Amount != 99 && book.Asks[0].Amount != 0 { + if cpy.Asks[0].Amount != 99 && cpy.Asks[0].Amount != 0 { t.Fatal("did not adjust ask item placement and details") } - book.Bids = append(bids[:0:0], bids...) // nolint:gocritic - book.Asks = append(asks[:0:0], asks...) // nolint:gocritic + book.LoadSnapshot(append(bids[:0:0], bids...), append(bids[:0:0], bids...)) // nolint:gocritic // Delete - not found err = holder.updateByIDAndAction(&Update{ @@ -1018,7 +956,7 @@ func TestUpdateByIDAndAction(t *testing.T) { t.Fatal(err) } - if len(book.Asks) != 99 { + if book.GetAskLength() != 99 { t.Fatal("element not deleted") } @@ -1033,7 +971,7 @@ func TestUpdateByIDAndAction(t *testing.T) { t.Fatal("error cannot be nil") } - update := book.Asks[0] + update := book.Retrieve().Asks[0] update.Amount = 1337 err = holder.updateByIDAndAction(&Update{ @@ -1046,20 +984,20 @@ func TestUpdateByIDAndAction(t *testing.T) { t.Fatal(err) } - if book.Asks[0].Amount != 1337 { + if book.Retrieve().Asks[0].Amount != 1337 { t.Fatal("element not updated") } } func TestFlushOrderbook(t *testing.T) { w := &Orderbook{} - err := w.Setup(5, false, false, false, false, "test", make(chan interface{}, 2)) + err := w.Setup(5, false, false, false, false, false, "test", make(chan interface{}, 2)) if err != nil { t.Fatal(err) } var snapShot1 orderbook.Base - snapShot1.ExchangeName = "Snapshooooot" + snapShot1.Exchange = "Snapshooooot" asks := []orderbook.Item{ {Price: 4000, Amount: 1, ID: 8}, } @@ -1068,7 +1006,7 @@ func TestFlushOrderbook(t *testing.T) { } snapShot1.Asks = asks snapShot1.Bids = bids - snapShot1.AssetType = asset.Spot + snapShot1.Asset = asset.Spot snapShot1.Pair = cp err = w.FlushOrderbook(cp, asset.Spot) @@ -1076,14 +1014,9 @@ func TestFlushOrderbook(t *testing.T) { t.Fatal("book not loaded error cannot be nil") } - o := w.GetOrderbook(cp, asset.Spot) - if o != nil { - t.Fatal("book not loaded, this should not happen") - } - - err = w.LoadSnapshot(&snapShot1) - if err != nil { - t.Fatal(err) + _, err = w.GetOrderbook(cp, asset.Spot) + if !errors.Is(err, errDepthNotFound) { + t.Fatalf("expected: %v but received: %v", errDepthNotFound, err) } err = w.LoadSnapshot(&snapShot1) @@ -1096,12 +1029,12 @@ func TestFlushOrderbook(t *testing.T) { t.Fatal(err) } - o = w.GetOrderbook(cp, asset.Spot) - if o == nil { - t.Fatal("cannot get book") + o, err := w.GetOrderbook(cp, asset.Spot) + if err != nil { + t.Fatal(err) } - if o.Bids != nil && o.Asks != nil { + if len(o.Bids) != 0 || len(o.Asks) != 0 { t.Fatal("orderbook items not flushed") } } diff --git a/exchanges/stream/buffer/buffer_types.go b/exchanges/stream/buffer/buffer_types.go index ec19cedc..e647faee 100644 --- a/exchanges/stream/buffer/buffer_types.go +++ b/exchanges/stream/buffer/buffer_types.go @@ -9,6 +9,10 @@ import ( "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" ) +// timerDefault defines the amount of time between alerting the sync manager of +// an update. +var timerDefault = time.Second * 10 + // Orderbook defines a local cache of orderbooks for amending, appending // and deleting changes and updates the main store for a stream type Orderbook struct { @@ -20,12 +24,20 @@ type Orderbook struct { updateEntriesByID bool // Use the update IDs to match ob entries exchangeName string dataHandler chan interface{} + verbose bool m sync.Mutex } +// orderbookHolder defines a store of pending updates and a pointer to the +// orderbook depth type orderbookHolder struct { - ob *orderbook.Base + ob *orderbook.Depth buffer *[]Update + // Reduces the amount of outbound alerts to the data handler for example + // coinbasepro can have up too 100 updates per second introducing overhead. + // The sync agent only requires an alert every 15 seconds for a specific + // currency. + ticker *time.Ticker } // Update stores orderbook updates and dictates what features to use when processing diff --git a/exchanges/stream/websocket.go b/exchanges/stream/websocket.go index 41799f55..66ca6d5b 100644 --- a/exchanges/stream/websocket.go +++ b/exchanges/stream/websocket.go @@ -113,6 +113,7 @@ func (w *Websocket) Setup(s *WebsocketSetup) error { s.SortBuffer, s.SortBufferByUpdateIDs, s.UpdateEntriesByID, + s.Verbose, w.exchangeName, w.DataHandler) } diff --git a/exchanges/yobit/yobit_wrapper.go b/exchanges/yobit/yobit_wrapper.go index 206b957c..24e5c8cb 100644 --- a/exchanges/yobit/yobit_wrapper.go +++ b/exchanges/yobit/yobit_wrapper.go @@ -240,10 +240,10 @@ func (y *Yobit) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderboo // UpdateOrderbook updates and returns the orderbook for a currency pair func (y *Yobit) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: y.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: y.OrderbookVerificationBypass, + Exchange: y.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: y.CanVerifyOrderbook, } fpair, err := y.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/zb/zb_websocket.go b/exchanges/zb/zb_websocket.go index 09c9f7c2..42a69538 100644 --- a/exchanges/zb/zb_websocket.go +++ b/exchanges/zb/zb_websocket.go @@ -143,11 +143,11 @@ func (z *ZB) wsHandleData(respRaw []byte) error { return err } - orderbook.Reverse(book.Asks) // Reverse asks for correct alignment - book.AssetType = asset.Spot + book.Asks.Reverse() // Reverse asks for correct alignment + book.Asset = asset.Spot book.Pair = cPair - book.ExchangeName = z.Name - book.VerificationBypass = z.OrderbookVerificationBypass + book.Exchange = z.Name + book.VerifyOrderbook = z.CanVerifyOrderbook err = z.Websocket.Orderbook.LoadSnapshot(&book) if err != nil { diff --git a/exchanges/zb/zb_wrapper.go b/exchanges/zb/zb_wrapper.go index 19089949..212328e3 100644 --- a/exchanges/zb/zb_wrapper.go +++ b/exchanges/zb/zb_wrapper.go @@ -301,10 +301,10 @@ func (z *ZB) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.B // UpdateOrderbook updates and returns the orderbook for a currency pair func (z *ZB) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: z.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: z.OrderbookVerificationBypass, + Exchange: z.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: z.CanVerifyOrderbook, } currFormat, err := z.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/gctscript/modules/gct/exchange.go b/gctscript/modules/gct/exchange.go index 6c9edcc0..676720d1 100644 --- a/gctscript/modules/gct/exchange.go +++ b/gctscript/modules/gct/exchange.go @@ -85,11 +85,11 @@ func ExchangeOrderbook(args ...objects.Object) (objects.Object, error) { } data := make(map[string]objects.Object, 5) - data["exchange"] = &objects.String{Value: ob.ExchangeName} + data["exchange"] = &objects.String{Value: ob.Exchange} data["pair"] = &objects.String{Value: ob.Pair.String()} data["asks"] = &asks data["bids"] = &bids - data["asset"] = &objects.String{Value: ob.AssetType.String()} + data["asset"] = &objects.String{Value: ob.Asset.String()} return &objects.Map{ Value: data, diff --git a/gctscript/wrappers/validator/validator.go b/gctscript/wrappers/validator/validator.go index dd469960..90cc31c5 100644 --- a/gctscript/wrappers/validator/validator.go +++ b/gctscript/wrappers/validator/validator.go @@ -49,9 +49,9 @@ func (w Wrapper) Orderbook(exch string, pair currency.Pair, item asset.Item) (*o } return &orderbook.Base{ - ExchangeName: exch, - AssetType: item, - Pair: pair, + Exchange: exch, + Asset: item, + Pair: pair, Bids: []orderbook.Item{ { Amount: 1, diff --git a/main.go b/main.go index 7cc203f5..4a1abce9 100644 --- a/main.go +++ b/main.go @@ -63,8 +63,10 @@ func main() { flag.BoolVar(&settings.EnableTradeSyncing, "tradesync", false, "enables trade syncing for all enabled exchanges") flag.IntVar(&settings.SyncWorkers, "syncworkers", engine.DefaultSyncerWorkers, "the amount of workers (goroutines) to use for syncing exchange data") flag.BoolVar(&settings.SyncContinuously, "synccontinuously", true, "whether to sync exchange data continuously (ticker, orderbook and trade history info") - flag.DurationVar(&settings.SyncTimeout, "synctimeout", engine.DefaultSyncerTimeout, - "the amount of time before the syncer will switch from one protocol to the other (e.g. from REST to websocket)") + flag.DurationVar(&settings.SyncTimeoutREST, "synctimeoutrest", engine.DefaultSyncerTimeoutREST, + "the amount of time before the syncer will switch from rest protocol to the streaming protocol (e.g. from REST to websocket)") + flag.DurationVar(&settings.SyncTimeoutWebsocket, "synctimeoutwebsocket", engine.DefaultSyncerTimeoutWebsocket, + "the amount of time before the syncer will switch from the websocket protocol to REST protocol (e.g. from websocket to REST)") // Forex provider settings flag.BoolVar(&settings.EnableCurrencyConverter, "currencyconverter", false, "overrides config and sets up foreign exchange Currency Converter")