diff --git a/.golangci.yml b/.golangci.yml index 6528b87f..c61cdb1a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,5 +1,5 @@ run: - timeout: 2m0s + timeout: 2m30s issues-exit-code: 1 tests: true skip-dirs: diff --git a/cmd/exchange_template/wrapper_file.tmpl b/cmd/exchange_template/wrapper_file.tmpl index 1fbbbd8c..af580c78 100644 --- a/cmd/exchange_template/wrapper_file.tmpl +++ b/cmd/exchange_template/wrapper_file.tmpl @@ -283,10 +283,10 @@ func ({{.Variable}} *{{.CapitalName}}) FetchOrderbook(currency currency.Pair, as // UpdateOrderbook updates and returns the orderbook for a currency pair func ({{.Variable}} *{{.CapitalName}}) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: {{.Variable}}.Name, + Exchange: {{.Variable}}.Name, Pair: p, - AssetType: assetType, - VerificationBypass: {{.Variable}}.OrderbookVerificationBypass, + Asset: assetType, + VerifyOrderbook: {{.Variable}}.CanVerifyOrderbook, } // NOTE: UPDATE ORDERBOOK EXAMPLE diff --git a/engine/engine.go b/engine/engine.go index ebcf7588..468c1d4a 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -208,7 +208,8 @@ func validateSettings(b *Engine, s *Settings, flagSet map[string]bool) { b.Settings.EnableOrderbookSyncing = s.EnableOrderbookSyncing b.Settings.EnableTradeSyncing = s.EnableTradeSyncing b.Settings.SyncWorkers = s.SyncWorkers - b.Settings.SyncTimeout = s.SyncTimeout + b.Settings.SyncTimeoutREST = s.SyncTimeoutREST + b.Settings.SyncTimeoutWebsocket = s.SyncTimeoutWebsocket b.Settings.SyncContinuously = s.SyncContinuously b.Settings.EnableDepositAddressManager = s.EnableDepositAddressManager b.Settings.EnableExchangeAutoPairUpdates = s.EnableExchangeAutoPairUpdates @@ -305,7 +306,8 @@ func PrintSettings(s *Settings) { gctlog.Debugf(gctlog.Global, "\t Enable ticker syncing: %v\n", s.EnableTickerSyncing) gctlog.Debugf(gctlog.Global, "\t Enable orderbook syncing: %v\n", s.EnableOrderbookSyncing) gctlog.Debugf(gctlog.Global, "\t Enable trade syncing: %v\n", s.EnableTradeSyncing) - gctlog.Debugf(gctlog.Global, "\t Exchange sync timeout: %v\n", s.SyncTimeout) + gctlog.Debugf(gctlog.Global, "\t Exchange REST sync timeout: %v\n", s.SyncTimeoutREST) + gctlog.Debugf(gctlog.Global, "\t Exchange Websocket sync timeout: %v\n", s.SyncTimeoutWebsocket) gctlog.Debugf(gctlog.Global, "- FOREX SETTINGS:") gctlog.Debugf(gctlog.Global, "\t Enable currency conveter: %v", s.EnableCurrencyConverter) gctlog.Debugf(gctlog.Global, "\t Enable currency layer: %v", s.EnableCurrencyLayer) @@ -464,13 +466,14 @@ func (bot *Engine) Start() error { if bot.Settings.EnableExchangeSyncManager { exchangeSyncCfg := CurrencyPairSyncerConfig{ - SyncTicker: bot.Settings.EnableTickerSyncing, - SyncOrderbook: bot.Settings.EnableOrderbookSyncing, - SyncTrades: bot.Settings.EnableTradeSyncing, - SyncContinuously: bot.Settings.SyncContinuously, - NumWorkers: bot.Settings.SyncWorkers, - Verbose: bot.Settings.Verbose, - SyncTimeout: bot.Settings.SyncTimeout, + SyncTicker: bot.Settings.EnableTickerSyncing, + SyncOrderbook: bot.Settings.EnableOrderbookSyncing, + SyncTrades: bot.Settings.EnableTradeSyncing, + SyncContinuously: bot.Settings.SyncContinuously, + NumWorkers: bot.Settings.SyncWorkers, + Verbose: bot.Settings.Verbose, + SyncTimeoutREST: bot.Settings.SyncTimeoutREST, + SyncTimeoutWebsocket: bot.Settings.SyncTimeoutWebsocket, } bot.ExchangeCurrencyPairManager, err = NewCurrencyPairSyncer(exchangeSyncCfg) diff --git a/engine/engine_types.go b/engine/engine_types.go index 1aaae2b7..d9bbf5d2 100644 --- a/engine/engine_types.go +++ b/engine/engine_types.go @@ -44,7 +44,8 @@ type Settings struct { EnableTradeSyncing bool SyncWorkers int SyncContinuously bool - SyncTimeout time.Duration + SyncTimeoutREST time.Duration + SyncTimeoutWebsocket time.Duration // Forex settings EnableCurrencyConverter bool diff --git a/engine/events_test.go b/engine/events_test.go index 0e88df76..b6016524 100644 --- a/engine/events_test.go +++ b/engine/events_test.go @@ -210,11 +210,11 @@ func TestProcessOrderbook(t *testing.T) { // now populate it with a 0 entry o := orderbook.Base{ - Pair: currency.NewPair(currency.BTC, currency.USD), - Bids: []orderbook.Item{{Amount: 24, Price: 23}}, - Asks: []orderbook.Item{{Amount: 24, Price: 23}}, - ExchangeName: e.Exchange, - AssetType: e.Asset, + Pair: currency.NewPair(currency.BTC, currency.USD), + Bids: []orderbook.Item{{Amount: 24, Price: 23}}, + Asks: []orderbook.Item{{Amount: 24, Price: 23}}, + Exchange: e.Exchange, + Asset: e.Asset, } if err := o.Process(); err != nil { t.Fatal("unexpected result:", err) diff --git a/engine/helpers_test.go b/engine/helpers_test.go index c8d554f0..07a2dca9 100644 --- a/engine/helpers_test.go +++ b/engine/helpers_test.go @@ -535,10 +535,10 @@ func TestGetSpecificOrderbook(t *testing.T) { bids = append(bids, orderbook.Item{Price: 1000, Amount: 1}) base := orderbook.Base{ - Pair: currency.NewPair(currency.BTC, currency.USD), - Bids: bids, - ExchangeName: "Bitstamp", - AssetType: asset.Spot, + Pair: currency.NewPair(currency.BTC, currency.USD), + Bids: bids, + Exchange: "Bitstamp", + Asset: asset.Spot, } err := base.Process() diff --git a/engine/routines.go b/engine/routines.go index 1789cec1..49deb313 100644 --- a/engine/routines.go +++ b/engine/routines.go @@ -132,17 +132,17 @@ func printOrderbookSummary(result *orderbook.Base, protocol string, bot *Engine, if err == common.ErrNotYetImplemented { log.Warnf(log.OrderBook, "Failed to get %s orderbook for %s %s %s. Error: %s\n", protocol, - result.ExchangeName, + result.Exchange, result.Pair, - result.AssetType, + result.Asset, err) return } log.Errorf(log.OrderBook, "Failed to get %s orderbook for %s %s %s. Error: %s\n", protocol, - result.ExchangeName, + result.Exchange, result.Pair, - result.AssetType, + result.Asset, err) return } @@ -165,10 +165,10 @@ func printOrderbookSummary(result *orderbook.Base, protocol string, bot *Engine, } log.Infof(log.OrderBook, book, - result.ExchangeName, + result.Exchange, protocol, bot.FormatCurrency(result.Pair), - strings.ToUpper(result.AssetType.String()), + strings.ToUpper(result.Asset.String()), len(result.Bids), bidsAmount, result.Pair.Base, @@ -317,7 +317,7 @@ func (bot *Engine) WebsocketDataHandler(exchName string, data interface{}) error if bot.Settings.EnableExchangeSyncManager && bot.ExchangeCurrencyPairManager != nil { bot.ExchangeCurrencyPairManager.update(exchName, d.Pair, - d.AssetType, + d.Asset, SyncItemOrderbook, nil) } diff --git a/engine/routines_test.go b/engine/routines_test.go index a691116d..af49f40a 100644 --- a/engine/routines_test.go +++ b/engine/routines_test.go @@ -118,8 +118,8 @@ func TestHandleData(t *testing.T) { } err = b.WebsocketDataHandler(exchName, &orderbook.Base{ - ExchangeName: fakePassExchange, - Pair: currency.NewPair(currency.BTC, currency.USD), + Exchange: fakePassExchange, + Pair: currency.NewPair(currency.BTC, currency.USD), }) if err != nil { t.Error(err) diff --git a/engine/rpcserver.go b/engine/rpcserver.go index 1548eb29..e550844d 100644 --- a/engine/rpcserver.go +++ b/engine/rpcserver.go @@ -1672,45 +1672,37 @@ func (s *RPCServer) GetOrderbookStream(r *gctrpc.GetOrderbookStreamRequest, stre return err } - pipe, err := orderbook.SubscribeOrderbook(r.Exchange, p, a) + depth, err := orderbook.GetDepth(r.Exchange, p, a) if err != nil { return err } - defer pipe.Release() - for { - data, ok := <-pipe.C - if !ok { - return errDispatchSystem + base := depth.Retrieve() + bids := make([]*gctrpc.OrderbookItem, len(base.Bids)) + for i := range base.Bids { + bids[i] = &gctrpc.OrderbookItem{ + Amount: base.Bids[i].Amount, + Price: base.Bids[i].Price, + Id: base.Bids[i].ID} } - - ob := (*data.(*interface{})).(orderbook.Base) - var bids, asks []*gctrpc.OrderbookItem - for i := range ob.Bids { - bids = append(bids, &gctrpc.OrderbookItem{ - Amount: ob.Bids[i].Amount, - Price: ob.Bids[i].Price, - Id: ob.Bids[i].ID, - }) - } - for i := range ob.Asks { - asks = append(asks, &gctrpc.OrderbookItem{ - Amount: ob.Asks[i].Amount, - Price: ob.Asks[i].Price, - Id: ob.Asks[i].ID, - }) + asks := make([]*gctrpc.OrderbookItem, len(base.Asks)) + for i := range base.Asks { + asks[i] = &gctrpc.OrderbookItem{ + Amount: base.Asks[i].Amount, + Price: base.Asks[i].Price, + Id: base.Asks[i].ID} } err := stream.Send(&gctrpc.OrderbookResponse{ - Pair: &gctrpc.CurrencyPair{Base: ob.Pair.Base.String(), - Quote: ob.Pair.Quote.String()}, + Pair: &gctrpc.CurrencyPair{Base: r.Pair.Base, Quote: r.Pair.Quote}, Bids: bids, Asks: asks, - AssetType: ob.AssetType.String(), + AssetType: r.AssetType, }) if err != nil { return err } + <-depth.Wait(nil) } } @@ -1734,27 +1726,26 @@ func (s *RPCServer) GetExchangeOrderbookStream(r *gctrpc.GetExchangeOrderbookStr } ob := (*data.(*interface{})).(orderbook.Base) - var bids, asks []*gctrpc.OrderbookItem + bids := make([]*gctrpc.OrderbookItem, len(ob.Bids)) for i := range ob.Bids { - bids = append(bids, &gctrpc.OrderbookItem{ + bids[i] = &gctrpc.OrderbookItem{ Amount: ob.Bids[i].Amount, Price: ob.Bids[i].Price, - Id: ob.Bids[i].ID, - }) + Id: ob.Bids[i].ID} } + asks := make([]*gctrpc.OrderbookItem, len(ob.Asks)) for i := range ob.Asks { - asks = append(asks, &gctrpc.OrderbookItem{ + asks[i] = &gctrpc.OrderbookItem{ Amount: ob.Asks[i].Amount, Price: ob.Asks[i].Price, - Id: ob.Asks[i].ID, - }) + Id: ob.Asks[i].ID} } err := stream.Send(&gctrpc.OrderbookResponse{ Pair: &gctrpc.CurrencyPair{Base: ob.Pair.Base.String(), Quote: ob.Pair.Quote.String()}, Bids: bids, Asks: asks, - AssetType: ob.AssetType.String(), + AssetType: ob.Asset.String(), }) if err != nil { return err diff --git a/engine/syncer.go b/engine/syncer.go index 87a1a65d..b14b68d2 100644 --- a/engine/syncer.go +++ b/engine/syncer.go @@ -18,8 +18,9 @@ const ( SyncItemOrderbook SyncItemTrade - DefaultSyncerWorkers = 15 - DefaultSyncerTimeout = time.Second * 15 + DefaultSyncerWorkers = 15 + DefaultSyncerTimeoutREST = time.Second * 15 + DefaultSyncerTimeoutWebsocket = time.Minute ) var ( @@ -37,28 +38,25 @@ func NewCurrencyPairSyncer(c CurrencyPairSyncerConfig) (*ExchangeCurrencyPairSyn c.NumWorkers = DefaultSyncerWorkers } - if c.SyncTimeout <= time.Duration(0) { - c.SyncTimeout = DefaultSyncerTimeout + if c.SyncTimeoutREST <= time.Duration(0) { + c.SyncTimeoutREST = DefaultSyncerTimeoutREST } - s := ExchangeCurrencyPairSyncer{ - Cfg: CurrencyPairSyncerConfig{ - SyncTicker: c.SyncTicker, - SyncOrderbook: c.SyncOrderbook, - SyncTrades: c.SyncTrades, - SyncContinuously: c.SyncContinuously, - SyncTimeout: c.SyncTimeout, - NumWorkers: c.NumWorkers, - }, + if c.SyncTimeoutWebsocket <= time.Duration(0) { + c.SyncTimeoutWebsocket = DefaultSyncerTimeoutWebsocket } + s := ExchangeCurrencyPairSyncer{Cfg: c} + s.tickerBatchLastRequested = make(map[string]time.Time) log.Debugf(log.SyncMgr, "Exchange currency pair syncer config: continuous: %v ticker: %v"+ - " orderbook: %v trades: %v workers: %v verbose: %v timeout: %v\n", + " orderbook: %v trades: %v workers: %v verbose: %v timeout REST: %v"+ + " timeout Websocket: %v\n", s.Cfg.SyncContinuously, s.Cfg.SyncTicker, s.Cfg.SyncOrderbook, - s.Cfg.SyncTrades, s.Cfg.NumWorkers, s.Cfg.Verbose, s.Cfg.SyncTimeout) + s.Cfg.SyncTrades, s.Cfg.NumWorkers, s.Cfg.Verbose, s.Cfg.SyncTimeoutREST, + s.Cfg.SyncTimeoutWebsocket) return &s, nil } @@ -138,20 +136,6 @@ func (e *ExchangeCurrencyPairSyncer) add(c *CurrencyPairSyncAgent) { e.CurrencyPairs = append(e.CurrencyPairs, *c) } -func (e *ExchangeCurrencyPairSyncer) remove(c *CurrencyPairSyncAgent) { - e.mux.Lock() - defer e.mux.Unlock() - - for x := range e.CurrencyPairs { - if e.CurrencyPairs[x].Exchange == c.Exchange && - e.CurrencyPairs[x].Pair.Equal(c.Pair) && - e.CurrencyPairs[x].AssetType == c.AssetType { - e.CurrencyPairs = append(e.CurrencyPairs[:x], e.CurrencyPairs[x+1:]...) - return - } - } -} - func (e *ExchangeCurrencyPairSyncer) isProcessing(exchangeName string, p currency.Pair, a asset.Item, syncType int) bool { e.mux.Lock() defer e.mux.Unlock() @@ -325,6 +309,8 @@ func (e *ExchangeCurrencyPairSyncer) worker() { if exchanges[x].GetBase().CurrencyPairs.IsAssetEnabled(assetTypes[y]) != nil { continue } + + wsAssetSupported := exchanges[x].IsAssetWebsocketSupported(assetTypes[y]) enabledPairs, err := exchanges[x].GetEnabledPairs(assetTypes[y]) if err != nil { log.Errorf(log.SyncMgr, @@ -345,25 +331,21 @@ func (e *ExchangeCurrencyPairSyncer) worker() { Pair: enabledPairs[i], } + sBase := SyncBase{ + IsUsingREST: usingREST || !wsAssetSupported, + IsUsingWebsocket: usingWebsocket && wsAssetSupported, + } + if e.Cfg.SyncTicker { - c.Ticker = SyncBase{ - IsUsingREST: usingREST, - IsUsingWebsocket: usingWebsocket, - } + c.Ticker = sBase } if e.Cfg.SyncOrderbook { - c.Orderbook = SyncBase{ - IsUsingREST: usingREST, - IsUsingWebsocket: usingWebsocket, - } + c.Orderbook = sBase } if e.Cfg.SyncTrades { - c.Trade = SyncBase{ - IsUsingREST: usingREST, - IsUsingWebsocket: usingWebsocket, - } + c.Trade = sBase } e.add(&c) @@ -382,9 +364,11 @@ func (e *ExchangeCurrencyPairSyncer) worker() { } if e.Cfg.SyncTicker { if !e.isProcessing(exchangeName, c.Pair, c.AssetType, SyncItemTicker) { - if c.Ticker.LastUpdated.IsZero() || time.Since(c.Ticker.LastUpdated) > e.Cfg.SyncTimeout { + if c.Ticker.LastUpdated.IsZero() || + (time.Since(c.Ticker.LastUpdated) > e.Cfg.SyncTimeoutREST && c.Ticker.IsUsingREST) || + (time.Since(c.Ticker.LastUpdated) > e.Cfg.SyncTimeoutWebsocket && c.Ticker.IsUsingWebsocket) { if c.Ticker.IsUsingWebsocket { - if time.Since(c.Created) < e.Cfg.SyncTimeout { + if time.Since(c.Created) < e.Cfg.SyncTimeoutWebsocket { continue } @@ -397,7 +381,7 @@ func (e *ExchangeCurrencyPairSyncer) worker() { c.Exchange, Bot.FormatCurrency(enabledPairs[i]).String(), strings.ToUpper(c.AssetType.String()), - e.Cfg.SyncTimeout, + e.Cfg.SyncTimeoutWebsocket, ) switchedToRest = true e.setProcessing(c.Exchange, c.Pair, c.AssetType, SyncItemTicker, false) @@ -417,7 +401,7 @@ func (e *ExchangeCurrencyPairSyncer) worker() { } e.mux.Unlock() - if batchLastDone.IsZero() || time.Since(batchLastDone) > e.Cfg.SyncTimeout { + if batchLastDone.IsZero() || time.Since(batchLastDone) > e.Cfg.SyncTimeoutREST { e.mux.Lock() if e.Cfg.Verbose { log.Debugf(log.SyncMgr, "%s Init'ing REST ticker batching\n", exchangeName) @@ -450,9 +434,11 @@ func (e *ExchangeCurrencyPairSyncer) worker() { if e.Cfg.SyncOrderbook { if !e.isProcessing(exchangeName, c.Pair, c.AssetType, SyncItemOrderbook) { - if c.Orderbook.LastUpdated.IsZero() || time.Since(c.Orderbook.LastUpdated) > e.Cfg.SyncTimeout { + if c.Orderbook.LastUpdated.IsZero() || + (time.Since(c.Orderbook.LastUpdated) > e.Cfg.SyncTimeoutREST && c.Orderbook.IsUsingREST) || + (time.Since(c.Orderbook.LastUpdated) > e.Cfg.SyncTimeoutWebsocket && c.Orderbook.IsUsingWebsocket) { if c.Orderbook.IsUsingWebsocket { - if time.Since(c.Created) < e.Cfg.SyncTimeout { + if time.Since(c.Created) < e.Cfg.SyncTimeoutWebsocket { continue } if supportsREST { @@ -464,7 +450,7 @@ func (e *ExchangeCurrencyPairSyncer) worker() { c.Exchange, Bot.FormatCurrency(c.Pair).String(), strings.ToUpper(c.AssetType.String()), - e.Cfg.SyncTimeout, + e.Cfg.SyncTimeoutWebsocket, ) switchedToRest = true e.setProcessing(c.Exchange, c.Pair, c.AssetType, SyncItemOrderbook, false) @@ -486,7 +472,7 @@ func (e *ExchangeCurrencyPairSyncer) worker() { } if e.Cfg.SyncTrades { if !e.isProcessing(exchangeName, c.Pair, c.AssetType, SyncItemTrade) { - if c.Trade.LastUpdated.IsZero() || time.Since(c.Trade.LastUpdated) > e.Cfg.SyncTimeout { + if c.Trade.LastUpdated.IsZero() || time.Since(c.Trade.LastUpdated) > e.Cfg.SyncTimeoutREST { e.setProcessing(c.Exchange, c.Pair, c.AssetType, SyncItemTrade, true) e.update(c.Exchange, c.Pair, c.AssetType, SyncItemTrade, nil) } @@ -560,6 +546,13 @@ func (e *ExchangeCurrencyPairSyncer) Start() { continue } + wsAssetSupported := exchanges[x].IsAssetWebsocketSupported(assetTypes[y]) + if !wsAssetSupported { + log.Warnf(log.SyncMgr, + "%s asset type %s websocket functionality is unsupported, REST fetching only.", + exchangeName, + assetTypes[y]) + } enabledPairs, err := exchanges[x].GetEnabledPairs(assetTypes[y]) if err != nil { log.Errorf(log.SyncMgr, @@ -579,25 +572,21 @@ func (e *ExchangeCurrencyPairSyncer) Start() { Pair: enabledPairs[i], } + sBase := SyncBase{ + IsUsingREST: usingREST || !wsAssetSupported, + IsUsingWebsocket: usingWebsocket && wsAssetSupported, + } + if e.Cfg.SyncTicker { - c.Ticker = SyncBase{ - IsUsingREST: usingREST, - IsUsingWebsocket: usingWebsocket, - } + c.Ticker = sBase } if e.Cfg.SyncOrderbook { - c.Orderbook = SyncBase{ - IsUsingREST: usingREST, - IsUsingWebsocket: usingWebsocket, - } + c.Orderbook = sBase } if e.Cfg.SyncTrades { - c.Trade = SyncBase{ - IsUsingREST: usingREST, - IsUsingWebsocket: usingWebsocket, - } + c.Trade = sBase } e.add(&c) diff --git a/engine/syncer_types.go b/engine/syncer_types.go index 4701b503..3a3c0f5e 100644 --- a/engine/syncer_types.go +++ b/engine/syncer_types.go @@ -10,13 +10,14 @@ import ( // CurrencyPairSyncerConfig stores the currency pair config type CurrencyPairSyncerConfig struct { - SyncTicker bool - SyncOrderbook bool - SyncTrades bool - SyncContinuously bool - SyncTimeout time.Duration - NumWorkers int - Verbose bool + SyncTicker bool + SyncOrderbook bool + SyncTrades bool + SyncContinuously bool + SyncTimeoutREST time.Duration + SyncTimeoutWebsocket time.Duration + NumWorkers int + Verbose bool } // ExchangeSyncerConfig stores the exchange syncer config diff --git a/exchanges/alphapoint/alphapoint_wrapper.go b/exchanges/alphapoint/alphapoint_wrapper.go index acd650d3..5e4f23ac 100644 --- a/exchanges/alphapoint/alphapoint_wrapper.go +++ b/exchanges/alphapoint/alphapoint_wrapper.go @@ -187,8 +187,8 @@ func (a *Alphapoint) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*or } orderBook.Pair = p - orderBook.ExchangeName = a.Name - orderBook.AssetType = assetType + orderBook.Exchange = a.Name + orderBook.Asset = assetType err = orderBook.Process() if err != nil { diff --git a/exchanges/binance/binance_test.go b/exchanges/binance/binance_test.go index 0ed59862..2e3bce4b 100644 --- a/exchanges/binance/binance_test.go +++ b/exchanges/binance/binance_test.go @@ -2128,7 +2128,10 @@ func TestWsDepthUpdate(t *testing.T) { t.Error(err) } - ob := b.Websocket.Orderbook.GetOrderbook(p, asset.Spot) + ob, err := b.Websocket.Orderbook.GetOrderbook(p, asset.Spot) + if err != nil { + t.Fatal(err) + } if exp, got := seedLastUpdateID, ob.LastUpdateID; got != exp { t.Fatalf("Unexpected Last update id of orderbook for old update. Exp: %d, got: %d", exp, got) } @@ -2154,11 +2157,14 @@ func TestWsDepthUpdate(t *testing.T) { ] }}`) - if err := b.wsHandleData(update2); err != nil { + if err = b.wsHandleData(update2); err != nil { t.Error(err) } - ob = b.Websocket.Orderbook.GetOrderbook(p, asset.Spot) + ob, err = b.Websocket.Orderbook.GetOrderbook(p, asset.Spot) + if err != nil { + t.Fatal(err) + } if exp, got := int64(165), ob.LastUpdateID; got != exp { t.Fatalf("Unexpected Last update id of orderbook for new update. Exp: %d, got: %d", exp, got) } diff --git a/exchanges/binance/binance_websocket.go b/exchanges/binance/binance_websocket.go index 10bd891c..7f6b4f5c 100644 --- a/exchanges/binance/binance_websocket.go +++ b/exchanges/binance/binance_websocket.go @@ -463,10 +463,10 @@ func (b *Binance) SeedLocalCacheWithBook(p currency.Pair, orderbookNew *OrderBoo } newOrderBook.Pair = p - newOrderBook.AssetType = asset.Spot - newOrderBook.ExchangeName = b.Name + newOrderBook.Asset = asset.Spot + newOrderBook.Exchange = b.Name newOrderBook.LastUpdateID = orderbookNew.LastUpdateID - newOrderBook.VerificationBypass = b.OrderbookVerificationBypass + newOrderBook.VerifyOrderbook = b.CanVerifyOrderbook return b.Websocket.Orderbook.LoadSnapshot(&newOrderBook) } @@ -632,8 +632,8 @@ func (b *Binance) applyBufferUpdate(pair currency.Pair) error { return nil } - recent := b.Websocket.Orderbook.GetOrderbook(pair, asset.Spot) - if recent == nil || (recent.Asks == nil && recent.Bids == nil) { + recent, err := b.Websocket.Orderbook.GetOrderbook(pair, asset.Spot) + if err != nil || (recent.Asks == nil && recent.Bids == nil) { return b.obm.fetchBookViaREST(pair) } diff --git a/exchanges/binance/binance_wrapper.go b/exchanges/binance/binance_wrapper.go index 6523d06c..301f8163 100644 --- a/exchanges/binance/binance_wrapper.go +++ b/exchanges/binance/binance_wrapper.go @@ -92,14 +92,26 @@ func (b *Binance) SetDefaults() { if err != nil { log.Errorln(log.ExchangeSys, err) } + err = b.DisableAssetWebsocketSupport(asset.Margin) + if err != nil { + log.Errorln(log.ExchangeSys, err) + } err = b.StoreAssetPairFormat(asset.CoinMarginedFutures, coinFutures) if err != nil { log.Errorln(log.ExchangeSys, err) } + err = b.DisableAssetWebsocketSupport(asset.CoinMarginedFutures) + if err != nil { + log.Errorln(log.ExchangeSys, err) + } err = b.StoreAssetPairFormat(asset.USDTMarginedFutures, usdtFutures) if err != nil { log.Errorln(log.ExchangeSys, err) } + err = b.DisableAssetWebsocketSupport(asset.USDTMarginedFutures) + if err != nil { + log.Errorln(log.ExchangeSys, err) + } b.Features = exchange.Features{ Supports: exchange.FeaturesSupported{ REST: true, @@ -520,10 +532,10 @@ func (b *Binance) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderb // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *Binance) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: b.CanVerifyOrderbook, } var orderbookNew OrderBook var err error diff --git a/exchanges/bitfinex/bitfinex_websocket.go b/exchanges/bitfinex/bitfinex_websocket.go index 1eac2b8d..a8cb90cb 100644 --- a/exchanges/bitfinex/bitfinex_websocket.go +++ b/exchanges/bitfinex/bitfinex_websocket.go @@ -962,13 +962,12 @@ func (b *Bitfinex) WsInsertSnapshot(p currency.Pair, assetType asset.Item, books } } - book.AssetType = assetType + book.Asset = assetType book.Pair = p - book.ExchangeName = b.Name - book.NotAggregated = true - book.HasChecksumValidation = true + book.Exchange = b.Name + book.PriceDuplication = true book.IsFundingRate = fundingRate - book.VerificationBypass = b.OrderbookVerificationBypass + book.VerifyOrderbook = b.CanVerifyOrderbook return b.Websocket.Orderbook.LoadSnapshot(&book) } @@ -1036,14 +1035,15 @@ func (b *Bitfinex) WsUpdateOrderbook(p currency.Pair, assetType asset.Item, book if checkme.Sequence+1 == sequenceNo { // Sequence numbers get dropped, if checksum is not in line with // sequence, do not check. - ob := b.Websocket.Orderbook.GetOrderbook(p, assetType) - if ob == nil { - return fmt.Errorf("cannot calculate websocket checksum: book not found for %s %s", + ob, err := b.Websocket.Orderbook.GetOrderbook(p, assetType) + if err != nil { + return fmt.Errorf("cannot calculate websocket checksum: book not found for %s %s %w", p, - assetType) + assetType, + err) } - err := validateCRC32(ob, checkme.Token) + err = validateCRC32(ob, checkme.Token) if err != nil { return err } @@ -1398,7 +1398,7 @@ func validateCRC32(book *orderbook.Base, token int) error { return nil } return fmt.Errorf("invalid checksum for %s %s: calculated [%d] does not match [%d]", - book.AssetType, + book.Asset, book.Pair, checksum, uint32(token)) diff --git a/exchanges/bitfinex/bitfinex_wrapper.go b/exchanges/bitfinex/bitfinex_wrapper.go index ab540aa4..050f2949 100644 --- a/exchanges/bitfinex/bitfinex_wrapper.go +++ b/exchanges/bitfinex/bitfinex_wrapper.go @@ -395,11 +395,11 @@ func (b *Bitfinex) FetchOrderbook(p currency.Pair, assetType asset.Item) (*order // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *Bitfinex) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { o := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - NotAggregated: true, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + PriceDuplication: true, + VerifyOrderbook: b.CanVerifyOrderbook, } fPair, err := b.FormatExchangeCurrency(p, assetType) diff --git a/exchanges/bitflyer/bitflyer_wrapper.go b/exchanges/bitflyer/bitflyer_wrapper.go index adf382d5..8a375aaa 100644 --- a/exchanges/bitflyer/bitflyer_wrapper.go +++ b/exchanges/bitflyer/bitflyer_wrapper.go @@ -254,10 +254,10 @@ func (b *Bitflyer) FetchOrderbook(p currency.Pair, assetType asset.Item) (*order // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *Bitflyer) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: b.CanVerifyOrderbook, } fPair, err := b.FormatExchangeCurrency(p, assetType) diff --git a/exchanges/bithumb/bithumb_wrapper.go b/exchanges/bithumb/bithumb_wrapper.go index ae685905..8aa2944b 100644 --- a/exchanges/bithumb/bithumb_wrapper.go +++ b/exchanges/bithumb/bithumb_wrapper.go @@ -246,10 +246,10 @@ func (b *Bithumb) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderb // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *Bithumb) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: b.CanVerifyOrderbook, } curr := p.Base.String() diff --git a/exchanges/bitmex/bitmex_test.go b/exchanges/bitmex/bitmex_test.go index a2de8cdf..d4868d8c 100644 --- a/exchanges/bitmex/bitmex_test.go +++ b/exchanges/bitmex/bitmex_test.go @@ -955,7 +955,7 @@ func TestWSOrderbookHandling(t *testing.T) { ] }`) err = b.wsHandleData(pressXToJSON) - if err != nil && err.Error() != "perpetualcontract ETHUSD update cannot be deleted id: 17999995000 not found" { + if err != nil && err.Error() != "delete error: cannot match ID on linked list 17999995000 not found" { t.Error(err) } if err == nil { diff --git a/exchanges/bitmex/bitmex_websocket.go b/exchanges/bitmex/bitmex_websocket.go index 8f4de3fe..e4f73618 100644 --- a/exchanges/bitmex/bitmex_websocket.go +++ b/exchanges/bitmex/bitmex_websocket.go @@ -508,11 +508,11 @@ func (b *Bitmex) processOrderbook(data []OrderBookL2, action string, p currency. data[i].Side) } } - orderbook.Reverse(book.Asks) // Reverse asks for correct alignment - book.AssetType = a + book.Asks.Reverse() // Reverse asks for correct alignment + book.Asset = a book.Pair = p - book.ExchangeName = b.Name - book.VerificationBypass = b.OrderbookVerificationBypass + book.Exchange = b.Name + book.VerifyOrderbook = b.CanVerifyOrderbook err := b.Websocket.Orderbook.LoadSnapshot(&book) if err != nil { diff --git a/exchanges/bitmex/bitmex_wrapper.go b/exchanges/bitmex/bitmex_wrapper.go index 40654faf..efa2a58f 100644 --- a/exchanges/bitmex/bitmex_wrapper.go +++ b/exchanges/bitmex/bitmex_wrapper.go @@ -69,6 +69,11 @@ func (b *Bitmex) SetDefaults() { log.Errorln(log.ExchangeSys, err) } + err = b.DisableAssetWebsocketSupport(asset.Index) + if err != nil { + log.Errorln(log.ExchangeSys, err) + } + b.Features = exchange.Features{ Supports: exchange.FeaturesSupported{ REST: true, @@ -346,10 +351,10 @@ func (b *Bitmex) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbo // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *Bitmex) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: b.CanVerifyOrderbook, } if assetType == asset.Index { @@ -384,7 +389,7 @@ func (b *Bitmex) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderb orderbookNew[i].Side) } } - orderbook.Reverse(book.Asks) + book.Asks.Reverse() // Reverse order of asks to ascending err = book.Process() if err != nil { diff --git a/exchanges/bitstamp/bitstamp_websocket.go b/exchanges/bitstamp/bitstamp_websocket.go index 58c56268..6f7032a2 100644 --- a/exchanges/bitstamp/bitstamp_websocket.go +++ b/exchanges/bitstamp/bitstamp_websocket.go @@ -239,13 +239,13 @@ func (b *Bitstamp) wsUpdateOrderbook(update websocketOrderBook, p currency.Pair, bids = append(bids, orderbook.Item{Price: target, Amount: amount}) } return b.Websocket.Orderbook.LoadSnapshot(&orderbook.Base{ - Bids: bids, - Asks: asks, - Pair: p, - LastUpdated: time.Unix(update.Timestamp, 0), - AssetType: assetType, - ExchangeName: b.Name, - VerificationBypass: b.OrderbookVerificationBypass, + Bids: bids, + Asks: asks, + Pair: p, + LastUpdated: time.Unix(update.Timestamp, 0), + Asset: assetType, + Exchange: b.Name, + VerifyOrderbook: b.CanVerifyOrderbook, }) } @@ -275,9 +275,9 @@ func (b *Bitstamp) seedOrderBook() error { }) } newOrderBook.Pair = p[x] - newOrderBook.AssetType = asset.Spot - newOrderBook.ExchangeName = b.Name - newOrderBook.VerificationBypass = b.OrderbookVerificationBypass + newOrderBook.Asset = asset.Spot + newOrderBook.Exchange = b.Name + newOrderBook.VerifyOrderbook = b.CanVerifyOrderbook err = b.Websocket.Orderbook.LoadSnapshot(&newOrderBook) if err != nil { diff --git a/exchanges/bitstamp/bitstamp_wrapper.go b/exchanges/bitstamp/bitstamp_wrapper.go index 54f3fd91..bfc6f1f7 100644 --- a/exchanges/bitstamp/bitstamp_wrapper.go +++ b/exchanges/bitstamp/bitstamp_wrapper.go @@ -324,10 +324,10 @@ func (b *Bitstamp) FetchOrderbook(p currency.Pair, assetType asset.Item) (*order // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *Bitstamp) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: b.CanVerifyOrderbook, } fPair, err := b.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/bittrex/bittrex_wrapper.go b/exchanges/bittrex/bittrex_wrapper.go index 87b402cf..a4cd72ca 100644 --- a/exchanges/bittrex/bittrex_wrapper.go +++ b/exchanges/bittrex/bittrex_wrapper.go @@ -333,10 +333,10 @@ func (b *Bittrex) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderb // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *Bittrex) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: b.CanVerifyOrderbook, } fpair, err := b.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/btcmarkets/btcmarkets_websocket.go b/exchanges/btcmarkets/btcmarkets_websocket.go index 5df16d99..76fc2d92 100644 --- a/exchanges/btcmarkets/btcmarkets_websocket.go +++ b/exchanges/btcmarkets/btcmarkets_websocket.go @@ -83,7 +83,7 @@ func (b *BTCMarkets) wsHandleData(respRaw []byte) error { return err } - var bids, asks []orderbook.Item + var bids, asks orderbook.Items for x := range ob.Bids { var price, amount float64 price, err = strconv.ParseFloat(ob.Bids[x][0].(string), 64) @@ -117,14 +117,15 @@ func (b *BTCMarkets) wsHandleData(respRaw []byte) error { }) } if ob.Snapshot { + bids.SortBids() // Alignment completely out, sort is needed. err = b.Websocket.Orderbook.LoadSnapshot(&orderbook.Base{ - Pair: p, - Bids: orderbook.SortBids(bids), // Alignment completely out sort is needed - Asks: asks, - LastUpdated: ob.Timestamp, - AssetType: asset.Spot, - ExchangeName: b.Name, - VerificationBypass: b.OrderbookVerificationBypass, + Pair: p, + Bids: bids, + Asks: asks, + LastUpdated: ob.Timestamp, + Asset: asset.Spot, + Exchange: b.Name, + VerifyOrderbook: b.CanVerifyOrderbook, }) } else { err = b.Websocket.Orderbook.Update(&buffer.Update{ diff --git a/exchanges/btcmarkets/btcmarkets_wrapper.go b/exchanges/btcmarkets/btcmarkets_wrapper.go index c598ff5a..bcd04d46 100644 --- a/exchanges/btcmarkets/btcmarkets_wrapper.go +++ b/exchanges/btcmarkets/btcmarkets_wrapper.go @@ -360,11 +360,11 @@ func (b *BTCMarkets) FetchOrderbook(p currency.Pair, assetType asset.Item) (*ord // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *BTCMarkets) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - NotAggregated: true, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + PriceDuplication: true, + VerifyOrderbook: b.CanVerifyOrderbook, } fpair, err := b.FormatExchangeCurrency(p, assetType) diff --git a/exchanges/btse/btse_websocket.go b/exchanges/btse/btse_websocket.go index 221cd23f..148afc8e 100644 --- a/exchanges/btse/btse_websocket.go +++ b/exchanges/btse/btse_websocket.go @@ -311,10 +311,10 @@ func (b *BTSE) wsHandleData(respRaw []byte) error { return err } newOB.Pair = p - newOB.AssetType = a - newOB.ExchangeName = b.Name - orderbook.Reverse(newOB.Asks) // Reverse asks for correct alignment - newOB.VerificationBypass = b.OrderbookVerificationBypass + newOB.Asset = a + newOB.Exchange = b.Name + newOB.Asks.Reverse() // Reverse asks for correct alignment + newOB.VerifyOrderbook = b.CanVerifyOrderbook err = b.Websocket.Orderbook.LoadSnapshot(&newOB) if err != nil { return err diff --git a/exchanges/btse/btse_wrapper.go b/exchanges/btse/btse_wrapper.go index 47215adb..dd8f7945 100644 --- a/exchanges/btse/btse_wrapper.go +++ b/exchanges/btse/btse_wrapper.go @@ -336,10 +336,10 @@ func (b *BTSE) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbook // UpdateOrderbook updates and returns the orderbook for a currency pair func (b *BTSE) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: b.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: b.OrderbookVerificationBypass, + Exchange: b.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: b.CanVerifyOrderbook, } fPair, err := b.FormatExchangeCurrency(p, assetType) if err != nil { @@ -366,10 +366,10 @@ func (b *BTSE) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderboo Price: a.SellQuote[x].Price, Amount: a.SellQuote[x].Size}) } - orderbook.Reverse(book.Asks) // Reverse asks for correct alignment + book.Asks.Reverse() // Reverse asks for correct alignment book.Pair = p - book.ExchangeName = b.Name - book.AssetType = assetType + book.Exchange = b.Name + book.Asset = assetType err = book.Process() if err != nil { return book, err diff --git a/exchanges/coinbasepro/coinbasepro_websocket.go b/exchanges/coinbasepro/coinbasepro_websocket.go index 621ab0bc..3b34ce76 100644 --- a/exchanges/coinbasepro/coinbasepro_websocket.go +++ b/exchanges/coinbasepro/coinbasepro_websocket.go @@ -309,10 +309,10 @@ func (c *CoinbasePro) ProcessSnapshot(snapshot *WebsocketOrderbookSnapshot) erro return err } - base.AssetType = asset.Spot + base.Asset = asset.Spot base.Pair = pair - base.ExchangeName = c.Name - base.VerificationBypass = c.OrderbookVerificationBypass + base.Exchange = c.Name + base.VerifyOrderbook = c.CanVerifyOrderbook return c.Websocket.Orderbook.LoadSnapshot(&base) } diff --git a/exchanges/coinbasepro/coinbasepro_wrapper.go b/exchanges/coinbasepro/coinbasepro_wrapper.go index 26cbb10d..4d4f57fc 100644 --- a/exchanges/coinbasepro/coinbasepro_wrapper.go +++ b/exchanges/coinbasepro/coinbasepro_wrapper.go @@ -411,10 +411,10 @@ func (c *CoinbasePro) FetchOrderbook(p currency.Pair, assetType asset.Item) (*or // UpdateOrderbook updates and returns the orderbook for a currency pair func (c *CoinbasePro) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: c.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: c.OrderbookVerificationBypass, + Exchange: c.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: c.CanVerifyOrderbook, } fpair, err := c.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/coinbene/coinbene_websocket.go b/exchanges/coinbene/coinbene_websocket.go index e769d598..3d5c9447 100644 --- a/exchanges/coinbene/coinbene_websocket.go +++ b/exchanges/coinbene/coinbene_websocket.go @@ -285,11 +285,11 @@ func (c *Coinbene) wsHandleData(respRaw []byte) error { var newOB orderbook.Base newOB.Asks = asks newOB.Bids = bids - newOB.AssetType = assetType + newOB.Asset = assetType newOB.Pair = newPair - newOB.ExchangeName = c.Name + newOB.Exchange = c.Name newOB.LastUpdated = time.Unix(orderBook.Data[0].Timestamp, 0) - newOB.VerificationBypass = c.OrderbookVerificationBypass + newOB.VerifyOrderbook = c.CanVerifyOrderbook err = c.Websocket.Orderbook.LoadSnapshot(&newOB) if err != nil { return err diff --git a/exchanges/coinbene/coinbene_wrapper.go b/exchanges/coinbene/coinbene_wrapper.go index 87262d68..2685ec63 100644 --- a/exchanges/coinbene/coinbene_wrapper.go +++ b/exchanges/coinbene/coinbene_wrapper.go @@ -411,10 +411,10 @@ func (c *Coinbene) FetchOrderbook(p currency.Pair, assetType asset.Item) (*order // UpdateOrderbook updates and returns the orderbook for a currency pair func (c *Coinbene) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: c.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: c.OrderbookVerificationBypass, + Exchange: c.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: c.CanVerifyOrderbook, } if !c.SupportsAsset(assetType) { return book, diff --git a/exchanges/coinut/coinut_websocket.go b/exchanges/coinut/coinut_websocket.go index 77ac3cdf..1143c8f7 100644 --- a/exchanges/coinut/coinut_websocket.go +++ b/exchanges/coinut/coinut_websocket.go @@ -530,7 +530,7 @@ func (c *COINUT) WsProcessOrderbookSnapshot(ob *WsOrderbookSnapshot) error { var newOrderBook orderbook.Base newOrderBook.Asks = asks newOrderBook.Bids = bids - newOrderBook.VerificationBypass = c.OrderbookVerificationBypass + newOrderBook.VerifyOrderbook = c.CanVerifyOrderbook pairs, err := c.GetEnabledPairs(asset.Spot) if err != nil { @@ -550,8 +550,8 @@ func (c *COINUT) WsProcessOrderbookSnapshot(ob *WsOrderbookSnapshot) error { return err } - newOrderBook.AssetType = asset.Spot - newOrderBook.ExchangeName = c.Name + newOrderBook.Asset = asset.Spot + newOrderBook.Exchange = c.Name return c.Websocket.Orderbook.LoadSnapshot(&newOrderBook) } diff --git a/exchanges/coinut/coinut_wrapper.go b/exchanges/coinut/coinut_wrapper.go index fefd2ef5..d310a7ab 100644 --- a/exchanges/coinut/coinut_wrapper.go +++ b/exchanges/coinut/coinut_wrapper.go @@ -463,10 +463,10 @@ func (c *COINUT) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbo // UpdateOrderbook updates and returns the orderbook for a currency pair func (c *COINUT) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: c.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: c.OrderbookVerificationBypass, + Exchange: c.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: c.CanVerifyOrderbook, } err := c.loadInstrumentsIfNotLoaded() if err != nil { diff --git a/exchanges/exchange.go b/exchanges/exchange.go index 2eb78946..969dd516 100644 --- a/exchanges/exchange.go +++ b/exchanges/exchange.go @@ -39,19 +39,19 @@ const ( DefaultWebsocketOrderbookBufferLimit = 5 ) -func (e *Base) checkAndInitRequester() { - if e.Requester == nil { - e.Requester = request.New(e.Name, +func (b *Base) checkAndInitRequester() { + if b.Requester == nil { + b.Requester = request.New(b.Name, &http.Client{Transport: new(http.Transport)}) } } // SetHTTPClientTimeout sets the timeout value for the exchanges HTTP Client and // also the underlying transports idle connection timeout -func (e *Base) SetHTTPClientTimeout(t time.Duration) error { - e.checkAndInitRequester() - e.Requester.HTTPClient.Timeout = t - tr, ok := e.Requester.HTTPClient.Transport.(*http.Transport) +func (b *Base) SetHTTPClientTimeout(t time.Duration) error { + b.checkAndInitRequester() + b.Requester.HTTPClient.Timeout = t + tr, ok := b.Requester.HTTPClient.Transport.(*http.Transport) if !ok { return errors.New("transport not set, cannot set timeout") } @@ -60,31 +60,31 @@ func (e *Base) SetHTTPClientTimeout(t time.Duration) error { } // SetHTTPClient sets exchanges HTTP client -func (e *Base) SetHTTPClient(h *http.Client) { - e.checkAndInitRequester() - e.Requester.HTTPClient = h +func (b *Base) SetHTTPClient(h *http.Client) { + b.checkAndInitRequester() + b.Requester.HTTPClient = h } // GetHTTPClient gets the exchanges HTTP client -func (e *Base) GetHTTPClient() *http.Client { - e.checkAndInitRequester() - return e.Requester.HTTPClient +func (b *Base) GetHTTPClient() *http.Client { + b.checkAndInitRequester() + return b.Requester.HTTPClient } // SetHTTPClientUserAgent sets the exchanges HTTP user agent -func (e *Base) SetHTTPClientUserAgent(ua string) { - e.checkAndInitRequester() - e.Requester.UserAgent = ua - e.HTTPUserAgent = ua +func (b *Base) SetHTTPClientUserAgent(ua string) { + b.checkAndInitRequester() + b.Requester.UserAgent = ua + b.HTTPUserAgent = ua } // GetHTTPClientUserAgent gets the exchanges HTTP user agent -func (e *Base) GetHTTPClientUserAgent() string { - return e.HTTPUserAgent +func (b *Base) GetHTTPClientUserAgent() string { + return b.HTTPUserAgent } // SetClientProxyAddress sets a proxy address for REST and websocket requests -func (e *Base) SetClientProxyAddress(addr string) error { +func (b *Base) SetClientProxyAddress(addr string) error { if addr == "" { return nil } @@ -94,13 +94,13 @@ func (e *Base) SetClientProxyAddress(addr string) error { err) } - err = e.Requester.SetProxy(proxy) + err = b.Requester.SetProxy(proxy) if err != nil { return err } - if e.Websocket != nil { - err = e.Websocket.SetProxyAddress(addr) + if b.Websocket != nil { + err = b.Websocket.SetProxyAddress(addr) if err != nil { return err } @@ -110,98 +110,98 @@ func (e *Base) SetClientProxyAddress(addr string) error { // SetFeatureDefaults sets the exchanges default feature // support set -func (e *Base) SetFeatureDefaults() { - if e.Config.Features == nil { +func (b *Base) SetFeatureDefaults() { + if b.Config.Features == nil { s := &config.FeaturesConfig{ Supports: config.FeaturesSupportedConfig{ - Websocket: e.Features.Supports.Websocket, - REST: e.Features.Supports.REST, + Websocket: b.Features.Supports.Websocket, + REST: b.Features.Supports.REST, RESTCapabilities: protocol.Features{ - AutoPairUpdates: e.Features.Supports.RESTCapabilities.AutoPairUpdates, + AutoPairUpdates: b.Features.Supports.RESTCapabilities.AutoPairUpdates, }, }, } - if e.Config.SupportsAutoPairUpdates != nil { - s.Supports.RESTCapabilities.AutoPairUpdates = *e.Config.SupportsAutoPairUpdates - s.Enabled.AutoPairUpdates = *e.Config.SupportsAutoPairUpdates + if b.Config.SupportsAutoPairUpdates != nil { + s.Supports.RESTCapabilities.AutoPairUpdates = *b.Config.SupportsAutoPairUpdates + s.Enabled.AutoPairUpdates = *b.Config.SupportsAutoPairUpdates } else { - s.Supports.RESTCapabilities.AutoPairUpdates = e.Features.Supports.RESTCapabilities.AutoPairUpdates - s.Enabled.AutoPairUpdates = e.Features.Supports.RESTCapabilities.AutoPairUpdates + s.Supports.RESTCapabilities.AutoPairUpdates = b.Features.Supports.RESTCapabilities.AutoPairUpdates + s.Enabled.AutoPairUpdates = b.Features.Supports.RESTCapabilities.AutoPairUpdates if !s.Supports.RESTCapabilities.AutoPairUpdates { - e.Config.CurrencyPairs.LastUpdated = time.Now().Unix() - e.CurrencyPairs.LastUpdated = e.Config.CurrencyPairs.LastUpdated + b.Config.CurrencyPairs.LastUpdated = time.Now().Unix() + b.CurrencyPairs.LastUpdated = b.Config.CurrencyPairs.LastUpdated } } - e.Config.Features = s - e.Config.SupportsAutoPairUpdates = nil + b.Config.Features = s + b.Config.SupportsAutoPairUpdates = nil } else { - if e.Features.Supports.RESTCapabilities.AutoPairUpdates != e.Config.Features.Supports.RESTCapabilities.AutoPairUpdates { - e.Config.Features.Supports.RESTCapabilities.AutoPairUpdates = e.Features.Supports.RESTCapabilities.AutoPairUpdates + if b.Features.Supports.RESTCapabilities.AutoPairUpdates != b.Config.Features.Supports.RESTCapabilities.AutoPairUpdates { + b.Config.Features.Supports.RESTCapabilities.AutoPairUpdates = b.Features.Supports.RESTCapabilities.AutoPairUpdates - if !e.Config.Features.Supports.RESTCapabilities.AutoPairUpdates { - e.Config.CurrencyPairs.LastUpdated = time.Now().Unix() + if !b.Config.Features.Supports.RESTCapabilities.AutoPairUpdates { + b.Config.CurrencyPairs.LastUpdated = time.Now().Unix() } } - if e.Features.Supports.REST != e.Config.Features.Supports.REST { - e.Config.Features.Supports.REST = e.Features.Supports.REST + if b.Features.Supports.REST != b.Config.Features.Supports.REST { + b.Config.Features.Supports.REST = b.Features.Supports.REST } - if e.Features.Supports.RESTCapabilities.TickerBatching != e.Config.Features.Supports.RESTCapabilities.TickerBatching { - e.Config.Features.Supports.RESTCapabilities.TickerBatching = e.Features.Supports.RESTCapabilities.TickerBatching + if b.Features.Supports.RESTCapabilities.TickerBatching != b.Config.Features.Supports.RESTCapabilities.TickerBatching { + b.Config.Features.Supports.RESTCapabilities.TickerBatching = b.Features.Supports.RESTCapabilities.TickerBatching } - if e.Features.Supports.Websocket != e.Config.Features.Supports.Websocket { - e.Config.Features.Supports.Websocket = e.Features.Supports.Websocket + if b.Features.Supports.Websocket != b.Config.Features.Supports.Websocket { + b.Config.Features.Supports.Websocket = b.Features.Supports.Websocket } - if e.IsSaveTradeDataEnabled() != e.Config.Features.Enabled.SaveTradeData { - e.SetSaveTradeDataStatus(e.Config.Features.Enabled.SaveTradeData) + if b.IsSaveTradeDataEnabled() != b.Config.Features.Enabled.SaveTradeData { + b.SetSaveTradeDataStatus(b.Config.Features.Enabled.SaveTradeData) } - e.Features.Enabled.AutoPairUpdates = e.Config.Features.Enabled.AutoPairUpdates + b.Features.Enabled.AutoPairUpdates = b.Config.Features.Enabled.AutoPairUpdates } } // SetAPICredentialDefaults sets the API Credential validator defaults -func (e *Base) SetAPICredentialDefaults() { +func (b *Base) SetAPICredentialDefaults() { // Exchange hardcoded settings take precedence and overwrite the config settings - if e.Config.API.CredentialsValidator == nil { - e.Config.API.CredentialsValidator = new(config.APICredentialsValidatorConfig) + if b.Config.API.CredentialsValidator == nil { + b.Config.API.CredentialsValidator = new(config.APICredentialsValidatorConfig) } - if e.Config.API.CredentialsValidator.RequiresKey != e.API.CredentialsValidator.RequiresKey { - e.Config.API.CredentialsValidator.RequiresKey = e.API.CredentialsValidator.RequiresKey + if b.Config.API.CredentialsValidator.RequiresKey != b.API.CredentialsValidator.RequiresKey { + b.Config.API.CredentialsValidator.RequiresKey = b.API.CredentialsValidator.RequiresKey } - if e.Config.API.CredentialsValidator.RequiresSecret != e.API.CredentialsValidator.RequiresSecret { - e.Config.API.CredentialsValidator.RequiresSecret = e.API.CredentialsValidator.RequiresSecret + if b.Config.API.CredentialsValidator.RequiresSecret != b.API.CredentialsValidator.RequiresSecret { + b.Config.API.CredentialsValidator.RequiresSecret = b.API.CredentialsValidator.RequiresSecret } - if e.Config.API.CredentialsValidator.RequiresBase64DecodeSecret != e.API.CredentialsValidator.RequiresBase64DecodeSecret { - e.Config.API.CredentialsValidator.RequiresBase64DecodeSecret = e.API.CredentialsValidator.RequiresBase64DecodeSecret + if b.Config.API.CredentialsValidator.RequiresBase64DecodeSecret != b.API.CredentialsValidator.RequiresBase64DecodeSecret { + b.Config.API.CredentialsValidator.RequiresBase64DecodeSecret = b.API.CredentialsValidator.RequiresBase64DecodeSecret } - if e.Config.API.CredentialsValidator.RequiresClientID != e.API.CredentialsValidator.RequiresClientID { - e.Config.API.CredentialsValidator.RequiresClientID = e.API.CredentialsValidator.RequiresClientID + if b.Config.API.CredentialsValidator.RequiresClientID != b.API.CredentialsValidator.RequiresClientID { + b.Config.API.CredentialsValidator.RequiresClientID = b.API.CredentialsValidator.RequiresClientID } - if e.Config.API.CredentialsValidator.RequiresPEM != e.API.CredentialsValidator.RequiresPEM { - e.Config.API.CredentialsValidator.RequiresPEM = e.API.CredentialsValidator.RequiresPEM + if b.Config.API.CredentialsValidator.RequiresPEM != b.API.CredentialsValidator.RequiresPEM { + b.Config.API.CredentialsValidator.RequiresPEM = b.API.CredentialsValidator.RequiresPEM } } // SupportsRESTTickerBatchUpdates returns whether or not the // exhange supports REST batch ticker fetching -func (e *Base) SupportsRESTTickerBatchUpdates() bool { - return e.Features.Supports.RESTCapabilities.TickerBatching +func (b *Base) SupportsRESTTickerBatchUpdates() bool { + return b.Features.Supports.RESTCapabilities.TickerBatching } // SupportsAutoPairUpdates returns whether or not the exchange supports // auto currency pair updating -func (e *Base) SupportsAutoPairUpdates() bool { - if e.Features.Supports.RESTCapabilities.AutoPairUpdates || - e.Features.Supports.WebsocketCapabilities.AutoPairUpdates { +func (b *Base) SupportsAutoPairUpdates() bool { + if b.Features.Supports.RESTCapabilities.AutoPairUpdates || + b.Features.Supports.WebsocketCapabilities.AutoPairUpdates { return true } return false @@ -209,22 +209,22 @@ func (e *Base) SupportsAutoPairUpdates() bool { // GetLastPairsUpdateTime returns the unix timestamp of when the exchanges // currency pairs were last updated -func (e *Base) GetLastPairsUpdateTime() int64 { - return e.CurrencyPairs.LastUpdated +func (b *Base) GetLastPairsUpdateTime() int64 { + return b.CurrencyPairs.LastUpdated } // GetAssetTypes returns the available asset types for an individual exchange -func (e *Base) GetAssetTypes() asset.Items { - return e.CurrencyPairs.GetAssetTypes() +func (b *Base) GetAssetTypes() asset.Items { + return b.CurrencyPairs.GetAssetTypes() } // GetPairAssetType returns the associated asset type for the currency pair // This method is only useful for exchanges that have pair names with multiple delimiters (BTC-USD-0626) // Helpful if the exchange has only a single asset type but in that case the asset type can be hard coded -func (e *Base) GetPairAssetType(c currency.Pair) (asset.Item, error) { - assetTypes := e.GetAssetTypes() +func (b *Base) GetPairAssetType(c currency.Pair) (asset.Item, error) { + assetTypes := b.GetAssetTypes() for i := range assetTypes { - avail, err := e.GetAvailablePairs(assetTypes[i]) + avail, err := b.GetAvailablePairs(assetTypes[i]) if err != nil { return "", err } @@ -237,137 +237,137 @@ func (e *Base) GetPairAssetType(c currency.Pair) (asset.Item, error) { // GetClientBankAccounts returns banking details associated with // a client for withdrawal purposes -func (e *Base) GetClientBankAccounts(exchangeName, withdrawalCurrency string) (*banking.Account, error) { +func (b *Base) GetClientBankAccounts(exchangeName, withdrawalCurrency string) (*banking.Account, error) { cfg := config.GetConfig() return cfg.GetClientBankAccounts(exchangeName, withdrawalCurrency) } // GetExchangeBankAccounts returns banking details associated with an // exchange for funding purposes -func (e *Base) GetExchangeBankAccounts(id, depositCurrency string) (*banking.Account, error) { +func (b *Base) GetExchangeBankAccounts(id, depositCurrency string) (*banking.Account, error) { cfg := config.GetConfig() - return cfg.GetExchangeBankAccounts(e.Name, id, depositCurrency) + return cfg.GetExchangeBankAccounts(b.Name, id, depositCurrency) } // SetCurrencyPairFormat checks the exchange request and config currency pair // formats and syncs it with the exchanges SetDefault settings -func (e *Base) SetCurrencyPairFormat() { - if e.Config.CurrencyPairs == nil { - e.Config.CurrencyPairs = new(currency.PairsManager) +func (b *Base) SetCurrencyPairFormat() { + if b.Config.CurrencyPairs == nil { + b.Config.CurrencyPairs = new(currency.PairsManager) } - e.Config.CurrencyPairs.UseGlobalFormat = e.CurrencyPairs.UseGlobalFormat - if e.Config.CurrencyPairs.UseGlobalFormat { - e.Config.CurrencyPairs.RequestFormat = e.CurrencyPairs.RequestFormat - e.Config.CurrencyPairs.ConfigFormat = e.CurrencyPairs.ConfigFormat + b.Config.CurrencyPairs.UseGlobalFormat = b.CurrencyPairs.UseGlobalFormat + if b.Config.CurrencyPairs.UseGlobalFormat { + b.Config.CurrencyPairs.RequestFormat = b.CurrencyPairs.RequestFormat + b.Config.CurrencyPairs.ConfigFormat = b.CurrencyPairs.ConfigFormat return } - if e.Config.CurrencyPairs.ConfigFormat != nil { - e.Config.CurrencyPairs.ConfigFormat = nil + if b.Config.CurrencyPairs.ConfigFormat != nil { + b.Config.CurrencyPairs.ConfigFormat = nil } - if e.Config.CurrencyPairs.RequestFormat != nil { - e.Config.CurrencyPairs.RequestFormat = nil + if b.Config.CurrencyPairs.RequestFormat != nil { + b.Config.CurrencyPairs.RequestFormat = nil } - assetTypes := e.GetAssetTypes() + assetTypes := b.GetAssetTypes() for x := range assetTypes { - if _, err := e.Config.CurrencyPairs.Get(assetTypes[x]); err != nil { - ps, err := e.CurrencyPairs.Get(assetTypes[x]) + if _, err := b.Config.CurrencyPairs.Get(assetTypes[x]); err != nil { + ps, err := b.CurrencyPairs.Get(assetTypes[x]) if err != nil { continue } - e.Config.CurrencyPairs.Store(assetTypes[x], *ps) + b.Config.CurrencyPairs.Store(assetTypes[x], *ps) } } } // SetConfigPairs sets the exchanges currency pairs to the pairs set in the config -func (e *Base) SetConfigPairs() error { - assetTypes := e.Config.CurrencyPairs.GetAssetTypes() - exchangeAssets := e.CurrencyPairs.GetAssetTypes() +func (b *Base) SetConfigPairs() error { + assetTypes := b.Config.CurrencyPairs.GetAssetTypes() + exchangeAssets := b.CurrencyPairs.GetAssetTypes() for x := range assetTypes { if !exchangeAssets.Contains(assetTypes[x]) { log.Warnf(log.ExchangeSys, "%s exchange asset type %s unsupported, please manually remove from configuration", - e.Name, + b.Name, assetTypes[x]) } - cfgPS, err := e.Config.CurrencyPairs.Get(assetTypes[x]) + cfgPS, err := b.Config.CurrencyPairs.Get(assetTypes[x]) if err != nil { return err } var enabledAsset bool - if e.Config.CurrencyPairs.IsAssetEnabled(assetTypes[x]) == nil { + if b.Config.CurrencyPairs.IsAssetEnabled(assetTypes[x]) == nil { enabledAsset = true } - e.CurrencyPairs.SetAssetEnabled(assetTypes[x], enabledAsset) + b.CurrencyPairs.SetAssetEnabled(assetTypes[x], enabledAsset) - if e.Config.CurrencyPairs.UseGlobalFormat { - e.CurrencyPairs.StorePairs(assetTypes[x], cfgPS.Available, false) - e.CurrencyPairs.StorePairs(assetTypes[x], cfgPS.Enabled, true) + if b.Config.CurrencyPairs.UseGlobalFormat { + b.CurrencyPairs.StorePairs(assetTypes[x], cfgPS.Available, false) + b.CurrencyPairs.StorePairs(assetTypes[x], cfgPS.Enabled, true) continue } - exchPS, err := e.CurrencyPairs.Get(assetTypes[x]) + exchPS, err := b.CurrencyPairs.Get(assetTypes[x]) if err != nil { return err } cfgPS.ConfigFormat = exchPS.ConfigFormat cfgPS.RequestFormat = exchPS.RequestFormat - e.CurrencyPairs.StorePairs(assetTypes[x], cfgPS.Available, false) - e.CurrencyPairs.StorePairs(assetTypes[x], cfgPS.Enabled, true) + b.CurrencyPairs.StorePairs(assetTypes[x], cfgPS.Available, false) + b.CurrencyPairs.StorePairs(assetTypes[x], cfgPS.Enabled, true) } return nil } // GetAuthenticatedAPISupport returns whether the exchange supports // authenticated API requests -func (e *Base) GetAuthenticatedAPISupport(endpoint uint8) bool { +func (b *Base) GetAuthenticatedAPISupport(endpoint uint8) bool { switch endpoint { case RestAuthentication: - return e.API.AuthenticatedSupport + return b.API.AuthenticatedSupport case WebsocketAuthentication: - return e.API.AuthenticatedWebsocketSupport + return b.API.AuthenticatedWebsocketSupport } return false } // GetName is a method that returns the name of the exchange base -func (e *Base) GetName() string { - return e.Name +func (b *Base) GetName() string { + return b.Name } // GetEnabledFeatures returns the exchanges enabled features -func (e *Base) GetEnabledFeatures() FeaturesEnabled { - return e.Features.Enabled +func (b *Base) GetEnabledFeatures() FeaturesEnabled { + return b.Features.Enabled } // GetSupportedFeatures returns the exchanges supported features -func (e *Base) GetSupportedFeatures() FeaturesSupported { - return e.Features.Supports +func (b *Base) GetSupportedFeatures() FeaturesSupported { + return b.Features.Supports } // GetPairFormat returns the pair format based on the exchange and // asset type -func (e *Base) GetPairFormat(assetType asset.Item, requestFormat bool) (currency.PairFormat, error) { - if e.CurrencyPairs.UseGlobalFormat { +func (b *Base) GetPairFormat(assetType asset.Item, requestFormat bool) (currency.PairFormat, error) { + if b.CurrencyPairs.UseGlobalFormat { if requestFormat { - if e.CurrencyPairs.RequestFormat == nil { + if b.CurrencyPairs.RequestFormat == nil { return currency.PairFormat{}, errors.New("global request format is nil") } - return *e.CurrencyPairs.RequestFormat, nil + return *b.CurrencyPairs.RequestFormat, nil } - if e.CurrencyPairs.ConfigFormat == nil { + if b.CurrencyPairs.ConfigFormat == nil { return currency.PairFormat{}, errors.New("global config format is nil") } - return *e.CurrencyPairs.ConfigFormat, nil + return *b.CurrencyPairs.ConfigFormat, nil } - ps, err := e.CurrencyPairs.Get(assetType) + ps, err := b.CurrencyPairs.Get(assetType) if err != nil { return currency.PairFormat{}, err } @@ -390,16 +390,16 @@ func (e *Base) GetPairFormat(assetType asset.Item, requestFormat bool) (currency // GetEnabledPairs is a method that returns the enabled currency pairs of // the exchange by asset type, if the asset type is disabled this will return no // enabled pairs -func (e *Base) GetEnabledPairs(a asset.Item) (currency.Pairs, error) { - err := e.CurrencyPairs.IsAssetEnabled(a) +func (b *Base) GetEnabledPairs(a asset.Item) (currency.Pairs, error) { + err := b.CurrencyPairs.IsAssetEnabled(a) if err != nil { return nil, nil } - format, err := e.GetPairFormat(a, false) + format, err := b.GetPairFormat(a, false) if err != nil { return nil, err } - enabledpairs, err := e.CurrencyPairs.GetPairs(a, true) + enabledpairs, err := b.CurrencyPairs.GetPairs(a, true) if err != nil { return nil, err } @@ -411,16 +411,16 @@ func (e *Base) GetEnabledPairs(a asset.Item) (currency.Pairs, error) { // GetRequestFormattedPairAndAssetType is a method that returns the enabled currency pair of // along with its asset type. Only use when there is no chance of the same name crossing over -func (e *Base) GetRequestFormattedPairAndAssetType(p string) (currency.Pair, asset.Item, error) { - assetTypes := e.GetAssetTypes() +func (b *Base) GetRequestFormattedPairAndAssetType(p string) (currency.Pair, asset.Item, error) { + assetTypes := b.GetAssetTypes() var response currency.Pair for i := range assetTypes { - format, err := e.GetPairFormat(assetTypes[i], true) + format, err := b.GetPairFormat(assetTypes[i], true) if err != nil { return response, assetTypes[i], err } - pairs, err := e.CurrencyPairs.GetPairs(assetTypes[i], true) + pairs, err := b.CurrencyPairs.GetPairs(assetTypes[i], true) if err != nil { return response, assetTypes[i], err } @@ -437,12 +437,12 @@ func (e *Base) GetRequestFormattedPairAndAssetType(p string) (currency.Pair, ass // GetAvailablePairs is a method that returns the available currency pairs // of the exchange by asset type -func (e *Base) GetAvailablePairs(assetType asset.Item) (currency.Pairs, error) { - format, err := e.GetPairFormat(assetType, false) +func (b *Base) GetAvailablePairs(assetType asset.Item) (currency.Pairs, error) { + format, err := b.GetPairFormat(assetType, false) if err != nil { return nil, err } - pairs, err := e.CurrencyPairs.GetPairs(assetType, false) + pairs, err := b.CurrencyPairs.GetPairs(assetType, false) if err != nil { return nil, err } @@ -451,9 +451,9 @@ func (e *Base) GetAvailablePairs(assetType asset.Item) (currency.Pairs, error) { // SupportsPair returns true or not whether a currency pair exists in the // exchange available currencies or not -func (e *Base) SupportsPair(p currency.Pair, enabledPairs bool, assetType asset.Item) error { +func (b *Base) SupportsPair(p currency.Pair, enabledPairs bool, assetType asset.Item) error { if enabledPairs { - pairs, err := e.GetEnabledPairs(assetType) + pairs, err := b.GetEnabledPairs(assetType) if err != nil { return err } @@ -463,7 +463,7 @@ func (e *Base) SupportsPair(p currency.Pair, enabledPairs bool, assetType asset. return errors.New("pair not supported") } - avail, err := e.GetAvailablePairs(assetType) + avail, err := b.GetAvailablePairs(assetType) if err != nil { return err } @@ -475,15 +475,15 @@ func (e *Base) SupportsPair(p currency.Pair, enabledPairs bool, assetType asset. // FormatExchangeCurrencies returns a string containing // the exchanges formatted currency pairs -func (e *Base) FormatExchangeCurrencies(pairs []currency.Pair, assetType asset.Item) (string, error) { +func (b *Base) FormatExchangeCurrencies(pairs []currency.Pair, assetType asset.Item) (string, error) { var currencyItems strings.Builder - pairFmt, err := e.GetPairFormat(assetType, true) + pairFmt, err := b.GetPairFormat(assetType, true) if err != nil { return "", err } for x := range pairs { - format, err := e.FormatExchangeCurrency(pairs[x], assetType) + format, err := b.FormatExchangeCurrency(pairs[x], assetType) if err != nil { return "", err } @@ -502,8 +502,8 @@ func (e *Base) FormatExchangeCurrencies(pairs []currency.Pair, assetType asset.I // FormatExchangeCurrency is a method that formats and returns a currency pair // based on the user currency display preferences -func (e *Base) FormatExchangeCurrency(p currency.Pair, assetType asset.Item) (currency.Pair, error) { - pairFmt, err := e.GetPairFormat(assetType, true) +func (b *Base) FormatExchangeCurrency(p currency.Pair, assetType asset.Item) (currency.Pair, error) { + pairFmt, err := b.GetPairFormat(assetType, true) if err != nil { return currency.Pair{}, err } @@ -511,47 +511,47 @@ func (e *Base) FormatExchangeCurrency(p currency.Pair, assetType asset.Item) (cu } // SetEnabled is a method that sets if the exchange is enabled -func (e *Base) SetEnabled(enabled bool) { - e.Enabled = enabled +func (b *Base) SetEnabled(enabled bool) { + b.Enabled = enabled } // IsEnabled is a method that returns if the current exchange is enabled -func (e *Base) IsEnabled() bool { - return e.Enabled +func (b *Base) IsEnabled() bool { + return b.Enabled } // SetAPIKeys is a method that sets the current API keys for the exchange -func (e *Base) SetAPIKeys(apiKey, apiSecret, clientID string) { - e.API.Credentials.Key = apiKey - e.API.Credentials.ClientID = clientID +func (b *Base) SetAPIKeys(apiKey, apiSecret, clientID string) { + b.API.Credentials.Key = apiKey + b.API.Credentials.ClientID = clientID - if e.API.CredentialsValidator.RequiresBase64DecodeSecret { + if b.API.CredentialsValidator.RequiresBase64DecodeSecret { result, err := crypto.Base64Decode(apiSecret) if err != nil { - e.API.AuthenticatedSupport = false - e.API.AuthenticatedWebsocketSupport = false + b.API.AuthenticatedSupport = false + b.API.AuthenticatedWebsocketSupport = false log.Warnf(log.ExchangeSys, warningBase64DecryptSecretKeyFailed, - e.Name) + b.Name) return } - e.API.Credentials.Secret = string(result) + b.API.Credentials.Secret = string(result) } else { - e.API.Credentials.Secret = apiSecret + b.API.Credentials.Secret = apiSecret } } // SetupDefaults sets the exchange settings based on the supplied config -func (e *Base) SetupDefaults(exch *config.ExchangeConfig) error { - e.Enabled = true - e.LoadedByConfig = true - e.Config = exch - e.Verbose = exch.Verbose +func (b *Base) SetupDefaults(exch *config.ExchangeConfig) error { + b.Enabled = true + b.LoadedByConfig = true + b.Config = exch + b.Verbose = exch.Verbose - e.API.AuthenticatedSupport = exch.API.AuthenticatedSupport - e.API.AuthenticatedWebsocketSupport = exch.API.AuthenticatedWebsocketSupport - if e.API.AuthenticatedSupport || e.API.AuthenticatedWebsocketSupport { - e.SetAPIKeys(exch.API.Credentials.Key, + b.API.AuthenticatedSupport = exch.API.AuthenticatedSupport + b.API.AuthenticatedWebsocketSupport = exch.API.AuthenticatedWebsocketSupport + if b.API.AuthenticatedSupport || b.API.AuthenticatedWebsocketSupport { + b.SetAPIKeys(exch.API.Credentials.Key, exch.API.Credentials.Secret, exch.API.Credentials.ClientID) } @@ -560,7 +560,7 @@ func (e *Base) SetupDefaults(exch *config.ExchangeConfig) error { exch.HTTPTimeout = DefaultHTTPTimeout } - err := e.SetHTTPClientTimeout(exch.HTTPTimeout) + err := b.SetHTTPClientTimeout(exch.HTTPTimeout) if err != nil { return err } @@ -569,109 +569,115 @@ func (e *Base) SetupDefaults(exch *config.ExchangeConfig) error { exch.CurrencyPairs = new(currency.PairsManager) } - e.HTTPDebugging = exch.HTTPDebugging - e.SetHTTPClientUserAgent(exch.HTTPUserAgent) - e.SetCurrencyPairFormat() + b.HTTPDebugging = exch.HTTPDebugging + b.SetHTTPClientUserAgent(exch.HTTPUserAgent) + b.SetCurrencyPairFormat() - err = e.SetConfigPairs() + err = b.SetConfigPairs() if err != nil { return err } - e.SetFeatureDefaults() + b.SetFeatureDefaults() - if e.API.Endpoints == nil { - e.API.Endpoints = e.NewEndpoints() + if b.API.Endpoints == nil { + b.API.Endpoints = b.NewEndpoints() } - err = e.SetAPIURL() + err = b.SetAPIURL() if err != nil { return err } - e.SetAPICredentialDefaults() + b.SetAPICredentialDefaults() - err = e.SetClientProxyAddress(exch.ProxyAddress) + err = b.SetClientProxyAddress(exch.ProxyAddress) if err != nil { return err } - e.BaseCurrencies = exch.BaseCurrencies - e.OrderbookVerificationBypass = exch.OrderbookConfig.VerificationBypass + b.BaseCurrencies = exch.BaseCurrencies + + if exch.OrderbookConfig.VerificationBypass { + log.Warnf(log.ExchangeSys, + "%s orderbook verification has been bypassed via config.", + b.Name) + } + b.CanVerifyOrderbook = !exch.OrderbookConfig.VerificationBypass return nil } // AllowAuthenticatedRequest checks to see if the required fields have been set // before sending an authenticated API request -func (e *Base) AllowAuthenticatedRequest() bool { - if e.SkipAuthCheck { +func (b *Base) AllowAuthenticatedRequest() bool { + if b.SkipAuthCheck { return true } // Individual package usage, allow request if API credentials are valid a // and without needing to set AuthenticatedSupport to true - if !e.LoadedByConfig { - return e.ValidateAPICredentials() + if !b.LoadedByConfig { + return b.ValidateAPICredentials() } // Bot usage, AuthenticatedSupport can be disabled by user if desired, so // don't allow authenticated requests. - if !e.API.AuthenticatedSupport && !e.API.AuthenticatedWebsocketSupport { + if !b.API.AuthenticatedSupport && !b.API.AuthenticatedWebsocketSupport { return false } // Check to see if the user has enabled AuthenticatedSupport, but has // invalid API credentials set and loaded by config - return e.ValidateAPICredentials() + return b.ValidateAPICredentials() } // ValidateAPICredentials validates the exchanges API credentials -func (e *Base) ValidateAPICredentials() bool { - if e.API.CredentialsValidator.RequiresKey { - if e.API.Credentials.Key == "" || - e.API.Credentials.Key == config.DefaultAPIKey { +func (b *Base) ValidateAPICredentials() bool { + if b.API.CredentialsValidator.RequiresKey { + if b.API.Credentials.Key == "" || + b.API.Credentials.Key == config.DefaultAPIKey { log.Warnf(log.ExchangeSys, "exchange %s requires API key but default/empty one set", - e.Name) + b.Name) return false } } - if e.API.CredentialsValidator.RequiresSecret { - if e.API.Credentials.Secret == "" || - e.API.Credentials.Secret == config.DefaultAPISecret { + if b.API.CredentialsValidator.RequiresSecret { + if b.API.Credentials.Secret == "" || + b.API.Credentials.Secret == config.DefaultAPISecret { log.Warnf(log.ExchangeSys, "exchange %s requires API secret but default/empty one set", - e.Name) + b.Name) return false } } - if e.API.CredentialsValidator.RequiresPEM { - if e.API.Credentials.PEMKey == "" || - strings.Contains(e.API.Credentials.PEMKey, "JUSTADUMMY") { + if b.API.CredentialsValidator.RequiresPEM { + if b.API.Credentials.PEMKey == "" || + strings.Contains(b.API.Credentials.PEMKey, "JUSTADUMMY") { log.Warnf(log.ExchangeSys, "exchange %s requires API PEM key but default/empty one set", - e.Name) + b.Name) return false } } - if e.API.CredentialsValidator.RequiresClientID { - if e.API.Credentials.ClientID == "" || - e.API.Credentials.ClientID == config.DefaultAPIClientID { + if b.API.CredentialsValidator.RequiresClientID { + if b.API.Credentials.ClientID == "" || + b.API.Credentials.ClientID == config.DefaultAPIClientID { log.Warnf(log.ExchangeSys, "exchange %s requires API ClientID but default/empty one set", - e.Name) + b.Name) return false } } - if e.API.CredentialsValidator.RequiresBase64DecodeSecret && !e.LoadedByConfig { - _, err := crypto.Base64Decode(e.API.Credentials.Secret) + if b.API.CredentialsValidator.RequiresBase64DecodeSecret && !b.LoadedByConfig { + _, err := crypto.Base64Decode(b.API.Credentials.Secret) if err != nil { log.Warnf(log.ExchangeSys, "exchange %s API secret base64 decode failed: %s", - e.Name, err) + b.Name, err) return false } } @@ -680,12 +686,12 @@ func (e *Base) ValidateAPICredentials() bool { // SetPairs sets the exchange currency pairs for either enabledPairs or // availablePairs -func (e *Base) SetPairs(pairs currency.Pairs, assetType asset.Item, enabled bool) error { +func (b *Base) SetPairs(pairs currency.Pairs, assetType asset.Item, enabled bool) error { if len(pairs) == 0 { - return fmt.Errorf("%s SetPairs error - pairs is empty", e.Name) + return fmt.Errorf("%s SetPairs error - pairs is empty", b.Name) } - pairFmt, err := e.GetPairFormat(assetType, false) + pairFmt, err := b.GetPairFormat(assetType, false) if err != nil { return err } @@ -696,14 +702,14 @@ func (e *Base) SetPairs(pairs currency.Pairs, assetType asset.Item, enabled bool pairFmt.Uppercase)) } - e.CurrencyPairs.StorePairs(assetType, newPairs, enabled) - e.Config.CurrencyPairs.StorePairs(assetType, newPairs, enabled) + b.CurrencyPairs.StorePairs(assetType, newPairs, enabled) + b.Config.CurrencyPairs.StorePairs(assetType, newPairs, enabled) return nil } // UpdatePairs updates the exchange currency pairs for either enabledPairs or // availablePairs -func (e *Base) UpdatePairs(exchangeProducts currency.Pairs, assetType asset.Item, enabled, force bool) error { +func (b *Base) UpdatePairs(exchangeProducts currency.Pairs, assetType asset.Item, enabled, force bool) error { exchangeProducts = exchangeProducts.Upper() var products currency.Pairs for x := range exchangeProducts { @@ -714,7 +720,7 @@ func (e *Base) UpdatePairs(exchangeProducts currency.Pairs, assetType asset.Item } var updateType string - targetPairs, err := e.CurrencyPairs.GetPairs(assetType, enabled) + targetPairs, err := b.CurrencyPairs.GetPairs(assetType, enabled) if err != nil { return err } @@ -730,14 +736,14 @@ func (e *Base) UpdatePairs(exchangeProducts currency.Pairs, assetType asset.Item if force { log.Debugf(log.ExchangeSys, "%s forced update of %s [%v] pairs.", - e.Name, + b.Name, updateType, strings.ToUpper(assetType.String())) } else { if len(newPairs) > 0 { log.Debugf(log.ExchangeSys, "%s Updating %s pairs [%v] - Added: %s.\n", - e.Name, + b.Name, updateType, strings.ToUpper(assetType.String()), newPairs) @@ -745,20 +751,20 @@ func (e *Base) UpdatePairs(exchangeProducts currency.Pairs, assetType asset.Item if len(removedPairs) > 0 { log.Debugf(log.ExchangeSys, "%s Updating %s pairs [%v] - Removed: %s.\n", - e.Name, + b.Name, updateType, strings.ToUpper(assetType.String()), removedPairs) } } - e.Config.CurrencyPairs.StorePairs(assetType, products, enabled) - e.CurrencyPairs.StorePairs(assetType, products, enabled) + b.Config.CurrencyPairs.StorePairs(assetType, products, enabled) + b.CurrencyPairs.StorePairs(assetType, products, enabled) if !enabled { // If available pairs are changed we will remove currency pair items // that are still included in the enabled pairs list. - enabledPairs, err := e.CurrencyPairs.GetPairs(assetType, true) + enabledPairs, err := b.CurrencyPairs.GetPairs(assetType, true) if err == nil { return nil } @@ -770,11 +776,11 @@ func (e *Base) UpdatePairs(exchangeProducts currency.Pairs, assetType asset.Item if len(remove) > 0 { log.Debugf(log.ExchangeSys, "%s Checked and updated enabled pairs [%v] - Removed: %s.\n", - e.Name, + b.Name, strings.ToUpper(assetType.String()), remove) - e.Config.CurrencyPairs.StorePairs(assetType, enabledPairs, true) - e.CurrencyPairs.StorePairs(assetType, enabledPairs, true) + b.Config.CurrencyPairs.StorePairs(assetType, enabledPairs, true) + b.CurrencyPairs.StorePairs(assetType, enabledPairs, true) } } } @@ -782,7 +788,7 @@ func (e *Base) UpdatePairs(exchangeProducts currency.Pairs, assetType asset.Item } // SetAPIURL sets configuration API URL for an exchange -func (e *Base) SetAPIURL() error { +func (b *Base) SetAPIURL() error { checkInsecureEndpoint := func(endpoint string) { if strings.Contains(endpoint, "https") || strings.Contains(endpoint, "wss") { return @@ -791,74 +797,74 @@ func (e *Base) SetAPIURL() error { "%s is using HTTP instead of HTTPS or WS instead of WSS [%s] for API functionality, an"+ " attacker could eavesdrop on this connection. Use at your"+ " own risk.", - e.Name, endpoint) + b.Name, endpoint) } var err error - if e.Config.API.OldEndPoints != nil { - if e.Config.API.OldEndPoints.URL != "" && e.Config.API.OldEndPoints.URL != config.APIURLNonDefaultMessage { - err = e.API.Endpoints.SetRunning(RestSpot.String(), e.Config.API.OldEndPoints.URL) + if b.Config.API.OldEndPoints != nil { + if b.Config.API.OldEndPoints.URL != "" && b.Config.API.OldEndPoints.URL != config.APIURLNonDefaultMessage { + err = b.API.Endpoints.SetRunning(RestSpot.String(), b.Config.API.OldEndPoints.URL) if err != nil { return err } - checkInsecureEndpoint(e.Config.API.OldEndPoints.URL) + checkInsecureEndpoint(b.Config.API.OldEndPoints.URL) } - if e.Config.API.OldEndPoints.URLSecondary != "" && e.Config.API.OldEndPoints.URLSecondary != config.APIURLNonDefaultMessage { - err = e.API.Endpoints.SetRunning(RestSpotSupplementary.String(), e.Config.API.OldEndPoints.URLSecondary) + if b.Config.API.OldEndPoints.URLSecondary != "" && b.Config.API.OldEndPoints.URLSecondary != config.APIURLNonDefaultMessage { + err = b.API.Endpoints.SetRunning(RestSpotSupplementary.String(), b.Config.API.OldEndPoints.URLSecondary) if err != nil { return err } - checkInsecureEndpoint(e.Config.API.OldEndPoints.URLSecondary) + checkInsecureEndpoint(b.Config.API.OldEndPoints.URLSecondary) } - if e.Config.API.OldEndPoints.WebsocketURL != "" && e.Config.API.OldEndPoints.WebsocketURL != config.WebsocketURLNonDefaultMessage { - err = e.API.Endpoints.SetRunning(WebsocketSpot.String(), e.Config.API.OldEndPoints.WebsocketURL) + if b.Config.API.OldEndPoints.WebsocketURL != "" && b.Config.API.OldEndPoints.WebsocketURL != config.WebsocketURLNonDefaultMessage { + err = b.API.Endpoints.SetRunning(WebsocketSpot.String(), b.Config.API.OldEndPoints.WebsocketURL) if err != nil { return err } - checkInsecureEndpoint(e.Config.API.OldEndPoints.WebsocketURL) + checkInsecureEndpoint(b.Config.API.OldEndPoints.WebsocketURL) } - e.Config.API.OldEndPoints = nil - } else if e.Config.API.Endpoints != nil { - for key, val := range e.Config.API.Endpoints { + b.Config.API.OldEndPoints = nil + } else if b.Config.API.Endpoints != nil { + for key, val := range b.Config.API.Endpoints { if val == "" || val == config.APIURLNonDefaultMessage || val == config.WebsocketURLNonDefaultMessage { continue } checkInsecureEndpoint(val) - err = e.API.Endpoints.SetRunning(key, val) + err = b.API.Endpoints.SetRunning(key, val) if err != nil { return err } } } - runningMap := e.API.Endpoints.GetURLMap() - e.Config.API.Endpoints = runningMap + runningMap := b.API.Endpoints.GetURLMap() + b.Config.API.Endpoints = runningMap return nil } // SupportsREST returns whether or not the exchange supports // REST -func (e *Base) SupportsREST() bool { - return e.Features.Supports.REST +func (b *Base) SupportsREST() bool { + return b.Features.Supports.REST } // GetWithdrawPermissions passes through the exchange's withdraw permissions -func (e *Base) GetWithdrawPermissions() uint32 { - return e.Features.Supports.WithdrawPermissions +func (b *Base) GetWithdrawPermissions() uint32 { + return b.Features.Supports.WithdrawPermissions } // SupportsWithdrawPermissions compares the supplied permissions with the exchange's to verify they're supported -func (e *Base) SupportsWithdrawPermissions(permissions uint32) bool { - exchangePermissions := e.GetWithdrawPermissions() +func (b *Base) SupportsWithdrawPermissions(permissions uint32) bool { + exchangePermissions := b.GetWithdrawPermissions() return permissions&exchangePermissions == permissions } // FormatWithdrawPermissions will return each of the exchange's compatible withdrawal methods in readable form -func (e *Base) FormatWithdrawPermissions() string { +func (b *Base) FormatWithdrawPermissions() string { var services []string for i := 0; i < 32; i++ { var check uint32 = 1 << uint32(i) - if e.GetWithdrawPermissions()&check != 0 { + if b.GetWithdrawPermissions()&check != 0 { switch check { case AutoWithdrawCrypto: services = append(services, AutoWithdrawCryptoText) @@ -912,29 +918,29 @@ func (e *Base) FormatWithdrawPermissions() string { // SupportsAsset whether or not the supplied asset is supported // by the exchange -func (e *Base) SupportsAsset(a asset.Item) bool { - _, ok := e.CurrencyPairs.Pairs[a] +func (b *Base) SupportsAsset(a asset.Item) bool { + _, ok := b.CurrencyPairs.Pairs[a] return ok } // PrintEnabledPairs prints the exchanges enabled asset pairs -func (e *Base) PrintEnabledPairs() { - for k, v := range e.CurrencyPairs.Pairs { +func (b *Base) PrintEnabledPairs() { + for k, v := range b.CurrencyPairs.Pairs { log.Infof(log.ExchangeSys, "%s Asset type %v:\n\t Enabled pairs: %v", - e.Name, strings.ToUpper(k.String()), v.Enabled) + b.Name, strings.ToUpper(k.String()), v.Enabled) } } // GetBase returns the exchange base -func (e *Base) GetBase() *Base { return e } +func (b *Base) GetBase() *Base { return b } // CheckTransientError catches transient errors and returns nil if found, used // for validation of API credentials -func (e *Base) CheckTransientError(err error) error { +func (b *Base) CheckTransientError(err error) error { if _, ok := err.(net.Error); ok { log.Warnf(log.ExchangeSys, "%s net error captured, will not disable authentication %s", - e.Name, + b.Name, err) return nil } @@ -942,20 +948,20 @@ func (e *Base) CheckTransientError(err error) error { } // DisableRateLimiter disables the rate limiting system for the exchange -func (e *Base) DisableRateLimiter() error { - return e.Requester.DisableRateLimiter() +func (b *Base) DisableRateLimiter() error { + return b.Requester.DisableRateLimiter() } // EnableRateLimiter enables the rate limiting system for the exchange -func (e *Base) EnableRateLimiter() error { - return e.Requester.EnableRateLimiter() +func (b *Base) EnableRateLimiter() error { + return b.Requester.EnableRateLimiter() } // StoreAssetPairFormat initialises and stores a defined asset format -func (e *Base) StoreAssetPairFormat(a asset.Item, f currency.PairStore) error { +func (b *Base) StoreAssetPairFormat(a asset.Item, f currency.PairStore) error { if a.String() == "" { return fmt.Errorf("%s cannot add to pairs manager, no asset provided", - e.Name) + b.Name) } if f.AssetEnabled == nil { @@ -964,152 +970,152 @@ func (e *Base) StoreAssetPairFormat(a asset.Item, f currency.PairStore) error { if f.RequestFormat == nil { return fmt.Errorf("%s cannot add to pairs manager, request pair format not provided", - e.Name) + b.Name) } if f.ConfigFormat == nil { return fmt.Errorf("%s cannot add to pairs manager, config pair format not provided", - e.Name) + b.Name) } - if e.CurrencyPairs.Pairs == nil { - e.CurrencyPairs.Pairs = make(map[asset.Item]*currency.PairStore) + if b.CurrencyPairs.Pairs == nil { + b.CurrencyPairs.Pairs = make(map[asset.Item]*currency.PairStore) } - e.CurrencyPairs.Pairs[a] = &f + b.CurrencyPairs.Pairs[a] = &f return nil } // SetGlobalPairsManager sets defined asset and pairs management system with // with global formatting -func (e *Base) SetGlobalPairsManager(request, config *currency.PairFormat, assets ...asset.Item) error { +func (b *Base) SetGlobalPairsManager(request, config *currency.PairFormat, assets ...asset.Item) error { if request == nil { return fmt.Errorf("%s cannot set pairs manager, request pair format not provided", - e.Name) + b.Name) } if config == nil { return fmt.Errorf("%s cannot set pairs manager, config pair format not provided", - e.Name) + b.Name) } if len(assets) == 0 { return fmt.Errorf("%s cannot set pairs manager, no assets provided", - e.Name) + b.Name) } - e.CurrencyPairs.UseGlobalFormat = true - e.CurrencyPairs.RequestFormat = request - e.CurrencyPairs.ConfigFormat = config + b.CurrencyPairs.UseGlobalFormat = true + b.CurrencyPairs.RequestFormat = request + b.CurrencyPairs.ConfigFormat = config - if e.CurrencyPairs.Pairs != nil { + if b.CurrencyPairs.Pairs != nil { return fmt.Errorf("%s cannot set pairs manager, pairs already set", - e.Name) + b.Name) } - e.CurrencyPairs.Pairs = make(map[asset.Item]*currency.PairStore) + b.CurrencyPairs.Pairs = make(map[asset.Item]*currency.PairStore) for i := range assets { if assets[i].String() == "" { - e.CurrencyPairs.Pairs = nil + b.CurrencyPairs.Pairs = nil return fmt.Errorf("%s cannot set pairs manager, asset is empty string", - e.Name) + b.Name) } - e.CurrencyPairs.Pairs[assets[i]] = new(currency.PairStore) - e.CurrencyPairs.Pairs[assets[i]].ConfigFormat = config - e.CurrencyPairs.Pairs[assets[i]].RequestFormat = request + b.CurrencyPairs.Pairs[assets[i]] = new(currency.PairStore) + b.CurrencyPairs.Pairs[assets[i]].ConfigFormat = config + b.CurrencyPairs.Pairs[assets[i]].RequestFormat = request } return nil } // GetWebsocket returns a pointer to the exchange websocket -func (e *Base) GetWebsocket() (*stream.Websocket, error) { - if e.Websocket == nil { +func (b *Base) GetWebsocket() (*stream.Websocket, error) { + if b.Websocket == nil { return nil, common.ErrFunctionNotSupported } - return e.Websocket, nil + return b.Websocket, nil } // SupportsWebsocket returns whether or not the exchange supports // websocket -func (e *Base) SupportsWebsocket() bool { - return e.Features.Supports.Websocket +func (b *Base) SupportsWebsocket() bool { + return b.Features.Supports.Websocket } // IsWebsocketEnabled returns whether or not the exchange has its // websocket client enabled -func (e *Base) IsWebsocketEnabled() bool { - if e.Websocket == nil { +func (b *Base) IsWebsocketEnabled() bool { + if b.Websocket == nil { return false } - return e.Websocket.IsEnabled() + return b.Websocket.IsEnabled() } // FlushWebsocketChannels refreshes websocket channel subscriptions based on // websocket features. Used in the event of a pair/asset or subscription change. -func (e *Base) FlushWebsocketChannels() error { - if e.Websocket == nil { +func (b *Base) FlushWebsocketChannels() error { + if b.Websocket == nil { return nil } - return e.Websocket.FlushChannels() + return b.Websocket.FlushChannels() } // SubscribeToWebsocketChannels appends to ChannelsToSubscribe // which lets websocket.manageSubscriptions handle subscribing -func (e *Base) SubscribeToWebsocketChannels(channels []stream.ChannelSubscription) error { - if e.Websocket == nil { +func (b *Base) SubscribeToWebsocketChannels(channels []stream.ChannelSubscription) error { + if b.Websocket == nil { return common.ErrFunctionNotSupported } - return e.Websocket.SubscribeToChannels(channels) + return b.Websocket.SubscribeToChannels(channels) } // UnsubscribeToWebsocketChannels removes from ChannelsToSubscribe // which lets websocket.manageSubscriptions handle unsubscribing -func (e *Base) UnsubscribeToWebsocketChannels(channels []stream.ChannelSubscription) error { - if e.Websocket == nil { +func (b *Base) UnsubscribeToWebsocketChannels(channels []stream.ChannelSubscription) error { + if b.Websocket == nil { return common.ErrFunctionNotSupported } - return e.Websocket.UnsubscribeChannels(channels) + return b.Websocket.UnsubscribeChannels(channels) } // GetSubscriptions returns a copied list of subscriptions -func (e *Base) GetSubscriptions() ([]stream.ChannelSubscription, error) { - if e.Websocket == nil { +func (b *Base) GetSubscriptions() ([]stream.ChannelSubscription, error) { + if b.Websocket == nil { return nil, common.ErrFunctionNotSupported } - return e.Websocket.GetSubscriptions(), nil + return b.Websocket.GetSubscriptions(), nil } // AuthenticateWebsocket sends an authentication message to the websocket -func (e *Base) AuthenticateWebsocket() error { +func (b *Base) AuthenticateWebsocket() error { return common.ErrFunctionNotSupported } // KlineIntervalEnabled returns if requested interval is enabled on exchange -func (e *Base) klineIntervalEnabled(in kline.Interval) bool { - return e.Features.Enabled.Kline.Intervals[in.Word()] +func (b *Base) klineIntervalEnabled(in kline.Interval) bool { + return b.Features.Enabled.Kline.Intervals[in.Word()] } // FormatExchangeKlineInterval returns Interval to string // Exchanges can override this if they require custom formatting -func (e *Base) FormatExchangeKlineInterval(in kline.Interval) string { +func (b *Base) FormatExchangeKlineInterval(in kline.Interval) string { return strconv.FormatFloat(in.Duration().Seconds(), 'f', 0, 64) } // ValidateKline confirms that the requested pair, asset & interval are supported and/or enabled by the requested exchange -func (e *Base) ValidateKline(pair currency.Pair, a asset.Item, interval kline.Interval) error { +func (b *Base) ValidateKline(pair currency.Pair, a asset.Item, interval kline.Interval) error { var errorList []string var err kline.ErrorKline - if e.CurrencyPairs.IsAssetEnabled(a) != nil { + if b.CurrencyPairs.IsAssetEnabled(a) != nil { err.Asset = a errorList = append(errorList, "asset not enabled") - } else if !e.CurrencyPairs.Pairs[a].Enabled.Contains(pair, true) { + } else if !b.CurrencyPairs.Pairs[a].Enabled.Contains(pair, true) { err.Pair = pair errorList = append(errorList, "pair not enabled") } - if !e.klineIntervalEnabled(interval) { + if !b.klineIntervalEnabled(interval) { err.Interval = interval errorList = append(errorList, "interval not supported") } @@ -1124,39 +1130,38 @@ func (e *Base) ValidateKline(pair currency.Pair, a asset.Item, interval kline.In // AddTradesToBuffer is a helper function that will only // add trades to the buffer if it is allowed -func (e *Base) AddTradesToBuffer(trades ...trade.Data) error { - if !e.IsSaveTradeDataEnabled() { +func (b *Base) AddTradesToBuffer(trades ...trade.Data) error { + if !b.IsSaveTradeDataEnabled() { return nil } - - return trade.AddTradesToBuffer(e.Name, trades...) + return trade.AddTradesToBuffer(b.Name, trades...) } // IsSaveTradeDataEnabled checks the state of // SaveTradeData in a concurrent-friendly manner -func (e *Base) IsSaveTradeDataEnabled() bool { - e.settingsMutex.RLock() - isEnabled := e.Features.Enabled.SaveTradeData - e.settingsMutex.RUnlock() +func (b *Base) IsSaveTradeDataEnabled() bool { + b.settingsMutex.RLock() + isEnabled := b.Features.Enabled.SaveTradeData + b.settingsMutex.RUnlock() return isEnabled } // SetSaveTradeDataStatus locks and sets the status of // the config and the exchange's setting for SaveTradeData -func (e *Base) SetSaveTradeDataStatus(enabled bool) { - e.settingsMutex.Lock() - defer e.settingsMutex.Unlock() - e.Features.Enabled.SaveTradeData = enabled - e.Config.Features.Enabled.SaveTradeData = enabled - if e.Verbose { - log.Debugf(log.Trade, "Set %v 'SaveTradeData' to %v", e.Name, enabled) +func (b *Base) SetSaveTradeDataStatus(enabled bool) { + b.settingsMutex.Lock() + defer b.settingsMutex.Unlock() + b.Features.Enabled.SaveTradeData = enabled + b.Config.Features.Enabled.SaveTradeData = enabled + if b.Verbose { + log.Debugf(log.Trade, "Set %v 'SaveTradeData' to %v", b.Name, enabled) } } // NewEndpoints declares default and running URLs maps -func (e *Base) NewEndpoints() *Endpoints { +func (b *Base) NewEndpoints() *Endpoints { return &Endpoints{ - Exchange: e.Name, + Exchange: b.Name, defaults: make(map[string]string), } } @@ -1182,7 +1187,11 @@ func (e *Endpoints) SetRunning(key, val string) error { } _, err = url.ParseRequestURI(val) if err != nil { - log.Warnf(log.ExchangeSys, "Could not set custom URL for %s to %s for exchange %s. invalid URI for request.", key, val, e.Exchange) + log.Warnf(log.ExchangeSys, + "Could not set custom URL for %s to %s for exchange %s. invalid URI for request.", + key, + val, + e.Exchange) return nil } e.defaults[key] = val @@ -1221,8 +1230,8 @@ func (e *Endpoints) GetURLMap() map[string]string { } // FormatSymbol formats the given pair to a string suitable for exchange API requests -func (e *Base) FormatSymbol(pair currency.Pair, assetType asset.Item) (string, error) { - pairFmt, err := e.GetPairFormat(assetType, true) +func (b *Base) FormatSymbol(pair currency.Pair, assetType asset.Item) (string, error) { + pairFmt, err := b.GetPairFormat(assetType, true) if err != nil { return pair.String(), err } @@ -1263,6 +1272,33 @@ func (u URL) String() string { } // UpdateOrderExecutionLimits updates order execution limits this is overridable -func (e *Base) UpdateOrderExecutionLimits(a asset.Item) error { +func (b *Base) UpdateOrderExecutionLimits(a asset.Item) error { return common.ErrNotYetImplemented } + +// DisableAssetWebsocketSupport disables websocket functionality for the +// supplied asset item. In the case that websocket functionality has not yet +// been implemented for that specific asset type. This is a base method to +// check availability of asset type. +func (b *Base) DisableAssetWebsocketSupport(aType asset.Item) error { + if !b.SupportsAsset(aType) { + return fmt.Errorf("%s %w", + aType, + asset.ErrNotSupported) + } + b.AssetWebsocketSupport.m.Lock() + if b.AssetWebsocketSupport.unsupported == nil { + b.AssetWebsocketSupport.unsupported = make(map[asset.Item]bool) + } + b.AssetWebsocketSupport.unsupported[aType] = true + b.AssetWebsocketSupport.m.Unlock() + return nil +} + +// IsAssetWebsocketSupported checks to see if the supplied asset type is +// supported by websocket. +func (a *AssetWebsocketSupport) IsAssetWebsocketSupported(aType asset.Item) bool { + a.m.RLock() + defer a.m.RUnlock() + return a.unsupported == nil || !a.unsupported[aType] +} diff --git a/exchanges/exchange_test.go b/exchanges/exchange_test.go index 2c76503d..71ac4f91 100644 --- a/exchanges/exchange_test.go +++ b/exchanges/exchange_test.go @@ -2323,3 +2323,49 @@ func TestSetRunning(t *testing.T) { t.Error(err) } } + +func TestAssetWebsocketFunctionality(t *testing.T) { + b := Base{} + if !b.IsAssetWebsocketSupported(asset.Spot) { + t.Fatal("error asset is not turned off, unexpected response") + } + + err := b.DisableAssetWebsocketSupport(asset.Spot) + if !errors.Is(err, asset.ErrNotSupported) { + t.Fatalf("expected error: %v but received: %v", asset.ErrNotSupported, err) + } + + err = b.StoreAssetPairFormat(asset.Spot, currency.PairStore{ + RequestFormat: ¤cy.PairFormat{ + Uppercase: true, + }, + ConfigFormat: ¤cy.PairFormat{ + Uppercase: true, + }, + }) + if err != nil { + log.Errorln(log.ExchangeSys, err) + } + + err = b.DisableAssetWebsocketSupport(asset.Spot) + if !errors.Is(err, nil) { + t.Fatalf("expected error: %v but received: %v", nil, err) + } + + if b.IsAssetWebsocketSupported(asset.Spot) { + t.Fatal("error asset is not turned off, unexpected response") + } + + // Edge case + b.AssetWebsocketSupport.unsupported = make(map[asset.Item]bool) + b.AssetWebsocketSupport.unsupported[asset.Spot] = true + b.AssetWebsocketSupport.unsupported[asset.Futures] = false + + if b.IsAssetWebsocketSupported(asset.Spot) { + t.Fatal("error asset is turned off, unexpected response") + } + + if !b.IsAssetWebsocketSupported(asset.Futures) { + t.Fatal("error asset is not turned off, unexpected response") + } +} diff --git a/exchanges/exchange_types.go b/exchanges/exchange_types.go index 153b6a36..dda436da 100644 --- a/exchanges/exchange_types.go +++ b/exchanges/exchange_types.go @@ -6,6 +6,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/config" "github.com/thrasher-corp/gocryptotrader/currency" + "github.com/thrasher-corp/gocryptotrader/exchanges/asset" "github.com/thrasher-corp/gocryptotrader/exchanges/kline" "github.com/thrasher-corp/gocryptotrader/exchanges/order" "github.com/thrasher-corp/gocryptotrader/exchanges/protocol" @@ -220,10 +221,15 @@ type Base struct { WebsocketOrderbookBufferLimit int64 Websocket *stream.Websocket *request.Requester - Config *config.ExchangeConfig - settingsMutex sync.RWMutex - OrderbookVerificationBypass bool + Config *config.ExchangeConfig + settingsMutex sync.RWMutex + // CanVerifyOrderbook determines if the orderbook verification can be bypassed, + // increasing potential update speed but decreasing confidence in orderbook + // integrity. + CanVerifyOrderbook bool order.ExecutionLimits + + AssetWebsocketSupport } // url lookup consts @@ -259,3 +265,11 @@ var keyURLs = []URL{RestSpot, // URL stores uint conversions type URL uint16 + +// AssetWebsocketSupport defines the availability of websocket functionality to +// the specific asset type. TODO: Deprecate as this is a temp item to address +// certain limitations quickly. +type AssetWebsocketSupport struct { + unsupported map[asset.Item]bool + m sync.RWMutex +} diff --git a/exchanges/exmo/exmo_wrapper.go b/exchanges/exmo/exmo_wrapper.go index 5379c604..df3088f4 100644 --- a/exchanges/exmo/exmo_wrapper.go +++ b/exchanges/exmo/exmo_wrapper.go @@ -240,10 +240,10 @@ func (e *EXMO) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbook // UpdateOrderbook updates and returns the orderbook for a currency pair func (e *EXMO) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { callingBook := &orderbook.Base{ - ExchangeName: e.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: e.OrderbookVerificationBypass, + Exchange: e.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: e.CanVerifyOrderbook, } enabledPairs, err := e.GetEnabledPairs(assetType) if err != nil { @@ -262,10 +262,10 @@ func (e *EXMO) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderboo for i := range enabledPairs { book := &orderbook.Base{ - ExchangeName: e.Name, - Pair: enabledPairs[i], - AssetType: assetType, - VerificationBypass: e.OrderbookVerificationBypass, + Exchange: e.Name, + Pair: enabledPairs[i], + Asset: assetType, + VerifyOrderbook: e.CanVerifyOrderbook, } curr, err := e.FormatExchangeCurrency(enabledPairs[i], assetType) diff --git a/exchanges/ftx/ftx_websocket.go b/exchanges/ftx/ftx_websocket.go index 13ee4aae..4a10e541 100644 --- a/exchanges/ftx/ftx_websocket.go +++ b/exchanges/ftx/ftx_websocket.go @@ -452,7 +452,10 @@ func (f *FTX) WsProcessUpdateOB(data *WsOrderbookData, p currency.Pair, a asset. return err } - updatedOb := f.Websocket.Orderbook.GetOrderbook(p, a) + updatedOb, err := f.Websocket.Orderbook.GetOrderbook(p, a) + if err != nil { + return err + } checksum := f.CalcUpdateOBChecksum(updatedOb) if checksum != data.Checksum { @@ -506,14 +509,13 @@ func (f *FTX) WsProcessPartialOB(data *WsOrderbookData, p currency.Pair, a asset } newOrderBook := orderbook.Base{ - Asks: asks, - Bids: bids, - AssetType: a, - LastUpdated: timestampFromFloat64(data.Time), - Pair: p, - ExchangeName: f.Name, - HasChecksumValidation: true, - VerificationBypass: f.OrderbookVerificationBypass, + Asks: asks, + Bids: bids, + Asset: a, + LastUpdated: timestampFromFloat64(data.Time), + Pair: p, + Exchange: f.Name, + VerifyOrderbook: f.CanVerifyOrderbook, } return f.Websocket.Orderbook.LoadSnapshot(&newOrderBook) } diff --git a/exchanges/ftx/ftx_wrapper.go b/exchanges/ftx/ftx_wrapper.go index 520d78ae..18c2e1f7 100644 --- a/exchanges/ftx/ftx_wrapper.go +++ b/exchanges/ftx/ftx_wrapper.go @@ -344,10 +344,10 @@ func (f *FTX) FetchOrderbook(currency currency.Pair, assetType asset.Item) (*ord // UpdateOrderbook updates and returns the orderbook for a currency pair func (f *FTX) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: f.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: f.OrderbookVerificationBypass, + Exchange: f.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: f.CanVerifyOrderbook, } formattedPair, err := f.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/gateio/gateio_websocket.go b/exchanges/gateio/gateio_websocket.go index d7c70cc4..0687d93e 100644 --- a/exchanges/gateio/gateio_websocket.go +++ b/exchanges/gateio/gateio_websocket.go @@ -350,10 +350,10 @@ func (g *Gateio) wsHandleData(respRaw []byte) error { var newOrderBook orderbook.Base newOrderBook.Asks = asks newOrderBook.Bids = bids - newOrderBook.AssetType = asset.Spot + newOrderBook.Asset = asset.Spot newOrderBook.Pair = p - newOrderBook.ExchangeName = g.Name - newOrderBook.VerificationBypass = g.OrderbookVerificationBypass + newOrderBook.Exchange = g.Name + newOrderBook.VerifyOrderbook = g.CanVerifyOrderbook err = g.Websocket.Orderbook.LoadSnapshot(&newOrderBook) if err != nil { diff --git a/exchanges/gateio/gateio_wrapper.go b/exchanges/gateio/gateio_wrapper.go index f55922a7..3cfe0aaa 100644 --- a/exchanges/gateio/gateio_wrapper.go +++ b/exchanges/gateio/gateio_wrapper.go @@ -287,10 +287,10 @@ func (g *Gateio) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbo // UpdateOrderbook updates and returns the orderbook for a currency pair func (g *Gateio) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: g.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: g.OrderbookVerificationBypass, + Exchange: g.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: g.CanVerifyOrderbook, } curr, err := g.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/gemini/gemini_websocket.go b/exchanges/gemini/gemini_websocket.go index 446e697c..be5047de 100644 --- a/exchanges/gemini/gemini_websocket.go +++ b/exchanges/gemini/gemini_websocket.go @@ -526,10 +526,10 @@ func (g *Gemini) wsProcessUpdate(result *wsL2MarketData) error { var newOrderBook orderbook.Base newOrderBook.Asks = asks newOrderBook.Bids = bids - newOrderBook.AssetType = asset.Spot + newOrderBook.Asset = asset.Spot newOrderBook.Pair = pair - newOrderBook.ExchangeName = g.Name - newOrderBook.VerificationBypass = g.OrderbookVerificationBypass + newOrderBook.Exchange = g.Name + newOrderBook.VerifyOrderbook = g.CanVerifyOrderbook err := g.Websocket.Orderbook.LoadSnapshot(&newOrderBook) if err != nil { return err diff --git a/exchanges/gemini/gemini_wrapper.go b/exchanges/gemini/gemini_wrapper.go index 22c9aa6a..4139a4fd 100644 --- a/exchanges/gemini/gemini_wrapper.go +++ b/exchanges/gemini/gemini_wrapper.go @@ -403,10 +403,10 @@ func (g *Gemini) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbo // UpdateOrderbook updates and returns the orderbook for a currency pair func (g *Gemini) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: g.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: g.OrderbookVerificationBypass, + Exchange: g.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: g.CanVerifyOrderbook, } fPair, err := g.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/hitbtc/hitbtc_websocket.go b/exchanges/hitbtc/hitbtc_websocket.go index 0a05e317..37f57eba 100644 --- a/exchanges/hitbtc/hitbtc_websocket.go +++ b/exchanges/hitbtc/hitbtc_websocket.go @@ -327,10 +327,10 @@ func (h *HitBTC) WsProcessOrderbookSnapshot(ob WsOrderbook) error { h.Websocket.DataHandler <- err return err } - newOrderBook.AssetType = asset.Spot + newOrderBook.Asset = asset.Spot newOrderBook.Pair = p - newOrderBook.ExchangeName = h.Name - newOrderBook.VerificationBypass = h.OrderbookVerificationBypass + newOrderBook.Exchange = h.Name + newOrderBook.VerifyOrderbook = h.CanVerifyOrderbook return h.Websocket.Orderbook.LoadSnapshot(&newOrderBook) } diff --git a/exchanges/hitbtc/hitbtc_wrapper.go b/exchanges/hitbtc/hitbtc_wrapper.go index b530cc27..b1d79531 100644 --- a/exchanges/hitbtc/hitbtc_wrapper.go +++ b/exchanges/hitbtc/hitbtc_wrapper.go @@ -376,10 +376,10 @@ func (h *HitBTC) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbo // UpdateOrderbook updates and returns the orderbook for a currency pair func (h *HitBTC) UpdateOrderbook(c currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: h.Name, - Pair: c, - AssetType: assetType, - VerificationBypass: h.OrderbookVerificationBypass, + Exchange: h.Name, + Pair: c, + Asset: assetType, + VerifyOrderbook: h.CanVerifyOrderbook, } fpair, err := h.FormatExchangeCurrency(c, assetType) if err != nil { diff --git a/exchanges/huobi/huobi_websocket.go b/exchanges/huobi/huobi_websocket.go index a77db319..5369f120 100644 --- a/exchanges/huobi/huobi_websocket.go +++ b/exchanges/huobi/huobi_websocket.go @@ -464,9 +464,9 @@ func (h *HUOBI) WsProcessOrderbook(update *WsDepth, symbol string) error { newOrderBook.Asks = asks newOrderBook.Bids = bids newOrderBook.Pair = p - newOrderBook.AssetType = asset.Spot - newOrderBook.ExchangeName = h.Name - newOrderBook.VerificationBypass = h.OrderbookVerificationBypass + newOrderBook.Asset = asset.Spot + newOrderBook.Exchange = h.Name + newOrderBook.VerifyOrderbook = h.CanVerifyOrderbook return h.Websocket.Orderbook.LoadSnapshot(&newOrderBook) } diff --git a/exchanges/huobi/huobi_wrapper.go b/exchanges/huobi/huobi_wrapper.go index 9c3161d0..5dc9c78d 100644 --- a/exchanges/huobi/huobi_wrapper.go +++ b/exchanges/huobi/huobi_wrapper.go @@ -522,10 +522,10 @@ func (h *HUOBI) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderboo // UpdateOrderbook updates and returns the orderbook for a currency pair func (h *HUOBI) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: h.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: h.OrderbookVerificationBypass, + Exchange: h.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: h.CanVerifyOrderbook, } var err error switch assetType { diff --git a/exchanges/interfaces.go b/exchanges/interfaces.go index 39b196d2..a956ba00 100644 --- a/exchanges/interfaces.go +++ b/exchanges/interfaces.go @@ -82,6 +82,7 @@ type IBotExchange interface { SupportsWebsocket() bool SubscribeToWebsocketChannels(channels []stream.ChannelSubscription) error UnsubscribeToWebsocketChannels(channels []stream.ChannelSubscription) error + IsAssetWebsocketSupported(aType asset.Item) bool // FlushWebsocketChannels checks and flushes subscriptions if there is a // pair,asset, url/proxy or subscription change FlushWebsocketChannels() error diff --git a/exchanges/itbit/itbit_wrapper.go b/exchanges/itbit/itbit_wrapper.go index 4269fce2..4c7aa5df 100644 --- a/exchanges/itbit/itbit_wrapper.go +++ b/exchanges/itbit/itbit_wrapper.go @@ -191,11 +191,11 @@ func (i *ItBit) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderboo // UpdateOrderbook updates and returns the orderbook for a currency pair func (i *ItBit) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: i.Name, - Pair: p, - AssetType: assetType, - NotAggregated: true, - VerificationBypass: i.OrderbookVerificationBypass, + Exchange: i.Name, + Pair: p, + Asset: assetType, + PriceDuplication: true, + VerifyOrderbook: i.CanVerifyOrderbook, } fpair, err := i.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/kraken/kraken_websocket.go b/exchanges/kraken/kraken_websocket.go index a3af907d..d768e7a4 100644 --- a/exchanges/kraken/kraken_websocket.go +++ b/exchanges/kraken/kraken_websocket.go @@ -795,9 +795,9 @@ func (k *Kraken) wsProcessOrderBook(channelData *WebsocketChannelData, data map[ // wsProcessOrderBookPartial creates a new orderbook entry for a given currency pair func (k *Kraken) wsProcessOrderBookPartial(channelData *WebsocketChannelData, askData, bidData []interface{}) error { base := orderbook.Base{ - Pair: channelData.Pair, - AssetType: asset.Spot, - VerificationBypass: k.OrderbookVerificationBypass, + Pair: channelData.Pair, + Asset: asset.Spot, + VerifyOrderbook: k.CanVerifyOrderbook, } // Kraken ob data is timestamped per price, GCT orderbook data is // timestamped per entry using the highest last update time, we can attempt @@ -851,8 +851,7 @@ func (k *Kraken) wsProcessOrderBookPartial(channelData *WebsocketChannelData, as } } base.LastUpdated = highestLastUpdate - base.ExchangeName = k.Name - base.HasChecksumValidation = true + base.Exchange = k.Name return k.Websocket.Orderbook.LoadSnapshot(&base) } @@ -995,11 +994,12 @@ func (k *Kraken) wsProcessOrderBookUpdate(channelData *WebsocketChannelData, ask return err } - book := k.Websocket.Orderbook.GetOrderbook(channelData.Pair, asset.Spot) - if book == nil { - return fmt.Errorf("cannot calculate websocket checksum: book not found for %s %s", + book, err := k.Websocket.Orderbook.GetOrderbook(channelData.Pair, asset.Spot) + if err != nil { + return fmt.Errorf("cannot calculate websocket checksum: book not found for %s %s %w", channelData.Pair, - asset.Spot) + asset.Spot, + err) } token, err := strconv.ParseInt(checksum, 10, 64) @@ -1014,12 +1014,13 @@ func validateCRC32(b *orderbook.Base, token uint32, decPrice, decAmount int) err if len(b.Asks) < 10 || len(b.Bids) < 10 { return fmt.Errorf("%s %s insufficient bid and asks to calculate checksum", b.Pair, - b.AssetType) + b.Asset) } if decPrice == 0 || decAmount == 0 { - return fmt.Errorf("%s %s trailing decimal count not calculated", b.Pair, - b.AssetType) + return fmt.Errorf("%s %s trailing decimal count not calculated", + b.Pair, + b.Asset) } var checkStr strings.Builder @@ -1040,7 +1041,7 @@ func validateCRC32(b *orderbook.Base, token uint32, decPrice, decAmount int) err if check := crc32.ChecksumIEEE([]byte(checkStr.String())); check != token { return fmt.Errorf("%s %s invalid checksum %d, expected %d", b.Pair, - b.AssetType, + b.Asset, check, token) } diff --git a/exchanges/kraken/kraken_wrapper.go b/exchanges/kraken/kraken_wrapper.go index 9fb0e8da..96db312c 100644 --- a/exchanges/kraken/kraken_wrapper.go +++ b/exchanges/kraken/kraken_wrapper.go @@ -93,6 +93,11 @@ func (k *Kraken) SetDefaults() { log.Errorln(log.ExchangeSys, err) } + err = k.DisableAssetWebsocketSupport(asset.Futures) + if err != nil { + log.Errorln(log.ExchangeSys, err) + } + k.Features = exchange.Features{ Supports: exchange.FeaturesSupported{ REST: true, @@ -497,10 +502,10 @@ func (k *Kraken) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbo // UpdateOrderbook updates and returns the orderbook for a currency pair func (k *Kraken) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: k.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: k.OrderbookVerificationBypass, + Exchange: k.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: k.CanVerifyOrderbook, } var err error switch assetType { diff --git a/exchanges/lakebtc/lakebtc_websocket.go b/exchanges/lakebtc/lakebtc_websocket.go index aadb8cb8..b4624eab 100644 --- a/exchanges/lakebtc/lakebtc_websocket.go +++ b/exchanges/lakebtc/lakebtc_websocket.go @@ -174,11 +174,11 @@ func (l *LakeBTC) processOrderbook(obUpdate, channel string) error { } book := orderbook.Base{ - Pair: p, - LastUpdated: time.Now(), - AssetType: asset.Spot, - ExchangeName: l.Name, - VerificationBypass: l.OrderbookVerificationBypass, + Pair: p, + LastUpdated: time.Now(), + Asset: asset.Spot, + Exchange: l.Name, + VerifyOrderbook: l.CanVerifyOrderbook, } for i := range update.Asks { diff --git a/exchanges/lakebtc/lakebtc_wrapper.go b/exchanges/lakebtc/lakebtc_wrapper.go index a8a12d8f..09c48329 100644 --- a/exchanges/lakebtc/lakebtc_wrapper.go +++ b/exchanges/lakebtc/lakebtc_wrapper.go @@ -279,10 +279,10 @@ func (l *LakeBTC) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderb // UpdateOrderbook updates and returns the orderbook for a currency pair func (l *LakeBTC) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: l.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: l.OrderbookVerificationBypass, + Exchange: l.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: l.CanVerifyOrderbook, } fPair, err := l.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/lbank/lbank_wrapper.go b/exchanges/lbank/lbank_wrapper.go index 1eb8ad96..0a1f366f 100644 --- a/exchanges/lbank/lbank_wrapper.go +++ b/exchanges/lbank/lbank_wrapper.go @@ -247,10 +247,10 @@ func (l *Lbank) FetchOrderbook(currency currency.Pair, assetType asset.Item) (*o // UpdateOrderbook updates and returns the orderbook for a currency pair func (l *Lbank) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: l.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: l.OrderbookVerificationBypass, + Exchange: l.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: l.CanVerifyOrderbook, } fpair, err := l.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/localbitcoins/localbitcoins_wrapper.go b/exchanges/localbitcoins/localbitcoins_wrapper.go index f0b7397a..ed162069 100644 --- a/exchanges/localbitcoins/localbitcoins_wrapper.go +++ b/exchanges/localbitcoins/localbitcoins_wrapper.go @@ -218,11 +218,12 @@ func (l *LocalBitcoins) FetchOrderbook(p currency.Pair, assetType asset.Item) (* // UpdateOrderbook updates and returns the orderbook for a currency pair func (l *LocalBitcoins) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: l.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: l.OrderbookVerificationBypass, + Exchange: l.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: l.CanVerifyOrderbook, } + orderbookNew, err := l.GetOrderbook(p.Quote.String()) if err != nil { return book, err @@ -242,7 +243,7 @@ func (l *LocalBitcoins) UpdateOrderbook(p currency.Pair, assetType asset.Item) ( }) } - book.NotAggregated = true + book.PriceDuplication = true err = book.Process() if err != nil { return book, err diff --git a/exchanges/okgroup/okgroup_websocket.go b/exchanges/okgroup/okgroup_websocket.go index bb3168cf..8e1a161a 100644 --- a/exchanges/okgroup/okgroup_websocket.go +++ b/exchanges/okgroup/okgroup_websocket.go @@ -678,14 +678,13 @@ func (o *OKGroup) WsProcessPartialOrderBook(wsEventData *WebsocketOrderBook, ins } newOrderBook := orderbook.Base{ - Asks: asks, - Bids: bids, - AssetType: a, - LastUpdated: wsEventData.Timestamp, - Pair: instrument, - ExchangeName: o.Name, - HasChecksumValidation: true, - VerificationBypass: o.OrderbookVerificationBypass, + Asks: asks, + Bids: bids, + Asset: a, + LastUpdated: wsEventData.Timestamp, + Pair: instrument, + Exchange: o.Name, + VerifyOrderbook: o.CanVerifyOrderbook, } return o.Websocket.Orderbook.LoadSnapshot(&newOrderBook) } @@ -715,7 +714,10 @@ func (o *OKGroup) WsProcessUpdateOrderbook(wsEventData *WebsocketOrderBook, inst return err } - updatedOb := o.Websocket.Orderbook.GetOrderbook(instrument, a) + updatedOb, err := o.Websocket.Orderbook.GetOrderbook(instrument, a) + if err != nil { + return err + } checksum := o.CalculateUpdateOrderbookChecksum(updatedOb) if checksum != wsEventData.Checksum { diff --git a/exchanges/okgroup/okgroup_wrapper.go b/exchanges/okgroup/okgroup_wrapper.go index 30bee556..4e3f234d 100644 --- a/exchanges/okgroup/okgroup_wrapper.go +++ b/exchanges/okgroup/okgroup_wrapper.go @@ -86,10 +86,10 @@ func (o *OKGroup) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderb // UpdateOrderbook updates and returns the orderbook for a currency pair func (o *OKGroup) UpdateOrderbook(p currency.Pair, a asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: o.Name, - Pair: p, - AssetType: a, - VerificationBypass: o.OrderbookVerificationBypass, + Exchange: o.Name, + Pair: p, + Asset: a, + VerifyOrderbook: o.CanVerifyOrderbook, } if a == asset.Index { diff --git a/exchanges/orderbook/calculator_test.go b/exchanges/orderbook/calculator_test.go index 410fa9f2..69fea175 100644 --- a/exchanges/orderbook/calculator_test.go +++ b/exchanges/orderbook/calculator_test.go @@ -8,8 +8,8 @@ import ( func testSetup() Base { return Base{ - ExchangeName: "a", - Pair: currency.NewPair(currency.BTC, currency.USD), + Exchange: "a", + Pair: currency.NewPair(currency.BTC, currency.USD), Asks: []Item{ {Price: 7000, Amount: 1}, {Price: 7001, Amount: 2}, diff --git a/exchanges/orderbook/depth.go b/exchanges/orderbook/depth.go new file mode 100644 index 00000000..c87fdab5 --- /dev/null +++ b/exchanges/orderbook/depth.go @@ -0,0 +1,353 @@ +package orderbook + +import ( + "sync" + "sync/atomic" + "time" + + "github.com/gofrs/uuid" + "github.com/thrasher-corp/gocryptotrader/dispatch" + "github.com/thrasher-corp/gocryptotrader/log" +) + +// Depth defines a linked list of orderbook items +type Depth struct { + asks + bids + + // unexported stack of nodes + stack *stack + + Alert + + mux *dispatch.Mux + id uuid.UUID + + options + m sync.Mutex +} + +// NewDepth returns a new depth item +func newDepth(id uuid.UUID) *Depth { + return &Depth{ + stack: newStack(), + id: id, + mux: service.Mux, + } +} + +// Publish alerts any subscribed routines using a dispatch mux +func (d *Depth) Publish() { + err := d.mux.Publish([]uuid.UUID{d.id}, d.Retrieve()) + if err != nil { + log.Errorf(log.ExchangeSys, "Cannot publish orderbook update to mux %v", err) + } +} + +// GetAskLength returns length of asks +func (d *Depth) GetAskLength() int { + d.m.Lock() + defer d.m.Unlock() + return d.asks.length +} + +// GetBidLength returns length of bids +func (d *Depth) GetBidLength() int { + d.m.Lock() + defer d.m.Unlock() + return d.bids.length +} + +// Retrieve returns the orderbook base a copy of the underlying linked list +// spread +func (d *Depth) Retrieve() *Base { + d.m.Lock() + defer d.m.Unlock() + return &Base{ + Bids: d.bids.retrieve(), + Asks: d.asks.retrieve(), + Exchange: d.exchange, + Asset: d.asset, + Pair: d.pair, + LastUpdated: d.lastUpdated, + LastUpdateID: d.lastUpdateID, + PriceDuplication: d.priceDuplication, + IsFundingRate: d.isFundingRate, + VerifyOrderbook: d.VerifyOrderbook, + } +} + +// TotalBidAmounts returns the total amount of bids and the total orderbook +// bids value +func (d *Depth) TotalBidAmounts() (liquidity, value float64) { + d.m.Lock() + defer d.m.Unlock() + return d.bids.amount() +} + +// TotalAskAmounts returns the total amount of asks and the total orderbook +// asks value +func (d *Depth) TotalAskAmounts() (liquidity, value float64) { + d.m.Lock() + defer d.m.Unlock() + return d.asks.amount() +} + +// LoadSnapshot flushes the bids and asks with a snapshot +func (d *Depth) LoadSnapshot(bids, asks []Item) { + d.m.Lock() + d.bids.load(bids, d.stack) + d.asks.load(asks, d.stack) + d.alert() + d.m.Unlock() +} + +// Flush flushes the bid and ask depths +func (d *Depth) Flush() { + d.m.Lock() + d.bids.load(nil, d.stack) + d.asks.load(nil, d.stack) + d.alert() + d.m.Unlock() +} + +// UpdateBidAskByPrice updates the bid and ask spread by supplied updates, this +// will trim total length of depth level to a specified supplied number +func (d *Depth) UpdateBidAskByPrice(bidUpdts, askUpdts Items, maxDepth int) { + if len(bidUpdts) == 0 && len(askUpdts) == 0 { + return + } + d.m.Lock() + tn := getNow() + if len(bidUpdts) != 0 { + d.bids.updateInsertByPrice(bidUpdts, d.stack, maxDepth, tn) + } + if len(askUpdts) != 0 { + d.asks.updateInsertByPrice(askUpdts, d.stack, maxDepth, tn) + } + d.alert() + d.m.Unlock() +} + +// UpdateBidAskByID amends details by ID +func (d *Depth) UpdateBidAskByID(bidUpdts, askUpdts Items) error { + if len(bidUpdts) == 0 && len(askUpdts) == 0 { + return nil + } + d.m.Lock() + defer d.m.Unlock() + if len(bidUpdts) != 0 { + err := d.bids.updateByID(bidUpdts) + if err != nil { + return err + } + } + if len(askUpdts) != 0 { + err := d.asks.updateByID(askUpdts) + if err != nil { + return err + } + } + d.alert() + return nil +} + +// DeleteBidAskByID deletes a price level by ID +func (d *Depth) DeleteBidAskByID(bidUpdts, askUpdts Items, bypassErr bool) error { + if len(bidUpdts) == 0 && len(askUpdts) == 0 { + return nil + } + d.m.Lock() + defer d.m.Unlock() + if len(bidUpdts) != 0 { + err := d.bids.deleteByID(bidUpdts, d.stack, bypassErr) + if err != nil { + return err + } + } + if len(askUpdts) != 0 { + err := d.asks.deleteByID(askUpdts, d.stack, bypassErr) + if err != nil { + return err + } + } + d.alert() + return nil +} + +// InsertBidAskByID inserts new updates +func (d *Depth) InsertBidAskByID(bidUpdts, askUpdts Items) error { + if len(bidUpdts) == 0 && len(askUpdts) == 0 { + return nil + } + d.m.Lock() + defer d.m.Unlock() + if len(bidUpdts) != 0 { + err := d.bids.insertUpdates(bidUpdts, d.stack) + if err != nil { + return err + } + } + if len(askUpdts) != 0 { + err := d.asks.insertUpdates(askUpdts, d.stack) + if err != nil { + return err + } + } + d.alert() + return nil +} + +// UpdateInsertByID updates or inserts by ID at current price level. +func (d *Depth) UpdateInsertByID(bidUpdts, askUpdts Items) error { + if len(bidUpdts) == 0 && len(askUpdts) == 0 { + return nil + } + d.m.Lock() + defer d.m.Unlock() + if len(bidUpdts) != 0 { + err := d.bids.updateInsertByID(bidUpdts, d.stack) + if err != nil { + return err + } + } + if len(askUpdts) != 0 { + err := d.asks.updateInsertByID(askUpdts, d.stack) + if err != nil { + return err + } + } + d.alert() + return nil +} + +// AssignOptions assigns the initial options for the depth instance +func (d *Depth) AssignOptions(b *Base) { + d.m.Lock() + d.options = options{ + exchange: b.Exchange, + pair: b.Pair, + asset: b.Asset, + lastUpdated: b.LastUpdated, + lastUpdateID: b.LastUpdateID, + priceDuplication: b.PriceDuplication, + isFundingRate: b.IsFundingRate, + VerifyOrderbook: b.VerifyOrderbook, + restSnapshot: b.RestSnapshot, + idAligned: b.IDAlignment, + } + d.m.Unlock() +} + +// SetLastUpdate sets details of last update information +func (d *Depth) SetLastUpdate(lastUpdate time.Time, lastUpdateID int64, updateByREST bool) { + d.m.Lock() + d.lastUpdated = lastUpdate + d.lastUpdateID = lastUpdateID + d.restSnapshot = updateByREST + d.m.Unlock() +} + +// GetName returns name of exchange +func (d *Depth) GetName() string { + d.m.Lock() + defer d.m.Unlock() + return d.exchange +} + +// IsRestSnapshot returns if the depth item was updated via REST +func (d *Depth) IsRestSnapshot() bool { + d.m.Lock() + defer d.m.Unlock() + return d.restSnapshot +} + +// LastUpdateID returns the last Update ID +func (d *Depth) LastUpdateID() int64 { + d.m.Lock() + defer d.m.Unlock() + return d.lastUpdateID +} + +// IsFundingRate returns if the depth is a funding rate +func (d *Depth) IsFundingRate() bool { + d.m.Lock() + defer d.m.Unlock() + return d.isFundingRate +} + +// Alert defines fields required to alert sub-systems of a change of state to +// re-check depth list +type Alert struct { + // Channel to wait for an alert on. + forAlert chan struct{} + // Lets the updater functions know if there are any routines waiting for an + // alert. + sema uint32 + // After closing the forAlert channel this will notify when all the routines + // that have waited, have either checked the orderbook depth or finished. + wg sync.WaitGroup + // Segregated lock only for waiting routines, so as this does not interfere + // with the main depth lock, acts as a rolling gate. + m sync.Mutex +} + +// alert establishes a state change on the orderbook depth. +func (a *Alert) alert() { + // CompareAndSwap is used to swap from 1 -> 2 so we don't keep actuating + // the opposing compare and swap in method wait. This function can return + // freely when an alert operation is in process. + if !atomic.CompareAndSwapUint32(&a.sema, 1, 2) { + // Return if no waiting routines or currently alerting. + return + } + + go func() { + // Actuate lock in a different routine, as alerting is a second order + // priority compared to updating and releasing calling routine. + a.m.Lock() + // Closing; alerts many waiting routines. + close(a.forAlert) + // Wait for waiting routines to receive alert and return. + a.wg.Wait() + atomic.SwapUint32(&a.sema, 0) // Swap back to neutral state. + a.m.Unlock() + }() +} + +// Wait pauses calling routine until depth change has been established via depth +// method alert. Kick allows for cancellation of waiting or when the caller has +// has been shut down, if this is not needed it can be set to nil. This +// returns a channel so strategies can cleanly wait on a select statement case. +func (a *Alert) Wait(kick <-chan struct{}) <-chan bool { + reply := make(chan bool) + a.m.Lock() + a.wg.Add(1) + if atomic.CompareAndSwapUint32(&a.sema, 0, 1) { + a.forAlert = make(chan struct{}) + } + go a.hold(reply, kick) + a.m.Unlock() + return reply +} + +// hold waits on either channel in the event that the routine has finished or an +// alert from a depth update has occurred. +func (a *Alert) hold(ch chan<- bool, kick <-chan struct{}) { + select { + // In a select statement, if by chance there is no receiver or its late, + // we can still close and return, limiting dead-lock potential. + case <-a.forAlert: // Main waiting channel from alert + select { + case ch <- false: + default: + } + case <-kick: // This can be nil. + select { + case ch <- true: + default: + } + } + a.wg.Done() + close(ch) +} diff --git a/exchanges/orderbook/depth_test.go b/exchanges/orderbook/depth_test.go new file mode 100644 index 00000000..ed49c23c --- /dev/null +++ b/exchanges/orderbook/depth_test.go @@ -0,0 +1,404 @@ +package orderbook + +import ( + "errors" + "log" + "reflect" + "sync" + "testing" + "time" + + "github.com/gofrs/uuid" + "github.com/thrasher-corp/gocryptotrader/currency" + "github.com/thrasher-corp/gocryptotrader/exchanges/asset" +) + +var id, _ = uuid.NewV4() + +func TestGetLength(t *testing.T) { + d := newDepth(id) + if d.GetAskLength() != 0 { + t.Errorf("expected len %v, but received %v", 0, d.GetAskLength()) + } + + d.asks.load([]Item{{Price: 1337}}, d.stack) + + if d.GetAskLength() != 1 { + t.Errorf("expected len %v, but received %v", 1, d.GetAskLength()) + } + + d = newDepth(id) + if d.GetBidLength() != 0 { + t.Errorf("expected len %v, but received %v", 0, d.GetBidLength()) + } + + d.bids.load([]Item{{Price: 1337}}, d.stack) + + if d.GetBidLength() != 1 { + t.Errorf("expected len %v, but received %v", 1, d.GetBidLength()) + } +} + +func TestRetrieve(t *testing.T) { + d := newDepth(id) + d.asks.load([]Item{{Price: 1337}}, d.stack) + d.bids.load([]Item{{Price: 1337}}, d.stack) + d.options = options{ + exchange: "THE BIG ONE!!!!!!", + pair: currency.NewPair(currency.THETA, currency.USD), + asset: "Silly asset", + lastUpdated: time.Now(), + lastUpdateID: 007, + priceDuplication: true, + isFundingRate: true, + VerifyOrderbook: true, + restSnapshot: true, + idAligned: true, + } + + // If we add anymore options to the options struct later this will complain + // generally want to return a full carbon copy + mirrored := reflect.Indirect(reflect.ValueOf(d.options)) + for n := 0; n < mirrored.NumField(); n++ { + structVal := mirrored.Field(n) + if structVal.IsZero() { + t.Fatalf("struct value options not set for field %v", + mirrored.Type().Field(n).Name) + } + } + theBigD := d.Retrieve() + if len(theBigD.Asks) != 1 { + t.Errorf("expected len %v, but received %v", 1, len(theBigD.Bids)) + } + + if len(theBigD.Bids) != 1 { + t.Errorf("expected len %v, but received %v", 1, len(theBigD.Bids)) + } +} + +func TestTotalAmounts(t *testing.T) { + d := newDepth(id) + + liquidity, value := d.TotalBidAmounts() + if liquidity != 0 || value != 0 { + t.Fatalf("liquidity expected %f received %f value expected %f received %f", + 0., + liquidity, + 0., + value) + } + + liquidity, value = d.TotalAskAmounts() + if liquidity != 0 || value != 0 { + t.Fatalf("liquidity expected %f received %f value expected %f received %f", + 0., + liquidity, + 0., + value) + } + + d.asks.load([]Item{{Price: 1337, Amount: 1}}, d.stack) + d.bids.load([]Item{{Price: 1337, Amount: 10}}, d.stack) + + liquidity, value = d.TotalBidAmounts() + if liquidity != 10 || value != 13370 { + t.Fatalf("liquidity expected %f received %f value expected %f received %f", + 10., + liquidity, + 13370., + value) + } + + liquidity, value = d.TotalAskAmounts() + if liquidity != 1 || value != 1337 { + t.Fatalf("liquidity expected %f received %f value expected %f received %f", + 1., + liquidity, + 1337., + value) + } +} + +func TestLoadSnapshot(t *testing.T) { + d := newDepth(id) + d.LoadSnapshot(Items{{Price: 1337, Amount: 1}}, Items{{Price: 1337, Amount: 10}}) + if d.Retrieve().Asks[0].Price != 1337 || d.Retrieve().Bids[0].Price != 1337 { + t.Fatal("not set") + } +} + +func TestFlush(t *testing.T) { + d := newDepth(id) + d.LoadSnapshot(Items{{Price: 1337, Amount: 1}}, Items{{Price: 1337, Amount: 10}}) + d.Flush() + if len(d.Retrieve().Asks) != 0 || len(d.Retrieve().Bids) != 0 { + t.Fatal("not flushed") + } + d.LoadSnapshot(Items{{Price: 1337, Amount: 1}}, Items{{Price: 1337, Amount: 10}}) + d.Flush() + if len(d.Retrieve().Asks) != 0 || len(d.Retrieve().Bids) != 0 { + t.Fatal("not flushed") + } +} + +func TestUpdateBidAskByPrice(t *testing.T) { + d := newDepth(id) + d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}) + d.UpdateBidAskByPrice(Items{{Price: 1337, Amount: 2, ID: 1}}, Items{{Price: 1337, Amount: 2, ID: 2}}, 0) + if d.Retrieve().Asks[0].Amount != 2 || d.Retrieve().Bids[0].Amount != 2 { + t.Fatal("orderbook amounts not updated correctly") + } + d.UpdateBidAskByPrice(Items{{Price: 1337, Amount: 0, ID: 1}}, Items{{Price: 1337, Amount: 0, ID: 2}}, 0) + if d.GetAskLength() != 0 || d.GetBidLength() != 0 { + t.Fatal("orderbook amounts not updated correctly") + } +} + +func TestDeleteBidAskByID(t *testing.T) { + d := newDepth(id) + d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}) + err := d.DeleteBidAskByID(Items{{Price: 1337, Amount: 2, ID: 1}}, Items{{Price: 1337, Amount: 2, ID: 2}}, false) + if err != nil { + t.Fatal(err) + } + if len(d.Retrieve().Asks) != 0 || len(d.Retrieve().Bids) != 0 { + t.Fatal("items not deleted") + } + + err = d.DeleteBidAskByID(Items{{Price: 1337, Amount: 2, ID: 1}}, nil, false) + if !errors.Is(err, errIDCannotBeMatched) { + t.Fatalf("error expected %v received %v", errIDCannotBeMatched, err) + } + + err = d.DeleteBidAskByID(nil, Items{{Price: 1337, Amount: 2, ID: 2}}, false) + if !errors.Is(err, errIDCannotBeMatched) { + t.Fatalf("error expected %v received %v", errIDCannotBeMatched, err) + } + + err = d.DeleteBidAskByID(nil, Items{{Price: 1337, Amount: 2, ID: 2}}, true) + if !errors.Is(err, nil) { + t.Fatalf("error expected %v received %v", nil, err) + } +} + +func TestUpdateBidAskByID(t *testing.T) { + d := newDepth(id) + d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}) + err := d.UpdateBidAskByID(Items{{Price: 1337, Amount: 2, ID: 1}}, Items{{Price: 1337, Amount: 2, ID: 2}}) + if err != nil { + t.Fatal(err) + } + if d.Retrieve().Asks[0].Amount != 2 || d.Retrieve().Bids[0].Amount != 2 { + t.Fatal("orderbook amounts not updated correctly") + } + + // random unmatching IDs + err = d.UpdateBidAskByID(Items{{Price: 1337, Amount: 2, ID: 666}}, nil) + if !errors.Is(err, errIDCannotBeMatched) { + t.Fatalf("error expected %v received %v", errIDCannotBeMatched, err) + } + + err = d.UpdateBidAskByID(nil, Items{{Price: 1337, Amount: 2, ID: 69}}) + if !errors.Is(err, errIDCannotBeMatched) { + t.Fatalf("error expected %v received %v", errIDCannotBeMatched, err) + } +} + +func TestInsertBidAskByID(t *testing.T) { + d := newDepth(id) + d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}) + err := d.InsertBidAskByID(Items{{Price: 1338, Amount: 2, ID: 3}}, Items{{Price: 1336, Amount: 2, ID: 4}}) + if err != nil { + t.Fatal(err) + } + if len(d.Retrieve().Asks) != 2 || len(d.Retrieve().Bids) != 2 { + t.Fatal("items not added correctly") + } +} + +func TestUpdateInsertByID(t *testing.T) { + d := newDepth(id) + d.LoadSnapshot(Items{{Price: 1337, Amount: 1, ID: 1}}, Items{{Price: 1337, Amount: 10, ID: 2}}) + + err := d.UpdateInsertByID(Items{{Price: 1338, Amount: 0, ID: 3}}, Items{{Price: 1336, Amount: 2, ID: 4}}) + if !errors.Is(err, errAmountCannotBeLessOrEqualToZero) { + t.Fatalf("expected: %v but received: %v", errAmountCannotBeLessOrEqualToZero, err) + } + + err = d.UpdateInsertByID(Items{{Price: 1338, Amount: 2, ID: 3}}, Items{{Price: 1336, Amount: 0, ID: 4}}) + if !errors.Is(err, errAmountCannotBeLessOrEqualToZero) { + t.Fatalf("expected: %v but received: %v", errAmountCannotBeLessOrEqualToZero, err) + } + + err = d.UpdateInsertByID(Items{{Price: 1338, Amount: 2, ID: 3}}, Items{{Price: 1336, Amount: 2, ID: 4}}) + if err != nil { + t.Fatal(err) + } + + if len(d.Retrieve().Asks) != 2 || len(d.Retrieve().Bids) != 2 { + t.Fatal("items not added correctly") + } +} + +func TestAssignOptions(t *testing.T) { + d := Depth{} + cp := currency.NewPair(currency.LINK, currency.BTC) + tn := time.Now() + d.AssignOptions(&Base{ + Exchange: "test", + Pair: cp, + Asset: asset.Spot, + LastUpdated: tn, + LastUpdateID: 1337, + PriceDuplication: true, + IsFundingRate: true, + VerifyOrderbook: true, + RestSnapshot: true, + IDAlignment: true, + }) + + if d.exchange != "test" || + d.pair != cp || + d.asset != asset.Spot || + d.lastUpdated != tn || + d.lastUpdateID != 1337 || + !d.priceDuplication || + !d.isFundingRate || + !d.VerifyOrderbook || + !d.restSnapshot || + !d.idAligned { + t.Fatal("failed to set correctly") + } +} + +func TestSetLastUpdate(t *testing.T) { + d := Depth{} + tn := time.Now() + d.SetLastUpdate(tn, 1337, true) + if d.lastUpdated != tn || + d.lastUpdateID != 1337 || + !d.restSnapshot { + t.Fatal("failed to set correctly") + } +} + +func TestGetName(t *testing.T) { + d := Depth{} + d.exchange = "test" + if d.GetName() != "test" { + t.Fatal("failed to get correct value") + } +} + +func TestIsRestSnapshot(t *testing.T) { + d := Depth{} + d.restSnapshot = true + if !d.IsRestSnapshot() { + t.Fatal("failed to set correctly") + } +} + +func TestLastUpdateID(t *testing.T) { + d := Depth{} + d.lastUpdateID = 1337 + if d.LastUpdateID() != 1337 { + t.Fatal("failed to get correct value") + } +} + +func TestIsFundingRate(t *testing.T) { + d := Depth{} + d.isFundingRate = true + if !d.IsFundingRate() { + t.Fatal("failed to get correct value") + } +} + +func TestPublish(t *testing.T) { + d := Depth{} + d.Publish() +} + +func TestWait(t *testing.T) { + wait := Alert{} + var wg sync.WaitGroup + + // standard alert + wg.Add(100) + for x := 0; x < 100; x++ { + go func() { + w := wait.Wait(nil) + wg.Done() + if <-w { + log.Fatal("incorrect routine wait response for alert expecting false") + } + wg.Done() + }() + } + + wg.Wait() + wg.Add(100) + isLeaky(&wait, nil, t) + wait.alert() + wg.Wait() + isLeaky(&wait, nil, t) + + // use kick + ch := make(chan struct{}) + wg.Add(100) + for x := 0; x < 100; x++ { + go func() { + w := wait.Wait(ch) + wg.Done() + if !<-w { + log.Fatal("incorrect routine wait response for kick expecting true") + } + wg.Done() + }() + } + wg.Wait() + wg.Add(100) + isLeaky(&wait, ch, t) + close(ch) + wg.Wait() + ch = make(chan struct{}) + isLeaky(&wait, ch, t) + + // late receivers + wg.Add(100) + for x := 0; x < 100; x++ { + go func(x int) { + bb := wait.Wait(ch) + wg.Done() + if x%2 == 0 { + time.Sleep(time.Millisecond * 5) + } + b := <-bb + if b { + log.Fatal("incorrect routine wait response since we call alert below; expecting false") + } + wg.Done() + }(x) + } + wg.Wait() + wg.Add(100) + isLeaky(&wait, ch, t) + wait.alert() + wg.Wait() + isLeaky(&wait, ch, t) +} + +// isLeaky tests to see if the wait functionality is returning an abnormal +// channel that is operational when it shouldn't be. +func isLeaky(a *Alert, ch chan struct{}, t *testing.T) { + t.Helper() + check := a.Wait(ch) + time.Sleep(time.Millisecond * 5) // When we call wait a routine for hold is + // spawned, so for a test we need to add in a time for goschedular to allow + // routine to actually wait on the forAlert and kick channels + select { + case <-check: + t.Fatal("leaky waiter") + default: + } +} diff --git a/exchanges/orderbook/linked_list.go b/exchanges/orderbook/linked_list.go new file mode 100644 index 00000000..d0050311 --- /dev/null +++ b/exchanges/orderbook/linked_list.go @@ -0,0 +1,530 @@ +package orderbook + +import ( + "errors" + "fmt" +) + +var errIDCannotBeMatched = errors.New("cannot match ID on linked list") +var errCollisionDetected = errors.New("cannot insert update collision detected") +var errAmountCannotBeLessOrEqualToZero = errors.New("amount cannot be less or equal to zero") + +// linkedList defines a linked list for a depth level, reutilisation of nodes +// to and from a stack. +type linkedList struct { + length int + head *node +} + +// comparison defines expected functionality to compare between two reference +// price levels +type comparison func(float64, float64) bool + +// load iterates across new items and refreshes linked list. It creates a linked +// list exactly the same as the item slice that is supplied, if items is of nil +// value it will flush entire list. +func (ll *linkedList) load(items Items, stack *stack) { + // Tip sets up a pointer to a struct field variable pointer. This is used + // so when a node is popped from the stack we can reference that current + // nodes' struct 'next' field and set on next iteration without utilising + // assignment for example `prev.next = *node`. + var tip = &ll.head + // Prev denotes a place holder to node and all of its next references need + // to be pushed back onto stack. + var prev *node + for i := range items { + if *tip == nil { + // Extend node chain + *tip = stack.Pop() + // Set current node prev to last node + (*tip).prev = prev + ll.length++ + } + // Set item value + (*tip).value = items[i] + // Set previous to current node + prev = *tip + // Set tip to next node + tip = &(*tip).next + } + + // Push has references to dangling nodes that need to be removed and pushed + // back onto stack for re-use + var push *node + // Cleave unused reference chain from main chain + if prev == nil { + // The entire chain will need to be pushed back on to stack + push = *tip + ll.head = nil + } else { + push = prev.next + prev.next = nil + } + + // Push unused pointers back on stack + for push != nil { + pending := push.next + stack.Push(push, getNow()) + ll.length-- + push = pending + } +} + +// updateByID amends price by corresponding ID and returns an error if not found +func (ll *linkedList) updateByID(updts []Item) error { +updates: + for x := range updts { + for tip := ll.head; tip != nil; tip = tip.next { + if updts[x].ID != tip.value.ID { // Filter IDs that don't match + continue + } + if updts[x].Price > 0 { + // Only apply changes when zero values are not present, Bitmex + // for example sends 0 price values. + tip.value.Price = updts[x].Price + } + tip.value.Amount = updts[x].Amount + continue updates + } + return fmt.Errorf("update error: %w %d not found", + errIDCannotBeMatched, + updts[x].ID) + } + return nil +} + +// deleteByID deletes reference by ID +func (ll *linkedList) deleteByID(updts Items, stack *stack, bypassErr bool) error { +updates: + for x := range updts { + for tip := &ll.head; *tip != nil; tip = &(*tip).next { + if updts[x].ID != (*tip).value.ID { + continue + } + stack.Push(deleteAtTip(ll, tip), getNow()) + continue updates + } + if !bypassErr { + return fmt.Errorf("delete error: %w %d not found", + errIDCannotBeMatched, + updts[x].ID) + } + } + return nil +} + +// cleanup reduces the max size of the depth length if exceeded. Is used after +// updates have been applied instead of adhoc, reason being its easier to prune +// at the end. (cant inline) +func (ll *linkedList) cleanup(maxChainLength int, stack *stack) { + // Reduces the max length of total linked list chain, occurs after updates + // have been implemented as updates can push length out of bounds, if + // cleaved after that update, new update might not applied correctly. + n := ll.head + for i := 0; i < maxChainLength; i++ { + if n.next == nil { + return + } + n = n.next + } + + // cleave reference to current node + if n.prev != nil { + n.prev.next = nil + } else { + ll.head = nil + } + + var pruned int + for n != nil { + pruned++ + pending := n.next + stack.Push(n, getNow()) + n = pending + } + ll.length -= pruned +} + +// amount returns total depth liquidity and value +func (ll *linkedList) amount() (liquidity, value float64) { + for tip := ll.head; tip != nil; tip = tip.next { + liquidity += tip.value.Amount + value += tip.value.Amount * tip.value.Price + } + return +} + +// retrieve returns a full slice of contents from the linked list +func (ll *linkedList) retrieve() Items { + depth := make(Items, ll.length) + iterator := 0 + for tip := ll.head; tip != nil; tip = tip.next { + depth[iterator] = tip.value + iterator++ + } + return depth +} + +// updateInsertByPrice amends, inserts, moves and cleaves length of depth by +// updates +func (ll *linkedList) updateInsertByPrice(updts Items, stack *stack, maxChainLength int, compare func(float64, float64) bool, tn now) { + for x := range updts { + for tip := &ll.head; ; tip = &(*tip).next { + if *tip == nil { + insertHeadSpecific(ll, updts[x], stack) + break + } + if (*tip).value.Price == updts[x].Price { // Match check + if updts[x].Amount <= 0 { // Capture delete update + stack.Push(deleteAtTip(ll, tip), tn) + } else { // Amend current amount value + (*tip).value.Amount = updts[x].Amount + } + break // Continue updates + } + + if compare((*tip).value.Price, updts[x].Price) { // Insert + // This check below filters zero values and provides an + // optimisation for when select exchanges send a delete update + // to a non-existent price level (OTC/Hidden order) so we can + // break instantly and reduce the traversal of the entire chain. + if updts[x].Amount > 0 { + insertAtTip(ll, tip, updts[x], stack) + } + break // Continue updates + } + + if (*tip).next == nil { // Tip is at tail + // This check below is just a catch all in the event the above + // zero value check fails + if updts[x].Amount > 0 { + insertAtTail(ll, tip, updts[x], stack) + } + break + } + } + } + // Reduces length of total linked list chain to a maxChainLength value + if maxChainLength != 0 && ll.length > maxChainLength { + ll.cleanup(maxChainLength, stack) + } +} + +// updateInsertByID updates or inserts if not found for a bid or ask depth +// 1) node ID found amount amended (best case) +// 2) node ID found amount and price amended and node moved to correct position +// (medium case) +// 3) Update price exceeds traversal node price before ID found, save node +// address for either; node ID matches then re-address node or end of depth pop +// a node from the stack (worst case) +func (ll *linkedList) updateInsertByID(updts Items, stack *stack, compare comparison) error { +updates: + for x := range updts { + if updts[x].Amount <= 0 { + return errAmountCannotBeLessOrEqualToZero + } + // bookmark allows for saving of a position of a node in the event that + // an update price exceeds the current node price. We can then match an + // ID and re-assign that ID's node to that positioning without popping + // from the stack and then pushing to the stack later for cleanup. + // If the ID is not found we can pop from stack then insert into that + // price level + var bookmark *node + for tip := ll.head; tip != nil; tip = tip.next { + if tip.value.ID == updts[x].ID { + if tip.value.Price != updts[x].Price { // Price level change + if tip.next == nil { + // no movement needed just a re-adjustment + tip.value.Price = updts[x].Price + tip.value.Amount = updts[x].Amount + continue updates + } + // bookmark tip to move this node to correct price level + bookmark = tip + continue // continue through node depth + } + // no price change, amend amount and continue update + tip.value.Amount = updts[x].Amount + continue updates // continue to next update + } + + if compare(tip.value.Price, updts[x].Price) { + if bookmark != nil { // shift bookmarked node to current tip + bookmark.value = updts[x] + move(&ll.head, bookmark, tip) + continue updates + } + + // search for ID + for n := tip.next; n != nil; n = n.next { + if n.value.ID == updts[x].ID { + n.value = updts[x] + // inserting before the tip + move(&ll.head, n, tip) + continue updates + } + } + // ID not matched in depth so add correct level for insert + if tip.next == nil { + n := stack.Pop() + n.value = updts[x] + ll.length++ + if tip.prev == nil { + tip.prev = n + n.next = tip + ll.head = n + continue updates + } + tip.prev.next = n + n.prev = tip.prev + tip.prev = n + n.next = tip + continue updates + } + bookmark = tip + break + } + + if tip.next == nil { + if shiftBookmark(tip, &bookmark, &ll.head, updts[x]) { + continue updates + } + } + } + n := stack.Pop() + n.value = updts[x] + insertNodeAtBookmark(ll, bookmark, n) // Won't inline with stack + } + return nil +} + +// insertUpdates inserts new updates for bids or asks based on price level +func (ll *linkedList) insertUpdates(updts Items, stack *stack, comp comparison) error { + for x := range updts { + var prev *node + for tip := &ll.head; ; tip = &(*tip).next { + if *tip == nil { // Head + n := stack.Pop() + n.value = updts[x] + n.prev = prev + ll.length++ + *tip = n + break // Continue updates + } + + if (*tip).value.Price == updts[x].Price { // Price already found + return fmt.Errorf("%w for price %f", + errCollisionDetected, + updts[x].Price) + } + + if comp((*tip).value.Price, updts[x].Price) { // Alignment + n := stack.Pop() + n.value = updts[x] + n.prev = prev + ll.length++ + // Reference current with new node + (*tip).prev = n + // Push tip to the right + n.next = *tip + // This is the same as prev.next = n + *tip = n + break // Continue updates + } + + if (*tip).next == nil { // Tail + insertAtTail(ll, tip, updts[x], stack) + break // Continue updates + } + prev = *tip + } + } + return nil +} + +// bids embed a linked list to attach methods for bid depth specific +// functionality +type bids struct { + linkedList +} + +// bidCompare ensures price is in correct descending alignment (can inline) +func bidCompare(left, right float64) bool { + return left < right +} + +// updateInsertByPrice amends, inserts, moves and cleaves length of depth by +// updates +func (ll *bids) updateInsertByPrice(updts Items, stack *stack, maxChainLength int, tn now) { + ll.linkedList.updateInsertByPrice(updts, stack, maxChainLength, bidCompare, tn) +} + +// updateInsertByID updates or inserts if not found +func (ll *bids) updateInsertByID(updts Items, stack *stack) error { + return ll.linkedList.updateInsertByID(updts, stack, bidCompare) +} + +// insertUpdates inserts new updates for bids based on price level +func (ll *bids) insertUpdates(updts Items, stack *stack) error { + return ll.linkedList.insertUpdates(updts, stack, bidCompare) +} + +// asks embed a linked list to attach methods for ask depth specific +// functionality +type asks struct { + linkedList +} + +// askCompare ensures price is in correct ascending alignment (can inline) +func askCompare(left, right float64) bool { + return left > right +} + +// updateInsertByPrice amends, inserts, moves and cleaves length of depth by +// updates +func (ll *asks) updateInsertByPrice(updts Items, stack *stack, maxChainLength int, tn now) { + ll.linkedList.updateInsertByPrice(updts, stack, maxChainLength, askCompare, tn) +} + +// updateInsertByID updates or inserts if not found +func (ll *asks) updateInsertByID(updts Items, stack *stack) error { + return ll.linkedList.updateInsertByID(updts, stack, askCompare) +} + +// insertUpdates inserts new updates for asks based on price level +func (ll *asks) insertUpdates(updts Items, stack *stack) error { + return ll.linkedList.insertUpdates(updts, stack, askCompare) +} + +// move moves a node from a point in a node chain to another node position, +// this left justified towards head as element zero is the top of the depth +// side. (can inline) +func move(head **node, from, to *node) { + if from.next != nil { // From is at tail + from.next.prev = from.prev + } + if from.prev == nil { // From is at head + (*head).next.prev = nil + *head = (*head).next + } else { + from.prev.next = from.next + } + // insert from node next to 'to' node + if to.prev == nil { // Destination is at head position + *head = from + } else { + to.prev.next = from + } + from.prev = to.prev + to.prev = from + from.next = to +} + +// deleteAtTip removes a node from tip target returns old node (can inline) +func deleteAtTip(ll *linkedList, tip **node) *node { + // Old is a placeholder for current tips node value to push + // back on to the stack. + old := *tip + switch { + case old.prev == nil: // At head position + // shift current tip head to the right + *tip = old.next + // Remove reference to node from chain + if old.next != nil { // This is when liquidity hits zero + old.next.prev = nil + } + case old.next == nil: // At tail position + // Remove reference to node from chain + old.prev.next = nil + default: + // Reference prior node in chain to next node in chain + // bypassing current node + old.prev.next = old.next + old.next.prev = old.prev + } + ll.length-- + return old +} + +// insertAtTip inserts at a tip target (can inline) +func insertAtTip(ll *linkedList, tip **node, updt Item, stack *stack) { + n := stack.Pop() + n.value = updt + n.next = *tip + n.prev = (*tip).prev + if (*tip).prev == nil { // Tip is at head + // Replace head which will push everything to the right + // when this node will reference new node below + *tip = n + } else { + // Reference new node to previous node + (*tip).prev.next = n + } + // Reference next node to new node + n.next.prev = n + ll.length++ +} + +// insertAtTail inserts at tail end of node chain (can inline) +func insertAtTail(ll *linkedList, tip **node, updt Item, stack *stack) { + n := stack.Pop() + n.value = updt + // Reference tip to new node + (*tip).next = n + // Reference new node with current tip + n.prev = *tip + ll.length++ +} + +// insertHeadSpecific inserts at head specifically there might be an instance +// where the liquidity on an exchange does fall to zero through a streaming +// endpoint then it comes back online. (can inline) +func insertHeadSpecific(ll *linkedList, updt Item, stack *stack) { + n := stack.Pop() + n.value = updt + ll.head = n + ll.length++ +} + +// insertNodeAtBookmark inserts a new node at a bookmarked node position +// returns if a node needs to replace head (can inline) +func insertNodeAtBookmark(ll *linkedList, bookmark, n *node) { + switch { + case bookmark == nil: // Zero liquidity and we are rebuilding from scratch + ll.head = n + case bookmark.prev == nil: + n.prev = bookmark.prev + bookmark.prev = n + n.next = bookmark + ll.head = n + case bookmark.next == nil: + n.prev = bookmark + bookmark.next = n + default: + bookmark.prev.next = n + n.prev = bookmark.prev + bookmark.prev = n + n.next = bookmark + } + ll.length++ +} + +// shiftBookmark moves a bookmarked node to the tip's next position or if nil, +// sets tip as bookmark (can inline) +func shiftBookmark(tip *node, bookmark, head **node, updt Item) bool { + if *bookmark == nil { // End of the chain and no bookmark set + *bookmark = tip // Set tip to bookmark so we can set a new node there + return false + } + (*bookmark).value = updt + (*bookmark).next.prev = (*bookmark).prev + if (*bookmark).prev == nil { // Bookmark is at head + *head = (*bookmark).next + } else { + (*bookmark).prev.next = (*bookmark).next + } + tip.next = *bookmark + (*bookmark).prev = tip + (*bookmark).next = nil + return true +} diff --git a/exchanges/orderbook/linked_list_test.go b/exchanges/orderbook/linked_list_test.go new file mode 100644 index 00000000..21d7df69 --- /dev/null +++ b/exchanges/orderbook/linked_list_test.go @@ -0,0 +1,1449 @@ +package orderbook + +import ( + "errors" + "fmt" + "testing" + "time" +) + +var ask = Items{ + Item{Price: 1337, Amount: 1}, + Item{Price: 1338, Amount: 1}, + Item{Price: 1339, Amount: 1}, + Item{Price: 1340, Amount: 1}, + Item{Price: 1341, Amount: 1}, + Item{Price: 1342, Amount: 1}, + Item{Price: 1343, Amount: 1}, + Item{Price: 1344, Amount: 1}, + Item{Price: 1345, Amount: 1}, + Item{Price: 1346, Amount: 1}, + Item{Price: 1347, Amount: 1}, + Item{Price: 1348, Amount: 1}, + Item{Price: 1349, Amount: 1}, + Item{Price: 1350, Amount: 1}, + Item{Price: 1351, Amount: 1}, + Item{Price: 1352, Amount: 1}, + Item{Price: 1353, Amount: 1}, + Item{Price: 1354, Amount: 1}, + Item{Price: 1355, Amount: 1}, + Item{Price: 1356, Amount: 1}, +} + +// Display displays depth content for tests +func (ll *linkedList) display() { + for tip := ll.head; tip != nil; tip = tip.next { + fmt.Printf("NODE: %+v %p \n", tip, tip) + } + fmt.Println() +} + +func TestLoad(t *testing.T) { + list := asks{} + Check(list, 0, 0, 0, t) + + stack := newStack() + list.load(Items{ + {Price: 1, Amount: 1}, + {Price: 3, Amount: 1}, + {Price: 5, Amount: 1}, + {Price: 7, Amount: 1}, + {Price: 9, Amount: 1}, + {Price: 11, Amount: 1}, + }, stack) + + if stack.getCount() != 0 { + t.Fatalf("incorrect stack count expected: %v received: %v", 0, stack.getCount()) + } + + Check(list, 6, 36, 6, t) + + list.load(Items{ + {Price: 1, Amount: 1}, + {Price: 3, Amount: 1}, + {Price: 5, Amount: 1}, + }, stack) + + if stack.getCount() != 3 { + t.Fatalf("incorrect stack count expected: %v received: %v", 3, stack.getCount()) + } + + Check(list, 3, 9, 3, t) + + list.load(Items{ + {Price: 1, Amount: 1}, + {Price: 3, Amount: 1}, + {Price: 5, Amount: 1}, + {Price: 7, Amount: 1}, + }, stack) + + if stack.getCount() != 2 { + t.Fatalf("incorrect stack count expected: %v received: %v", 2, stack.getCount()) + } + + Check(list, 4, 16, 4, t) + + // purge entire list + list.load(nil, stack) + + if stack.getCount() != 6 { + t.Fatalf("incorrect stack count expected: %v received: %v", 6, stack.getCount()) + } + + Check(list, 0, 0, 0, t) +} + +// 22222386 57.3 ns/op 0 B/op 0 allocs/op (old) +// 27906781 42.4 ns/op 0 B/op 0 allocs/op (new) +func BenchmarkLoad(b *testing.B) { + ll := linkedList{} + s := newStack() + for i := 0; i < b.N; i++ { + ll.load(ask, s) + } +} + +func TestUpdateInsertByPrice(t *testing.T) { + a := asks{} + stack := newStack() + asksSnapshot := Items{ + {Price: 1, Amount: 1}, + {Price: 3, Amount: 1}, + {Price: 5, Amount: 1}, + {Price: 7, Amount: 1}, + {Price: 9, Amount: 1}, + {Price: 11, Amount: 1}, + } + a.load(asksSnapshot, stack) + + // Update one instance with matching price + a.updateInsertByPrice(Items{ + {Price: 1, Amount: 2}, + }, stack, 0, getNow()) + + Check(a, 7, 37, 6, t) + + if stack.getCount() != 0 { + t.Fatalf("incorrect stack count expected: %v received: %v", 0, stack.getCount()) + } + + // Insert at head + a.updateInsertByPrice(Items{ + {Price: 0.5, Amount: 2}, + }, stack, 0, getNow()) + + Check(a, 9, 38, 7, t) + + if stack.getCount() != 0 { + t.Fatalf("incorrect stack count expected: %v received: %v", 0, stack.getCount()) + } + + // Insert at tail + a.updateInsertByPrice(Items{ + {Price: 12, Amount: 2}, + }, stack, 0, getNow()) + + Check(a, 11, 62, 8, t) + + if stack.getCount() != 0 { + t.Fatalf("incorrect stack count expected: %v received: %v", 0, stack.getCount()) + } + + // Insert between price and up to and beyond max allowable depth level + a.updateInsertByPrice(Items{ + {Price: 11.5, Amount: 2}, + {Price: 10.5, Amount: 2}, + {Price: 13, Amount: 2}, + }, stack, 10, getNow()) + + Check(a, 15, 106, 10, t) + + if stack.getCount() != 1 { + t.Fatalf("incorrect stack count expected: %v received: %v", 1, stack.getCount()) + } + + // delete at tail + a.updateInsertByPrice(Items{ + {Price: 12, Amount: 0}, + }, stack, 0, getNow()) + + Check(a, 13, 82, 9, t) + + if stack.getCount() != 2 { + t.Fatalf("incorrect stack count expected: %v received: %v", 2, stack.getCount()) + } + + // delete at mid + a.updateInsertByPrice(Items{ + {Price: 7, Amount: 0}, + }, stack, 0, getNow()) + + Check(a, 12, 75, 8, t) + + if stack.getCount() != 3 { + t.Fatalf("incorrect stack count expected: %v received: %v", 3, stack.getCount()) + } + + // delete at head + a.updateInsertByPrice(Items{ + {Price: 0.5, Amount: 0}, + }, stack, 0, getNow()) + + Check(a, 10, 74, 7, t) + + if stack.getCount() != 4 { + t.Fatalf("incorrect stack count expected: %v received: %v", 4, stack.getCount()) + } + + // purge if liquidity plunges to zero + a.load(nil, stack) + + // rebuild everything again + a.updateInsertByPrice(Items{ + {Price: 1, Amount: 1}, + {Price: 3, Amount: 1}, + {Price: 5, Amount: 1}, + {Price: 7, Amount: 1}, + {Price: 9, Amount: 1}, + {Price: 11, Amount: 1}, + }, stack, 0, getNow()) + + Check(a, 6, 36, 6, t) + + if stack.getCount() != 5 { + t.Fatalf("incorrect stack count expected: %v received: %v", 4, stack.getCount()) + } + + b := bids{} + bidsSnapshot := Items{ + {Price: 11, Amount: 1}, + {Price: 9, Amount: 1}, + {Price: 7, Amount: 1}, + {Price: 5, Amount: 1}, + {Price: 3, Amount: 1}, + {Price: 1, Amount: 1}, + } + b.load(bidsSnapshot, stack) + + // Update one instance with matching price + b.updateInsertByPrice(Items{ + {Price: 11, Amount: 2}, + }, stack, 0, getNow()) + + Check(b, 7, 47, 6, t) + + if stack.getCount() != 0 { + t.Fatalf("incorrect stack count expected: %v received: %v", 0, stack.getCount()) + } + + // Insert at head + b.updateInsertByPrice(Items{ + {Price: 12, Amount: 2}, + }, stack, 0, getNow()) + + Check(b, 9, 71, 7, t) + + if stack.getCount() != 0 { + t.Fatalf("incorrect stack count expected: %v received: %v", 0, stack.getCount()) + } + + // Insert at tail + b.updateInsertByPrice(Items{ + {Price: 0.5, Amount: 2}, + }, stack, 0, getNow()) + + Check(b, 11, 72, 8, t) + + if stack.getCount() != 0 { + t.Fatalf("incorrect stack count expected: %v received: %v", 0, stack.getCount()) + } + + // Insert between price and up to and beyond max allowable depth level + b.updateInsertByPrice(Items{ + {Price: 11.5, Amount: 2}, + {Price: 10.5, Amount: 2}, + {Price: 13, Amount: 2}, + }, stack, 10, getNow()) + + Check(b, 15, 141, 10, t) + + if stack.getCount() != 1 { + t.Fatalf("incorrect stack count expected: %v received: %v", 0, stack.getCount()) + } + + // Insert between price and up to and beyond max allowable depth level + b.updateInsertByPrice(Items{ + {Price: 1, Amount: 0}, + }, stack, 0, getNow()) + + Check(b, 14, 140, 9, t) + + if stack.getCount() != 2 { + t.Fatalf("incorrect stack count expected: %v received: %v", 2, stack.getCount()) + } + + // delete at mid + b.updateInsertByPrice(Items{ + {Price: 10.5, Amount: 0}, + }, stack, 0, getNow()) + + Check(b, 12, 119, 8, t) + + if stack.getCount() != 3 { + t.Fatalf("incorrect stack count expected: %v received: %v", 3, stack.getCount()) + } + + // delete at head + b.updateInsertByPrice(Items{ + {Price: 13, Amount: 0}, + }, stack, 0, getNow()) + + Check(b, 10, 93, 7, t) + + if stack.getCount() != 4 { + t.Fatalf("incorrect stack count expected: %v received: %v", 4, stack.getCount()) + } + + // purge if liquidity plunges to zero + b.load(nil, stack) + + // rebuild everything again + b.updateInsertByPrice(Items{ + {Price: 1, Amount: 1}, + {Price: 3, Amount: 1}, + {Price: 5, Amount: 1}, + {Price: 7, Amount: 1}, + {Price: 9, Amount: 1}, + {Price: 11, Amount: 1}, + }, stack, 0, getNow()) + + Check(b, 6, 36, 6, t) + + if stack.getCount() != 5 { + t.Fatalf("incorrect stack count expected: %v received: %v", 4, stack.getCount()) + } +} + +func TestCleanup(t *testing.T) { + a := asks{} + stack := newStack() + asksSnapshot := Items{ + {Price: 1, Amount: 1}, + {Price: 3, Amount: 1}, + {Price: 5, Amount: 1}, + {Price: 7, Amount: 1}, + {Price: 9, Amount: 1}, + {Price: 11, Amount: 1}, + } + a.load(asksSnapshot, stack) + + a.cleanup(6, stack) + Check(a, 6, 36, 6, t) + a.cleanup(5, stack) + Check(a, 5, 25, 5, t) + a.cleanup(1, stack) + Check(a, 1, 1, 1, t) + a.cleanup(10, stack) + Check(a, 1, 1, 1, t) + a.cleanup(0, stack) // will purge, underlying checks are done elseware to prevent this + Check(a, 0, 0, 0, t) +} + +// 46154023 24.0 ns/op 0 B/op 0 allocs/op (old) +// 134830672 9.83 ns/op 0 B/op 0 allocs/op (new) +func BenchmarkUpdateInsertByPrice_Amend(b *testing.B) { + a := asks{} + stack := newStack() + + a.load(ask, stack) + + updates := Items{ + { + Price: 1337, // Amend + Amount: 2, + }, + { + Price: 1337, // Amend + Amount: 1, + }, + } + + for i := 0; i < b.N; i++ { + a.updateInsertByPrice(updates, stack, 0, getNow()) + } +} + +// 49763002 24.9 ns/op 0 B/op 0 allocs/op +func BenchmarkUpdateInsertByPrice_Insert_Delete(b *testing.B) { + a := asks{} + stack := newStack() + + a.load(ask, stack) + + updates := Items{ + { + Price: 1337.5, // Insert + Amount: 2, + }, + { + Price: 1337.5, // Delete + Amount: 0, + }, + } + + for i := 0; i < b.N; i++ { + a.updateInsertByPrice(updates, stack, 0, getNow()) + } +} + +func TestUpdateByID(t *testing.T) { + a := asks{} + s := newStack() + asksSnapshot := Items{ + {Price: 1, Amount: 1, ID: 1}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 11, Amount: 1, ID: 11}, + } + a.load(asksSnapshot, s) + + err := a.updateByID(Items{ + {Price: 1, Amount: 1, ID: 1}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 11, Amount: 1, ID: 11}, + }) + if err != nil { + t.Fatal(err) + } + + Check(a, 6, 36, 6, t) + + err = a.updateByID(Items{ + {Price: 11, Amount: 1, ID: 1337}, + }) + if !errors.Is(err, errIDCannotBeMatched) { + t.Fatalf("expecting %s but received %v", errIDCannotBeMatched, err) + } + + err = a.updateByID(Items{ // Simulate Bitmex updating + {Price: 0, Amount: 1337, ID: 3}, + }) + if !errors.Is(err, nil) { + t.Fatalf("expecting %v but received %v", nil, err) + } + + if a.retrieve()[1].Price == 0 { + t.Fatal("price should not be replaced with zero") + } + + if a.retrieve()[1].Amount != 1337 { + t.Fatal("unexpected value for update") + } +} + +// 46043871 25.9 ns/op 0 B/op 0 allocs/op +func BenchmarkUpdateByID(b *testing.B) { + asks := linkedList{} + s := newStack() + asksSnapshot := Items{ + {Price: 1, Amount: 1, ID: 1}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 11, Amount: 1, ID: 11}, + } + asks.load(asksSnapshot, s) + + for i := 0; i < b.N; i++ { + err := asks.updateByID(Items{ + {Price: 1, Amount: 1, ID: 1}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 11, Amount: 1, ID: 11}, + }) + if err != nil { + b.Fatal(err) + } + } +} + +func TestDeleteByID(t *testing.T) { + a := asks{} + s := newStack() + asksSnapshot := Items{ + {Price: 1, Amount: 1, ID: 1}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 11, Amount: 1, ID: 11}, + } + a.load(asksSnapshot, s) + + // Delete at head + err := a.deleteByID(Items{ + {Price: 1, Amount: 1, ID: 1}, + }, s, false) + if err != nil { + t.Fatal(err) + } + + Check(a, 5, 35, 5, t) + + // Delete at tail + err = a.deleteByID(Items{ + {Price: 1, Amount: 1, ID: 11}, + }, s, false) + if err != nil { + t.Fatal(err) + } + + Check(a, 4, 24, 4, t) + + // Delete in middle + err = a.deleteByID(Items{ + {Price: 1, Amount: 1, ID: 5}, + }, s, false) + if err != nil { + t.Fatal(err) + } + + Check(a, 3, 19, 3, t) + + // Intentional error + err = a.deleteByID(Items{ + {Price: 11, Amount: 1, ID: 1337}, + }, s, false) + if !errors.Is(err, errIDCannotBeMatched) { + t.Fatalf("expecting %s but received %v", errIDCannotBeMatched, err) + } + + // Error bypass + err = a.deleteByID(Items{ + {Price: 11, Amount: 1, ID: 1337}, + }, s, true) + if err != nil { + t.Fatal(err) + } +} + +func TestUpdateInsertByIDAsk(t *testing.T) { + a := asks{} + s := newStack() + asksSnapshot := Items{ + {Price: 1, Amount: 1, ID: 1}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 11, Amount: 1, ID: 11}, + } + a.load(asksSnapshot, s) + + // Update one instance with matching ID + err := a.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 7, 37, 6, t) + + // Reset + a.load(asksSnapshot, s) + + // Update all instances with matching ID in order + err = a.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 5, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 12, 72, 6, t) + + // Update all instances with matching ID in backwards + err = a.updateInsertByID(Items{ + {Price: 11, Amount: 2, ID: 11}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 5, Amount: 2, ID: 5}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 1, Amount: 2, ID: 1}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 12, 72, 6, t) + + // Update all instances with matching ID all over the ship + err = a.updateInsertByID(Items{ + {Price: 11, Amount: 2, ID: 11}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 1, Amount: 2, ID: 1}, + {Price: 5, Amount: 2, ID: 5}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 12, 72, 6, t) + + // Update all instances move one before ID in middle + err = a.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 2, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 12, 66, 6, t) + + // Update all instances move one before ID at head + err = a.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: .5, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 12, 63, 6, t) + + // Reset + a.load(asksSnapshot, s) + + // Update all instances move one after ID + err = a.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 8, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 12, 78, 6, t) + + // Reset + a.load(asksSnapshot, s) + + // Update all instances move one after ID to tail + err = a.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 12, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 12, 86, 6, t) + + // Update all instances then pop new instance + err = a.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 12, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + {Price: 10, Amount: 2, ID: 10}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 14, 106, 7, t) + + // Reset + a.load(asksSnapshot, s) + + // Update all instances pop at head + err = a.updateInsertByID(Items{ + {Price: 0.5, Amount: 2, ID: 0}, + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 12, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 14, 87, 7, t) + + // bookmark head and move to mid + err = a.updateInsertByID(Items{ + {Price: 7.5, Amount: 2, ID: 0}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 14, 101, 7, t) + + // bookmark head and move to tail + err = a.updateInsertByID(Items{ + {Price: 12.5, Amount: 2, ID: 1}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 14, 124, 7, t) + + // move tail location to head + err = a.updateInsertByID(Items{ + {Price: 2.5, Amount: 2, ID: 1}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 14, 104, 7, t) + + // move tail location to mid + err = a.updateInsertByID(Items{ + {Price: 8, Amount: 2, ID: 5}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 14, 96, 7, t) + + // insert at tail dont match + err = a.updateInsertByID(Items{ + {Price: 30, Amount: 2, ID: 1234}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 16, 156, 8, t) + + // insert between last and 2nd last + err = a.updateInsertByID(Items{ + {Price: 12, Amount: 2, ID: 12345}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 18, 180, 9, t) + + // readjust at end + err = a.updateInsertByID(Items{ + {Price: 29, Amount: 3, ID: 1234}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 19, 207, 9, t) + + // readjust further and decrease price past tail + err = a.updateInsertByID(Items{ + {Price: 31, Amount: 3, ID: 1234}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 19, 213, 9, t) + + // purge + a.load(nil, s) + + // insert with no liquidity and jumbled + err = a.updateInsertByID(Items{ + {Price: 11, Amount: 2, ID: 11}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 0.5, Amount: 2, ID: 0}, + {Price: 12, Amount: 2, ID: 5}, + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 14, 87, 7, t) +} + +func TestUpdateInsertByIDBids(t *testing.T) { + b := bids{} + s := newStack() + bidsSnapshot := Items{ + {Price: 11, Amount: 1, ID: 11}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 1, Amount: 1, ID: 1}, + } + b.load(bidsSnapshot, s) + + // Update one instance with matching ID + err := b.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 7, 37, 6, t) + + // Reset + b.load(bidsSnapshot, s) + + // Update all instances with matching ID in order + err = b.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 5, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 12, 72, 6, t) + + // Update all instances with matching ID in backwards + err = b.updateInsertByID(Items{ + {Price: 11, Amount: 2, ID: 11}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 5, Amount: 2, ID: 5}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 1, Amount: 2, ID: 1}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 12, 72, 6, t) + + // Update all instances with matching ID all over the ship + err = b.updateInsertByID(Items{ + {Price: 11, Amount: 2, ID: 11}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 1, Amount: 2, ID: 1}, + {Price: 5, Amount: 2, ID: 5}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 12, 72, 6, t) + + // Update all instances move one before ID in middle + err = b.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 2, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 12, 66, 6, t) + + // Update all instances move one before ID at head + err = b.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: .5, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 12, 63, 6, t) + + // Reset + b.load(bidsSnapshot, s) + + // Update all instances move one after ID + err = b.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 8, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 12, 78, 6, t) + + // Reset + b.load(bidsSnapshot, s) + + // Update all instances move one after ID to tail + err = b.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 12, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 12, 86, 6, t) + + // Update all instances then pop new instance + err = b.updateInsertByID(Items{ + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 12, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + {Price: 10, Amount: 2, ID: 10}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 14, 106, 7, t) + + // Reset + b.load(bidsSnapshot, s) + + // Update all instances pop at tail + err = b.updateInsertByID(Items{ + {Price: 0.5, Amount: 2, ID: 0}, + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 12, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 14, 87, 7, t) + + // bookmark head and move to mid + err = b.updateInsertByID(Items{ + {Price: 9.5, Amount: 2, ID: 5}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 14, 82, 7, t) + + // bookmark head and move to tail + err = b.updateInsertByID(Items{ + {Price: 0.25, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 14, 60.5, 7, t) + + // move tail location to head + err = b.updateInsertByID(Items{ + {Price: 10, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 14, 80, 7, t) + + // move tail location to mid + err = b.updateInsertByID(Items{ + {Price: 7.5, Amount: 2, ID: 0}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 14, 94, 7, t) + + // insert at head dont match + err = b.updateInsertByID(Items{ + {Price: 30, Amount: 2, ID: 1234}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 16, 154, 8, t) + + // insert between last and 2nd last + err = b.updateInsertByID(Items{ + {Price: 1.5, Amount: 2, ID: 12345}, + }, s) + if err != nil { + t.Fatal(err) + } + Check(b, 18, 157, 9, t) + + // readjust at end + err = b.updateInsertByID(Items{ + {Price: 1, Amount: 3, ID: 1}, + }, s) + if err != nil { + t.Fatal(err) + } + Check(b, 19, 158, 9, t) + + // readjust further and decrease price past tail + err = b.updateInsertByID(Items{ + {Price: .9, Amount: 3, ID: 1}, + }, s) + if err != nil { + t.Fatal(err) + } + Check(b, 19, 157.7, 9, t) + + // purge + b.load(nil, s) + + // insert with no liquidity and jumbled + err = b.updateInsertByID(Items{ + {Price: 0.5, Amount: 2, ID: 0}, + {Price: 1, Amount: 2, ID: 1}, + {Price: 3, Amount: 2, ID: 3}, + {Price: 12, Amount: 2, ID: 5}, + {Price: 7, Amount: 2, ID: 7}, + {Price: 9, Amount: 2, ID: 9}, + {Price: 11, Amount: 2, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 14, 87, 7, t) +} + +func TestInsertUpdatesBid(t *testing.T) { + b := bids{} + s := newStack() + bidsSnapshot := Items{ + {Price: 11, Amount: 1, ID: 11}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 1, Amount: 1, ID: 1}, + } + b.load(bidsSnapshot, s) + + err := b.insertUpdates(Items{ + {Price: 11, Amount: 1, ID: 11}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 1, Amount: 1, ID: 1}, + }, s) + if !errors.Is(err, errCollisionDetected) { + t.Fatalf("expected error %s but received %v", errCollisionDetected, err) + } + + Check(b, 6, 36, 6, t) + + // Insert at head + err = b.insertUpdates(Items{ + {Price: 12, Amount: 1, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 7, 48, 7, t) + + // Insert at tail + err = b.insertUpdates(Items{ + {Price: 0.5, Amount: 1, ID: 12}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 8, 48.5, 8, t) + + // Insert at mid + err = b.insertUpdates(Items{ + {Price: 5.5, Amount: 1, ID: 13}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 9, 54, 9, t) + + // purge + b.load(nil, s) + + // Add one at head + err = b.insertUpdates(Items{ + {Price: 5.5, Amount: 1, ID: 13}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(b, 1, 5.5, 1, t) +} + +func TestInsertUpdatesAsk(t *testing.T) { + a := asks{} + s := newStack() + askSnapshot := Items{ + {Price: 1, Amount: 1, ID: 1}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 11, Amount: 1, ID: 11}, + } + a.load(askSnapshot, s) + + err := a.insertUpdates(Items{ + {Price: 11, Amount: 1, ID: 11}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 1, Amount: 1, ID: 1}, + }, s) + if !errors.Is(err, errCollisionDetected) { + t.Fatalf("expected error %s but received %v", errCollisionDetected, err) + } + + Check(a, 6, 36, 6, t) + + // Insert at tail + err = a.insertUpdates(Items{ + {Price: 12, Amount: 1, ID: 11}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 7, 48, 7, t) + + // Insert at head + err = a.insertUpdates(Items{ + {Price: 0.5, Amount: 1, ID: 12}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 8, 48.5, 8, t) + + // Insert at mid + err = a.insertUpdates(Items{ + {Price: 5.5, Amount: 1, ID: 13}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 9, 54, 9, t) + + // purge + a.load(nil, s) + + // Add one at head + err = a.insertUpdates(Items{ + {Price: 5.5, Amount: 1, ID: 13}, + }, s) + if err != nil { + t.Fatal(err) + } + + Check(a, 1, 5.5, 1, t) +} + +// check checks depth values after an update has taken place +func Check(depth interface{}, liquidity, value float64, nodeCount int, t *testing.T) { + t.Helper() + b, isBid := depth.(bids) + a, isAsk := depth.(asks) + + var ll linkedList + switch { + case isBid: + ll = b.linkedList + case isAsk: + ll = a.linkedList + default: + t.Fatal("value passed in is not of type bids or asks") + } + + liquidityTotal, valueTotal := ll.amount() + + if liquidityTotal != liquidity { + ll.display() + t.Fatalf("mismatched liquidity expecting %v but received %v", + liquidity, + liquidityTotal) + } + + if valueTotal != value { + ll.display() + t.Fatalf("mismatched total value expecting %v but received %v", + value, + valueTotal) + } + + if ll.length != nodeCount { + ll.display() + t.Fatalf("mismatched node count expecting %v but received %v", + nodeCount, + ll.length) + } + + if ll.head == nil { + return + } + + var tail *node + var price float64 + for tip := ll.head; ; tip = tip.next { + switch { + case price == 0: + price = tip.value.Price + case isBid && price < tip.value.Price: + ll.display() + t.Fatal("Bid pricing out of order should be descending") + case isAsk && price > tip.value.Price: + ll.display() + t.Fatal("Ask pricing out of order should be ascending") + default: + price = tip.value.Price + } + + if tip.next == nil { + tail = tip + break + } + } + + var liqReversed, valReversed float64 + var nodeReversed int + for tip := tail; tip != nil; tip = tip.prev { + liqReversed += tip.value.Amount + valReversed += tip.value.Amount * tip.value.Price + nodeReversed++ + } + + if liquidity-liqReversed != 0 { + ll.display() + fmt.Println(liquidity, liqReversed) + t.Fatalf("mismatched liquidity when reversing direction expecting %v but received %v", + 0, + liquidity-liqReversed) + } + + if nodeCount-nodeReversed != 0 { + ll.display() + t.Fatalf("mismatched node count when reversing direction expecting %v but received %v", + 0, + nodeCount-nodeReversed) + } + + if value-valReversed != 0 { + ll.display() + fmt.Println(valReversed, value) + t.Fatalf("mismatched total book value when reversing direction expecting %v but received %v", + 0, + value-valReversed) + } +} + +func TestAmount(t *testing.T) { + a := asks{} + s := newStack() + askSnapshot := Items{ + {Price: 1, Amount: 1, ID: 1}, + {Price: 3, Amount: 1, ID: 3}, + {Price: 5, Amount: 1, ID: 5}, + {Price: 7, Amount: 1, ID: 7}, + {Price: 9, Amount: 1, ID: 9}, + {Price: 11, Amount: 1, ID: 11}, + } + a.load(askSnapshot, s) + + liquidity, value := a.amount() + if liquidity != 6 { + t.Fatalf("incorrect liquidity calculation expected 6 but received %f", liquidity) + } + + if value != 36 { + t.Fatalf("incorrect value calculation expected 36 but received %f", value) + } +} + +func TestShiftBookmark(t *testing.T) { + bookmarkedNode := &node{ + value: Item{ + ID: 1337, + Amount: 1, + Price: 2, + }, + next: nil, + prev: nil, + shelved: time.Time{}, + } + + originalBookmarkPrev := &node{ + value: Item{ + ID: 1336, + }, + next: bookmarkedNode, + prev: nil, // At head + shelved: time.Time{}, + } + originalBookmarkNext := &node{ + value: Item{ + ID: 1338, + }, + next: nil, // This can be left nil in actuality this will be + // populated + prev: bookmarkedNode, + shelved: time.Time{}, + } + + // associate previous and next nodes to bookmarked node + bookmarkedNode.prev = originalBookmarkPrev + bookmarkedNode.next = originalBookmarkNext + + tip := &node{ + value: Item{ + ID: 69420, + }, + next: nil, // In this case tip will be at tail + prev: nil, + shelved: time.Time{}, + } + + tipprev := &node{ + value: Item{ + ID: 69419, + }, + next: tip, + prev: nil, // This can be left nil in actuality this will be + // populated + shelved: time.Time{}, + } + + // associate tips prev field with the correct prev node + tip.prev = tipprev + + if !shiftBookmark(tip, &bookmarkedNode, nil, Item{Amount: 1336, ID: 1337, Price: 9999}) { + t.Fatal("There should be liquidity so we don't need to set tip to bookmark") + } + + if bookmarkedNode.value.Price != 9999 || + bookmarkedNode.value.Amount != 1336 || + bookmarkedNode.value.ID != 1337 { + t.Fatal("bookmarked details are not set correctly with shift") + } + + if bookmarkedNode.prev != tip { + t.Fatal("bookmarked prev memory address does not point to tip") + } + + if bookmarkedNode.next != nil { + t.Fatal("bookmarked next is at tail and should be nil") + } + + if bookmarkedNode.next != nil { + t.Fatal("bookmarked next is at tail and should be nil") + } + + if originalBookmarkPrev.next != originalBookmarkNext { + t.Fatal("original bookmarked prev node should be associated with original bookmarked next node") + } + + if originalBookmarkNext.prev != originalBookmarkPrev { + t.Fatal("original bookmarked next node should be associated with original bookmarked prev node") + } + + var nilBookmark *node + + if shiftBookmark(tip, &nilBookmark, nil, Item{Amount: 1336, ID: 1337, Price: 9999}) { + t.Fatal("there should not be a bookmarked node") + } + + if tip != nilBookmark { + t.Fatal("nilBookmark not reassigned") + } + + head := bookmarkedNode + bookmarkedNode.prev = nil + bookmarkedNode.next = originalBookmarkNext + tip.next = nil + + if !shiftBookmark(tip, &bookmarkedNode, &head, Item{Amount: 1336, ID: 1337, Price: 9999}) { + t.Fatal("There should be liquidity so we don't need to set tip to bookmark") + } + + if head != originalBookmarkNext { + t.Fatal("unexpected pointer variable") + } +} diff --git a/exchanges/orderbook/node.go b/exchanges/orderbook/node.go new file mode 100644 index 00000000..c35b72bc --- /dev/null +++ b/exchanges/orderbook/node.go @@ -0,0 +1,135 @@ +package orderbook + +import ( + "sync/atomic" + "time" +) + +const ( + neutral uint32 = iota + active +) + +var ( + defaultInterval = time.Minute + defaultAllowance = time.Second * 30 +) + +// node defines a linked list node for an orderbook item +type node struct { + value Item + next *node + prev *node + // Denotes time pushed to stack, this will influence cleanup routine when + // there is a pause or minimal actions during period + shelved time.Time +} + +// stack defines a FILO list of reusable nodes +type stack struct { + nodes []*node + sema uint32 + count int32 +} + +// newStack returns a ptr to a new stack instance, also starts the cleaning +// service +func newStack() *stack { + s := &stack{} + go s.cleaner() + return s +} + +// now defines a time which is now to ensure no other values get passed in +type now time.Time + +// getNow returns the time at which it is called +func getNow() now { + return now(time.Now()) +} + +// Push pushes a node pointer into the stack to be reused the time is passed in +// to allow for inlining which sets the time at which the node is theoretically +// pushed to a stack. +func (s *stack) Push(n *node, tn now) { + if !atomic.CompareAndSwapUint32(&s.sema, neutral, active) { + // Stack is in use, for now we can dereference pointer + n = nil + return + } + // Adds a time when its placed back on to stack. + n.shelved = time.Time(tn) + n.next = nil + n.prev = nil + n.value = Item{} + + // Allows for resize when overflow TODO: rethink this + s.nodes = append(s.nodes[:s.count], n) + s.count++ + atomic.StoreUint32(&s.sema, neutral) +} + +// Pop returns the last pointer off the stack and reduces the count and if empty +// will produce a lovely fresh node +func (s *stack) Pop() *node { + if !atomic.CompareAndSwapUint32(&s.sema, neutral, active) { + // Stack is in use, for now we can allocate a new node pointer + return &node{} + } + + if s.count == 0 { + // Create an empty node when no nodes are in slice or when cleaning + // service is running + atomic.StoreUint32(&s.sema, neutral) + return &node{} + } + s.count-- + n := s.nodes[s.count] + atomic.StoreUint32(&s.sema, neutral) + return n +} + +// cleaner (POC) runs to the defaultTimer to clean excess nodes (nodes not being +// utilised) TODO: Couple time parameters to check for a reduction in activity. +// Add in counter per second function (?) so if there is a lot of activity don't +// inhibit stack performance. +func (s *stack) cleaner() { + tt := time.NewTimer(defaultInterval) +sleeperino: + for range tt.C { + if !atomic.CompareAndSwapUint32(&s.sema, neutral, active) { + // Stack is in use, reset timer to zero to recheck for neutral state. + tt.Reset(0) + continue + } + // As the old nodes are going to be left justified on this slice we + // should just be able to shift the nodes that are still within time + // allowance all the way to the left. Not going to resize capacity + // because if it can get this big, it might as well stay this big. + // TODO: Test and rethink if sizing is an issue + for x := int32(0); x < s.count; x++ { + if time.Since(s.nodes[x].shelved) > defaultAllowance { + // Old node found continue + continue + } + // First good node found, everything to the left of this on the + // slice can be reassigned + var counter int32 + for y := int32(0); y+x < s.count; y++ { // Go through good nodes + // Reassign + s.nodes[y] = s.nodes[y+x] + // Add to the changed counter to remove from main + // counter + counter++ + } + s.count -= counter + atomic.StoreUint32(&s.sema, neutral) + tt.Reset(defaultInterval) + continue sleeperino + } + // Nodes are old, flush entirety. + s.count = 0 + atomic.StoreUint32(&s.sema, neutral) + tt.Reset(defaultInterval) + } +} diff --git a/exchanges/orderbook/node_test.go b/exchanges/orderbook/node_test.go new file mode 100644 index 00000000..553a675c --- /dev/null +++ b/exchanges/orderbook/node_test.go @@ -0,0 +1,101 @@ +package orderbook + +import ( + "fmt" + "sync/atomic" + "testing" + "time" +) + +func TestPushPop(t *testing.T) { + s := newStack() + var nSlice []*node + for i := 0; i < 100; i++ { + nSlice = append(nSlice, s.Pop()) + } + + if s.getCount() != 0 { + t.Fatalf("incorrect stack count expected %v but received %v", 0, s.getCount()) + } + + for i := 0; i < 100; i++ { + s.Push(nSlice[i], getNow()) + } + + if s.getCount() != 100 { + t.Fatalf("incorrect stack count expected %v but received %v", 100, s.getCount()) + } +} + +func TestCleaner(t *testing.T) { + s := newStack() + var nSlice []*node + for i := 0; i < 100; i++ { + nSlice = append(nSlice, s.Pop()) + } + + tn := getNow() + for i := 0; i < 50; i++ { + s.Push(nSlice[i], tn) + } + // Makes all the 50 pushed nodes invalid + time.Sleep(time.Millisecond * 550) + tn = getNow() + for i := 50; i < 100; i++ { + s.Push(nSlice[i], tn) + } + time.Sleep(time.Millisecond * 550) + if s.getCount() != 50 { + t.Fatalf("incorrect stack count expected %v but received %v", 50, s.getCount()) + } + time.Sleep(time.Second) + if s.getCount() != 0 { + t.Fatalf("incorrect stack count expected %v but received %v", 0, s.getCount()) + } +} + +// Display nodes for testing purposes +func (s *stack) Display() { + for i := int32(0); i < s.getCount(); i++ { + fmt.Printf("NODE IN STACK: %+v %p \n", s.nodes[i], s.nodes[i]) + } + fmt.Println("TOTAL COUNT:", s.getCount()) +} + +// 158 9,521,717 ns/op 9600104 B/op 100001 allocs/op +func BenchmarkWithoutStack(b *testing.B) { + var n *node + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := 0; j < 100000; j++ { + n = new(node) + n.value.Price = 1337 + } + } +} + +// 316 3,485,211 ns/op 1 B/op 0 allocs/op +func BenchmarkWithStack(b *testing.B) { + var n *node + stack := newStack() + b.ReportAllocs() + b.ResetTimer() + tn := getNow() + for i := 0; i < b.N; i++ { + for j := 0; j < 100000; j++ { + n = stack.Pop() + n.value.Price = 1337 + stack.Push(n, tn) + } + } +} + +// getCount is a test helper function to derive the count that does not race. +func (s *stack) getCount() int32 { + if !atomic.CompareAndSwapUint32(&s.sema, neutral, active) { + return -1 + } + defer atomic.StoreUint32(&s.sema, neutral) + return s.count +} diff --git a/exchanges/orderbook/orderbook.go b/exchanges/orderbook/orderbook.go index f5e664be..4c242ac3 100644 --- a/exchanges/orderbook/orderbook.go +++ b/exchanges/orderbook/orderbook.go @@ -18,144 +18,181 @@ func Get(exchange string, p currency.Pair, a asset.Item) (*Base, error) { return service.Retrieve(exchange, p, a) } -// SubscribeOrderbook subcribes to an orderbook and returns a communication -// channel to stream orderbook data updates -func SubscribeOrderbook(exchange string, p currency.Pair, a asset.Item) (dispatch.Pipe, error) { - exchange = strings.ToLower(exchange) - service.Lock() - defer service.Unlock() - book, ok := service.Books[exchange][a][p.Base.Item][p.Quote.Item] - if !ok { - return dispatch.Pipe{}, - fmt.Errorf("orderbook item not found for %s %s %s", - exchange, - p, - a) - } - return service.mux.Subscribe(book.Main) +// GetDepth returns a Depth pointer allowing the caller to stream orderbook +// changes +func GetDepth(exchange string, p currency.Pair, a asset.Item) (*Depth, error) { + return service.GetDepth(exchange, p, a) } -// SubscribeToExchangeOrderbooks subcribes to all orderbooks on an exchange +// DeployDepth sets a depth struct and returns a depth pointer. This allows for +// the loading of a new orderbook snapshot and incremental updates via the +// streaming package. +func DeployDepth(exchange string, p currency.Pair, a asset.Item) (*Depth, error) { + return service.DeployDepth(exchange, p, a) +} + +// SubscribeToExchangeOrderbooks returns a pipe to an exchange feed func SubscribeToExchangeOrderbooks(exchange string) (dispatch.Pipe, error) { service.Lock() defer service.Unlock() - id, ok := service.Exchange[strings.ToLower(exchange)] + exch, ok := service.books[strings.ToLower(exchange)] if !ok { - return dispatch.Pipe{}, fmt.Errorf("%s exchange orderbooks not found", - exchange) + return dispatch.Pipe{}, fmt.Errorf("%w for %s exchange", + errCannotFindOrderbook, exchange) } - return service.mux.Subscribe(id) + return service.Mux.Subscribe(exch.ID) } // Update stores orderbook data func (s *Service) Update(b *Base) error { - name := strings.ToLower(b.ExchangeName) + name := strings.ToLower(b.Exchange) s.Lock() - m1, ok := s.Books[name] + m1, ok := s.books[name] if !ok { - m1 = make(map[asset.Item]map[*currency.Item]map[*currency.Item]*Book) - s.Books[name] = m1 + id, err := s.Mux.GetID() + if err != nil { + s.Unlock() + return err + } + m1 = Exchange{ + m: make(map[asset.Item]map[*currency.Item]map[*currency.Item]*Depth), + ID: id, + } + s.books[name] = m1 } - m2, ok := m1[b.AssetType] + m2, ok := m1.m[b.Asset] if !ok { - m2 = make(map[*currency.Item]map[*currency.Item]*Book) - m1[b.AssetType] = m2 + m2 = make(map[*currency.Item]map[*currency.Item]*Depth) + m1.m[b.Asset] = m2 } m3, ok := m2[b.Pair.Base.Item] if !ok { - m3 = make(map[*currency.Item]*Book) + m3 = make(map[*currency.Item]*Depth) m2[b.Pair.Base.Item] = m3 } book, ok := m3[b.Pair.Quote.Item] if !ok { - book = new(Book) + book = newDepth(m1.ID) + book.AssignOptions(b) m3[b.Pair.Quote.Item] = book - err := s.SetNewData(b, book, name) - s.Unlock() - return err } - - book.b.Bids = append(b.Bids[:0:0], b.Bids...) // nolint:gocritic // Short hand to not use make and copy - book.b.Asks = append(b.Asks[:0:0], b.Asks...) // nolint:gocritic // Short hand to not use make and copy - book.b.LastUpdated = b.LastUpdated - ids := append(book.Assoc, book.Main) + book.SetLastUpdate(b.LastUpdated, b.LastUpdateID, true) + book.LoadSnapshot(b.Bids, b.Asks) s.Unlock() - return s.mux.Publish(ids, b) + return s.Mux.Publish([]uuid.UUID{m1.ID}, book.Retrieve()) } -// SetNewData sets new data -func (s *Service) SetNewData(ob *Base, book *Book, exch string) error { - var err error - book.Assoc, err = s.getAssociations(strings.ToLower(exch)) - if err != nil { - return err +// DeployDepth used for subsystem deployment creates a depth item in the struct +// then returns a ptr to that Depth item +func (s *Service) DeployDepth(exchange string, p currency.Pair, a asset.Item) (*Depth, error) { + if exchange == "" { + return nil, errExchangeNameUnset } - book.Main, err = s.mux.GetID() - if err != nil { - return err + if p.IsEmpty() { + return nil, errPairNotSet } - - // Below instigates orderbook item separation so we can ensure, in the event - // of a simultaneous update via websocket/rest/fix, we don't affect package - // scoped orderbook data which could result in a potential panic - cpy := *ob - cpy.Bids = append(cpy.Bids[:0:0], cpy.Bids...) - cpy.Asks = append(cpy.Asks[:0:0], cpy.Asks...) - book.b = &cpy - return nil -} - -// GetAssociations links a singular book with it's dispatch associations -func (s *Service) getAssociations(exch string) ([]uuid.UUID, error) { - var ids []uuid.UUID - exchangeID, ok := s.Exchange[exch] + if !a.IsValid() { + return nil, errAssetTypeNotSet + } + s.Lock() + m1, ok := s.books[strings.ToLower(exchange)] if !ok { - var err error - exchangeID, err = s.mux.GetID() + id, err := s.Mux.GetID() if err != nil { return nil, err } - s.Exchange[exch] = exchangeID + m1 = Exchange{ + m: make(map[asset.Item]map[*currency.Item]map[*currency.Item]*Depth), + ID: id, + } + s.books[strings.ToLower(exchange)] = m1 } - - ids = append(ids, exchangeID) - return ids, nil + m2, ok := m1.m[a] + if !ok { + m2 = make(map[*currency.Item]map[*currency.Item]*Depth) + m1.m[a] = m2 + } + m3, ok := m2[p.Base.Item] + if !ok { + m3 = make(map[*currency.Item]*Depth) + m2[p.Base.Item] = m3 + } + book, ok := m3[p.Quote.Item] + if !ok { + book = newDepth(m1.ID) + m3[p.Quote.Item] = book + } + s.Unlock() + return book, nil } -// Retrieve gets orderbook data from the slice -func (s *Service) Retrieve(exchange string, p currency.Pair, a asset.Item) (*Base, error) { +// GetDepth returns the actual depth struct for potential subsystems and +// strategies to interact with +func (s *Service) GetDepth(exchange string, p currency.Pair, a asset.Item) (*Depth, error) { s.Lock() defer s.Unlock() - m1, ok := s.Books[strings.ToLower(exchange)] + m1, ok := s.books[strings.ToLower(exchange)] if !ok { - return nil, fmt.Errorf("no orderbooks for %s exchange", exchange) + return nil, fmt.Errorf("%w for %s exchange", + errCannotFindOrderbook, exchange) } - m2, ok := m1[a] + m2, ok := m1.m[a] if !ok { - return nil, fmt.Errorf("no orderbooks associated with asset type %s", + return nil, fmt.Errorf("%w associated with asset type %s", + errCannotFindOrderbook, a) } m3, ok := m2[p.Base.Item] if !ok { - return nil, fmt.Errorf("no orderbooks associated with base currency %s", + return nil, fmt.Errorf("%w associated with base currency %s", + errCannotFindOrderbook, p.Base) } book, ok := m3[p.Quote.Item] if !ok { - return nil, fmt.Errorf("no orderbooks associated with base currency %s", + return nil, fmt.Errorf("%w associated with base currency %s", + errCannotFindOrderbook, p.Quote) } + return book, nil +} - ob := *book.b - ob.Bids = append(ob.Bids[:0:0], ob.Bids...) - ob.Asks = append(ob.Asks[:0:0], ob.Asks...) - return &ob, nil +// Retrieve gets orderbook depth data from the associated linked list and +// returns the base equivalent copy +func (s *Service) Retrieve(exchange string, p currency.Pair, a asset.Item) (*Base, error) { + s.Lock() + defer s.Unlock() + m1, ok := s.books[strings.ToLower(exchange)] + if !ok { + return nil, fmt.Errorf("%w for %s exchange", + errCannotFindOrderbook, + exchange) + } + m2, ok := m1.m[a] + if !ok { + return nil, fmt.Errorf("%w associated with asset type %s", + errCannotFindOrderbook, + a) + } + m3, ok := m2[p.Base.Item] + if !ok { + return nil, fmt.Errorf("%w associated with base currency %s", + errCannotFindOrderbook, + p.Base) + } + book, ok := m3[p.Quote.Item] + if !ok { + return nil, fmt.Errorf("%w associated with base currency %s", + errCannotFindOrderbook, + p.Quote) + } + return book.Retrieve(), nil } // TotalBidsAmount returns the total amount of bids and the total orderbook @@ -178,18 +215,15 @@ func (b *Base) TotalAsksAmount() (amountCollated, total float64) { return amountCollated, total } -// Update updates the bids and asks -func (b *Base) Update(bids, asks []Item) { - b.Bids = bids - b.Asks = asks - b.LastUpdated = time.Now() -} - // Verify ensures that the orderbook items are correctly sorted prior to being // set and will reject any book with incorrect values. // Bids should always go from a high price to a low price and // Asks should always go from a low price to a higher price func (b *Base) Verify() error { + if !b.VerifyOrderbook { + return nil + } + // Checking for both ask and bid lengths being zero has been removed and // a warning has been put in place some exchanges e.g. LakeBTC return zero // level books. In the event that there is a massive liquidity change where @@ -198,58 +232,68 @@ func (b *Base) Verify() error { if len(b.Asks) == 0 || len(b.Bids) == 0 { log.Warnf(log.OrderBook, bookLengthIssue, - b.ExchangeName, + b.Exchange, b.Pair, - b.AssetType, + b.Asset, len(b.Bids), len(b.Asks)) } - for i := range b.Bids { - if b.Bids[i].Price == 0 { - return fmt.Errorf(bidLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errPriceNotSet) - } - if b.Bids[i].Amount <= 0 { - return fmt.Errorf(bidLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errAmountInvalid) - } - if b.IsFundingRate && b.Bids[i].Period == 0 { - return fmt.Errorf(bidLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errPeriodUnset) - } - if i != 0 { - if b.Bids[i].Price > b.Bids[i-1].Price { - return fmt.Errorf(bidLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errOutOfOrder) - } - - if !b.NotAggregated && b.Bids[i].Price == b.Bids[i-1].Price { - return fmt.Errorf(bidLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errDuplication) - } - - if b.Bids[i].ID != 0 && b.Bids[i].ID == b.Bids[i-1].ID { - return fmt.Errorf(bidLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errIDDuplication) - } - } + err := checkAlignment(b.Bids, b.IsFundingRate, b.PriceDuplication, b.IDAlignment, dsc) + if err != nil { + return fmt.Errorf(bidLoadBookFailure, b.Exchange, b.Pair, b.Asset, err) } + err = checkAlignment(b.Asks, b.IsFundingRate, b.PriceDuplication, b.IDAlignment, asc) + if err != nil { + return fmt.Errorf(askLoadBookFailure, b.Exchange, b.Pair, b.Asset, err) + } + return nil +} - for i := range b.Asks { - if b.Asks[i].Price == 0 { - return fmt.Errorf(askLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errPriceNotSet) +// checker defines specific functionality to determine ascending/descending +// validation +type checker func(current Item, previous Item) error + +// asc specifically defines ascending price check +var asc = func(current Item, previous Item) error { + if current.Price < previous.Price { + return errPriceOutOfOrder + } + return nil +} + +// dsc specifically defines descending price check +var dsc = func(current Item, previous Item) error { + if current.Price > previous.Price { + return errPriceOutOfOrder + } + return nil +} + +// checkAlignment validates full orderbook +func checkAlignment(depth Items, fundingRate, priceDuplication, isIDAligned bool, c checker) error { + for i := range depth { + if depth[i].Price == 0 { + return errPriceNotSet } - if b.Asks[i].Amount <= 0 { - return fmt.Errorf(askLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errAmountInvalid) + if depth[i].Amount <= 0 { + return errAmountInvalid } - if b.IsFundingRate && b.Asks[i].Period == 0 { - return fmt.Errorf(askLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errPeriodUnset) + if fundingRate && depth[i].Period == 0 { + return errPeriodUnset } if i != 0 { - if b.Asks[i].Price < b.Asks[i-1].Price { - return fmt.Errorf(askLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errOutOfOrder) + prev := i - 1 + if err := c(depth[i], depth[prev]); err != nil { + return err } - - if !b.NotAggregated && b.Asks[i].Price == b.Asks[i-1].Price { - return fmt.Errorf(askLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errDuplication) + if isIDAligned && depth[i].ID < depth[prev].ID { + return errIDOutOfOrder } - - if b.Asks[i].ID != 0 && b.Asks[i].ID == b.Asks[i-1].ID { - return fmt.Errorf(askLoadBookFailure, b.ExchangeName, b.Pair, b.AssetType, errIDDuplication) + if !priceDuplication && depth[i].Price == depth[prev].Price { + return errDuplication + } + if depth[i].ID != 0 && depth[i].ID == depth[prev].ID { + return errIDDuplication } } } @@ -259,7 +303,7 @@ func (b *Base) Verify() error { // Process processes incoming orderbooks, creating or updating the orderbook // list func (b *Base) Process() error { - if b.ExchangeName == "" { + if b.Exchange == "" { return errExchangeNameUnset } @@ -267,7 +311,7 @@ func (b *Base) Process() error { return errPairNotSet } - if b.AssetType.String() == "" { + if b.Asset.String() == "" { return errAssetTypeNotSet } @@ -275,11 +319,9 @@ func (b *Base) Process() error { b.LastUpdated = time.Now() } - if !b.VerificationBypass && !b.HasChecksumValidation { - err := b.Verify() - if err != nil { - return err - } + err := b.Verify() + if err != nil { + return err } return service.Update(b) } @@ -290,27 +332,25 @@ func (b *Base) Process() error { // using a sort algorithm as the algorithm could be impeded by a worst case time // complexity when elements are shifted as opposed to just swapping element // values. -func Reverse(elem []Item) { - eLen := len(elem) +func (elem *Items) Reverse() { + eLen := len(*elem) var target int for i := eLen/2 - 1; i >= 0; i-- { target = eLen - 1 - i - elem[i], elem[target] = elem[target], elem[i] + (*elem)[i], (*elem)[target] = (*elem)[target], (*elem)[i] } } // SortAsks sorts ask items to the correct ascending order if pricing values are // scattered. If order from exchange is descending consider using the Reverse // function. -func SortAsks(d []Item) []Item { - sort.Sort(byOBPrice(d)) - return d +func (elem *Items) SortAsks() { + sort.Sort(byOBPrice(*elem)) } // SortBids sorts bid items to the correct descending order if pricing values // are scattered. If order from exchange is ascending consider using the Reverse // function. -func SortBids(d []Item) []Item { - sort.Sort(sort.Reverse(byOBPrice(d))) - return d +func (elem *Items) SortBids() { + sort.Sort(sort.Reverse(byOBPrice(*elem))) } diff --git a/exchanges/orderbook/orderbook_test.go b/exchanges/orderbook/orderbook_test.go index 06eb5af5..760f3d04 100644 --- a/exchanges/orderbook/orderbook_test.go +++ b/exchanges/orderbook/orderbook_test.go @@ -16,105 +16,29 @@ import ( ) func TestMain(m *testing.M) { + // Sets up lower values for test environment + defaultInterval = time.Second + defaultAllowance = time.Millisecond * 500 err := dispatch.Start(1, dispatch.DefaultJobsLimit) if err != nil { log.Fatal(err) } - - cpyMux = service.mux - os.Exit(m.Run()) } -var cpyMux *dispatch.Mux - -func TestSubscribeOrderbook(t *testing.T) { - _, err := SubscribeOrderbook("", currency.Pair{}, asset.Item("")) - if err == nil { - t.Error("error cannot be nil") - } - - p := currency.NewPair(currency.BTC, currency.USD) - - b := Base{ - Pair: p, - AssetType: asset.Spot, - } - - err = b.Process() - if err == nil { - t.Error("error cannot be nil") - } - - b.ExchangeName = "SubscribeOBTest" - b.Bids = []Item{{Price: 100, Amount: 1}, {Price: 99, Amount: 1}} - err = b.Process() - if err != nil { - t.Error("process error", err) - } - - _, err = SubscribeOrderbook("SubscribeOBTest", p, asset.Spot) - if err != nil { - t.Error(err) - } - - // process redundant update - err = b.Process() - if err != nil { - t.Error("process error", err) - } -} - -func TestUpdateBooks(t *testing.T) { - p := currency.NewPair(currency.BTC, currency.USD) - - b := Base{ - Pair: p, - AssetType: asset.Spot, - ExchangeName: "UpdateTest", - } - - service.mux = nil - - err := service.Update(&b) - if err == nil { - t.Error("error cannot be nil") - } - - b.Pair.Base = currency.CYC - err = service.Update(&b) - if err == nil { - t.Error("error cannot be nil") - } - - b.Pair.Quote = currency.ENAU - err = service.Update(&b) - if err == nil { - t.Error("error cannot be nil") - } - - b.AssetType = "unicorns" - err = service.Update(&b) - if err == nil { - t.Error("error cannot be nil") - } - - service.mux = cpyMux -} - func TestSubscribeToExchangeOrderbooks(t *testing.T) { _, err := SubscribeToExchangeOrderbooks("") - if err == nil { - t.Error("error cannot be nil") + if !errors.Is(err, errCannotFindOrderbook) { + t.Fatalf("expected: %v but received: %v", errCannotFindOrderbook, err) } p := currency.NewPair(currency.BTC, currency.USD) b := Base{ - Pair: p, - AssetType: asset.Spot, - ExchangeName: "SubscribeToExchangeOrderbooks", - Bids: []Item{{Price: 100, Amount: 1}, {Price: 99, Amount: 1}}, + Pair: p, + Asset: asset.Spot, + Exchange: "SubscribeToExchangeOrderbooks", + Bids: []Item{{Price: 100, Amount: 1}, {Price: 99, Amount: 1}}, } err = b.Process() @@ -131,9 +55,10 @@ func TestSubscribeToExchangeOrderbooks(t *testing.T) { func TestVerify(t *testing.T) { t.Parallel() b := Base{ - ExchangeName: "TestExchange", - AssetType: asset.Spot, - Pair: currency.NewPair(currency.BTC, currency.USD), + Exchange: "TestExchange", + Asset: asset.Spot, + Pair: currency.NewPair(currency.BTC, currency.USD), + VerifyOrderbook: true, } err := b.Verify() @@ -143,75 +68,75 @@ func TestVerify(t *testing.T) { b.Asks = []Item{{ID: 1337, Price: 99, Amount: 1}, {ID: 1337, Price: 100, Amount: 1}} err = b.Verify() - if err == nil || !errors.Is(err, errIDDuplication) { + if !errors.Is(err, errIDDuplication) { t.Fatalf("expecting %s error but received %v", errIDDuplication, err) } b.Asks = []Item{{Price: 100, Amount: 1}, {Price: 100, Amount: 1}} err = b.Verify() - if err == nil || !errors.Is(err, errDuplication) { + if !errors.Is(err, errDuplication) { t.Fatalf("expecting %s error but received %v", errDuplication, err) } b.Asks = []Item{{Price: 100, Amount: 1}, {Price: 99, Amount: 1}} b.IsFundingRate = true err = b.Verify() - if err == nil || !errors.Is(err, errPeriodUnset) { + if !errors.Is(err, errPeriodUnset) { t.Fatalf("expecting %s error but received %v", errPeriodUnset, err) } b.IsFundingRate = false err = b.Verify() - if err == nil || !errors.Is(err, errOutOfOrder) { - t.Fatalf("expecting %s error but received %v", errOutOfOrder, err) + if !errors.Is(err, errPriceOutOfOrder) { + t.Fatalf("expecting %s error but received %v", errPriceOutOfOrder, err) } b.Asks = []Item{{Price: 100, Amount: 1}, {Price: 100, Amount: 0}} err = b.Verify() - if err == nil || !errors.Is(err, errAmountInvalid) { + if !errors.Is(err, errAmountInvalid) { t.Fatalf("expecting %s error but received %v", errAmountInvalid, err) } b.Asks = []Item{{Price: 100, Amount: 1}, {Price: 0, Amount: 100}} err = b.Verify() - if err == nil || !errors.Is(err, errPriceNotSet) { + if !errors.Is(err, errPriceNotSet) { t.Fatalf("expecting %s error but received %v", errPriceNotSet, err) } b.Bids = []Item{{ID: 1337, Price: 100, Amount: 1}, {ID: 1337, Price: 99, Amount: 1}} err = b.Verify() - if err == nil || !errors.Is(err, errIDDuplication) { + if !errors.Is(err, errIDDuplication) { t.Fatalf("expecting %s error but received %v", errIDDuplication, err) } b.Bids = []Item{{Price: 100, Amount: 1}, {Price: 100, Amount: 1}} err = b.Verify() - if err == nil || !errors.Is(err, errDuplication) { + if !errors.Is(err, errDuplication) { t.Fatalf("expecting %s error but received %v", errDuplication, err) } b.Bids = []Item{{Price: 99, Amount: 1}, {Price: 100, Amount: 1}} b.IsFundingRate = true err = b.Verify() - if err == nil || !errors.Is(err, errPeriodUnset) { + if !errors.Is(err, errPeriodUnset) { t.Fatalf("expecting %s error but received %v", errPeriodUnset, err) } b.IsFundingRate = false err = b.Verify() - if err == nil || !errors.Is(err, errOutOfOrder) { - t.Fatalf("expecting %s error but received %v", errOutOfOrder, err) + if !errors.Is(err, errPriceOutOfOrder) { + t.Fatalf("expecting %s error but received %v", errPriceOutOfOrder, err) } b.Bids = []Item{{Price: 100, Amount: 1}, {Price: 100, Amount: 0}} err = b.Verify() - if err == nil || !errors.Is(err, errAmountInvalid) { + if !errors.Is(err, errAmountInvalid) { t.Fatalf("expecting %s error but received %v", errAmountInvalid, err) } b.Bids = []Item{{Price: 100, Amount: 1}, {Price: 0, Amount: 100}} err = b.Verify() - if err == nil || !errors.Is(err, errPriceNotSet) { + if !errors.Is(err, errPriceNotSet) { t.Fatalf("expecting %s error but received %v", errPriceNotSet, err) } } @@ -251,51 +176,17 @@ func TestCalculateTotaAsks(t *testing.T) { } } -func TestUpdate(t *testing.T) { - t.Parallel() - curr, err := currency.NewPairFromStrings("BTC", "USD") - if err != nil { - t.Fatal(err) - } - timeNow := time.Now() - base := Base{ - Pair: curr, - Asks: []Item{{Price: 100, Amount: 10}}, - Bids: []Item{{Price: 200, Amount: 10}}, - LastUpdated: timeNow, - } - - asks := []Item{{Price: 200, Amount: 101}} - bids := []Item{{Price: 201, Amount: 100}} - time.Sleep(time.Millisecond * 50) - base.Update(bids, asks) - - if !base.LastUpdated.After(timeNow) { - t.Fatal("TestUpdate expected LastUpdated to be greater then original time") - } - - a, b := base.TotalAsksAmount() - if a != 100 && b != 20200 { - t.Fatal("TestUpdate expected a = 100 and b = 20100") - } - - a, b = base.TotalBidsAmount() - if a != 100 && b != 20100 { - t.Fatal("TestUpdate expected a = 100 and b = 20100") - } -} - func TestGetOrderbook(t *testing.T) { c, err := currency.NewPairFromStrings("BTC", "USD") if err != nil { t.Fatal(err) } base := &Base{ - Pair: c, - Asks: []Item{{Price: 100, Amount: 10}}, - Bids: []Item{{Price: 200, Amount: 10}}, - ExchangeName: "Exchange", - AssetType: asset.Spot, + Pair: c, + Asks: []Item{{Price: 100, Amount: 10}}, + Bids: []Item{{Price: 200, Amount: 10}}, + Exchange: "Exchange", + Asset: asset.Spot, } err = base.Process() @@ -344,17 +235,102 @@ func TestGetOrderbook(t *testing.T) { } } +func TestGetDepth(t *testing.T) { + c, err := currency.NewPairFromStrings("BTC", "USD") + if err != nil { + t.Fatal(err) + } + base := &Base{ + Pair: c, + Asks: []Item{{Price: 100, Amount: 10}}, + Bids: []Item{{Price: 200, Amount: 10}}, + Exchange: "Exchange", + Asset: asset.Spot, + } + + err = base.Process() + if err != nil { + t.Fatal(err) + } + + result, err := GetDepth("Exchange", c, asset.Spot) + if err != nil { + t.Fatalf("TestGetOrderbook failed to get orderbook. Error %s", + err) + } + if !result.pair.Equal(c) { + t.Fatal("TestGetOrderbook failed. Mismatched pairs") + } + + _, err = GetDepth("nonexistent", c, asset.Spot) + if !errors.Is(err, errCannotFindOrderbook) { + t.Fatalf("expecting %s error but received %v", errCannotFindOrderbook, err) + } + + c.Base = currency.NewCode("blah") + _, err = GetDepth("Exchange", c, asset.Spot) + if !errors.Is(err, errCannotFindOrderbook) { + t.Fatalf("expecting %s error but received %v", errCannotFindOrderbook, err) + } + + newCurrency, err := currency.NewPairFromStrings("BTC", "AUD") + if err != nil { + t.Fatal(err) + } + _, err = GetDepth("Exchange", newCurrency, asset.Futures) + if !errors.Is(err, errCannotFindOrderbook) { + t.Fatalf("expecting %s error but received %v", errCannotFindOrderbook, err) + } + + base.Pair = newCurrency + err = base.Process() + if err != nil { + t.Error(err) + } + + _, err = GetDepth("Exchange", newCurrency, "meowCats") + if !errors.Is(err, errCannotFindOrderbook) { + t.Fatalf("expecting %s error but received %v", errCannotFindOrderbook, err) + } +} + +func TestDeployDepth(t *testing.T) { + c, err := currency.NewPairFromStrings("BTC", "USD") + if err != nil { + t.Fatal(err) + } + _, err = DeployDepth("", c, asset.Spot) + if !errors.Is(err, errExchangeNameUnset) { + t.Fatalf("expecting %s error but received %v", errExchangeNameUnset, err) + } + _, err = DeployDepth("test", currency.Pair{}, asset.Spot) + if !errors.Is(err, errPairNotSet) { + t.Fatalf("expecting %s error but received %v", errPairNotSet, err) + } + _, err = DeployDepth("test", c, "") + if !errors.Is(err, errAssetTypeNotSet) { + t.Fatalf("expecting %s error but received %v", errAssetTypeNotSet, err) + } + d, err := DeployDepth("test", c, asset.Spot) + if err != nil { + t.Fatal(err) + } + if d == nil { + t.Fatal("depth ptr shall not be nill") + } +} + func TestCreateNewOrderbook(t *testing.T) { c, err := currency.NewPairFromStrings("BTC", "USD") if err != nil { t.Fatal(err) } base := &Base{ - Pair: c, - Asks: []Item{{Price: 100, Amount: 10}}, - Bids: []Item{{Price: 200, Amount: 10}}, - ExchangeName: "testCreateNewOrderbook", - AssetType: asset.Spot, + Pair: c, + Asks: []Item{{Price: 100, Amount: 10}}, + Bids: []Item{{Price: 200, Amount: 10}}, + Exchange: "testCreateNewOrderbook", + Asset: asset.Spot, } err = base.Process() @@ -388,9 +364,9 @@ func TestProcessOrderbook(t *testing.T) { t.Fatal(err) } base := Base{ - Asks: []Item{{Price: 100, Amount: 10}}, - Bids: []Item{{Price: 200, Amount: 10}}, - ExchangeName: "ProcessOrderbook", + Asks: []Item{{Price: 100, Amount: 10}}, + Bids: []Item{{Price: 200, Amount: 10}}, + Exchange: "ProcessOrderbook", } // test for empty pair @@ -408,7 +384,7 @@ func TestProcessOrderbook(t *testing.T) { } // now process a valid orderbook - base.AssetType = asset.Spot + base.Asset = asset.Spot err = base.Process() if err != nil { t.Error("unexpcted result: ", err) @@ -458,7 +434,7 @@ func TestProcessOrderbook(t *testing.T) { } base.Asks = []Item{{Price: 200, Amount: 200}} - base.AssetType = "monthly" + base.Asset = "monthly" err = base.Process() if err != nil { t.Error("Process() error", err) @@ -475,8 +451,8 @@ func TestProcessOrderbook(t *testing.T) { } base.Bids = []Item{{Price: 420, Amount: 200}} - base.ExchangeName = "Blah" - base.AssetType = "quarterly" + base.Exchange = "Blah" + base.Asset = "quarterly" err = base.Process() if err != nil { t.Error("Process() error", err) @@ -521,11 +497,11 @@ func TestProcessOrderbook(t *testing.T) { asks := []Item{{Price: rand.Float64(), Amount: rand.Float64()}} // nolint:gosec // no need to import crypo/rand for testing bids := []Item{{Price: rand.Float64(), Amount: rand.Float64()}} // nolint:gosec // no need to import crypo/rand for testing base := &Base{ - Pair: newPairs, - Asks: asks, - Bids: bids, - ExchangeName: newName, - AssetType: asset.Spot, + Pair: newPairs, + Asks: asks, + Bids: bids, + Exchange: newName, + Asset: asset.Spot, } m.Lock() @@ -577,7 +553,7 @@ func TestProcessOrderbook(t *testing.T) { wg.Wait() } -func deployUnorderedSlice() []Item { +func deployUnorderedSlice() Items { var items []Item rand.Seed(time.Now().UnixNano()) for i := 0; i < 1000; i++ { @@ -588,16 +564,15 @@ func deployUnorderedSlice() []Item { func TestSorting(t *testing.T) { var b Base + b.VerifyOrderbook = true b.Asks = deployUnorderedSlice() err := b.Verify() - if err == nil { - t.Fatal("error cannot be nil") + if !errors.Is(err, errPriceOutOfOrder) { + t.Fatalf("error expected %v received %v", errPriceOutOfOrder, err) } - SortAsks(nil) - - SortAsks(b.Asks) + b.Asks.SortAsks() err = b.Verify() if err != nil { t.Fatal(err) @@ -605,20 +580,18 @@ func TestSorting(t *testing.T) { b.Bids = deployUnorderedSlice() err = b.Verify() - if err == nil { - t.Fatal("error cannot be nil") + if !errors.Is(err, errPriceOutOfOrder) { + t.Fatalf("error expected %v received %v", errPriceOutOfOrder, err) } - SortBids(nil) - - SortBids(b.Bids) + b.Bids.SortBids() err = b.Verify() if err != nil { t.Fatal(err) } } -func deploySliceOrdered() []Item { +func deploySliceOrdered() Items { rand.Seed(time.Now().UnixNano()) var items []Item for i := 0; i < 1000; i++ { @@ -629,6 +602,7 @@ func deploySliceOrdered() []Item { func TestReverse(t *testing.T) { var b Base + b.VerifyOrderbook = true length := 1000 b.Bids = deploySliceOrdered() @@ -637,12 +611,11 @@ func TestReverse(t *testing.T) { } err := b.Verify() - if err == nil { - t.Fatal("error cannot be nil") + if !errors.Is(err, errPriceOutOfOrder) { + t.Fatalf("error expected %v received %v", errPriceOutOfOrder, err) } - Reverse(nil) - Reverse(b.Bids) + b.Bids.Reverse() err = b.Verify() if err != nil { t.Fatal(err) @@ -650,11 +623,11 @@ func TestReverse(t *testing.T) { b.Asks = append(b.Bids[:0:0], b.Bids...) // nolint:gocritic // Short hand err = b.Verify() - if err == nil { - t.Fatal("error cannot be nil") + if !errors.Is(err, errPriceOutOfOrder) { + t.Fatalf("error expected %v received %v", errPriceOutOfOrder, err) } - Reverse(b.Asks) + b.Asks.Reverse() err = b.Verify() if err != nil { t.Fatal(err) @@ -670,7 +643,7 @@ func BenchmarkReverse(b *testing.B) { } for i := 0; i < b.N; i++ { - Reverse(s) + s.Reverse() } } @@ -679,17 +652,17 @@ func BenchmarkSortAsksDecending(b *testing.B) { s := deploySliceOrdered() for i := 0; i < b.N; i++ { ts := append(s[:0:0], s...) - SortAsks(ts) + ts.SortAsks() } } // 14924 79199 ns/op 49206 B/op 3 allocs/op func BenchmarkSortBidsAscending(b *testing.B) { s := deploySliceOrdered() - Reverse(s) + s.Reverse() for i := 0; i < b.N; i++ { ts := append(s[:0:0], s...) - SortBids(ts) + ts.SortBids() } } @@ -698,7 +671,7 @@ func BenchmarkSortAsksStandard(b *testing.B) { s := deployUnorderedSlice() for i := 0; i < b.N; i++ { ts := append(s[:0:0], s...) - SortAsks(ts) + ts.SortAsks() } } @@ -707,7 +680,7 @@ func BenchmarkSortBidsStandard(b *testing.B) { s := deployUnorderedSlice() for i := 0; i < b.N; i++ { ts := append(s[:0:0], s...) - SortBids(ts) + ts.SortBids() } } @@ -716,21 +689,21 @@ func BenchmarkSortAsksAscending(b *testing.B) { s := deploySliceOrdered() for i := 0; i < b.N; i++ { ts := append(s[:0:0], s...) - SortAsks(ts) + ts.SortAsks() } } // 12565 97257 ns/op 49208 B/op 3 allocs/op func BenchmarkSortBidsDescending(b *testing.B) { s := deploySliceOrdered() - Reverse(s) + s.Reverse() for i := 0; i < b.N; i++ { ts := append(s[:0:0], s...) - SortBids(ts) + ts.SortBids() } } -// 923154 1169 ns/op 4096 B/op 1 allocs/op +// 124867 8480 ns/op 49152 B/op 1 allocs/op func BenchmarkDuplicatingSlice(b *testing.B) { s := deploySliceOrdered() for i := 0; i < b.N; i++ { @@ -738,7 +711,7 @@ func BenchmarkDuplicatingSlice(b *testing.B) { } } -// 705922 1546 ns/op 4096 B/op 1 allocs/op +// 122998 8441 ns/op 49152 B/op 1 allocs/op func BenchmarkCopySlice(b *testing.B) { s := deploySliceOrdered() for i := 0; i < b.N; i++ { diff --git a/exchanges/orderbook/orderbook_types.go b/exchanges/orderbook/orderbook_types.go index 48f53342..2e21a6b6 100644 --- a/exchanges/orderbook/orderbook_types.go +++ b/exchanges/orderbook/orderbook_types.go @@ -20,42 +20,38 @@ const ( // Vars for the orderbook package var ( - service *Service - - errExchangeNameUnset = errors.New("orderbook exchange name not set") - errPairNotSet = errors.New("orderbook currency pair not set") - errAssetTypeNotSet = errors.New("orderbook asset type not set") - errNoOrderbook = errors.New("orderbook bids and asks are empty") - errPriceNotSet = errors.New("price cannot be zero") - errAmountInvalid = errors.New("amount cannot be less or equal to zero") - errOutOfOrder = errors.New("pricing out of order") - errDuplication = errors.New("price duplication") - errIDDuplication = errors.New("id duplication") - errPeriodUnset = errors.New("funding rate period is unset") + errExchangeNameUnset = errors.New("orderbook exchange name not set") + errPairNotSet = errors.New("orderbook currency pair not set") + errAssetTypeNotSet = errors.New("orderbook asset type not set") + errCannotFindOrderbook = errors.New("cannot find orderbook(s)") + errPriceNotSet = errors.New("price cannot be zero") + errAmountInvalid = errors.New("amount cannot be less or equal to zero") + errPriceOutOfOrder = errors.New("pricing out of order") + errIDOutOfOrder = errors.New("ID out of order") + errDuplication = errors.New("price duplication") + errIDDuplication = errors.New("id duplication") + errPeriodUnset = errors.New("funding rate period is unset") ) -func init() { - service = new(Service) - service.mux = dispatch.GetNewMux() - service.Books = make(map[string]map[asset.Item]map[*currency.Item]map[*currency.Item]*Book) - service.Exchange = make(map[string]uuid.UUID) +var service = Service{ + books: make(map[string]Exchange), + Mux: dispatch.GetNewMux(), } -// Book defines an orderbook with its links to different dispatch outputs -type Book struct { - b *Base - Main uuid.UUID - Assoc []uuid.UUID -} - -// Service holds orderbook information for each individual exchange +// Service provides a store for difference exchange orderbooks type Service struct { - Books map[string]map[asset.Item]map[*currency.Item]map[*currency.Item]*Book - Exchange map[string]uuid.UUID - mux *dispatch.Mux + books map[string]Exchange + *dispatch.Mux sync.Mutex } +// Exchange defines a holder for the exchange specific depth items with a +// specific ID associated with that exchange +type Exchange struct { + m map[asset.Item]map[*currency.Item]map[*currency.Item]*Depth + ID uuid.UUID +} + // Item stores the amount and price values type Item struct { Amount float64 @@ -70,27 +66,34 @@ type Item struct { OrderCount int64 } +// Items defines a slice of orderbook items +type Items []Item + // Base holds the fields for the orderbook base type Base struct { - Pair currency.Pair `json:"pair"` - Bids []Item `json:"bids"` - Asks []Item `json:"asks"` - LastUpdated time.Time `json:"lastUpdated"` - LastUpdateID int64 `json:"lastUpdateId"` - AssetType asset.Item `json:"assetType"` - ExchangeName string `json:"exchangeName"` + Bids Items + Asks Items - // NotAggregated defines whether an orderbook can contain duplicate prices - // in a payload - NotAggregated bool `json:"-"` - IsFundingRate bool `json:"fundingRate"` + Exchange string + Pair currency.Pair + Asset asset.Item - // VerificationBypass is a complete orderbook verification bypass set by - // user configuration - VerificationBypass bool `json:"-"` - // HasChecksumValidation defines an allowance to bypass internal - // verification if the book has been verified by checksum. - HasChecksumValidation bool `json:"-"` + LastUpdated time.Time + LastUpdateID int64 + // PriceDuplication defines whether an orderbook can contain duplicate + // prices in a payload + PriceDuplication bool + IsFundingRate bool + // VerifyOrderbook allows for a toggle between orderbook verification set by + // user configuration, this allows for a potential processing boost but + // a potential for orderbook integrity being deminished. + VerifyOrderbook bool `json:"-"` + // RestSnapshot defines if the depth was applied via the REST protocol thus + // an update cannot be applied via websocket mechanics and a resubscription + // would need to take place to maintain book integrity + RestSnapshot bool + // Checks if the orderbook needs ID alignment as well as price alignment + IDAlignment bool } type byOBPrice []Item @@ -98,3 +101,16 @@ type byOBPrice []Item func (a byOBPrice) Len() int { return len(a) } func (a byOBPrice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a byOBPrice) Less(i, j int) bool { return a[i].Price < a[j].Price } + +type options struct { + exchange string + pair currency.Pair + asset asset.Item + lastUpdated time.Time + lastUpdateID int64 + priceDuplication bool + isFundingRate bool + VerifyOrderbook bool + restSnapshot bool + idAligned bool +} diff --git a/exchanges/poloniex/poloniex_websocket.go b/exchanges/poloniex/poloniex_websocket.go index 7ff3f25c..af26205e 100644 --- a/exchanges/poloniex/poloniex_websocket.go +++ b/exchanges/poloniex/poloniex_websocket.go @@ -503,17 +503,17 @@ func (p *Poloniex) WsProcessOrderbookSnapshot(ob []interface{}, symbol string) e } // Both sides are completely out of order - sort needs to be used - book.Asks = orderbook.SortAsks(book.Asks) - book.Bids = orderbook.SortBids(book.Bids) - book.AssetType = asset.Spot - book.VerificationBypass = p.OrderbookVerificationBypass + book.Asks.SortAsks() + book.Bids.SortBids() + book.Asset = asset.Spot + book.VerifyOrderbook = p.CanVerifyOrderbook var err error book.Pair, err = currency.NewPairFromString(symbol) if err != nil { return err } - book.ExchangeName = p.Name + book.Exchange = p.Name return p.Websocket.Orderbook.LoadSnapshot(&book) } diff --git a/exchanges/poloniex/poloniex_wrapper.go b/exchanges/poloniex/poloniex_wrapper.go index ff7dee1b..c6974774 100644 --- a/exchanges/poloniex/poloniex_wrapper.go +++ b/exchanges/poloniex/poloniex_wrapper.go @@ -330,10 +330,10 @@ func (p *Poloniex) FetchOrderbook(currencyPair currency.Pair, assetType asset.It // UpdateOrderbook updates and returns the orderbook for a currency pair func (p *Poloniex) UpdateOrderbook(c currency.Pair, assetType asset.Item) (*orderbook.Base, error) { callingBook := &orderbook.Base{ - ExchangeName: p.Name, - Pair: c, - AssetType: assetType, - VerificationBypass: p.OrderbookVerificationBypass, + Exchange: p.Name, + Pair: c, + Asset: assetType, + VerifyOrderbook: p.CanVerifyOrderbook, } orderbookNew, err := p.GetOrderbook("", poloniexMaxOrderbookDepth) if err != nil { @@ -346,10 +346,10 @@ func (p *Poloniex) UpdateOrderbook(c currency.Pair, assetType asset.Item) (*orde } for i := range enabledPairs { book := &orderbook.Base{ - ExchangeName: p.Name, - Pair: enabledPairs[i], - AssetType: assetType, - VerificationBypass: p.OrderbookVerificationBypass, + Exchange: p.Name, + Pair: enabledPairs[i], + Asset: assetType, + VerifyOrderbook: p.CanVerifyOrderbook, } fpair, err := p.FormatExchangeCurrency(enabledPairs[i], assetType) diff --git a/exchanges/stream/buffer/buffer.go b/exchanges/stream/buffer/buffer.go index d275ee6c..a85bfb5c 100644 --- a/exchanges/stream/buffer/buffer.go +++ b/exchanges/stream/buffer/buffer.go @@ -4,10 +4,12 @@ import ( "errors" "fmt" "sort" + "time" "github.com/thrasher-corp/gocryptotrader/currency" "github.com/thrasher-corp/gocryptotrader/exchanges/asset" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" + "github.com/thrasher-corp/gocryptotrader/log" ) const packageError = "websocket orderbook buffer error: %w" @@ -18,6 +20,8 @@ var ( errIssueBufferEnabledButNoLimit = errors.New("buffer enabled but no limit set") errUpdateIsNil = errors.New("update is nil") errUpdateNoTargets = errors.New("update bid/ask targets cannot be nil") + errDepthNotFound = errors.New("orderbook depth not found") + errRESTOverwrite = errors.New("orderbook has been overwritten by REST protocol") ) // Setup sets private variables @@ -25,7 +29,10 @@ func (w *Orderbook) Setup(obBufferLimit int, bufferEnabled, sortBuffer, sortBufferByUpdateIDs, - updateEntriesByID bool, exchangeName string, dataHandler chan interface{}) error { + updateEntriesByID, + verbose bool, + exchangeName string, + dataHandler chan interface{}) error { if exchangeName == "" { return fmt.Errorf(packageError, errUnsetExchangeName) } @@ -43,6 +50,7 @@ func (w *Orderbook) Setup(obBufferLimit int, w.exchangeName = exchangeName w.dataHandler = dataHandler w.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder) + w.verbose = verbose return nil } @@ -57,27 +65,48 @@ func (w *Orderbook) validate(u *Update) error { return nil } -// Update updates a local buffer using bid targets and ask targets then updates -// main orderbook -// Volume == 0; deletion at price target -// Price target not found; append of price target -// Price target found; amend volume of price target +// Update updates a stored pointer to an orderbook.Depth struct containing a +// linked list, this switches between the usage of a buffered update func (w *Orderbook) Update(u *Update) error { if err := w.validate(u); err != nil { return err } w.m.Lock() defer w.m.Unlock() - obLookup, ok := w.ob[u.Pair.Base][u.Pair.Quote][u.Asset] + book, ok := w.ob[u.Pair.Base][u.Pair.Quote][u.Asset] if !ok { - return fmt.Errorf("ob.Base could not be found for Exchange %s CurrencyPair: %s AssetType: %s", + return fmt.Errorf("%w for Exchange %s CurrencyPair: %s AssetType: %s", + errDepthNotFound, w.exchangeName, u.Pair, u.Asset) } + // Checks for when the rest protocol overwrites a streaming dominated book + // will stop updating book via incremental updates. This occurs because our + // sync manager (engine/sync.go) timer has elapsed for streaming. Usually + // because the book is highly illiquid. TODO: Book resubscribe on websocket. + if book.ob.IsRestSnapshot() { + if w.verbose { + log.Warnf(log.WebsocketMgr, + "%s for Exchange %s CurrencyPair: %s AssetType: %s consider extending synctimeoutwebsocket", + errRESTOverwrite, + w.exchangeName, + u.Pair, + u.Asset) + } + return fmt.Errorf("%w for Exchange %s CurrencyPair: %s AssetType: %s", + errRESTOverwrite, + w.exchangeName, + u.Pair, + u.Asset) + } + + // Apply new update information + book.ob.SetLastUpdate(u.UpdateTime, u.UpdateID, false) + if w.bufferEnabled { - processed, err := w.processBufferUpdate(obLookup, u) + processed, err := w.processBufferUpdate(book, u) if err != nil { return err } @@ -86,21 +115,36 @@ func (w *Orderbook) Update(u *Update) error { return nil } } else { - err := w.processObUpdate(obLookup, u) + err := w.processObUpdate(book, u) if err != nil { return err } } - err := obLookup.ob.Process() - if err != nil { - return err + if book.ob.VerifyOrderbook { // This is used here so as to not retrieve + // book if verification is off. + // On every update, this will retrieve and verify orderbook depths + err := book.ob.Retrieve().Verify() + if err != nil { + return err + } } - // Process in data handler select { - case w.dataHandler <- obLookup.ob: + case <-book.ticker.C: + // Opted to wait for receiver because we are limiting here and the sync + // manager requires update + go func() { + w.dataHandler <- book.ob.Retrieve() + book.ob.Publish() + }() default: + // We do not need to send an update to the sync manager within this time + // window unless verbose is turned on + if w.verbose { + w.dataHandler <- book.ob.Retrieve() + book.ob.Publish() + } } return nil } @@ -139,230 +183,42 @@ func (w *Orderbook) processBufferUpdate(o *orderbookHolder, u *Update) (bool, er // processObUpdate processes updates either by its corresponding id or by // price level func (w *Orderbook) processObUpdate(o *orderbookHolder, u *Update) error { - o.ob.LastUpdateID = u.UpdateID if w.updateEntriesByID { return o.updateByIDAndAction(u) } - return o.updateByPrice(u) + o.updateByPrice(u) + return nil } // updateByPrice ammends amount if match occurs by price, deletes if amount is // zero or less and inserts if not found. -func (o *orderbookHolder) updateByPrice(updts *Update) error { -askUpdates: - for j := range updts.Asks { - for target := range o.ob.Asks { - if o.ob.Asks[target].Price == updts.Asks[j].Price { - if updts.Asks[j].Amount == 0 { - o.ob.Asks = append(o.ob.Asks[:target], o.ob.Asks[target+1:]...) - continue askUpdates - } - o.ob.Asks[target].Amount = updts.Asks[j].Amount - continue askUpdates - } - } - if updts.Asks[j].Amount <= 0 { - continue - } - insertAsk(updts.Asks[j], &o.ob.Asks) - if updts.MaxDepth != 0 && len(o.ob.Asks) > updts.MaxDepth { - o.ob.Asks = o.ob.Asks[:updts.MaxDepth] - } - } -bidUpdates: - for j := range updts.Bids { - for target := range o.ob.Bids { - if o.ob.Bids[target].Price == updts.Bids[j].Price { - if updts.Bids[j].Amount == 0 { - o.ob.Bids = append(o.ob.Bids[:target], o.ob.Bids[target+1:]...) - continue bidUpdates - } - o.ob.Bids[target].Amount = updts.Bids[j].Amount - continue bidUpdates - } - } - if updts.Bids[j].Amount <= 0 { - continue - } - insertBid(updts.Bids[j], &o.ob.Bids) - if updts.MaxDepth != 0 && len(o.ob.Bids) > updts.MaxDepth { - o.ob.Bids = o.ob.Bids[:updts.MaxDepth] - } - } - return nil +func (o *orderbookHolder) updateByPrice(updts *Update) { + o.ob.UpdateBidAskByPrice(updts.Bids, updts.Asks, updts.MaxDepth) } // updateByIDAndAction will receive an action to execute against the orderbook // it will then match by IDs instead of price to perform the action -func (o *orderbookHolder) updateByIDAndAction(updts *Update) (err error) { +func (o *orderbookHolder) updateByIDAndAction(updts *Update) error { switch updts.Action { case Amend: - err = applyUpdates(updts.Bids, o.ob.Bids) - if err != nil { - return err - } - err = applyUpdates(updts.Asks, o.ob.Asks) - if err != nil { - return err - } + return o.ob.UpdateBidAskByID(updts.Bids, updts.Asks) case Delete: // edge case for Bitfinex as their streaming endpoint duplicates deletes - bypassErr := o.ob.ExchangeName == "Bitfinex" && o.ob.IsFundingRate - err = deleteUpdates(updts.Bids, &o.ob.Bids, bypassErr) - if err != nil { - return fmt.Errorf("%s %s %v", o.ob.AssetType, o.ob.Pair, err) - } - err = deleteUpdates(updts.Asks, &o.ob.Asks, bypassErr) - if err != nil { - return fmt.Errorf("%s %s %v", o.ob.AssetType, o.ob.Pair, err) - } + bypassErr := o.ob.GetName() == "Bitfinex" && o.ob.IsFundingRate() + return o.ob.DeleteBidAskByID(updts.Bids, updts.Asks, bypassErr) case Insert: - insertUpdatesBid(updts.Bids, &o.ob.Bids) - insertUpdatesAsk(updts.Asks, &o.ob.Asks) + return o.ob.InsertBidAskByID(updts.Bids, updts.Asks) case UpdateInsert: - updateBids: - for x := range updts.Bids { - for target := range o.ob.Bids { // First iteration finds ID matches - if o.ob.Bids[target].ID == updts.Bids[x].ID { - if o.ob.Bids[target].Price != updts.Bids[x].Price { - // Price change occurred so correct bid alignment is - // needed - delete instance and insert into correct - // price level - o.ob.Bids = append(o.ob.Bids[:target], o.ob.Bids[target+1:]...) - break - } - o.ob.Bids[target].Amount = updts.Bids[x].Amount - continue updateBids - } - } - insertBid(updts.Bids[x], &o.ob.Bids) - } - updateAsks: - for x := range updts.Asks { - for target := range o.ob.Asks { - if o.ob.Asks[target].ID == updts.Asks[x].ID { - if o.ob.Asks[target].Price != updts.Asks[x].Price { - // Price change occurred so correct ask alignment is - // needed - delete instance and insert into correct - // price level - o.ob.Asks = append(o.ob.Asks[:target], o.ob.Asks[target+1:]...) - break - } - o.ob.Asks[target].Amount = updts.Asks[x].Amount - continue updateAsks - } - } - insertAsk(updts.Asks[x], &o.ob.Asks) - } + return o.ob.UpdateInsertByID(updts.Bids, updts.Asks) default: return fmt.Errorf("invalid action [%s]", updts.Action) } - return nil } -// applyUpdates amends amount by ID and returns an error if not found -func applyUpdates(updts, book []orderbook.Item) error { -updates: - for x := range updts { - for y := range book { - if book[y].ID == updts[x].ID { - book[y].Amount = updts[x].Amount - continue updates - } - } - return fmt.Errorf("update cannot be applied id: %d not found", - updts[x].ID) - } - return nil -} - -// deleteUpdates removes updates from orderbook and returns an error if not -// found -func deleteUpdates(updt []orderbook.Item, book *[]orderbook.Item, bypassErr bool) error { -updates: - for x := range updt { - for y := range *book { - if (*book)[y].ID == updt[x].ID { - *book = append((*book)[:y], (*book)[y+1:]...) // nolint:gocritic - continue updates - } - } - // bypassErr is for expected duplication from endpoint. - if !bypassErr { - return fmt.Errorf("update cannot be deleted id: %d not found", - updt[x].ID) - } - } - return nil -} - -func insertAsk(updt orderbook.Item, book *[]orderbook.Item) { - for target := range *book { - if updt.Price < (*book)[target].Price { - insertItem(updt, book, target) - return - } - } - *book = append(*book, updt) -} - -func insertBid(updt orderbook.Item, book *[]orderbook.Item) { - for target := range *book { - if updt.Price > (*book)[target].Price { - insertItem(updt, book, target) - return - } - } - *book = append(*book, updt) -} - -// insertUpdatesBid inserts on **correctly aligned** book at price level -func insertUpdatesBid(updt []orderbook.Item, book *[]orderbook.Item) { -updates: - for x := range updt { - for target := range *book { - if updt[x].Price > (*book)[target].Price { - insertItem(updt[x], book, target) - continue updates - } - } - *book = append(*book, updt[x]) - } -} - -// insertUpdatesBid inserts on **correctly aligned** book at price level -func insertUpdatesAsk(updt []orderbook.Item, book *[]orderbook.Item) { -updates: - for x := range updt { - for target := range *book { - if updt[x].Price < (*book)[target].Price { - insertItem(updt[x], book, target) - continue updates - } - } - *book = append(*book, updt[x]) - } -} - -// insertItem inserts item in slice by target element this is an optimization -// to reduce the need for sorting algorithms -func insertItem(update orderbook.Item, book *[]orderbook.Item, target int) { - // TODO: extend slice by incoming update length before this gets hit - *book = append(*book, orderbook.Item{}) - copy((*book)[target+1:], (*book)[target:]) - (*book)[target] = update -} - -// LoadSnapshot loads initial snapshot of ob data from websocket +// LoadSnapshot loads initial snapshot of orderbook data from websocket func (w *Orderbook) LoadSnapshot(book *orderbook.Base) error { w.m.Lock() defer w.m.Unlock() - - err := book.Process() - if err != nil { - return err - } - m1, ok := w.ob[book.Pair.Base] if !ok { m1 = make(map[currency.Code]map[asset.Item]*orderbookHolder) @@ -373,31 +229,60 @@ func (w *Orderbook) LoadSnapshot(book *orderbook.Base) error { m2 = make(map[asset.Item]*orderbookHolder) m1[book.Pair.Quote] = m2 } - m3, ok := m2[book.AssetType] + holder, ok := m2[book.Asset] if !ok { - m3 = &orderbookHolder{ob: book, buffer: &[]Update{}} - m2[book.AssetType] = m3 - } else { - m3.ob.LastUpdateID = book.LastUpdateID - m3.ob.Bids = book.Bids - m3.ob.Asks = book.Asks + // Associate orderbook pointer with local exchange depth map + depth, err := orderbook.DeployDepth(book.Exchange, book.Pair, book.Asset) + if err != nil { + return err + } + depth.AssignOptions(book) + buffer := make([]Update, w.obBufferLimit) + ticker := time.NewTicker(timerDefault) + holder = &orderbookHolder{ + ob: depth, + buffer: &buffer, + ticker: ticker, + } + m2[book.Asset] = holder } - w.dataHandler <- book + + // Checks if book can deploy to linked list + err := book.Verify() + if err != nil { + return err + } + + holder.ob.LoadSnapshot(book.Bids, book.Asks) + + if holder.ob.VerifyOrderbook { // This is used here so as to not retrieve + // book if verification is off. + // Checks to see if orderbook snapshot that was deployed has not been + // altered in any way + err = holder.ob.Retrieve().Verify() + if err != nil { + return err + } + } + + w.dataHandler <- holder.ob.Retrieve() + holder.ob.Publish() return nil } -// GetOrderbook returns orderbook stored in current buffer -func (w *Orderbook) GetOrderbook(p currency.Pair, a asset.Item) *orderbook.Base { +// GetOrderbook returns an orderbook copy as orderbook.Base +func (w *Orderbook) GetOrderbook(p currency.Pair, a asset.Item) (*orderbook.Base, error) { w.m.Lock() defer w.m.Unlock() - ptr, ok := w.ob[p.Base][p.Quote][a] + book, ok := w.ob[p.Base][p.Quote][a] if !ok { - return nil + return nil, fmt.Errorf("%s %s %s %w", + w.exchangeName, + p, + a, + errDepthNotFound) } - cpy := *ptr.ob - cpy.Asks = append(cpy.Asks[:0:0], cpy.Asks...) - cpy.Bids = append(cpy.Bids[:0:0], cpy.Bids...) - return &cpy + return book.ob.Retrieve(), nil } // FlushBuffer flushes w.ob data to be garbage collected and refreshed when a @@ -414,9 +299,12 @@ func (w *Orderbook) FlushOrderbook(p currency.Pair, a asset.Item) error { defer w.m.Unlock() book, ok := w.ob[p.Base][p.Quote][a] if !ok { - return fmt.Errorf("orderbook not associated with pair: [%s] and asset [%s]", p, a) + return fmt.Errorf("cannot flush orderbook %s %s %s %w", + w.exchangeName, + p, + a, + errDepthNotFound) } - book.ob.Bids = nil - book.ob.Asks = nil + book.ob.Flush() return nil } diff --git a/exchanges/stream/buffer/buffer_test.go b/exchanges/stream/buffer/buffer_test.go index e7b10aeb..e5a96d72 100644 --- a/exchanges/stream/buffer/buffer_test.go +++ b/exchanges/stream/buffer/buffer_test.go @@ -26,26 +26,27 @@ const ( exchangeName = "exchangeTest" ) -func createSnapshot() (obl *Orderbook, asks, bids []orderbook.Item, err error) { - var snapShot1 orderbook.Base - snapShot1.ExchangeName = exchangeName - asks = []orderbook.Item{ - {Price: 4000, Amount: 1, ID: 6}, +func createSnapshot() (holder *Orderbook, asks, bids orderbook.Items, err error) { + asks = orderbook.Items{{Price: 4000, Amount: 1, ID: 6}} + bids = orderbook.Items{{Price: 4000, Amount: 1, ID: 6}} + + book := &orderbook.Base{ + Exchange: exchangeName, + Asks: asks, + Bids: bids, + Asset: asset.Spot, + Pair: cp, + PriceDuplication: true, } - bids = []orderbook.Item{ - {Price: 4000, Amount: 1, ID: 6}, - } - snapShot1.Asks = asks - snapShot1.Bids = bids - snapShot1.AssetType = asset.Spot - snapShot1.Pair = cp - snapShot1.NotAggregated = true - obl = &Orderbook{ + + newBook := make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder) + + holder = &Orderbook{ exchangeName: exchangeName, dataHandler: make(chan interface{}, 100), - ob: make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder), + ob: newBook, } - err = obl.LoadSnapshot(&snapShot1) + err = holder.LoadSnapshot(book) return } @@ -108,20 +109,13 @@ func BenchmarkUpdateAsksByPrice(b *testing.B) { // BenchmarkBufferPerformance demonstrates buffer more performant than multi // process calls +// 4219518 287 ns/op 176 B/op 1 allocs/op func BenchmarkBufferPerformance(b *testing.B) { - obl, asks, bids, err := createSnapshot() + holder, asks, bids, err := createSnapshot() if err != nil { b.Fatal(err) } - obl.bufferEnabled = true - // This is to ensure we do not send in zero orderbook info to our main book - // in orderbook.go, orderbooks should not be zero even after an update. - dummyItem := orderbook.Item{ - Amount: 1333337, - Price: 1337.1337, - ID: 1337, - } - obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids = append(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids, dummyItem) + holder.bufferEnabled = true update := &Update{ Bids: bids, Asks: asks, @@ -134,7 +128,7 @@ func BenchmarkBufferPerformance(b *testing.B) { randomIndex := rand.Intn(4) // nolint:gosec // no need to import crypo/rand for testing update.Asks = itemArray[randomIndex] update.Bids = itemArray[randomIndex] - err = obl.Update(update) + err = holder.Update(update) if err != nil { b.Fatal(err) } @@ -142,21 +136,14 @@ func BenchmarkBufferPerformance(b *testing.B) { } // BenchmarkBufferSortingPerformance benchmark +// 2693391 467 ns/op 208 B/op 2 allocs/op func BenchmarkBufferSortingPerformance(b *testing.B) { - obl, asks, bids, err := createSnapshot() + holder, asks, bids, err := createSnapshot() if err != nil { b.Fatal(err) } - obl.bufferEnabled = true - obl.sortBuffer = true - // This is to ensure we do not send in zero orderbook info to our main book - // in orderbook.go, orderbooks should not be zero even after an update. - dummyItem := orderbook.Item{ - Amount: 1333337, - Price: 1337.1337, - ID: 1337, - } - obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids = append(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids, dummyItem) + holder.bufferEnabled = true + holder.sortBuffer = true update := &Update{ Bids: bids, Asks: asks, @@ -169,7 +156,7 @@ func BenchmarkBufferSortingPerformance(b *testing.B) { randomIndex := rand.Intn(4) // nolint:gosec // no need to import crypo/rand for testing update.Asks = itemArray[randomIndex] update.Bids = itemArray[randomIndex] - err = obl.Update(update) + err = holder.Update(update) if err != nil { b.Fatal(err) } @@ -177,22 +164,15 @@ func BenchmarkBufferSortingPerformance(b *testing.B) { } // BenchmarkBufferSortingPerformance benchmark +// 1000000 1019 ns/op 208 B/op 2 allocs/op func BenchmarkBufferSortingByIDPerformance(b *testing.B) { - obl, asks, bids, err := createSnapshot() + holder, asks, bids, err := createSnapshot() if err != nil { b.Fatal(err) } - obl.bufferEnabled = true - obl.sortBuffer = true - obl.sortBufferByUpdateIDs = true - // This is to ensure we do not send in zero orderbook info to our main book - // in orderbook.go, orderbooks should not be zero even after an update. - dummyItem := orderbook.Item{ - Amount: 1333337, - Price: 1337.1337, - ID: 1337, - } - obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids = append(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids, dummyItem) + holder.bufferEnabled = true + holder.sortBuffer = true + holder.sortBufferByUpdateIDs = true update := &Update{ Bids: bids, Asks: asks, @@ -205,28 +185,21 @@ func BenchmarkBufferSortingByIDPerformance(b *testing.B) { randomIndex := rand.Intn(4) // nolint:gosec // no need to import crypo/rand for testing update.Asks = itemArray[randomIndex] update.Bids = itemArray[randomIndex] - err = obl.Update(update) + err = holder.Update(update) if err != nil { b.Fatal(err) } } } -// BenchmarkNoBufferPerformance demonstrates orderbook process less performant +// BenchmarkNoBufferPerformance demonstrates orderbook process more performant // than buffer +// 9516966 141 ns/op 0 B/op 0 allocs/op func BenchmarkNoBufferPerformance(b *testing.B) { obl, asks, bids, err := createSnapshot() if err != nil { b.Fatal(err) } - // This is to ensure we do not send in zero orderbook info to our main book - // in orderbook.go, orderbooks should not be zero even after an update. - dummyItem := orderbook.Item{ - Amount: 1333337, - Price: 1337.1337, - ID: 1337, - } - obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids = append(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids, dummyItem) update := &Update{ Bids: bids, Asks: asks, @@ -247,13 +220,13 @@ func BenchmarkNoBufferPerformance(b *testing.B) { } func TestUpdates(t *testing.T) { - obl, _, _, err := createSnapshot() + holder, _, _, err := createSnapshot() if err != nil { t.Error(err) } - holder := obl.ob[cp.Base][cp.Quote][asset.Spot] - holder.updateByPrice(&Update{ + book := holder.ob[cp.Base][cp.Quote][asset.Spot] + book.updateByPrice(&Update{ Bids: itemArray[5], Asks: itemArray[5], Pair: cp, @@ -264,7 +237,7 @@ func TestUpdates(t *testing.T) { t.Error(err) } - holder.updateByPrice(&Update{ + book.updateByPrice(&Update{ Bids: itemArray[0], Asks: itemArray[0], Pair: cp, @@ -275,23 +248,23 @@ func TestUpdates(t *testing.T) { t.Error(err) } - if len(holder.ob.Asks) != 3 { + if book.ob.GetAskLength() != 3 { t.Error("Did not update") } } // TestHittingTheBuffer logic test func TestHittingTheBuffer(t *testing.T) { - obl, _, _, err := createSnapshot() + holder, _, _, err := createSnapshot() if err != nil { t.Fatal(err) } - obl.bufferEnabled = true - obl.obBufferLimit = 5 + holder.bufferEnabled = true + holder.obBufferLimit = 5 for i := range itemArray { asks := itemArray[i] bids := itemArray[i] - err = obl.Update(&Update{ + err = holder.Update(&Update{ Bids: bids, Asks: asks, Pair: cp, @@ -302,67 +275,67 @@ func TestHittingTheBuffer(t *testing.T) { t.Fatal(err) } } - if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks) != 3 { - t.Errorf("expected 3 entries, received: %v", - len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks)) + + book := holder.ob[cp.Base][cp.Quote][asset.Spot] + if book.ob.GetAskLength() != 3 { + t.Errorf("expected 3 entries, received: %v", book.ob.GetAskLength()) } - if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids) != 3 { - t.Errorf("expected 3 entries, received: %v", - len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids)) + if book.ob.GetBidLength() != 3 { + t.Errorf("expected 3 entries, received: %v", book.ob.GetBidLength()) } } // TestInsertWithIDs logic test func TestInsertWithIDs(t *testing.T) { - obl, _, _, err := createSnapshot() + holder, _, _, err := createSnapshot() if err != nil { t.Fatal(err) } - obl.bufferEnabled = true - obl.updateEntriesByID = true - obl.obBufferLimit = 5 + holder.bufferEnabled = true + holder.updateEntriesByID = true + holder.obBufferLimit = 5 for i := range itemArray { asks := itemArray[i] if asks[0].Amount <= 0 { continue } bids := itemArray[i] - err = obl.Update(&Update{ + err = holder.Update(&Update{ Bids: bids, Asks: asks, Pair: cp, UpdateTime: time.Now(), Asset: asset.Spot, - Action: "insert", + Action: UpdateInsert, }) if err != nil { t.Fatal(err) } } - if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks) != 6 { - t.Errorf("expected 6 entries, received: %v", - len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks)) + + book := holder.ob[cp.Base][cp.Quote][asset.Spot] + if book.ob.GetAskLength() != 6 { + t.Errorf("expected 5 entries, received: %v", book.ob.GetAskLength()) } - if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids) != 6 { - t.Errorf("expected 6 entries, received: %v", - len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids)) + if book.ob.GetBidLength() != 6 { + t.Errorf("expected 5 entries, received: %v", book.ob.GetBidLength()) } } // TestSortIDs logic test func TestSortIDs(t *testing.T) { - obl, _, _, err := createSnapshot() + holder, _, _, err := createSnapshot() if err != nil { t.Fatal(err) } - obl.bufferEnabled = true - obl.sortBufferByUpdateIDs = true - obl.sortBuffer = true - obl.obBufferLimit = 5 + holder.bufferEnabled = true + holder.sortBufferByUpdateIDs = true + holder.sortBuffer = true + holder.obBufferLimit = 5 for i := range itemArray { asks := itemArray[i] bids := itemArray[i] - err = obl.Update(&Update{ + err = holder.Update(&Update{ Bids: bids, Asks: asks, Pair: cp, @@ -373,19 +346,18 @@ func TestSortIDs(t *testing.T) { t.Fatal(err) } } - if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks) != 3 { - t.Errorf("expected 3 entries, received: %v", - len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks)) + book := holder.ob[cp.Base][cp.Quote][asset.Spot] + if book.ob.GetAskLength() != 3 { + t.Errorf("expected 3 entries, received: %v", book.ob.GetAskLength()) } - if len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids) != 3 { - t.Errorf("expected 3 entries, received: %v", - len(obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Bids)) + if book.ob.GetAskLength() != 3 { + t.Errorf("expected 3 entries, received: %v", book.ob.GetAskLength()) } } // TestOutOfOrderIDs logic test func TestOutOfOrderIDs(t *testing.T) { - obl, _, _, err := createSnapshot() + holder, _, _, err := createSnapshot() if err != nil { t.Fatal(err) } @@ -394,12 +366,12 @@ func TestOutOfOrderIDs(t *testing.T) { t.Errorf("expected sorted price to be 3000, received: %v", itemArray[1][0].Price) } - obl.bufferEnabled = true - obl.sortBuffer = true - obl.obBufferLimit = 5 + holder.bufferEnabled = true + holder.sortBuffer = true + holder.obBufferLimit = 5 for i := range itemArray { asks := itemArray[i] - err = obl.Update(&Update{ + err = holder.Update(&Update{ Asks: asks, Pair: cp, UpdateID: outOFOrderIDs[i], @@ -409,15 +381,16 @@ func TestOutOfOrderIDs(t *testing.T) { t.Fatal(err) } } + book := holder.ob[cp.Base][cp.Quote][asset.Spot] + cpy := book.ob.Retrieve() // Index 1 since index 0 is price 7000 - if obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks[1].Price != 2000 { - t.Errorf("expected sorted price to be 3000, received: %v", - obl.ob[cp.Base][cp.Quote][asset.Spot].ob.Asks[1].Price) + if cpy.Asks[1].Price != 2000 { + t.Errorf("expected sorted price to be 2000, received: %v", cpy.Asks[1].Price) } } func TestOrderbookLastUpdateID(t *testing.T) { - obl, _, _, err := createSnapshot() + holder, _, _, err := createSnapshot() if err != nil { t.Fatal(err) } @@ -428,7 +401,7 @@ func TestOrderbookLastUpdateID(t *testing.T) { for i := range itemArray { asks := itemArray[i] - err = obl.Update(&Update{ + err = holder.Update(&Update{ Asks: asks, Pair: cp, UpdateID: int64(i) + 1, @@ -439,7 +412,10 @@ func TestOrderbookLastUpdateID(t *testing.T) { } } - ob := obl.GetOrderbook(cp, asset.Spot) + ob, err := holder.GetOrderbook(cp, asset.Spot) + if err != nil { + t.Fatal(err) + } if exp := len(itemArray); ob.LastUpdateID != int64(exp) { t.Errorf("expected last update id to be %d, received: %v", exp, ob.LastUpdateID) } @@ -447,7 +423,7 @@ func TestOrderbookLastUpdateID(t *testing.T) { // TestRunUpdateWithoutSnapshot logic test func TestRunUpdateWithoutSnapshot(t *testing.T) { - var obl Orderbook + var holder Orderbook var snapShot1 orderbook.Base asks := []orderbook.Item{ {Price: 4000, Amount: 1, ID: 8}, @@ -458,21 +434,18 @@ func TestRunUpdateWithoutSnapshot(t *testing.T) { } snapShot1.Asks = asks snapShot1.Bids = bids - snapShot1.AssetType = asset.Spot + snapShot1.Asset = asset.Spot snapShot1.Pair = cp - obl.exchangeName = exchangeName - err := obl.Update(&Update{ + holder.exchangeName = exchangeName + err := holder.Update(&Update{ Bids: bids, Asks: asks, Pair: cp, UpdateTime: time.Now(), Asset: asset.Spot, }) - if err == nil { - t.Fatal("expected an error running update with no snapshot loaded") - } - if err.Error() != "ob.Base could not be found for Exchange exchangeTest CurrencyPair: BTCUSD AssetType: spot" { - t.Fatal(err) + if !errors.Is(err, errDepthNotFound) { + t.Fatalf("expected %v but received %v", errDepthNotFound, err) } } @@ -482,7 +455,7 @@ func TestRunUpdateWithoutAnyUpdates(t *testing.T) { var snapShot1 orderbook.Base snapShot1.Asks = []orderbook.Item{} snapShot1.Bids = []orderbook.Item{} - snapShot1.AssetType = asset.Spot + snapShot1.Asset = asset.Spot snapShot1.Pair = cp obl.exchangeName = exchangeName err := obl.Update(&Update{ @@ -492,11 +465,8 @@ func TestRunUpdateWithoutAnyUpdates(t *testing.T) { UpdateTime: time.Now(), Asset: asset.Spot, }) - if err == nil { - t.Fatal("expected an error running update with no snapshot loaded") - } - if err.Error() != "websocket orderbook buffer error: update bid/ask targets cannot be nil" { - t.Fatal("expected nil asks and bids error") + if !errors.Is(err, errUpdateNoTargets) { + t.Fatalf("expected %v but received %v", errUpdateNoTargets, err) } } @@ -506,9 +476,9 @@ func TestRunSnapshotWithNoData(t *testing.T) { obl.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder) obl.dataHandler = make(chan interface{}, 1) var snapShot1 orderbook.Base - snapShot1.AssetType = asset.Spot + snapShot1.Asset = asset.Spot snapShot1.Pair = cp - snapShot1.ExchangeName = "test" + snapShot1.Exchange = "test" obl.exchangeName = "test" err := obl.LoadSnapshot(&snapShot1) if err != nil { @@ -522,7 +492,7 @@ func TestLoadSnapshot(t *testing.T) { obl.dataHandler = make(chan interface{}, 100) obl.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder) var snapShot1 orderbook.Base - snapShot1.ExchangeName = "SnapshotWithOverride" + snapShot1.Exchange = "SnapshotWithOverride" asks := []orderbook.Item{ {Price: 4000, Amount: 1, ID: 8}, } @@ -531,7 +501,7 @@ func TestLoadSnapshot(t *testing.T) { } snapShot1.Asks = asks snapShot1.Bids = bids - snapShot1.AssetType = asset.Spot + snapShot1.Asset = asset.Spot snapShot1.Pair = cp err := obl.LoadSnapshot(&snapShot1) if err != nil { @@ -556,11 +526,11 @@ func TestFlushbuffer(t *testing.T) { // TestInsertingSnapShots logic test func TestInsertingSnapShots(t *testing.T) { - var obl Orderbook - obl.dataHandler = make(chan interface{}, 100) - obl.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder) + var holder Orderbook + holder.dataHandler = make(chan interface{}, 100) + holder.ob = make(map[currency.Code]map[currency.Code]map[asset.Item]*orderbookHolder) var snapShot1 orderbook.Base - snapShot1.ExchangeName = "WSORDERBOOKTEST1" + snapShot1.Exchange = "WSORDERBOOKTEST1" asks := []orderbook.Item{ {Price: 6000, Amount: 1, ID: 1}, {Price: 6001, Amount: 0.5, ID: 2}, @@ -591,14 +561,14 @@ func TestInsertingSnapShots(t *testing.T) { snapShot1.Asks = asks snapShot1.Bids = bids - snapShot1.AssetType = asset.Spot + snapShot1.Asset = asset.Spot snapShot1.Pair = cp - err := obl.LoadSnapshot(&snapShot1) + err := holder.LoadSnapshot(&snapShot1) if err != nil { t.Fatal(err) } var snapShot2 orderbook.Base - snapShot2.ExchangeName = "WSORDERBOOKTEST2" + snapShot2.Exchange = "WSORDERBOOKTEST2" asks = []orderbook.Item{ {Price: 51, Amount: 1, ID: 1}, {Price: 52, Amount: 0.5, ID: 2}, @@ -627,19 +597,21 @@ func TestInsertingSnapShots(t *testing.T) { {Price: 39, Amount: 7, ID: 22}, } - snapShot2.Asks = orderbook.SortAsks(asks) - snapShot2.Bids = orderbook.SortBids(bids) - snapShot2.AssetType = asset.Spot + snapShot2.Asks = asks + snapShot2.Asks.SortAsks() + snapShot2.Bids = bids + snapShot2.Bids.SortBids() + snapShot2.Asset = asset.Spot snapShot2.Pair, err = currency.NewPairFromString("LTCUSD") if err != nil { t.Fatal(err) } - err = obl.LoadSnapshot(&snapShot2) + err = holder.LoadSnapshot(&snapShot2) if err != nil { t.Fatal(err) } var snapShot3 orderbook.Base - snapShot3.ExchangeName = "WSORDERBOOKTEST3" + snapShot3.Exchange = "WSORDERBOOKTEST3" asks = []orderbook.Item{ {Price: 511, Amount: 1, ID: 1}, {Price: 52, Amount: 0.5, ID: 2}, @@ -668,74 +640,88 @@ func TestInsertingSnapShots(t *testing.T) { {Price: 39, Amount: 7, ID: 22}, } - snapShot3.Asks = orderbook.SortAsks(asks) - snapShot3.Bids = orderbook.SortBids(bids) - snapShot3.AssetType = "FUTURES" + snapShot3.Asks = asks + snapShot3.Asks.SortAsks() + snapShot3.Bids = bids + snapShot3.Bids.SortBids() + snapShot3.Asset = asset.Futures snapShot3.Pair, err = currency.NewPairFromString("LTCUSD") if err != nil { t.Fatal(err) } - err = obl.LoadSnapshot(&snapShot3) + err = holder.LoadSnapshot(&snapShot3) if err != nil { t.Fatal(err) } - if obl.ob[snapShot1.Pair.Base][snapShot1.Pair.Quote][snapShot1.AssetType].ob.Asks[0] != snapShot1.Asks[0] { + ob, err := holder.GetOrderbook(snapShot1.Pair, snapShot1.Asset) + if err != nil { + t.Fatal(err) + } + if ob.Asks[0] != snapShot1.Asks[0] { t.Errorf("loaded data mismatch. Expected %v, received %v", snapShot1.Asks[0], - obl.ob[snapShot1.Pair.Base][snapShot1.Pair.Quote][snapShot1.AssetType].ob.Asks[0]) + ob.Asks[0]) } - if obl.ob[snapShot2.Pair.Base][snapShot1.Pair.Quote][snapShot2.AssetType].ob.Asks[0] != snapShot2.Asks[0] { + ob, err = holder.GetOrderbook(snapShot2.Pair, snapShot2.Asset) + if err != nil { + t.Fatal(err) + } + if ob.Asks[0] != snapShot2.Asks[0] { t.Errorf("loaded data mismatch. Expected %v, received %v", snapShot2.Asks[0], - obl.ob[snapShot2.Pair.Base][snapShot1.Pair.Quote][snapShot2.AssetType].ob.Asks[0]) + ob.Asks[0]) } - if obl.ob[snapShot3.Pair.Base][snapShot1.Pair.Quote][snapShot3.AssetType].ob.Asks[0] != snapShot3.Asks[0] { + ob, err = holder.GetOrderbook(snapShot3.Pair, snapShot3.Asset) + if err != nil { + t.Fatal(err) + } + if ob.Asks[0] != snapShot3.Asks[0] { t.Errorf("loaded data mismatch. Expected %v, received %v", snapShot3.Asks[0], - obl.ob[snapShot3.Pair.Base][snapShot1.Pair.Quote][snapShot3.AssetType].ob.Asks[0]) + ob.Asks[0]) } } func TestGetOrderbook(t *testing.T) { - obl, _, _, err := createSnapshot() + holder, _, _, err := createSnapshot() if err != nil { t.Fatal(err) } - ob := obl.GetOrderbook(cp, asset.Spot) - bufferOb := obl.ob[cp.Base][cp.Quote][asset.Spot] - if bufferOb.ob == ob { - t.Error("orderbooks should be separate in pointer value and not linked to orderbook package") + ob, err := holder.GetOrderbook(cp, asset.Spot) + if err != nil { + t.Fatal(err) } - - if len(bufferOb.ob.Asks) != len(ob.Asks) || - len(bufferOb.ob.Bids) != len(ob.Bids) || - bufferOb.ob.AssetType != ob.AssetType || - bufferOb.ob.ExchangeName != ob.ExchangeName || - bufferOb.ob.LastUpdateID != ob.LastUpdateID || - bufferOb.ob.NotAggregated != ob.NotAggregated || - bufferOb.ob.Pair != ob.Pair { + bufferOb := holder.ob[cp.Base][cp.Quote][asset.Spot] + b := bufferOb.ob.Retrieve() + if bufferOb.ob.GetAskLength() != len(ob.Asks) || + bufferOb.ob.GetBidLength() != len(ob.Bids) || + b.Asset != ob.Asset || + b.Exchange != ob.Exchange || + b.LastUpdateID != ob.LastUpdateID || + b.PriceDuplication != ob.PriceDuplication || + b.Pair != ob.Pair { t.Fatal("data on both books should be the same") } } func TestSetup(t *testing.T) { w := Orderbook{} - err := w.Setup(0, false, false, false, false, "", nil) - if err == nil || !errors.Is(err, errUnsetExchangeName) { + err := w.Setup(0, false, false, false, false, true, "", nil) + if !errors.Is(err, errUnsetExchangeName) { t.Fatalf("expected error %v but received %v", errUnsetExchangeName, err) } - err = w.Setup(0, false, false, false, false, "test", nil) - if err == nil || !errors.Is(err, errUnsetDataHandler) { + err = w.Setup(0, false, false, false, false, false, "test", nil) + if !errors.Is(err, errUnsetDataHandler) { t.Fatalf("expected error %v but received %v", errUnsetDataHandler, err) } - err = w.Setup(0, true, false, false, false, "test", make(chan interface{})) - if err == nil || !errors.Is(err, errIssueBufferEnabledButNoLimit) { + err = w.Setup(0, true, false, false, false, true, "test", make(chan interface{})) + if !errors.Is(err, errIssueBufferEnabledButNoLimit) { t.Fatalf("expected error %v but received %v", errIssueBufferEnabledButNoLimit, err) } - err = w.Setup(1337, true, true, true, true, "test", make(chan interface{})) + err = w.Setup(1337, true, true, true, true, false, "test", make(chan interface{})) if err != nil { t.Fatal(err) } @@ -752,25 +738,25 @@ func TestSetup(t *testing.T) { func TestValidate(t *testing.T) { w := Orderbook{} err := w.validate(nil) - if err == nil || !errors.Is(err, errUpdateIsNil) { + if !errors.Is(err, errUpdateIsNil) { t.Fatalf("expected error %v but received %v", errUpdateIsNil, err) } err = w.validate(&Update{}) - if err == nil || !errors.Is(err, errUpdateNoTargets) { + if !errors.Is(err, errUpdateNoTargets) { t.Fatalf("expected error %v but received %v", errUpdateNoTargets, err) } } func TestEnsureMultipleUpdatesViaPrice(t *testing.T) { - obl, _, _, err := createSnapshot() + holder, _, _, err := createSnapshot() if err != nil { t.Error(err) } asks := bidAskGenerator() - holder := obl.ob[cp.Base][cp.Quote][asset.Spot] - holder.updateByPrice(&Update{ + book := holder.ob[cp.Base][cp.Quote][asset.Spot] + book.updateByPrice(&Update{ Bids: asks, Asks: asks, Pair: cp, @@ -781,69 +767,12 @@ func TestEnsureMultipleUpdatesViaPrice(t *testing.T) { t.Error(err) } - if len(holder.ob.Asks) <= 3 { + if book.ob.GetAskLength() <= 3 { t.Errorf("Insufficient updates") } } -func TestInsertItem(t *testing.T) { - update := []orderbook.Item{{Price: 4}} - - // Correctly aligned - asks := []orderbook.Item{ - { - Price: 1, - }, - { - Price: 2, - }, - { - Price: 3, - }, - { - Price: 5, - }, - { - Price: 6, - }, - { - Price: 7, - }, - } - - insertUpdatesAsk(update, &asks) - if asks[3].Price != 4 { - t.Fatal("incorrect insertion") - } - - bids := []orderbook.Item{ - { - Price: 7, - }, - { - Price: 6, - }, - { - Price: 5, - }, - { - Price: 3, - }, - { - Price: 2, - }, - { - Price: 1, - }, - } - - insertUpdatesBid(update, &bids) - if asks[3].Price != 4 { - t.Fatal("incorrect insertion") - } -} - -func deploySliceOrdered(size int) []orderbook.Item { +func deploySliceOrdered(size int) orderbook.Items { rand.Seed(time.Now().UnixNano()) var items []orderbook.Item for i := 0; i < size; i++ { @@ -857,14 +786,16 @@ func TestUpdateByIDAndAction(t *testing.T) { asks := deploySliceOrdered(100) bids := append(asks[:0:0], asks...) - orderbook.Reverse(bids) + bids.Reverse() - book := &orderbook.Base{ - Bids: append(bids[:0:0], bids...), - Asks: append(asks[:0:0], asks...), + book, err := orderbook.DeployDepth("test", cp, asset.Spot) + if err != nil { + t.Fatal(err) } - err := book.Verify() + book.LoadSnapshot(append(bids[:0:0], bids...), append(asks[:0:0], asks...)) + + err = book.Retrieve().Verify() if err != nil { t.Fatal(err) } @@ -894,14 +825,16 @@ func TestUpdateByIDAndAction(t *testing.T) { Action: UpdateInsert, Bids: []orderbook.Item{ { - Price: 0, - ID: 1337, + Price: 0, + ID: 1337, + Amount: 1, }, }, Asks: []orderbook.Item{ { - Price: 100, - ID: 1337, + Price: 100, + ID: 1337, + Amount: 1, }, }, }) @@ -909,10 +842,12 @@ func TestUpdateByIDAndAction(t *testing.T) { t.Fatal(err) } - if book.Bids[len(book.Bids)-1].Price != 0 { + cpy := book.Retrieve() + + if cpy.Bids[len(cpy.Bids)-1].Price != 0 { t.Fatal("did not append bid item") } - if book.Asks[len(book.Asks)-1].Price != 100 { + if cpy.Asks[len(cpy.Asks)-1].Price != 100 { t.Fatal("did not append ask item") } @@ -938,11 +873,13 @@ func TestUpdateByIDAndAction(t *testing.T) { t.Fatal(err) } - if book.Bids[len(book.Bids)-1].Amount != 100 { - t.Fatal("did not update bid amount") + cpy = book.Retrieve() + + if cpy.Bids[len(cpy.Bids)-1].Amount != 100 { + t.Fatal("did not update bid amount", cpy.Bids[len(cpy.Bids)-1].Amount) } - if book.Asks[len(book.Asks)-1].Amount != 100 { + if cpy.Asks[len(cpy.Asks)-1].Amount != 100 { t.Fatal("did not update ask amount") } @@ -968,16 +905,17 @@ func TestUpdateByIDAndAction(t *testing.T) { t.Fatal(err) } - if book.Bids[0].Amount != 99 && book.Bids[0].Price != 100 { + cpy = book.Retrieve() + + if cpy.Bids[0].Amount != 99 && cpy.Bids[0].Price != 100 { t.Fatal("did not adjust bid item placement and details") } - if book.Asks[0].Amount != 99 && book.Asks[0].Amount != 0 { + if cpy.Asks[0].Amount != 99 && cpy.Asks[0].Amount != 0 { t.Fatal("did not adjust ask item placement and details") } - book.Bids = append(bids[:0:0], bids...) // nolint:gocritic - book.Asks = append(asks[:0:0], asks...) // nolint:gocritic + book.LoadSnapshot(append(bids[:0:0], bids...), append(bids[:0:0], bids...)) // nolint:gocritic // Delete - not found err = holder.updateByIDAndAction(&Update{ @@ -1018,7 +956,7 @@ func TestUpdateByIDAndAction(t *testing.T) { t.Fatal(err) } - if len(book.Asks) != 99 { + if book.GetAskLength() != 99 { t.Fatal("element not deleted") } @@ -1033,7 +971,7 @@ func TestUpdateByIDAndAction(t *testing.T) { t.Fatal("error cannot be nil") } - update := book.Asks[0] + update := book.Retrieve().Asks[0] update.Amount = 1337 err = holder.updateByIDAndAction(&Update{ @@ -1046,20 +984,20 @@ func TestUpdateByIDAndAction(t *testing.T) { t.Fatal(err) } - if book.Asks[0].Amount != 1337 { + if book.Retrieve().Asks[0].Amount != 1337 { t.Fatal("element not updated") } } func TestFlushOrderbook(t *testing.T) { w := &Orderbook{} - err := w.Setup(5, false, false, false, false, "test", make(chan interface{}, 2)) + err := w.Setup(5, false, false, false, false, false, "test", make(chan interface{}, 2)) if err != nil { t.Fatal(err) } var snapShot1 orderbook.Base - snapShot1.ExchangeName = "Snapshooooot" + snapShot1.Exchange = "Snapshooooot" asks := []orderbook.Item{ {Price: 4000, Amount: 1, ID: 8}, } @@ -1068,7 +1006,7 @@ func TestFlushOrderbook(t *testing.T) { } snapShot1.Asks = asks snapShot1.Bids = bids - snapShot1.AssetType = asset.Spot + snapShot1.Asset = asset.Spot snapShot1.Pair = cp err = w.FlushOrderbook(cp, asset.Spot) @@ -1076,14 +1014,9 @@ func TestFlushOrderbook(t *testing.T) { t.Fatal("book not loaded error cannot be nil") } - o := w.GetOrderbook(cp, asset.Spot) - if o != nil { - t.Fatal("book not loaded, this should not happen") - } - - err = w.LoadSnapshot(&snapShot1) - if err != nil { - t.Fatal(err) + _, err = w.GetOrderbook(cp, asset.Spot) + if !errors.Is(err, errDepthNotFound) { + t.Fatalf("expected: %v but received: %v", errDepthNotFound, err) } err = w.LoadSnapshot(&snapShot1) @@ -1096,12 +1029,12 @@ func TestFlushOrderbook(t *testing.T) { t.Fatal(err) } - o = w.GetOrderbook(cp, asset.Spot) - if o == nil { - t.Fatal("cannot get book") + o, err := w.GetOrderbook(cp, asset.Spot) + if err != nil { + t.Fatal(err) } - if o.Bids != nil && o.Asks != nil { + if len(o.Bids) != 0 || len(o.Asks) != 0 { t.Fatal("orderbook items not flushed") } } diff --git a/exchanges/stream/buffer/buffer_types.go b/exchanges/stream/buffer/buffer_types.go index ec19cedc..e647faee 100644 --- a/exchanges/stream/buffer/buffer_types.go +++ b/exchanges/stream/buffer/buffer_types.go @@ -9,6 +9,10 @@ import ( "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" ) +// timerDefault defines the amount of time between alerting the sync manager of +// an update. +var timerDefault = time.Second * 10 + // Orderbook defines a local cache of orderbooks for amending, appending // and deleting changes and updates the main store for a stream type Orderbook struct { @@ -20,12 +24,20 @@ type Orderbook struct { updateEntriesByID bool // Use the update IDs to match ob entries exchangeName string dataHandler chan interface{} + verbose bool m sync.Mutex } +// orderbookHolder defines a store of pending updates and a pointer to the +// orderbook depth type orderbookHolder struct { - ob *orderbook.Base + ob *orderbook.Depth buffer *[]Update + // Reduces the amount of outbound alerts to the data handler for example + // coinbasepro can have up too 100 updates per second introducing overhead. + // The sync agent only requires an alert every 15 seconds for a specific + // currency. + ticker *time.Ticker } // Update stores orderbook updates and dictates what features to use when processing diff --git a/exchanges/stream/websocket.go b/exchanges/stream/websocket.go index 41799f55..66ca6d5b 100644 --- a/exchanges/stream/websocket.go +++ b/exchanges/stream/websocket.go @@ -113,6 +113,7 @@ func (w *Websocket) Setup(s *WebsocketSetup) error { s.SortBuffer, s.SortBufferByUpdateIDs, s.UpdateEntriesByID, + s.Verbose, w.exchangeName, w.DataHandler) } diff --git a/exchanges/yobit/yobit_wrapper.go b/exchanges/yobit/yobit_wrapper.go index 206b957c..24e5c8cb 100644 --- a/exchanges/yobit/yobit_wrapper.go +++ b/exchanges/yobit/yobit_wrapper.go @@ -240,10 +240,10 @@ func (y *Yobit) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderboo // UpdateOrderbook updates and returns the orderbook for a currency pair func (y *Yobit) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: y.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: y.OrderbookVerificationBypass, + Exchange: y.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: y.CanVerifyOrderbook, } fpair, err := y.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/exchanges/zb/zb_websocket.go b/exchanges/zb/zb_websocket.go index 09c9f7c2..42a69538 100644 --- a/exchanges/zb/zb_websocket.go +++ b/exchanges/zb/zb_websocket.go @@ -143,11 +143,11 @@ func (z *ZB) wsHandleData(respRaw []byte) error { return err } - orderbook.Reverse(book.Asks) // Reverse asks for correct alignment - book.AssetType = asset.Spot + book.Asks.Reverse() // Reverse asks for correct alignment + book.Asset = asset.Spot book.Pair = cPair - book.ExchangeName = z.Name - book.VerificationBypass = z.OrderbookVerificationBypass + book.Exchange = z.Name + book.VerifyOrderbook = z.CanVerifyOrderbook err = z.Websocket.Orderbook.LoadSnapshot(&book) if err != nil { diff --git a/exchanges/zb/zb_wrapper.go b/exchanges/zb/zb_wrapper.go index 19089949..212328e3 100644 --- a/exchanges/zb/zb_wrapper.go +++ b/exchanges/zb/zb_wrapper.go @@ -301,10 +301,10 @@ func (z *ZB) FetchOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.B // UpdateOrderbook updates and returns the orderbook for a currency pair func (z *ZB) UpdateOrderbook(p currency.Pair, assetType asset.Item) (*orderbook.Base, error) { book := &orderbook.Base{ - ExchangeName: z.Name, - Pair: p, - AssetType: assetType, - VerificationBypass: z.OrderbookVerificationBypass, + Exchange: z.Name, + Pair: p, + Asset: assetType, + VerifyOrderbook: z.CanVerifyOrderbook, } currFormat, err := z.FormatExchangeCurrency(p, assetType) if err != nil { diff --git a/gctscript/modules/gct/exchange.go b/gctscript/modules/gct/exchange.go index 6c9edcc0..676720d1 100644 --- a/gctscript/modules/gct/exchange.go +++ b/gctscript/modules/gct/exchange.go @@ -85,11 +85,11 @@ func ExchangeOrderbook(args ...objects.Object) (objects.Object, error) { } data := make(map[string]objects.Object, 5) - data["exchange"] = &objects.String{Value: ob.ExchangeName} + data["exchange"] = &objects.String{Value: ob.Exchange} data["pair"] = &objects.String{Value: ob.Pair.String()} data["asks"] = &asks data["bids"] = &bids - data["asset"] = &objects.String{Value: ob.AssetType.String()} + data["asset"] = &objects.String{Value: ob.Asset.String()} return &objects.Map{ Value: data, diff --git a/gctscript/wrappers/validator/validator.go b/gctscript/wrappers/validator/validator.go index dd469960..90cc31c5 100644 --- a/gctscript/wrappers/validator/validator.go +++ b/gctscript/wrappers/validator/validator.go @@ -49,9 +49,9 @@ func (w Wrapper) Orderbook(exch string, pair currency.Pair, item asset.Item) (*o } return &orderbook.Base{ - ExchangeName: exch, - AssetType: item, - Pair: pair, + Exchange: exch, + Asset: item, + Pair: pair, Bids: []orderbook.Item{ { Amount: 1, diff --git a/main.go b/main.go index 7cc203f5..4a1abce9 100644 --- a/main.go +++ b/main.go @@ -63,8 +63,10 @@ func main() { flag.BoolVar(&settings.EnableTradeSyncing, "tradesync", false, "enables trade syncing for all enabled exchanges") flag.IntVar(&settings.SyncWorkers, "syncworkers", engine.DefaultSyncerWorkers, "the amount of workers (goroutines) to use for syncing exchange data") flag.BoolVar(&settings.SyncContinuously, "synccontinuously", true, "whether to sync exchange data continuously (ticker, orderbook and trade history info") - flag.DurationVar(&settings.SyncTimeout, "synctimeout", engine.DefaultSyncerTimeout, - "the amount of time before the syncer will switch from one protocol to the other (e.g. from REST to websocket)") + flag.DurationVar(&settings.SyncTimeoutREST, "synctimeoutrest", engine.DefaultSyncerTimeoutREST, + "the amount of time before the syncer will switch from rest protocol to the streaming protocol (e.g. from REST to websocket)") + flag.DurationVar(&settings.SyncTimeoutWebsocket, "synctimeoutwebsocket", engine.DefaultSyncerTimeoutWebsocket, + "the amount of time before the syncer will switch from the websocket protocol to REST protocol (e.g. from websocket to REST)") // Forex provider settings flag.BoolVar(&settings.EnableCurrencyConverter, "currencyconverter", false, "overrides config and sets up foreign exchange Currency Converter")