d0zingcat/gocryptotrader
1
0
Fork 0
mirror of https://github.com/d0zingcat/gocryptotrader.git synced 2026-05-13 23:16:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

orderbook: Implement initial linked list (#643)

Browse Source 
* Exchanges: Initial implementation after rebase of depth (WIP)

* orderbook/buffer: convert and couple orderbook interaction functionality from buffer to orderbook linked list - Use single point reference for orderbook depth

* buffer/orderbook: conversion continued (WIP)

* exchange: buffer/linkedlist handover (WIP)

* Added some tests for yesterday

* linkedList: added more testing and trying to figure out broken things

* Started tying everything in

* continuous integration and testing

* orderbook: expanded tests

* go mod tidy

* Add in different synchornisation levels for protocols
Add in timer for the streaming system to reduce updates to datahandler
Add in more test code as I integrate more exchanges

* Depth: Add tests, add length check to call linked list updating, add in constructor.
Linked List: Improve tests, add in checks for zero liquidity on books.
Node: Added in cleaner POC, add in contructor.
Buffer: Fixed tests, checked benchmarks.

* orderbook: reinstate dispatch calls

* Addr glorious & madcozbad nits

* fix functionality and add tests

* Address linterinos

* remove label

* expanded comment

* fix races and and bitmex test

* reinstate go routine for alerting changes

* rm line :D

* fix more tests

* Addr glorious nits

* rm glorious field

* depth: defer unlock to stop deadlock

* orderbook: remove unused vars

* buffer: fix test to what it should be

* nits: madcosbad addr

* nits: glorious nits

* linkedlist: remove unused params

* orderbook: shift time call to outside of push to inline, add in case for update inster price for zero liquidity, nits

* orderbook: nits addressed

* engine: change stream -> websocket convention and remove unused function

* nits: glorious nits

* Websocket Buffer: Add verbosity switch

* linked list: Add comment

* linked list: fix spelling

* nits: glorious nits

* orderbook: Adds in test and explicit time type with constructor, fix nits

* linter

* spelling: removed the dere fence

* depth: Update alerting mechanism to a more battle tested state

* depth: spelling

* nits: glorious nits

* linked list: match cases

* buffer: fix linter issue

* golangci: increase timeout by 30 seconds

* nodes: update atomic checks

* spelling: fix

* node: add in commentary

* exchanges/syncer: add function to switch over to REST when websocket functionality is not available for a specific asset type

* linter: exchange linter issues

* syncer: Add in warning

* nits: glorious nits

* AssetWebsocketSupport: unexport map

* Nits: Adrr

* rm letter

* exchanges: Orderbook verification change for naming, deprecate checksum bypass as it has the potential to obfuscate errors that are at the tail end of the book, add in verification for websocket stream updates

* general: fix spelling remove breakpoint

* nits: fix more glorious nits until more are found

* orderbook: fix tests

* orderbook: fix wait tests and add in more checks

* nits: addr

* orderbook: remove dispatch reference

* linkedlist: consolidate bid/ask functions

* linked lisdt: remove words

* fix spelling
This commit is contained in:
Ryan O'Hara-Reid
2021-04-23 15:16:01 +10:00
committed by GitHub
parent 9973523f1e
commit 7b718700f7
81 changed files with 4546 additions and 1592 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
engine/engine.go
View File

@@ -208,7 +208,8 @@ func validateSettings(b *Engine, s *Settings, flagSet map[string]bool) {
b.Settings.EnableOrderbookSyncing = s.EnableOrderbookSyncing
b.Settings.EnableTradeSyncing = s.EnableTradeSyncing
b.Settings.SyncWorkers = s.SyncWorkers
b.Settings.SyncTimeout = s.SyncTimeout
b.Settings.SyncTimeoutREST = s.SyncTimeoutREST
b.Settings.SyncTimeoutWebsocket = s.SyncTimeoutWebsocket
b.Settings.SyncContinuously = s.SyncContinuously
b.Settings.EnableDepositAddressManager = s.EnableDepositAddressManager
b.Settings.EnableExchangeAutoPairUpdates = s.EnableExchangeAutoPairUpdates
@@ -305,7 +306,8 @@ func PrintSettings(s *Settings) {
gctlog.Debugf(gctlog.Global, "\t Enable ticker syncing: %v\n", s.EnableTickerSyncing)
gctlog.Debugf(gctlog.Global, "\t Enable orderbook syncing: %v\n", s.EnableOrderbookSyncing)
gctlog.Debugf(gctlog.Global, "\t Enable trade syncing: %v\n", s.EnableTradeSyncing)
gctlog.Debugf(gctlog.Global, "\t Exchange sync timeout: %v\n", s.SyncTimeout)
gctlog.Debugf(gctlog.Global, "\t Exchange REST sync timeout: %v\n", s.SyncTimeoutREST)
gctlog.Debugf(gctlog.Global, "\t Exchange Websocket sync timeout: %v\n", s.SyncTimeoutWebsocket)
gctlog.Debugf(gctlog.Global, "- FOREX SETTINGS:")
gctlog.Debugf(gctlog.Global, "\t Enable currency conveter: %v", s.EnableCurrencyConverter)
gctlog.Debugf(gctlog.Global, "\t Enable currency layer: %v", s.EnableCurrencyLayer)
@@ -464,13 +466,14 @@ func (bot *Engine) Start() error {
if bot.Settings.EnableExchangeSyncManager {
exchangeSyncCfg := CurrencyPairSyncerConfig{
SyncTicker: bot.Settings.EnableTickerSyncing,
SyncOrderbook: bot.Settings.EnableOrderbookSyncing,
SyncTrades: bot.Settings.EnableTradeSyncing,
SyncContinuously: bot.Settings.SyncContinuously,
NumWorkers: bot.Settings.SyncWorkers,
Verbose: bot.Settings.Verbose,
SyncTimeout: bot.Settings.SyncTimeout,
SyncTicker: bot.Settings.EnableTickerSyncing,
SyncOrderbook: bot.Settings.EnableOrderbookSyncing,
SyncTrades: bot.Settings.EnableTradeSyncing,
SyncContinuously: bot.Settings.SyncContinuously,
NumWorkers: bot.Settings.SyncWorkers,
Verbose: bot.Settings.Verbose,
SyncTimeoutREST: bot.Settings.SyncTimeoutREST,
SyncTimeoutWebsocket: bot.Settings.SyncTimeoutWebsocket,
}
bot.ExchangeCurrencyPairManager, err = NewCurrencyPairSyncer(exchangeSyncCfg)

3
engine/engine_types.go
View File

@@ -44,7 +44,8 @@ type Settings struct {
EnableTradeSyncing bool
SyncWorkers int
SyncContinuously bool
SyncTimeout time.Duration
SyncTimeoutREST time.Duration
SyncTimeoutWebsocket time.Duration
// Forex settings
EnableCurrencyConverter bool

10
engine/events_test.go
View File

@@ -210,11 +210,11 @@ func TestProcessOrderbook(t *testing.T) {
// now populate it with a 0 entry
o := orderbook.Base{
Pair: currency.NewPair(currency.BTC, currency.USD),
Bids: []orderbook.Item{{Amount: 24, Price: 23}},
Asks: []orderbook.Item{{Amount: 24, Price: 23}},
ExchangeName: e.Exchange,
AssetType: e.Asset,
Pair: currency.NewPair(currency.BTC, currency.USD),
Bids: []orderbook.Item{{Amount: 24, Price: 23}},
Asks: []orderbook.Item{{Amount: 24, Price: 23}},
Exchange: e.Exchange,
Asset: e.Asset,
}
if err := o.Process(); err != nil {
t.Fatal("unexpected result:", err)

8
engine/helpers_test.go
View File

@@ -535,10 +535,10 @@ func TestGetSpecificOrderbook(t *testing.T) {
bids = append(bids, orderbook.Item{Price: 1000, Amount: 1})
base := orderbook.Base{
Pair: currency.NewPair(currency.BTC, currency.USD),
Bids: bids,
ExchangeName: "Bitstamp",
AssetType: asset.Spot,
Pair: currency.NewPair(currency.BTC, currency.USD),
Bids: bids,
Exchange: "Bitstamp",
Asset: asset.Spot,
}
err := base.Process()

14
engine/routines.go
View File

@@ -132,17 +132,17 @@ func printOrderbookSummary(result *orderbook.Base, protocol string, bot *Engine,
if err == common.ErrNotYetImplemented {
log.Warnf(log.OrderBook, "Failed to get %s orderbook for %s %s %s. Error: %s\n",
protocol,
result.ExchangeName,
result.Exchange,
result.Pair,
result.AssetType,
result.Asset,
err)
return
}
log.Errorf(log.OrderBook, "Failed to get %s orderbook for %s %s %s. Error: %s\n",
protocol,
result.ExchangeName,
result.Exchange,
result.Pair,
result.AssetType,
result.Asset,
err)
return
}
@@ -165,10 +165,10 @@ func printOrderbookSummary(result *orderbook.Base, protocol string, bot *Engine,
}
log.Infof(log.OrderBook, book,
result.ExchangeName,
result.Exchange,
protocol,
bot.FormatCurrency(result.Pair),
strings.ToUpper(result.AssetType.String()),
strings.ToUpper(result.Asset.String()),
len(result.Bids),
bidsAmount,
result.Pair.Base,
@@ -317,7 +317,7 @@ func (bot *Engine) WebsocketDataHandler(exchName string, data interface{}) error
if bot.Settings.EnableExchangeSyncManager && bot.ExchangeCurrencyPairManager != nil {
bot.ExchangeCurrencyPairManager.update(exchName,
d.Pair,
d.AssetType,
d.Asset,
SyncItemOrderbook,
nil)
}

4
engine/routines_test.go
View File

@@ -118,8 +118,8 @@ func TestHandleData(t *testing.T) {
}
err = b.WebsocketDataHandler(exchName, &orderbook.Base{
ExchangeName: fakePassExchange,
Pair: currency.NewPair(currency.BTC, currency.USD),
Exchange: fakePassExchange,
Pair: currency.NewPair(currency.BTC, currency.USD),
})
if err != nil {
t.Error(err)

57
engine/rpcserver.go
View File

@@ -1672,45 +1672,37 @@ func (s *RPCServer) GetOrderbookStream(r *gctrpc.GetOrderbookStreamRequest, stre
return err
}
pipe, err := orderbook.SubscribeOrderbook(r.Exchange, p, a)
depth, err := orderbook.GetDepth(r.Exchange, p, a)
if err != nil {
return err
}
defer pipe.Release()
for {
data, ok := <-pipe.C
if !ok {
return errDispatchSystem
base := depth.Retrieve()
bids := make([]*gctrpc.OrderbookItem, len(base.Bids))
for i := range base.Bids {
bids[i] = &gctrpc.OrderbookItem{
Amount: base.Bids[i].Amount,
Price: base.Bids[i].Price,
Id: base.Bids[i].ID}
}
ob := (*data.(*interface{})).(orderbook.Base)
var bids, asks []*gctrpc.OrderbookItem
for i := range ob.Bids {
bids = append(bids, &gctrpc.OrderbookItem{
Amount: ob.Bids[i].Amount,
Price: ob.Bids[i].Price,
Id: ob.Bids[i].ID,
})
}
for i := range ob.Asks {
asks = append(asks, &gctrpc.OrderbookItem{
Amount: ob.Asks[i].Amount,
Price: ob.Asks[i].Price,
Id: ob.Asks[i].ID,
})
asks := make([]*gctrpc.OrderbookItem, len(base.Asks))
for i := range base.Asks {
asks[i] = &gctrpc.OrderbookItem{
Amount: base.Asks[i].Amount,
Price: base.Asks[i].Price,
Id: base.Asks[i].ID}
}
err := stream.Send(&gctrpc.OrderbookResponse{
Pair: &gctrpc.CurrencyPair{Base: ob.Pair.Base.String(),
Quote: ob.Pair.Quote.String()},
Pair: &gctrpc.CurrencyPair{Base: r.Pair.Base, Quote: r.Pair.Quote},
Bids: bids,
Asks: asks,
AssetType: ob.AssetType.String(),
AssetType: r.AssetType,
})
if err != nil {
return err
}
<-depth.Wait(nil)
}
}
@@ -1734,27 +1726,26 @@ func (s *RPCServer) GetExchangeOrderbookStream(r *gctrpc.GetExchangeOrderbookStr
}
ob := (*data.(*interface{})).(orderbook.Base)
var bids, asks []*gctrpc.OrderbookItem
bids := make([]*gctrpc.OrderbookItem, len(ob.Bids))
for i := range ob.Bids {
bids = append(bids, &gctrpc.OrderbookItem{
bids[i] = &gctrpc.OrderbookItem{
Amount: ob.Bids[i].Amount,
Price: ob.Bids[i].Price,
Id: ob.Bids[i].ID,
})
Id: ob.Bids[i].ID}
}
asks := make([]*gctrpc.OrderbookItem, len(ob.Asks))
for i := range ob.Asks {
asks = append(asks, &gctrpc.OrderbookItem{
asks[i] = &gctrpc.OrderbookItem{
Amount: ob.Asks[i].Amount,
Price: ob.Asks[i].Price,
Id: ob.Asks[i].ID,
})
Id: ob.Asks[i].ID}
}
err := stream.Send(&gctrpc.OrderbookResponse{
Pair: &gctrpc.CurrencyPair{Base: ob.Pair.Base.String(),
Quote: ob.Pair.Quote.String()},
Bids: bids,
Asks: asks,
AssetType: ob.AssetType.String(),
AssetType: ob.Asset.String(),
})
if err != nil {
return err

111
engine/syncer.go
View File

@@ -18,8 +18,9 @@ const (
SyncItemOrderbook
SyncItemTrade
DefaultSyncerWorkers = 15
DefaultSyncerTimeout = time.Second * 15
DefaultSyncerWorkers = 15
DefaultSyncerTimeoutREST = time.Second * 15
DefaultSyncerTimeoutWebsocket = time.Minute
)
var (
@@ -37,28 +38,25 @@ func NewCurrencyPairSyncer(c CurrencyPairSyncerConfig) (*ExchangeCurrencyPairSyn
c.NumWorkers = DefaultSyncerWorkers
}
if c.SyncTimeout <= time.Duration(0) {
c.SyncTimeout = DefaultSyncerTimeout
if c.SyncTimeoutREST <= time.Duration(0) {
c.SyncTimeoutREST = DefaultSyncerTimeoutREST
}
s := ExchangeCurrencyPairSyncer{
Cfg: CurrencyPairSyncerConfig{
SyncTicker: c.SyncTicker,
SyncOrderbook: c.SyncOrderbook,
SyncTrades: c.SyncTrades,
SyncContinuously: c.SyncContinuously,
SyncTimeout: c.SyncTimeout,
NumWorkers: c.NumWorkers,
},
if c.SyncTimeoutWebsocket <= time.Duration(0) {
c.SyncTimeoutWebsocket = DefaultSyncerTimeoutWebsocket
}
s := ExchangeCurrencyPairSyncer{Cfg: c}
s.tickerBatchLastRequested = make(map[string]time.Time)
log.Debugf(log.SyncMgr,
"Exchange currency pair syncer config: continuous: %v ticker: %v"+
" orderbook: %v trades: %v workers: %v verbose: %v timeout: %v\n",
" orderbook: %v trades: %v workers: %v verbose: %v timeout REST: %v"+
" timeout Websocket: %v\n",
s.Cfg.SyncContinuously, s.Cfg.SyncTicker, s.Cfg.SyncOrderbook,
s.Cfg.SyncTrades, s.Cfg.NumWorkers, s.Cfg.Verbose, s.Cfg.SyncTimeout)
s.Cfg.SyncTrades, s.Cfg.NumWorkers, s.Cfg.Verbose, s.Cfg.SyncTimeoutREST,
s.Cfg.SyncTimeoutWebsocket)
return &s, nil
}
@@ -138,20 +136,6 @@ func (e *ExchangeCurrencyPairSyncer) add(c *CurrencyPairSyncAgent) {
e.CurrencyPairs = append(e.CurrencyPairs, *c)
}
func (e *ExchangeCurrencyPairSyncer) remove(c *CurrencyPairSyncAgent) {
e.mux.Lock()
defer e.mux.Unlock()
for x := range e.CurrencyPairs {
if e.CurrencyPairs[x].Exchange == c.Exchange &&
e.CurrencyPairs[x].Pair.Equal(c.Pair) &&
e.CurrencyPairs[x].AssetType == c.AssetType {
e.CurrencyPairs = append(e.CurrencyPairs[:x], e.CurrencyPairs[x+1:]...)
return
}
}
}
func (e *ExchangeCurrencyPairSyncer) isProcessing(exchangeName string, p currency.Pair, a asset.Item, syncType int) bool {
e.mux.Lock()
defer e.mux.Unlock()
@@ -325,6 +309,8 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
if exchanges[x].GetBase().CurrencyPairs.IsAssetEnabled(assetTypes[y]) != nil {
continue
}
wsAssetSupported := exchanges[x].IsAssetWebsocketSupported(assetTypes[y])
enabledPairs, err := exchanges[x].GetEnabledPairs(assetTypes[y])
if err != nil {
log.Errorf(log.SyncMgr,
@@ -345,25 +331,21 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
Pair: enabledPairs[i],
}
sBase := SyncBase{
IsUsingREST: usingREST || !wsAssetSupported,
IsUsingWebsocket: usingWebsocket && wsAssetSupported,
}
if e.Cfg.SyncTicker {
c.Ticker = SyncBase{
IsUsingREST: usingREST,
IsUsingWebsocket: usingWebsocket,
}
c.Ticker = sBase
}
if e.Cfg.SyncOrderbook {
c.Orderbook = SyncBase{
IsUsingREST: usingREST,
IsUsingWebsocket: usingWebsocket,
}
c.Orderbook = sBase
}
if e.Cfg.SyncTrades {
c.Trade = SyncBase{
IsUsingREST: usingREST,
IsUsingWebsocket: usingWebsocket,
}
c.Trade = sBase
}
e.add(&c)
@@ -382,9 +364,11 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
}
if e.Cfg.SyncTicker {
if !e.isProcessing(exchangeName, c.Pair, c.AssetType, SyncItemTicker) {
if c.Ticker.LastUpdated.IsZero() || time.Since(c.Ticker.LastUpdated) > e.Cfg.SyncTimeout {
if c.Ticker.LastUpdated.IsZero() ||
(time.Since(c.Ticker.LastUpdated) > e.Cfg.SyncTimeoutREST && c.Ticker.IsUsingREST) ||
(time.Since(c.Ticker.LastUpdated) > e.Cfg.SyncTimeoutWebsocket && c.Ticker.IsUsingWebsocket) {
if c.Ticker.IsUsingWebsocket {
if time.Since(c.Created) < e.Cfg.SyncTimeout {
if time.Since(c.Created) < e.Cfg.SyncTimeoutWebsocket {
continue
}
@@ -397,7 +381,7 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
c.Exchange,
Bot.FormatCurrency(enabledPairs[i]).String(),
strings.ToUpper(c.AssetType.String()),
e.Cfg.SyncTimeout,
e.Cfg.SyncTimeoutWebsocket,
)
switchedToRest = true
e.setProcessing(c.Exchange, c.Pair, c.AssetType, SyncItemTicker, false)
@@ -417,7 +401,7 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
}
e.mux.Unlock()
if batchLastDone.IsZero() || time.Since(batchLastDone) > e.Cfg.SyncTimeout {
if batchLastDone.IsZero() || time.Since(batchLastDone) > e.Cfg.SyncTimeoutREST {
e.mux.Lock()
if e.Cfg.Verbose {
log.Debugf(log.SyncMgr, "%s Init'ing REST ticker batching\n", exchangeName)
@@ -450,9 +434,11 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
if e.Cfg.SyncOrderbook {
if !e.isProcessing(exchangeName, c.Pair, c.AssetType, SyncItemOrderbook) {
if c.Orderbook.LastUpdated.IsZero() || time.Since(c.Orderbook.LastUpdated) > e.Cfg.SyncTimeout {
if c.Orderbook.LastUpdated.IsZero() ||
(time.Since(c.Orderbook.LastUpdated) > e.Cfg.SyncTimeoutREST && c.Orderbook.IsUsingREST) ||
(time.Since(c.Orderbook.LastUpdated) > e.Cfg.SyncTimeoutWebsocket && c.Orderbook.IsUsingWebsocket) {
if c.Orderbook.IsUsingWebsocket {
if time.Since(c.Created) < e.Cfg.SyncTimeout {
if time.Since(c.Created) < e.Cfg.SyncTimeoutWebsocket {
continue
}
if supportsREST {
@@ -464,7 +450,7 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
c.Exchange,
Bot.FormatCurrency(c.Pair).String(),
strings.ToUpper(c.AssetType.String()),
e.Cfg.SyncTimeout,
e.Cfg.SyncTimeoutWebsocket,
)
switchedToRest = true
e.setProcessing(c.Exchange, c.Pair, c.AssetType, SyncItemOrderbook, false)
@@ -486,7 +472,7 @@ func (e *ExchangeCurrencyPairSyncer) worker() {
}
if e.Cfg.SyncTrades {
if !e.isProcessing(exchangeName, c.Pair, c.AssetType, SyncItemTrade) {
if c.Trade.LastUpdated.IsZero() || time.Since(c.Trade.LastUpdated) > e.Cfg.SyncTimeout {
if c.Trade.LastUpdated.IsZero() || time.Since(c.Trade.LastUpdated) > e.Cfg.SyncTimeoutREST {
e.setProcessing(c.Exchange, c.Pair, c.AssetType, SyncItemTrade, true)
e.update(c.Exchange, c.Pair, c.AssetType, SyncItemTrade, nil)
}
@@ -560,6 +546,13 @@ func (e *ExchangeCurrencyPairSyncer) Start() {
continue
}
wsAssetSupported := exchanges[x].IsAssetWebsocketSupported(assetTypes[y])
if !wsAssetSupported {
log.Warnf(log.SyncMgr,
"%s asset type %s websocket functionality is unsupported, REST fetching only.",
exchangeName,
assetTypes[y])
}
enabledPairs, err := exchanges[x].GetEnabledPairs(assetTypes[y])
if err != nil {
log.Errorf(log.SyncMgr,
@@ -579,25 +572,21 @@ func (e *ExchangeCurrencyPairSyncer) Start() {
Pair: enabledPairs[i],
}
sBase := SyncBase{
IsUsingREST: usingREST || !wsAssetSupported,
IsUsingWebsocket: usingWebsocket && wsAssetSupported,
}
if e.Cfg.SyncTicker {
c.Ticker = SyncBase{
IsUsingREST: usingREST,
IsUsingWebsocket: usingWebsocket,
}
c.Ticker = sBase
}
if e.Cfg.SyncOrderbook {
c.Orderbook = SyncBase{
IsUsingREST: usingREST,
IsUsingWebsocket: usingWebsocket,
}
c.Orderbook = sBase
}
if e.Cfg.SyncTrades {
c.Trade = SyncBase{
IsUsingREST: usingREST,
IsUsingWebsocket: usingWebsocket,
}
c.Trade = sBase
}
e.add(&c)

15
engine/syncer_types.go
View File

@@ -10,13 +10,14 @@ import (
// CurrencyPairSyncerConfig stores the currency pair config
type CurrencyPairSyncerConfig struct {
SyncTicker bool
SyncOrderbook bool
SyncTrades bool
SyncContinuously bool
SyncTimeout time.Duration
NumWorkers int
Verbose bool
SyncTicker bool
SyncOrderbook bool
SyncTrades bool
SyncContinuously bool
SyncTimeoutREST time.Duration
SyncTimeoutWebsocket time.Duration
NumWorkers int
Verbose bool
}
// ExchangeSyncerConfig stores the exchange syncer config