Engine: Dispatch service (#346)

* Added dispatch service * Added orderbook streaming capabilities * Assigned correct orderbook.base exchange name * Fixed Requested niterinos Add in cli orderbook QA tool to gctcli Add exchange orderbook streaming * Add ticker streaming support through dispatch package * Added in some more info on error returns for orderbook.go * fix linter issues * Fix some issues * Update * Fix requested * move dispatch out of exchanges folder to its own independant folder * Fix requested * change orderbook string to tickers * Limit orderbooks to 50 and made dispatch system more stateless in operation * lower cases for update/retrieve/sub exchange name * Adds in asset validation and lower case conversion on cli * Remove comment * Moved timer to a higher scope so its not constantly initialised just reset per instance and removed returning unused channel on error * Rm unused release function in dispatch.go Reset timer and bleed buffered timer chan if needed in dispatch.go Added in ticker.Stop() and timer.Stop() functions for worker routine return in dispatch.go Index aggregated bid and ask functions for orderbook.go Added in dummy slice for wsorderbook_test.go * Moved drain to above Reset so potential race would not occur in dispatch.go Fix various linter issues dispatch.go * Fix some issues * change to start/stop service, added in service state change via cli, updated logger * fix requested * Add worker amount init spawning * fix linter issues * Fix more linter issues * More fixes * Fix race issue on releasing pipe channel on a close after shutting down dispatcher system * Moved all types to dispatch_types.go && remove panic * Moved types into serperate file && improve test coverage * RM unnecessary select case for draining channel && fixed error string * Added orderbook_types file and improved code coverage * gofmt file * reinstated select cases on drain because I am silly * Remove error for drop worker * Added more test cases * not checking error issue fix * remove func causing race in test, this has required protection via an exported function * set Gemini websocket orderbook exchange name
2026-05-31 23:16:54 +00:00 · 2019-10-03 09:47:37 +10:00
parent 4a0fcc7f0f
commit db317a2447
42 changed files with 3802 additions and 1019 deletions
--- a/engine/engine.go
+++ b/engine/engine.go
@@ -11,6 +11,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/config"
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	"github.com/thrasher-corp/gocryptotrader/currency/coinmarketcap"
+	"github.com/thrasher-corp/gocryptotrader/dispatch"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
@@ -111,6 +112,7 @@ func ValidateSettings(b *Engine, s *Settings) {
 	b.Settings.EnablePortfolioManager = s.EnablePortfolioManager
 	b.Settings.EnableCoinmarketcapAnalysis = s.EnableCoinmarketcapAnalysis
 	b.Settings.EnableDatabaseManager = s.EnableDatabaseManager
+	b.Settings.EnableDispatcher = s.EnableDispatcher

 	// TO-DO: FIXME
 	if flag.Lookup("grpc") != nil {
@@ -203,6 +205,7 @@ func ValidateSettings(b *Engine, s *Settings) {
 	}

 	b.Settings.GlobalHTTPProxy = s.GlobalHTTPProxy
+	b.Settings.DispatchMaxWorkerAmount = s.DispatchMaxWorkerAmount
 }

 // PrintSettings returns the engine settings
@@ -230,6 +233,8 @@ func PrintSettings(s *Settings) {
 	log.Debugf(log.Global, "\t Enable orderbook syncing: %v", s.EnableOrderbookSyncing)
 	log.Debugf(log.Global, "\t Enable websocket routine: %v\n", s.EnableWebsocketRoutine)
 	log.Debugf(log.Global, "\t Enable NTP client: %v", s.EnableNTPClient)
+	log.Debugf(log.Global, "\t Enable dispatcher: %v", s.EnableDispatcher)
+	log.Debugf(log.Global, "\t Dispatch package max worker amount: %d", s.DispatchMaxWorkerAmount)
 	log.Debugf(log.Global, "- FOREX SETTINGS:")
 	log.Debugf(log.Global, "\t Enable currency conveter: %v", s.EnableCurrencyConverter)
 	log.Debugf(log.Global, "\t Enable currency layer: %v", s.EnableCurrencyLayer)
@@ -266,6 +271,12 @@ func (e *Engine) Start() error {
 		}
 	}

+	if e.Settings.EnableDispatcher {
+		if err := dispatch.Start(e.Settings.DispatchMaxWorkerAmount); err != nil {
+			log.Errorf(log.DispatchMgr, "Dispatcher unable to start: %v", err)
+		}
+	}
+
 	// Sets up internet connectivity monitor
 	if e.Settings.EnableConnectivityMonitor {
 		if err := e.ConnectionManager.Start(); err != nil {
@@ -436,6 +447,12 @@ func (e *Engine) Stop() {
 		}
 	}

+	if dispatch.IsRunning() {
+		if err := dispatch.Stop(); err != nil {
+			log.Errorf(log.DispatchMgr, "Dispatch system unable to stop. Error: %v", err)
+		}
+	}
+
 	if !e.Settings.EnableDryRun {
 		err := e.Config.SaveConfig(e.Settings.ConfigFile, false)
 		if err != nil {
--- a/engine/engine_types.go
+++ b/engine/engine_types.go
@@ -61,4 +61,8 @@ type Settings struct {
 	ExchangeHTTPTimeout   time.Duration
 	ExchangeHTTPUserAgent string
 	ExchangeHTTPProxy     string
+
+	// Dispatch system settings
+	EnableDispatcher        bool
+	DispatchMaxWorkerAmount int64
 }
--- a/engine/events_test.go
+++ b/engine/events_test.go
@@ -149,12 +149,6 @@ func TestProcessTicker(t *testing.T) {
 		},
 	}

-	// this will throw an err with an unpopulated ticker
-	ticker.Tickers = nil
-	if r := e.processTicker(); r {
-		t.Error("unexpected result")
-	}
-
 	// now populate it with a 0 entry
 	tick := ticker.Price{
 		Pair: currency.NewPair(currency.BTC, currency.USD),
@@ -222,12 +216,6 @@ func TestProcessOrderbook(t *testing.T) {
 		},
 	}

-	// this will throw an err with an unpopulated orderbook
-	orderbook.Orderbooks = nil
-	if r := e.processOrderbook(); r {
-		t.Error("unexpected result")
-	}
-
 	// now populate it with a 0 entry
 	o := orderbook.Base{
 		Pair:         currency.NewPair(currency.BTC, currency.USD),
--- a/engine/helpers.go
+++ b/engine/helpers.go
@@ -19,6 +19,7 @@ import (
 	"github.com/pquerna/otp/totp"
 	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/currency"
+	"github.com/thrasher-corp/gocryptotrader/dispatch"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
@@ -42,6 +43,7 @@ func GetSubsystemsStatus() map[string]bool {
 	systems["grpc_proxy"] = Bot.Settings.EnableGRPCProxy
 	systems["deprecated_rpc"] = Bot.Settings.EnableDeprecatedRPC
 	systems["websocket_rpc"] = Bot.Settings.EnableWebsocketRPC
+	systems["dispatch"] = dispatch.IsRunning()
 	return systems
 }

@@ -106,6 +108,11 @@ func SetSubsystem(subsys string, enable bool) error {
 			Bot.ExchangeCurrencyPairManager.Start()
 		}
 		Bot.ExchangeCurrencyPairManager.Stop()
+	case "dispatch":
+		if enable {
+			return dispatch.Start(Bot.Settings.DispatchMaxWorkerAmount)
+		}
+		return dispatch.Stop()
 	}
 	return errors.New("subsystem not found")
 }
--- a/engine/routines.go
+++ b/engine/routines.go
@@ -9,7 +9,6 @@ import (

 	"github.com/thrasher-corp/gocryptotrader/common"
 	"github.com/thrasher-corp/gocryptotrader/currency"
-	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/stats"
@@ -186,101 +185,6 @@ func relayWebsocketEvent(result interface{}, event, assetType, exchangeName stri
 	}
 }

-// TickerUpdaterRoutine fetches and updates the ticker for all enabled
-// currency pairs and exchanges
-func TickerUpdaterRoutine() {
-	log.Debugln(log.Ticker, "Starting ticker updater routine.")
-	var wg sync.WaitGroup
-	for {
-		wg.Add(len(Bot.Exchanges))
-		for x := range Bot.Exchanges {
-			go func(x int, wg *sync.WaitGroup) {
-				defer wg.Done()
-
-				if Bot.Exchanges[x] == nil || !Bot.Exchanges[x].SupportsREST() {
-					return
-				}
-
-				exchangeName := Bot.Exchanges[x].GetName()
-				supportsBatching := Bot.Exchanges[x].SupportsRESTTickerBatchUpdates()
-				assetTypes := Bot.Exchanges[x].GetAssetTypes()
-
-				processTicker := func(exch exchange.IBotExchange, update bool, c currency.Pair, assetType asset.Item) {
-					var result ticker.Price
-					var err error
-					if update {
-						result, err = exch.UpdateTicker(c, assetType)
-					} else {
-						result, err = exch.FetchTicker(c, assetType)
-					}
-					printTickerSummary(&result, c, assetType, exchangeName, err)
-					if err == nil {
-						if Bot.Config.RemoteControl.WebsocketRPC.Enabled {
-							relayWebsocketEvent(result, "ticker_update", assetType.String(), exchangeName)
-						}
-					}
-				}
-
-				for y := range assetTypes {
-					enabledCurrencies := Bot.Exchanges[x].GetEnabledPairs(assetTypes[y])
-					for z := range enabledCurrencies {
-						if supportsBatching && z > 0 {
-							processTicker(Bot.Exchanges[x], false, enabledCurrencies[z], assetTypes[y])
-							continue
-						}
-						processTicker(Bot.Exchanges[x], true, enabledCurrencies[z], assetTypes[y])
-					}
-				}
-			}(x, &wg)
-		}
-		wg.Wait()
-		log.Debugln(log.Ticker, "All enabled currency tickers fetched.")
-		time.Sleep(time.Second * 10)
-	}
-}
-
-// OrderbookUpdaterRoutine fetches and updates the orderbooks for all enabled
-// currency pairs and exchanges
-func OrderbookUpdaterRoutine() {
-	log.Debugln(log.OrderBook, "Starting orderbook updater routine.")
-	var wg sync.WaitGroup
-	for {
-		wg.Add(len(Bot.Exchanges))
-		for x := range Bot.Exchanges {
-			go func(x int, wg *sync.WaitGroup) {
-				defer wg.Done()
-
-				if Bot.Exchanges[x] == nil || !Bot.Exchanges[x].SupportsREST() {
-					return
-				}
-
-				exchangeName := Bot.Exchanges[x].GetName()
-				assetTypes := Bot.Exchanges[x].GetAssetTypes()
-
-				processOrderbook := func(exch exchange.IBotExchange, c currency.Pair, assetType asset.Item) {
-					result, err := exch.UpdateOrderbook(c, assetType)
-					printOrderbookSummary(&result, c, assetType, exchangeName, err)
-					if err == nil {
-						if Bot.Config.RemoteControl.WebsocketRPC.Enabled {
-							relayWebsocketEvent(result, "orderbook_update", assetType.String(), exchangeName)
-						}
-					}
-				}
-
-				for y := range assetTypes {
-					enabledCurrencies := Bot.Exchanges[x].GetEnabledPairs(assetTypes[y])
-					for z := range enabledCurrencies {
-						processOrderbook(Bot.Exchanges[x], enabledCurrencies[z], assetTypes[y])
-					}
-				}
-			}(x, &wg)
-		}
-		wg.Wait()
-		log.Debugln(log.OrderBook, "All enabled currency orderbooks fetched.")
-		time.Sleep(time.Second * 10)
-	}
-}
-
 // WebsocketRoutine Initial routine management system for websocket
 func WebsocketRoutine() {
 	if Bot.Settings.Verbose {
--- a/engine/rpcserver.go
+++ b/engine/rpcserver.go
@@ -17,6 +17,8 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
 	"github.com/thrasher-corp/gocryptotrader/gctrpc"
 	"github.com/thrasher-corp/gocryptotrader/gctrpc/auth"
 	log "github.com/thrasher-corp/gocryptotrader/logger"
@@ -27,6 +29,13 @@ import (
 	"google.golang.org/grpc/metadata"
 )

+const (
+	errExchangeNameUnset = "exchange name unset"
+	errCurrencyPairUnset = "currency pair unset"
+	errAssetTypeUnset    = "asset type unset"
+	errDispatchSystem    = "dispatch system offline"
+)
+
 // RPCServer struct
 type RPCServer struct{}

@@ -953,5 +962,200 @@ func (s *RPCServer) DisableExchangePair(ctx context.Context, r *gctrpc.ExchangeP
 	err = GetExchangeByName(r.Exchange).GetBase().CurrencyPairs.DisablePair(
 		asset.Item(r.AssetType), p)
 	return &gctrpc.GenericExchangeNameResponse{}, err
-
+}
+
+// GetOrderbookStream streams the requested updated orderbook
+func (s *RPCServer) GetOrderbookStream(r *gctrpc.GetOrderbookStreamRequest, stream gctrpc.GoCryptoTrader_GetOrderbookStreamServer) error {
+	if r.Exchange == "" {
+		return errors.New(errExchangeNameUnset)
+	}
+
+	if r.Pair.String() == "" {
+		return errors.New(errCurrencyPairUnset)
+	}
+
+	if r.AssetType == "" {
+		return errors.New(errAssetTypeUnset)
+	}
+
+	p := currency.NewPairFromStrings(r.Pair.Base, r.Pair.Quote)
+
+	pipe, err := orderbook.SubscribeOrderbook(r.Exchange, p, asset.Item(r.AssetType))
+	if err != nil {
+		return err
+	}
+
+	defer pipe.Release()
+
+	for {
+		data, ok := <-pipe.C
+		if !ok {
+			return errors.New(errDispatchSystem)
+		}
+
+		ob := (*data.(*interface{})).(orderbook.Base)
+		var bids, asks []*gctrpc.OrderbookItem
+		for i := range ob.Bids {
+			bids = append(bids, &gctrpc.OrderbookItem{
+				Amount: ob.Bids[i].Amount,
+				Price:  ob.Bids[i].Price,
+				Id:     ob.Bids[i].ID,
+			})
+		}
+		for i := range ob.Asks {
+			asks = append(asks, &gctrpc.OrderbookItem{
+				Amount: ob.Asks[i].Amount,
+				Price:  ob.Asks[i].Price,
+				Id:     ob.Asks[i].ID,
+			})
+		}
+		err := stream.Send(&gctrpc.OrderbookResponse{
+			Pair: &gctrpc.CurrencyPair{Base: ob.Pair.Base.String(),
+				Quote: ob.Pair.Quote.String()},
+			Bids:      bids,
+			Asks:      asks,
+			AssetType: ob.AssetType.String(),
+		})
+		if err != nil {
+			return err
+		}
+	}
+}
+
+// GetExchangeOrderbookStream streams all orderbooks associated with an exchange
+func (s *RPCServer) GetExchangeOrderbookStream(r *gctrpc.GetExchangeOrderbookStreamRequest, stream gctrpc.GoCryptoTrader_GetExchangeOrderbookStreamServer) error {
+	if r.Exchange == "" {
+		return errors.New(errExchangeNameUnset)
+	}
+
+	pipe, err := orderbook.SubscribeToExchangeOrderbooks(r.Exchange)
+	if err != nil {
+		return err
+	}
+
+	defer pipe.Release()
+
+	for {
+		data, ok := <-pipe.C
+		if !ok {
+			return errors.New(errDispatchSystem)
+		}
+
+		ob := (*data.(*interface{})).(orderbook.Base)
+		var bids, asks []*gctrpc.OrderbookItem
+		for i := range ob.Bids {
+			bids = append(bids, &gctrpc.OrderbookItem{
+				Amount: ob.Bids[i].Amount,
+				Price:  ob.Bids[i].Price,
+				Id:     ob.Bids[i].ID,
+			})
+		}
+		for i := range ob.Asks {
+			asks = append(asks, &gctrpc.OrderbookItem{
+				Amount: ob.Asks[i].Amount,
+				Price:  ob.Asks[i].Price,
+				Id:     ob.Asks[i].ID,
+			})
+		}
+		err := stream.Send(&gctrpc.OrderbookResponse{
+			Pair: &gctrpc.CurrencyPair{Base: ob.Pair.Base.String(),
+				Quote: ob.Pair.Quote.String()},
+			Bids:      bids,
+			Asks:      asks,
+			AssetType: ob.AssetType.String(),
+		})
+		if err != nil {
+			return err
+		}
+	}
+}
+
+// GetTickerStream streams the requested updated ticker
+func (s *RPCServer) GetTickerStream(r *gctrpc.GetTickerStreamRequest, stream gctrpc.GoCryptoTrader_GetTickerStreamServer) error {
+	if r.Exchange == "" {
+		return errors.New(errExchangeNameUnset)
+	}
+
+	if r.Pair.String() == "" {
+		return errors.New(errCurrencyPairUnset)
+	}
+
+	if r.AssetType == "" {
+		return errors.New(errAssetTypeUnset)
+	}
+
+	p := currency.NewPairFromStrings(r.Pair.Base, r.Pair.Quote)
+
+	pipe, err := ticker.SubscribeTicker(r.Exchange, p, asset.Item(r.AssetType))
+	if err != nil {
+		return err
+	}
+
+	defer pipe.Release()
+
+	for {
+		data, ok := <-pipe.C
+		if !ok {
+			return errors.New(errDispatchSystem)
+		}
+		t := (*data.(*interface{})).(ticker.Price)
+
+		err := stream.Send(&gctrpc.TickerResponse{
+			Pair: &gctrpc.CurrencyPair{
+				Base:      t.Pair.Base.String(),
+				Quote:     t.Pair.Quote.String(),
+				Delimiter: t.Pair.Delimiter},
+			LastUpdated: t.LastUpdated.Unix(),
+			Last:        t.Last,
+			High:        t.High,
+			Low:         t.Low,
+			Bid:         t.Bid,
+			Ask:         t.Ask,
+			Volume:      t.Volume,
+			PriceAth:    t.PriceATH,
+		})
+		if err != nil {
+			return err
+		}
+	}
+}
+
+// GetExchangeTickerStream streams all tickers associated with an exchange
+func (s *RPCServer) GetExchangeTickerStream(r *gctrpc.GetExchangeTickerStreamRequest, stream gctrpc.GoCryptoTrader_GetExchangeTickerStreamServer) error {
+	if r.Exchange == "" {
+		return errors.New(errExchangeNameUnset)
+	}
+
+	pipe, err := ticker.SubscribeToExchangeTickers(r.Exchange)
+	if err != nil {
+		return err
+	}
+
+	defer pipe.Release()
+
+	for {
+		data, ok := <-pipe.C
+		if !ok {
+			return errors.New(errDispatchSystem)
+		}
+		t := (*data.(*interface{})).(ticker.Price)
+
+		err := stream.Send(&gctrpc.TickerResponse{
+			Pair: &gctrpc.CurrencyPair{
+				Base:      t.Pair.Base.String(),
+				Quote:     t.Pair.Quote.String(),
+				Delimiter: t.Pair.Delimiter},
+			LastUpdated: t.LastUpdated.Unix(),
+			Last:        t.Last,
+			High:        t.High,
+			Low:         t.Low,
+			Bid:         t.Bid,
+			Ask:         t.Ask,
+			Volume:      t.Volume,
+			PriceAth:    t.PriceATH,
+		})
+		if err != nil {
+			return err
+		}
+	}
 }