GCT: general updates across codebase (#699)

* orderbook: export orderbook nodes for external strategy inspection

* orderbook: Add in methods for locking and unlocking multiple books at the same time e.g. book1.LockWith(book2); defer book1.UnlockWith(book2)

* include waiting functionality for depth change alert

* backtester: add word.

* log: include logger changes to impl with downstream integration

* engine: reduce params for loading exchange

* assort: rm verbose in tests, change wording in ob, expose sync.waitgroup for ext. sync options

* ticker: reduce map look ups and contention when using RW mutex when there are over 80% writes adds find last function to get the latest rate

* engine/syncmanager: add in waitgroup for step over for external package calls

* cleaup

* engine: linter fix

* currency/fx: include all references to fiat currencies to default

* orderbook: Add in fields to Unsafe type for strategies to detect potential out of sync book operations

* syncmanager: changed config variable to display correct time

* ordermanager: Add time when none provided

* currency/manager: update getasset param to get enabled assets for minor optimizations

* ftx: use get all wallet balances for a better accounts breakdown

* orderbook: unlock in reverse order

* bithumb: fixes bug on market buy and sell orders

* bithumb: fix bug for nonce is also time window sensitive

* bithumb: get orders add required parameter

* bithumb: Add asset type to account struct

* currency: improve log output when checking currency and it fails

* bithumb: Add error return on incomplete pair

* ticker:unexport all service related methods

* ticker/currency: fixes

* orderbook: fix comment

* engine: revert variable name in LoadExchange method

* sync_manager: fix panic when enabling disabling manager

* engine: fix naming convention of exported function and comments

* engine: update comment

* orderbook: fix comment for unsafe type

exchanges/ticker/ticker.go

@@ -12,6 +12,12 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
 )
 
+var (
+	errInvalidTicker       = errors.New("invalid ticker")
+	errTickerNotFound      = errors.New("ticker not found")
+	errExchangeNameIsEmpty = errors.New("exchange name is empty")
+)
+
 func init() {
 	service = new(Service)
 	service.Tickers = make(map[string]map[*currency.Item]map[*currency.Item]map[asset.Item]*Ticker)
@@ -23,8 +29,8 @@ func init() {
 // stream new ticker updates
 func SubscribeTicker(exchange string, p currency.Pair, a asset.Item) (dispatch.Pipe, error) {
 	exchange = strings.ToLower(exchange)
-	service.RLock()
-	defer service.RUnlock()
+	service.Lock()
+	defer service.Unlock()
 
 	tick, ok := service.Tickers[exchange][p.Base.Item][p.Quote.Item][a]
 	if !ok {
@@ -39,8 +45,8 @@ func SubscribeTicker(exchange string, p currency.Pair, a asset.Item) (dispatch.P
 // SubscribeToExchangeTickers subcribes to all tickers on an exchange
 func SubscribeToExchangeTickers(exchange string) (dispatch.Pipe, error) {
 	exchange = strings.ToLower(exchange)
-	service.RLock()
-	defer service.RUnlock()
+	service.Lock()
+	defer service.Unlock()
 	id, ok := service.Exchange[exchange]
 	if !ok {
 		return dispatch.Pipe{}, fmt.Errorf("%s exchange tickers not found",
@@ -51,108 +57,137 @@ func SubscribeToExchangeTickers(exchange string) (dispatch.Pipe, error) {
 }
 
 // GetTicker checks and returns a requested ticker if it exists
-func GetTicker(exchange string, p currency.Pair, tickerType asset.Item) (*Price, error) {
+func GetTicker(exchange string, p currency.Pair, a asset.Item) (*Price, error) {
 	exchange = strings.ToLower(exchange)
-	service.RLock()
-	defer service.RUnlock()
-	if service.Tickers[exchange] == nil {
+	service.Lock()
+	defer service.Unlock()
+	m1, ok := service.Tickers[exchange]
+	if !ok {
 		return nil, fmt.Errorf("no tickers for %s exchange", exchange)
 	}
 
-	if service.Tickers[exchange][p.Base.Item] == nil {
+	m2, ok := m1[p.Base.Item]
+	if !ok {
 		return nil, fmt.Errorf("no tickers associated with base currency %s",
 			p.Base)
 	}
 
-	if service.Tickers[exchange][p.Base.Item][p.Quote.Item] == nil {
+	m3, ok := m2[p.Quote.Item]
+	if !ok {
 		return nil, fmt.Errorf("no tickers associated with quote currency %s",
 			p.Quote)
 	}
 
-	if service.Tickers[exchange][p.Base.Item][p.Quote.Item][tickerType] == nil {
+	t, ok := m3[a]
+	if !ok {
 		return nil, fmt.Errorf("no tickers associated with asset type %s",
-			tickerType)
+			a)
 	}
 
-	return &service.Tickers[exchange][p.Base.Item][p.Quote.Item][tickerType].Price, nil
+	cpy := t.Price // Don't let external functions have access to underlying
+	return &cpy, nil
+}
+
+// FindLast searches for a currency pair and returns the first available
+func FindLast(p currency.Pair, a asset.Item) (float64, error) {
+	service.Lock()
+	defer service.Unlock()
+	for _, m1 := range service.Tickers {
+		m2, ok := m1[p.Base.Item]
+		if !ok {
+			continue
+		}
+		m3, ok := m2[p.Quote.Item]
+		if !ok {
+			continue
+		}
+		t, ok := m3[a]
+		if !ok {
+			continue
+		}
+
+		if t.Last == 0 {
+			return 0, errInvalidTicker
+		}
+		return t.Last, nil
+	}
+	return 0, fmt.Errorf("%w %s %s", errTickerNotFound, p, a)
 }
 
 // ProcessTicker processes incoming tickers, creating or updating the Tickers
 // list
-func ProcessTicker(tickerNew *Price) error {
-	if tickerNew.ExchangeName == "" {
-		return fmt.Errorf(ErrExchangeNameUnset)
-	}
-
-	if tickerNew.Pair.IsEmpty() {
-		return fmt.Errorf("%s %s", tickerNew.ExchangeName, errPairNotSet)
-	}
-
-	if tickerNew.AssetType == "" {
-		return fmt.Errorf("%s %s %s",
-			tickerNew.ExchangeName,
-			tickerNew.Pair,
-			errAssetTypeNotSet)
-	}
-
-	if tickerNew.LastUpdated.IsZero() {
-		tickerNew.LastUpdated = time.Now()
-	}
-
-	return service.Update(tickerNew)
-}
-
-// Update updates ticker price
-func (s *Service) Update(p *Price) error {
-	name := strings.ToLower(p.ExchangeName)
-	s.Lock()
-
-	ticker, ok := s.Tickers[name][p.Pair.Base.Item][p.Pair.Quote.Item][p.AssetType]
-	if ok {
-		ticker.Last = p.Last
-		ticker.High = p.High
-		ticker.Low = p.Low
-		ticker.Bid = p.Bid
-		ticker.Ask = p.Ask
-		ticker.Volume = p.Volume
-		ticker.QuoteVolume = p.QuoteVolume
-		ticker.PriceATH = p.PriceATH
-		ticker.Open = p.Open
-		ticker.Close = p.Close
-		ticker.LastUpdated = p.LastUpdated
-		ids := append(ticker.Assoc, ticker.Main)
-		s.Unlock()
-		return s.mux.Publish(ids, p)
-	}
-
-	switch {
-	case s.Tickers[name] == nil:
-		s.Tickers[name] = make(map[*currency.Item]map[*currency.Item]map[asset.Item]*Ticker)
-		fallthrough
-	case s.Tickers[name][p.Pair.Base.Item] == nil:
-		s.Tickers[name][p.Pair.Base.Item] = make(map[*currency.Item]map[asset.Item]*Ticker)
-		fallthrough
-	case s.Tickers[name][p.Pair.Base.Item][p.Pair.Quote.Item] == nil:
-		s.Tickers[name][p.Pair.Base.Item][p.Pair.Quote.Item] = make(map[asset.Item]*Ticker)
-	}
-
-	err := s.SetItemID(p, name)
-	if err != nil {
-		s.Unlock()
-		return err
-	}
-
-	s.Unlock()
-	return nil
-}
-
-// SetItemID retrieves and sets dispatch mux publish IDs
-func (s *Service) SetItemID(p *Price, fmtName string) error {
+func ProcessTicker(p *Price) error {
 	if p == nil {
 		return errors.New(errTickerPriceIsNil)
 	}
 
-	ids, err := s.GetAssociations(p, fmtName)
+	if p.ExchangeName == "" {
+		return fmt.Errorf(ErrExchangeNameUnset)
+	}
+
+	if p.Pair.IsEmpty() {
+		return fmt.Errorf("%s %s", p.ExchangeName, errPairNotSet)
+	}
+
+	if p.AssetType == "" {
+		return fmt.Errorf("%s %s %s",
+			p.ExchangeName,
+			p.Pair,
+			errAssetTypeNotSet)
+	}
+
+	if p.LastUpdated.IsZero() {
+		p.LastUpdated = time.Now()
+	}
+
+	return service.update(p)
+}
+
+// update updates ticker price
+func (s *Service) update(p *Price) error {
+	name := strings.ToLower(p.ExchangeName)
+	s.Lock()
+
+	m1, ok := service.Tickers[name]
+	if !ok {
+		m1 = make(map[*currency.Item]map[*currency.Item]map[asset.Item]*Ticker)
+		service.Tickers[name] = m1
+	}
+
+	m2, ok := m1[p.Pair.Base.Item]
+	if !ok {
+		m2 = make(map[*currency.Item]map[asset.Item]*Ticker)
+		m1[p.Pair.Base.Item] = m2
+	}
+
+	m3, ok := m2[p.Pair.Quote.Item]
+	if !ok {
+		m3 = make(map[asset.Item]*Ticker)
+		m2[p.Pair.Quote.Item] = m3
+	}
+
+	t, ok := m3[p.AssetType]
+	if !ok || t == nil {
+		newTicker := &Ticker{}
+		err := s.setItemID(newTicker, p, name)
+		if err != nil {
+			s.Unlock()
+			return err
+		}
+		m3[p.AssetType] = newTicker
+		s.Unlock()
+		return nil
+	}
+
+	t.Price = *p
+	ids := append(t.Assoc, t.Main)
+	s.Unlock()
+	return s.mux.Publish(ids, p)
+}
+
+// setItemID retrieves and sets dispatch mux publish IDs
+func (s *Service) setItemID(t *Ticker, p *Price, exch string) error {
+	ids, err := s.getAssociations(exch)
 	if err != nil {
 		return err
 	}
@@ -161,28 +196,27 @@ func (s *Service) SetItemID(p *Price, fmtName string) error {
 		return err
 	}
 
-	s.Tickers[fmtName][p.Pair.Base.Item][p.Pair.Quote.Item][p.AssetType] = &Ticker{Price: *p,
-		Main:  singleID,
-		Assoc: ids}
+	t.Price = *p
+	t.Main = singleID
+	t.Assoc = ids
 	return nil
 }
 
-// GetAssociations links a singular book with it's dispatch associations
-func (s *Service) GetAssociations(p *Price, fmtName string) ([]uuid.UUID, error) {
-	if p == nil || *p == (Price{}) {
-		return nil, errors.New(errTickerPriceIsNil)
+// getAssociations links a singular book with it's dispatch associations
+func (s *Service) getAssociations(exch string) ([]uuid.UUID, error) {
+	if exch == "" {
+		return nil, errExchangeNameIsEmpty
 	}
 	var ids []uuid.UUID
-	exchangeID, ok := s.Exchange[fmtName]
+	exchangeID, ok := s.Exchange[exch]
 	if !ok {
 		var err error
 		exchangeID, err = s.mux.GetID()
 		if err != nil {
 			return nil, err
 		}
-		s.Exchange[fmtName] = exchangeID
+		s.Exchange[exch] = exchangeID
 	}
-
 	ids = append(ids, exchangeID)
 	return ids, nil
 }

exchanges/ticker/ticker_test.go

@@ -1,6 +1,7 @@
 package ticker
 
 import (
+	"errors"
 	"log"
 	"math/rand"
 	"os"
@@ -80,7 +81,7 @@ func TestSubscribeTicker(t *testing.T) {
 		ExchangeName: "subscribetest",
 		AssetType:    asset.Spot})
 	if err != nil {
-		t.Error("error cannot be nil")
+		t.Fatal(err)
 	}
 
 	_, err = SubscribeTicker("subscribetest", p, asset.Spot)
@@ -198,6 +199,38 @@ func TestGetTicker(t *testing.T) {
 	}
 }
 
+func TestFindLast(t *testing.T) {
+	cp := currency.NewPair(currency.BTC, currency.XRP)
+	_, err := FindLast(cp, asset.Spot)
+	if !errors.Is(err, errTickerNotFound) {
+		t.Errorf("received: %v but expected: %v", err, errTickerNotFound)
+	}
+
+	err = service.update(&Price{Last: 0, ExchangeName: "testerinos", Pair: cp, AssetType: asset.Spot})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = FindLast(cp, asset.Spot)
+	if !errors.Is(err, errInvalidTicker) {
+		t.Errorf("received: %v but expected: %v", err, errInvalidTicker)
+	}
+
+	err = service.update(&Price{Last: 1337, ExchangeName: "testerinos", Pair: cp, AssetType: asset.Spot})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	last, err := FindLast(cp, asset.Spot)
+	if !errors.Is(err, nil) {
+		t.Errorf("received: %v but expected: %v", err, nil)
+	}
+
+	if last != 1337 {
+		t.Fatal("unexpected value")
+	}
+}
+
 func TestProcessTicker(t *testing.T) { // non-appending function to tickers
 	exchName := "bitstamp"
 	newPair, err := currency.NewPairFromStrings("BTC", "USD")
@@ -368,39 +401,15 @@ func TestProcessTicker(t *testing.T) { // non-appending function to tickers
 	wg.Wait()
 }
 
-func TestSetItemID(t *testing.T) {
-	err := service.SetItemID(nil, "")
-	if err == nil {
-		t.Error("error cannot be nil")
-	}
-
-	err = service.SetItemID(&Price{}, "")
-	if err == nil {
-		t.Error("error cannot be nil")
-	}
-
-	p := currency.NewPair(currency.CYC, currency.CYG)
-
-	service.mux = nil
-	err = service.SetItemID(&Price{Pair: p, ExchangeName: "SetItemID"}, "setitemid")
-	if err == nil {
-		t.Error("error cannot be nil")
-	}
-
-	service.mux = cpyMux
-}
-
 func TestGetAssociation(t *testing.T) {
-	_, err := service.GetAssociations(nil, "")
-	if err == nil {
-		t.Error("error cannot be nil")
+	_, err := service.getAssociations("")
+	if !errors.Is(err, errExchangeNameIsEmpty) {
+		t.Errorf("received: %v but expected: %v", err, errExchangeNameIsEmpty)
 	}
 
-	p := currency.NewPair(currency.CYC, currency.CYG)
-
 	service.mux = nil
 
-	_, err = service.GetAssociations(&Price{Pair: p, ExchangeName: "GetAssociation"}, "getassociation")
+	_, err = service.getAssociations("getassociation")
 	if err == nil {
 		t.Error("error cannot be nil")
 	}

exchanges/ticker/ticker_types.go

@@ -28,7 +28,7 @@ type Service struct {
 	Tickers  map[string]map[*currency.Item]map[*currency.Item]map[asset.Item]*Ticker
 	Exchange map[string]uuid.UUID
 	mux      *dispatch.Mux
-	sync.RWMutex
+	sync.Mutex
 }
 
 // Price struct stores the currency pair and pricing information