BTSE: Fix duplicate pair errors on Million pairs (M_*) (#1401)

* BTSE: Fix duplicate error on Million pairs (M_*)

BTSE has listed Pitbull token with two symbols:
PIT-USD and M_PIT-USD for millons of PIT / USD.
The native token is not tradable, so we ignore them and
get a base of M_PIT because that's what later APIs will accept

* BTSE: Fix test errors on locked market

* Common: Improve AppendError and ExcludeError

This change switches from a stateful multiError to caring more about the
Unwrap() []error interface, the same as [go standard
lib](https://github.com/golang/go/blob/go1.21.4/src/errors/wrap.go#L54-L68)

Notably, if we implement Unwrap() []error and do NOT implement Is() then
we get free compatibility with the core functions.

The only distateful thing here is needing to deeply unwrap fmt.Errorf
errors, since they don't flatten. I can't see any way around that

* Pairs: Fix exchange config Pairs loading

When a pair string contained two punctuation runes, the first one is used,
and the configFormat is ignored.

This fix checks the list and corrects any with the wrong delimiter, or
errors if the format is inconsistent.

* BTSE: Fix all tickers retrieved by GetTicker

PR #764 introduced GetTickers, but it wasn't rolled out to BTSE.
This fix ensures that when one ticker is a locked market, the rest continue to
function. Particularly important if the locked market wasn't even
enabled anyway.

* Kucoin: Fix test config future pairs

* BTSE: Remove PIT tests; Token removed

BTSE have removed the PIT token pairs

All these changes stand, and this just removes the test

* ITBit: Fix fatal error on second run

This fix removes incorrect config pair delimiter, because it would be
re-inserted into config the first run, and then error the second time.

This delimiter doesn't match the config we have.
There's no implementation of fetching pairs, so what's in config files
now is all that matters

* Engine: Fix TestConfigAllJsonResponse

* Clarity of non-matching json improved
* Handling for fixing pair delimiters

exchanges/btse/btse.go

@@ -118,7 +118,7 @@ func (b *BTSE) GetTrades(ctx context.Context, symbol string, start, end time.Tim
 		urlValues.Add("end", strconv.FormatInt(end.Unix(), 10))
 	}
 	if !start.IsZero() && !end.IsZero() && start.After(end) {
-		return t, errors.New("start cannot be after end time")
+		return t, common.ErrStartAfterEnd
 	}
 	if beforeSerialID > 0 {
 		urlValues.Add("beforeSerialId", strconv.Itoa(beforeSerialID))
@@ -141,7 +141,7 @@ func (b *BTSE) GetOHLCV(ctx context.Context, symbol string, start, end time.Time
 
 	if !start.IsZero() && !end.IsZero() {
 		if start.After(end) {
-			return o, errors.New("start cannot be after end time")
+			return o, common.ErrStartAfterEnd
 		}
 		urlValues.Add("start", strconv.FormatInt(start.Unix(), 10))
 		urlValues.Add("end", strconv.FormatInt(end.Unix(), 10))
@@ -195,7 +195,7 @@ func (b *BTSE) GetWalletHistory(ctx context.Context, symbol string, start, end t
 	}
 	if !start.IsZero() && !end.IsZero() {
 		if start.After(end) || end.Before(start) {
-			return resp, errors.New("start cannot be after end time")
+			return resp, common.ErrStartAfterEnd
 		}
 		urlValues.Add("start", strconv.FormatInt(start.Unix(), 10))
 		urlValues.Add("end", strconv.FormatInt(end.Unix(), 10))
@@ -615,3 +615,24 @@ func (b *BTSE) calculateTradingFee(ctx context.Context, feeBuilder *exchange.Fee
 func parseOrderTime(timeStr string) (time.Time, error) {
 	return time.Parse(time.DateTime, timeStr)
 }
+
+// MillionPairs returns a map of symbol names which have a IsMillion equivalent
+func (m *MarketSummary) MillionPairs() map[string]bool {
+	pairs := map[string]bool{}
+	for _, s := range *m {
+		if s.Active && s.HasLiquidity() && s.IsMillions() {
+			pairs[strings.TrimPrefix(s.Symbol, "M_")] = true
+		}
+	}
+	return pairs
+}
+
+// HasLiquidity returns if a market pair has a bid or ask != 0
+func (m *MarketPair) HasLiquidity() bool {
+	return m.LowestAsk != 0 || m.HighestBid != 0
+}
+
+// IsMillions returns if a market pair represents a million of Base / Quote
+func (m *MarketPair) IsMillions() bool {
+	return strings.HasPrefix(m.Symbol, "M_")
+}

exchanges/btse/btse_types.go

@@ -21,7 +21,10 @@ type FundingHistoryData struct {
 }
 
 // MarketSummary response data
-type MarketSummary []struct {
+type MarketSummary []*MarketPair
+
+// MarketPair is a single pair in Market Summary
+type MarketPair struct {
 	Symbol              string   `json:"symbol"`
 	Last                float64  `json:"last"`
 	LowestAsk           float64  `json:"lowestAsk"`
@@ -56,7 +59,7 @@ type MarketSummary []struct {
 	AvailableSettlement []string `json:"availableSettlement"`
 	Futures             bool     `json:"futures"`
 	IsMarketOpenToSpot  bool     `json:"isMarketOpenToSpot"`
-	IsMarketOpentoOTC   bool     `json:"isMarketOpenToOtc"`
+	IsMarketOpenToOTC   bool     `json:"isMarketOpenToOtc"`
 }
 
 // OHLCV holds Open, High Low, Close, Volume data for set symbol

exchanges/btse/btse_wrapper.go

@@ -258,31 +258,35 @@ func (b *BTSE) FetchTradablePairs(ctx context.Context, a asset.Item) (currency.P
 	if err != nil {
 		return nil, err
 	}
-	pairs := make([]currency.Pair, 0, len(m))
-	for x := range m {
-		if !m[x].Active ||
-			// BTSE returns 0 for both highest bid and lowest ask if there is
-			// no order book data, so we skip those pairs. There is no way to
-			// take or provide liquidity for these pairs.
-
-			// TODO: Add support for an OTC asset as this eliminates many valid
-			// tradable pairs which are active, OTC only and available on the
-			// front-end.
-			(m[x].LowestAsk == 0 && m[x].HighestBid == 0) {
+	pairs := make(currency.Pairs, 0, len(m))
+	mPairs := m.MillionPairs()
+	for _, l := range m {
+		if !l.Active || !l.HasLiquidity() ||
+			(a == asset.Spot && !l.IsMarketOpenToSpot) { // Skip OTC assets only tradable on web UI
 			continue
 		}
-
-		var pair currency.Pair
-		quote := m[x].Quote
+		if mPairs[l.Symbol] {
+			// BTSE lists M_ symbols for very small pairs, in millions. For those listings, we want to take the M_ listing in preference
+			// to the native listing, since they're often going to appear as locked markets due to size (bid == ask, e.g. 0.0000000003)
+			continue
+		}
+		baseCurr := l.Base
+		var quoteCurr string
 		if a == asset.Futures {
-			symSplit := strings.Split(m[x].Symbol, m[x].Base)
-			if len(symSplit) <= 1 {
+			s := strings.Split(l.Symbol, l.Base) // e.g. RUNEPFC for RUNE-USD futures pair
+			if len(s) <= 1 {
 				continue
 			}
-			quote = symSplit[1]
+			quoteCurr = s[1]
+		} else {
+			s := strings.Split(l.Symbol, currency.DashDelimiter)
+			if len(s) != 2 {
+				continue
+			}
+			baseCurr = s[0]
+			quoteCurr = s[1]
 		}
-
-		pair, err = currency.NewPairFromStrings(m[x].Base, quote)
+		pair, err := currency.NewPairFromStrings(baseCurr, quoteCurr)
 		if err != nil {
 			return nil, err
 		}
@@ -317,29 +321,27 @@ func (b *BTSE) UpdateTickers(ctx context.Context, a asset.Item) error {
 	if err != nil {
 		return err
 	}
+	var errs error
 	for x := range tickers {
-		var pair currency.Pair
-		pair, err = currency.NewPairFromString(tickers[x].Symbol)
-		if err != nil {
-			return err
+		pair, err := currency.NewPairFromString(tickers[x].Symbol)
+		if err == nil {
+			err = ticker.ProcessTicker(&ticker.Price{
+				Pair:         pair,
+				Ask:          tickers[x].LowestAsk,
+				Bid:          tickers[x].HighestBid,
+				Low:          tickers[x].Low24Hr,
+				Last:         tickers[x].Last,
+				Volume:       tickers[x].Volume,
+				High:         tickers[x].High24Hr,
+				ExchangeName: b.Name,
+				AssetType:    a})
 		}
-
-		err = ticker.ProcessTicker(&ticker.Price{
-			Pair:         pair,
-			Ask:          tickers[x].LowestAsk,
-			Bid:          tickers[x].HighestBid,
-			Low:          tickers[x].Low24Hr,
-			Last:         tickers[x].Last,
-			Volume:       tickers[x].Volume,
-			High:         tickers[x].High24Hr,
-			ExchangeName: b.Name,
-			AssetType:    a})
 		if err != nil {
-			return err
+			errs = common.AppendError(errs, err)
 		}
 	}
 
-	return nil
+	return errs
 }
 
 // UpdateTicker updates and returns the ticker for a currency pair
@@ -350,7 +352,24 @@ func (b *BTSE) UpdateTicker(ctx context.Context, p currency.Pair, a asset.Item)
 	if !b.SupportsAsset(a) {
 		return nil, fmt.Errorf("%w %v", asset.ErrNotSupported, a)
 	}
-	if err := b.UpdateTickers(ctx, a); err != nil {
+	ticks, err := b.GetMarketSummary(ctx, p.String(), a == asset.Spot)
+	if err != nil {
+		return nil, err
+	}
+	if len(ticks) != 1 {
+		return nil, errors.New("market_summary should return 1 tick for a single ticker")
+	}
+	err = ticker.ProcessTicker(&ticker.Price{
+		Pair:         p,
+		Ask:          ticks[0].LowestAsk,
+		Bid:          ticks[0].HighestBid,
+		Low:          ticks[0].Low24Hr,
+		Last:         ticks[0].Last,
+		Volume:       ticks[0].Volume,
+		High:         ticks[0].High24Hr,
+		ExchangeName: b.Name,
+		AssetType:    a})
+	if err != nil {
 		return nil, err
 	}
 	return ticker.GetTicker(b.Name, p, a)

exchanges/itbit/itbit_wrapper.go

@@ -61,7 +61,7 @@ func (i *ItBit) SetDefaults() {
 	i.API.CredentialsValidator.RequiresSecret = true
 
 	requestFmt := &currency.PairFormat{Uppercase: true}
-	configFmt := &currency.PairFormat{Uppercase: true, Delimiter: currency.DashDelimiter}
+	configFmt := &currency.PairFormat{Uppercase: true}
 	err := i.SetGlobalPairsManager(requestFmt, configFmt, asset.Spot)
 	if err != nil {
 		log.Errorln(log.ExchangeSys, err)

exchanges/ticker/ticker.go

@@ -14,10 +14,11 @@ import (
 )
 
 var (
+	// ErrBidEqualsAsk error for locked markets
+	ErrBidEqualsAsk        = errors.New("bid equals ask this is a crossed or locked market")
 	errInvalidTicker       = errors.New("invalid ticker")
 	errTickerNotFound      = errors.New("ticker not found")
 	errExchangeNameIsEmpty = errors.New("exchange name is empty")
-	errBidEqualsAsk        = errors.New("bid equals ask this is a crossed or locked market")
 	errBidGreaterThanAsk   = errors.New("bid greater than ask this is a crossed or locked market")
 )
 
@@ -132,7 +133,7 @@ func ProcessTicker(p *Price) error {
 				return fmt.Errorf("%s %s %w",
 					p.ExchangeName,
 					p.Pair,
-					errBidEqualsAsk)
+					ErrBidEqualsAsk)
 			}
 
 			if p.Bid > p.Ask {

exchanges/ticker/ticker_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	"github.com/thrasher-corp/gocryptotrader/dispatch"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
@@ -288,9 +289,7 @@ func TestProcessTicker(t *testing.T) { // non-appending function to tickers
 		Bid:          1337,
 		Ask:          1337,
 	})
-	if !errors.Is(err, errBidEqualsAsk) {
-		t.Errorf("received: %v but expected: %v", err, errBidEqualsAsk)
-	}
+	assert.ErrorIs(t, err, ErrBidEqualsAsk, "ProcessTicker should error locked market")
 
 	err = ProcessTicker(&Price{
 		ExchangeName: "Bitfinex",