subscriptions: Add templating support and integrate with Binance (#1568)

* Subscriptions: Add List.AssetPairs

* Subscriptions: Add Template and QualifiedChannel

These fields separate the concept of what the channel is from the
qualified resource name

* Subscriptions: Add List.SetStates()

* Subscriptions: Add List.QualifiedChannels

* Subscriptions: Rename testsubs.EqualLists

* Binance: Switch to ExpandTemplates

* Binance: Update ConfigTest format

* Subscriptions: Test Coverage improvements

* Subscriptions: Reenterant List.ExpandTemplates

* Subscriptions: Move templates from subscriptions to exchanges

* Binance: Inline subscription template and improvements

exchanges/binance/binance.go

@@ -21,7 +21,6 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/order"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/subscription"
 )
 
 // Binance is the overarching type across the Binance package
@@ -111,13 +110,6 @@ var (
 	errEitherLoanOrCollateralAmountsMustBeSet = errors.New("either loan or collateral amounts must be set")
 )
 
-var subscriptionNames = map[string]string{
-	subscription.TickerChannel:    "ticker",
-	subscription.OrderbookChannel: "depth",
-	subscription.CandlesChannel:   "kline",
-	subscription.AllTradesChannel: "trade",
-}
-
 // GetExchangeInfo returns exchange information. Check binance_types for more
 // information
 func (b *Binance) GetExchangeInfo(ctx context.Context) (ExchangeInfo, error) {

exchanges/binance/binance_test.go

@@ -28,9 +28,9 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/exchanges/margin"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/order"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues"
-	"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/subscription"
 	testexch "github.com/thrasher-corp/gocryptotrader/internal/testing/exchange"
+	testsubs "github.com/thrasher-corp/gocryptotrader/internal/testing/subscriptions"
 	"github.com/thrasher-corp/gocryptotrader/portfolio/withdraw"
 )
 
@@ -1981,26 +1981,23 @@ func BenchmarkWsHandleData(bb *testing.B) {
 
 func TestSubscribe(t *testing.T) {
 	t.Parallel()
-	b := b
-	channels := subscription.List{
-		{Channel: "btcusdt@ticker"},
-		{Channel: "btcusdt@trade"},
-	}
+	b := new(Binance) //nolint:govet // Intentional shadow to avoid future copy/paste mistakes
+	require.NoError(t, testexch.Setup(b), "Test instance Setup must not error")
+	channels, err := b.generateSubscriptions() // Note: We grab this before it's overwritten by MockWsInstance below
+	require.NoError(t, err, "generateSubscriptions must not error")
 	if mockTests {
+		exp := []string{"btcusdt@depth@100ms", "btcusdt@kline_1m", "btcusdt@ticker", "btcusdt@trade", "dogeusdt@depth@100ms", "dogeusdt@kline_1m", "dogeusdt@ticker", "dogeusdt@trade"}
 		mock := func(msg []byte, w *websocket.Conn) error {
 			var req WsPayload
-			err := json.Unmarshal(msg, &req)
-			require.NoError(t, err, "Unmarshal should not error")
-			require.Len(t, req.Params, len(channels), "Params should only have 2 channel") // Failure might mean mockWSInstance default Subs is not empty
-			assert.Equal(t, req.Params[0], channels[0].Channel, "Channel name should be correct")
-			assert.Equal(t, req.Params[1], channels[1].Channel, "Channel name should be correct")
+			require.NoError(t, json.Unmarshal(msg, &req), "Unmarshal should not error")
+			require.ElementsMatch(t, req.Params, exp, "Params should have correct channels")
 			return w.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf(`{"result":null,"id":%d}`, req.ID)))
 		}
 		b = testexch.MockWsInstance[Binance](t, testexch.CurryWsMockUpgrader(t, mock))
 	} else {
 		testexch.SetupWs(t, b)
 	}
-	err := b.Subscribe(channels)
+	err = b.Subscribe(channels)
 	require.NoError(t, err, "Subscribe should not error")
 	err = b.Unsubscribe(channels)
 	require.NoError(t, err, "Unsubscribe should not error")
@@ -2019,7 +2016,6 @@ func TestSubscribeBadResp(t *testing.T) {
 	}
 	b := testexch.MockWsInstance[Binance](t, testexch.CurryWsMockUpgrader(t, mock)) //nolint:govet // Intentional shadow to avoid future copy/paste mistakes
 	err := b.Subscribe(channels)
-	assert.ErrorIs(t, err, stream.ErrSubscriptionFailure, "Subscribe should error ErrSubscriptionFailure")
 	assert.ErrorIs(t, err, errUnknownError, "Subscribe should error errUnknownError")
 	assert.ErrorContains(t, err, "carrots", "Subscribe should error containing the carrots")
 }
@@ -2434,61 +2430,42 @@ func TestSeedLocalCache(t *testing.T) {
 
 func TestGenerateSubscriptions(t *testing.T) {
 	t.Parallel()
-	expected := subscription.List{}
+	exp := subscription.List{}
 	pairs, err := b.GetEnabledPairs(asset.Spot)
 	assert.NoError(t, err, "GetEnabledPairs should not error")
+	wsFmt := currency.PairFormat{Uppercase: false, Delimiter: ""}
+	baseExp := subscription.List{
+		{Channel: subscription.CandlesChannel, QualifiedChannel: "kline_1m", Asset: asset.Spot, Interval: kline.OneMin},
+		{Channel: subscription.OrderbookChannel, QualifiedChannel: "depth@100ms", Asset: asset.Spot, Interval: kline.HundredMilliseconds},
+		{Channel: subscription.TickerChannel, QualifiedChannel: "ticker", Asset: asset.Spot},
+		{Channel: subscription.AllTradesChannel, QualifiedChannel: "trade", Asset: asset.Spot},
+	}
 	for _, p := range pairs {
-		for _, c := range []string{"kline_1m", "depth@100ms", "ticker", "trade"} {
-			expected = append(expected, &subscription.Subscription{
-				Channel: p.Format(currency.PairFormat{Delimiter: "", Uppercase: false}).String() + "@" + c,
-				Pairs:   currency.Pairs{p},
-				Asset:   asset.Spot,
-			})
+		for _, baseSub := range baseExp {
+			sub := baseSub.Clone()
+			sub.Pairs = currency.Pairs{p}
+			sub.QualifiedChannel = wsFmt.Format(p) + "@" + sub.QualifiedChannel
+			exp = append(exp, sub)
 		}
 	}
 	subs, err := b.generateSubscriptions()
-	assert.NoError(t, err, "GenerateSubscriptions should not error")
-	if assert.Len(t, subs, len(expected), "Should have the correct number of subs") {
-		assert.ElementsMatch(t, subs, expected, "Should get the correct subscriptions")
-	}
+	require.NoError(t, err, "generateSubscriptions should not error")
+	testsubs.EqualLists(t, exp, subs)
 }
 
-func TestChannelName(t *testing.T) {
-	_, err := channelName(&subscription.Subscription{Channel: "Wobbegongs"})
-	assert.ErrorIs(t, err, stream.ErrSubscriptionNotSupported, "Invalid channel name should return ErrSubNotSupported")
-	assert.ErrorContains(t, err, "Wobbegong", "Invalid channel name error should contain at least one shark")
+// TestFormatChannelInterval exercises formatChannelInterval
+func TestFormatChannelInterval(t *testing.T) {
+	t.Parallel()
+	assert.Equal(t, "@1000ms", formatChannelInterval(&subscription.Subscription{Channel: subscription.OrderbookChannel, Interval: kline.ThousandMilliseconds}), "1s should format correctly for Orderbook")
+	assert.Equal(t, "@1m", formatChannelInterval(&subscription.Subscription{Channel: subscription.OrderbookChannel, Interval: kline.OneMin}), "Orderbook should format correctly")
+	assert.Equal(t, "_15m", formatChannelInterval(&subscription.Subscription{Channel: subscription.CandlesChannel, Interval: kline.FifteenMin}), "Candles should format correctly")
+}
 
-	n, err := channelName(&subscription.Subscription{Channel: subscription.TickerChannel})
-	assert.NoError(t, err, "Ticker channel should not error")
-	assert.Equal(t, "ticker", n, "Ticker channel name should be correct")
-
-	n, err = channelName(&subscription.Subscription{Channel: subscription.AllTradesChannel})
-	assert.NoError(t, err, "AllTrades channel should not error")
-	assert.Equal(t, "trade", n, "Trades channel name should be correct")
-
-	n, err = channelName(&subscription.Subscription{Channel: subscription.OrderbookChannel})
-	assert.NoError(t, err, "Orderbook channel should not error")
-	assert.Equal(t, "depth@0s", n, "Orderbook with no update rate should return 0s") // It's not channelName's job to supply defaults
-
-	n, err = channelName(&subscription.Subscription{Channel: subscription.OrderbookChannel, Interval: kline.Interval(time.Second)})
-	assert.NoError(t, err, "Orderbook channel should not error")
-	assert.Equal(t, "depth@1000ms", n, "Orderbook with 1s update rate should 1000ms")
-
-	n, err = channelName(&subscription.Subscription{Channel: subscription.OrderbookChannel, Interval: kline.HundredMilliseconds})
-	assert.NoError(t, err, "Orderbook channel should not error")
-	assert.Equal(t, "depth@100ms", n, "Orderbook with update rate should return it in the depth channel name")
-
-	n, err = channelName(&subscription.Subscription{Channel: subscription.OrderbookChannel, Interval: kline.HundredMilliseconds, Levels: 5})
-	assert.NoError(t, err, "Orderbook channel should not error")
-	assert.Equal(t, "depth@5@100ms", n, "Orderbook with Level should return it in the depth channel name")
-
-	n, err = channelName(&subscription.Subscription{Channel: subscription.CandlesChannel, Interval: kline.FifteenMin})
-	assert.NoError(t, err, "Candles channel should not error")
-	assert.Equal(t, "kline_15m", n, "Candles with interval should return it in the depth channel name")
-
-	n, err = channelName(&subscription.Subscription{Channel: subscription.CandlesChannel})
-	assert.NoError(t, err, "Candles channel should not error")
-	assert.Equal(t, "kline_0s", n, "Candles with no interval should return 0s") // It's not channelName's job to supply defaults
+// TestFormatChannelLevels exercises formatChannelLevels
+func TestFormatChannelLevels(t *testing.T) {
+	t.Parallel()
+	assert.Equal(t, "10", formatChannelLevels(&subscription.Subscription{Channel: subscription.OrderbookChannel, Levels: 10}), "Levels should format correctly")
+	assert.Empty(t, formatChannelLevels(&subscription.Subscription{Channel: subscription.OrderbookChannel, Levels: 0}), "Levels should format correctly")
 }
 
 var websocketDepthUpdate = []byte(`{"E":1608001030784,"U":7145637266,"a":[["19455.19000000","0.59490200"],["19455.37000000","0.00000000"],["19456.11000000","0.00000000"],["19456.16000000","0.00000000"],["19458.67000000","0.06400000"],["19460.73000000","0.05139800"],["19461.43000000","0.00000000"],["19464.59000000","0.00000000"],["19466.03000000","0.45000000"],["19466.36000000","0.00000000"],["19508.67000000","0.00000000"],["19572.96000000","0.00217200"],["24386.00000000","0.00256600"]],"b":[["19455.18000000","2.94649200"],["19453.15000000","0.01233600"],["19451.18000000","0.00000000"],["19446.85000000","0.11427900"],["19446.74000000","0.00000000"],["19446.73000000","0.00000000"],["19444.45000000","0.14937800"],["19426.75000000","0.00000000"],["19416.36000000","0.36052100"]],"e":"depthUpdate","s":"BTCUSDT","u":7145637297}`)

exchanges/binance/binance_websocket.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+	"text/template"
 	"time"
 
 	"github.com/buger/jsonparser"
@@ -503,130 +504,82 @@ func (b *Binance) UpdateLocalBuffer(wsdp *WebsocketDepthStream) (bool, error) {
 	return false, err
 }
 
-// generateSubscriptions generates the default subscription set
 func (b *Binance) generateSubscriptions() (subscription.List, error) {
-	var channels = make([]string, 0, len(b.Features.Subscriptions))
-	for i := range b.Features.Subscriptions {
-		name, err := channelName(b.Features.Subscriptions[i])
-		if err != nil {
-			return nil, err
-		}
-		channels = append(channels, name)
-	}
-	var subscriptions subscription.List
-	pairs, err := b.GetEnabledPairs(asset.Spot)
-	if err != nil {
-		return nil, err
-	}
-	for y := range pairs {
-		for z := range channels {
-			lp := pairs[y].Lower()
-			lp.Delimiter = ""
-			subscriptions = append(subscriptions, &subscription.Subscription{
-				Channel: lp.String() + "@" + channels[z],
-				Pairs:   currency.Pairs{pairs[y]},
-				Asset:   asset.Spot,
-			})
+	for _, s := range b.Features.Subscriptions {
+		if s.Asset == asset.Empty {
+			// Handle backwards compatibility with config without assets, all binance subs are spot
+			s.Asset = asset.Spot
 		}
 	}
-	return subscriptions, nil
+	return b.Features.Subscriptions.ExpandTemplates(b)
 }
 
-// channelName converts a Subscription Config into binance format channel suffix
-func channelName(s *subscription.Subscription) (string, error) {
-	name, ok := subscriptionNames[s.Channel]
-	if !ok {
-		return name, fmt.Errorf("%w: %s", stream.ErrSubscriptionNotSupported, s.Channel)
-	}
+var subTemplate *template.Template
 
+// GetSubscriptionTemplate returns a subscription channel template
+func (b *Binance) GetSubscriptionTemplate(_ *subscription.Subscription) (*template.Template, error) {
+	var err error
+	if subTemplate == nil {
+		subTemplate, err = template.New("subscriptions.tmpl").
+			Funcs(template.FuncMap{
+				"interval": formatChannelInterval,
+				"levels":   formatChannelLevels,
+				"fmt":      currency.EMPTYFORMAT.Format,
+			}).
+			Parse(subTplText)
+	}
+	return subTemplate, err
+}
+
+func formatChannelLevels(s *subscription.Subscription) string {
+	if s.Levels != 0 {
+		return strconv.Itoa(s.Levels)
+	}
+	return ""
+}
+
+func formatChannelInterval(s *subscription.Subscription) string {
 	switch s.Channel {
 	case subscription.OrderbookChannel:
-		if s.Levels != 0 {
-			name += "@" + strconv.Itoa(s.Levels)
-		}
 		if s.Interval.Duration() == time.Second {
-			name += "@1000ms"
-		} else {
-			name += "@" + s.Interval.Short()
+			return "@1000ms"
 		}
+		return "@" + s.Interval.Short()
 	case subscription.CandlesChannel:
-		name += "_" + s.Interval.Short()
+		return "_" + s.Interval.Short()
 	}
-	return name, nil
+	return ""
 }
 
 // Subscribe subscribes to a set of channels
 func (b *Binance) Subscribe(channels subscription.List) error {
-	return b.ParallelChanOp(channels, b.subscribeToChan, 50)
-}
-
-// subscribeToChan handles a single subscription and parses the result
-// on success it adds the subscription to the websocket
-func (b *Binance) subscribeToChan(chans subscription.List) error {
-	id := b.Websocket.Conn.GenerateMessageID(false)
-
-	cNames := make([]string, len(chans))
-	for i := range chans {
-		c := chans[i]
-		cNames[i] = c.Channel
-		if err := b.Websocket.AddSubscriptions(c); err != nil {
-			return fmt.Errorf("%w Channel: %s Pair: %s Error: %w", stream.ErrSubscriptionFailure, c.Channel, c.Pairs, err)
-		}
-	}
-
-	req := WsPayload{
-		Method: wsSubscribeMethod,
-		Params: cNames,
-		ID:     id,
-	}
-
-	respRaw, err := b.Websocket.Conn.SendMessageReturnResponse(id, req)
-	if err == nil {
-		if v, d, _, rErr := jsonparser.Get(respRaw, "result"); rErr != nil {
-			err = rErr
-		} else if d != jsonparser.Null { // null is the only expected and acceptable response
-			err = fmt.Errorf("%w: %s", errUnknownError, v)
-		}
-	}
-
-	if err != nil {
-		if err2 := b.Websocket.RemoveSubscriptions(chans...); err2 != nil {
-			err = common.AppendError(err, err2)
-		}
-		err = fmt.Errorf("%w: %w; Channels: %s", stream.ErrSubscriptionFailure, err, strings.Join(cNames, ", "))
-		b.Websocket.DataHandler <- err
-	} else {
-		for _, s := range chans {
-			if sErr := s.SetState(subscription.SubscribedState); sErr != nil {
-				err = common.AppendError(err, sErr)
-			}
-		}
-	}
-
-	return err
+	return b.ParallelChanOp(channels, func(l subscription.List) error { return b.manageSubs(wsSubscribeMethod, l) }, 50)
 }
 
 // Unsubscribe unsubscribes from a set of channels
 func (b *Binance) Unsubscribe(channels subscription.List) error {
-	return b.ParallelChanOp(channels, b.unsubscribeFromChan, 50)
+	return b.ParallelChanOp(channels, func(l subscription.List) error { return b.manageSubs(wsUnsubscribeMethod, l) }, 50)
 }
 
-// unsubscribeFromChan sends a websocket message to stop receiving data from a channel
-func (b *Binance) unsubscribeFromChan(chans subscription.List) error {
-	id := b.Websocket.Conn.GenerateMessageID(false)
-
-	cNames := make([]string, len(chans))
-	for i := range chans {
-		cNames[i] = chans[i].Channel
+// manageSubs subscribes or unsubscribes from a list of subscriptions
+func (b *Binance) manageSubs(op string, subs subscription.List) error {
+	if op == wsSubscribeMethod {
+		if err := b.Websocket.AddSubscriptions(subs...); err != nil { // Note: AddSubscription will set state to subscribing
+			return err
+		}
+	} else {
+		if err := subs.SetStates(subscription.UnsubscribingState); err != nil {
+			return err
+		}
 	}
 
 	req := WsPayload{
-		Method: wsUnsubscribeMethod,
-		Params: cNames,
-		ID:     id,
+		ID:     b.Websocket.Conn.GenerateMessageID(false),
+		Method: op,
+		Params: subs.QualifiedChannels(),
 	}
 
-	respRaw, err := b.Websocket.Conn.SendMessageReturnResponse(id, req)
+	respRaw, err := b.Websocket.Conn.SendMessageReturnResponse(req.ID, req)
 	if err == nil {
 		if v, d, _, rErr := jsonparser.Get(respRaw, "result"); rErr != nil {
 			err = rErr
@@ -636,10 +589,20 @@ func (b *Binance) unsubscribeFromChan(chans subscription.List) error {
 	}
 
 	if err != nil {
-		err = fmt.Errorf("%w: %w; Channels: %s", stream.ErrUnsubscribeFailure, err, strings.Join(cNames, ", "))
+		err = fmt.Errorf("%w; Channels: %s", err, strings.Join(subs.QualifiedChannels(), ", "))
 		b.Websocket.DataHandler <- err
+
+		if op == wsSubscribeMethod {
+			if err2 := b.Websocket.RemoveSubscriptions(subs...); err2 != nil {
+				err = common.AppendError(err, err2)
+			}
+		}
 	} else {
-		err = b.Websocket.RemoveSubscriptions(chans...)
+		if op == wsSubscribeMethod {
+			err = common.AppendError(err, subs.SetStates(subscription.SubscribedState))
+		} else {
+			err = b.Websocket.RemoveSubscriptions(subs...)
+		}
 	}
 
 	return err
@@ -1051,3 +1014,16 @@ func (o *orderbookManager) stopNeedsFetchingBook(pair currency.Pair) error {
 	state.needsFetchingBook = false
 	return nil
 }
+
+const subTplText = `
+{{ range $pair := index $.AssetPairs $.S.Asset }}
+  {{ fmt $pair -}} @
+  {{- with $c := $.S.Channel -}}
+  {{ if eq $c "ticker"         -}} ticker
+  {{ else if eq $c "allTrades" -}} trade
+  {{ else if eq $c "candles"   -}} kline  {{- interval $.S }}
+  {{ else if eq $c "orderbook" -}} depth  {{- levels $.S }}{{ interval $.S }}
+  {{- end }}{{ end }}
+  {{ $.PairSeparator }}
+{{end}}
+`

exchanges/binance/binance_wrapper.go

@@ -189,10 +189,10 @@ func (b *Binance) SetDefaults() {
 			},
 		},
 		Subscriptions: subscription.List{
-			{Enabled: true, Channel: subscription.TickerChannel},
-			{Enabled: true, Channel: subscription.AllTradesChannel},
-			{Enabled: true, Channel: subscription.CandlesChannel, Interval: kline.OneMin},
-			{Enabled: true, Channel: subscription.OrderbookChannel, Interval: kline.HundredMilliseconds},
+			{Enabled: true, Asset: asset.Spot, Channel: subscription.TickerChannel},
+			{Enabled: true, Asset: asset.Spot, Channel: subscription.AllTradesChannel},
+			{Enabled: true, Asset: asset.Spot, Channel: subscription.CandlesChannel, Interval: kline.OneMin},
+			{Enabled: true, Asset: asset.Spot, Channel: subscription.OrderbookChannel, Interval: kline.HundredMilliseconds},
 		},
 	}
 
@@ -222,16 +222,14 @@ func (b *Binance) SetDefaults() {
 
 // Setup takes in the supplied exchange configuration details and sets params
 func (b *Binance) Setup(exch *config.Exchange) error {
-	err := exch.Validate()
-	if err != nil {
+	if err := exch.Validate(); err != nil {
 		return err
 	}
 	if !exch.Enabled {
 		b.SetEnabled(false)
 		return nil
 	}
-	err = b.SetupDefaults(exch)
-	if err != nil {
+	if err := b.SetupDefaults(exch); err != nil {
 		return err
 	}
 	ePoint, err := b.API.Endpoints.GetURL(exchange.WebsocketSpot)

exchanges/exchange.go

@@ -10,6 +10,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"text/template"
 	"time"
 	"unicode"
 
@@ -1150,11 +1151,22 @@ func (b *Base) GetSubscriptions() (subscription.List, error) {
 	return b.Websocket.GetSubscriptions(), nil
 }
 
+// GetSubscriptionTemplate returns a template for a given subscription; See exchange/subscription/README.md for more information
+func (b *Base) GetSubscriptionTemplate(*subscription.Subscription) (*template.Template, error) {
+	return nil, common.ErrFunctionNotSupported
+}
+
 // AuthenticateWebsocket sends an authentication message to the websocket
 func (b *Base) AuthenticateWebsocket(_ context.Context) error {
 	return common.ErrFunctionNotSupported
 }
 
+// CanUseAuthenticatedWebsocketEndpoints calls b.Websocket.CanUseAuthenticatedEndpoints
+// Used to avoid import cycles on stream.websocket
+func (b *Base) CanUseAuthenticatedWebsocketEndpoints() bool {
+	return b.Websocket != nil && b.Websocket.CanUseAuthenticatedEndpoints()
+}
+
 // KlineIntervalEnabled returns if requested interval is enabled on exchange
 func (b *Base) klineIntervalEnabled(in kline.Interval) bool {
 	// TODO: Add in the ability to use custom klines

exchanges/exchange_test.go

@@ -2818,11 +2818,7 @@ func TestGetCachedOpenInterest(t *testing.T) {
 // TestSetSubscriptionsFromConfig tests the setting and loading of subscriptions from config and exchange defaults
 func TestSetSubscriptionsFromConfig(t *testing.T) {
 	t.Parallel()
-	b := Base{
-		Config: &config.Exchange{
-			Features: &config.FeaturesConfig{},
-		},
-	}
+	b := Base{Config: &config.Exchange{Features: &config.FeaturesConfig{}}}
 	subs := subscription.List{
 		{Channel: subscription.CandlesChannel, Interval: kline.OneDay, Enabled: true},
 		{Channel: subscription.OrderbookChannel, Enabled: false},
@@ -2900,6 +2896,17 @@ func TestGetDefaultConfig(t *testing.T) {
 	assert.Equal(t, cpy, exch.Requester)
 }
 
+// TestCanUseAuthenticatedWebsocketEndpoints exercises CanUseAuthenticatedWebsocketEndpoints
+func TestCanUseAuthenticatedWebsocketEndpoints(t *testing.T) {
+	t.Parallel()
+	e := &FakeBase{}
+	assert.False(t, e.CanUseAuthenticatedWebsocketEndpoints(), "CanUseAuthenticatedWebsocketEndpoints should return false with nil websocket")
+	e.Websocket = stream.NewWebsocket()
+	assert.False(t, e.CanUseAuthenticatedWebsocketEndpoints())
+	e.Websocket.SetCanUseAuthenticatedEndpoints(true)
+	assert.True(t, e.CanUseAuthenticatedWebsocketEndpoints())
+}
+
 // FakeBase is used to override functions
 type FakeBase struct{ Base }
 

exchanges/interfaces.go

@@ -2,6 +2,7 @@ package exchange
 
 import (
 	"context"
+	"text/template"
 	"time"
 
 	"github.com/thrasher-corp/gocryptotrader/common/key"
@@ -78,8 +79,10 @@ type IBotExchange interface {
 	SubscribeToWebsocketChannels(channels subscription.List) error
 	UnsubscribeToWebsocketChannels(channels subscription.List) error
 	GetSubscriptions() (subscription.List, error)
+	GetSubscriptionTemplate(*subscription.Subscription) (*template.Template, error)
 	FlushWebsocketChannels() error
 	AuthenticateWebsocket(ctx context.Context) error
+	CanUseAuthenticatedWebsocketEndpoints() bool
 	GetOrderExecutionLimits(a asset.Item, cp currency.Pair) (order.MinMaxLevel, error)
 	CheckOrderExecutionLimits(a asset.Item, cp currency.Pair, price, amount float64, orderType order.Type) error
 	UpdateOrderExecutionLimits(ctx context.Context, a asset.Item) error

exchanges/subscription/README.md

@@ -0,0 +1,72 @@
+# GoCryptoTrader package Subscription
+
+<img src="/common/gctlogo.png?raw=true" width="350px" height="350px" hspace="70">
+
+
+[![Build Status](https://github.com/thrasher-corp/gocryptotrader/actions/workflows/tests.yml/badge.svg?branch=master)](https://github.com/thrasher-corp/gocryptotrader/actions/workflows/tests.yml)
+[![Software License](https://img.shields.io/badge/License-MIT-orange.svg?style=flat-square)](https://github.com/thrasher-corp/gocryptotrader/blob/master/LICENSE)
+[![GoDoc](https://godoc.org/github.com/thrasher-corp/gocryptotrader?status.svg)](https://godoc.org/github.com/thrasher-corp/gocryptotrader/exchanges/subscription)
+[![Coverage Status](http://codecov.io/github/thrasher-corp/gocryptotrader/coverage.svg?branch=master)](http://codecov.io/github/thrasher-corp/gocryptotrader?branch=master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/thrasher-corp/gocryptotrader)](https://goreportcard.com/report/github.com/thrasher-corp/gocryptotrader)
+
+
+This subscription package is part of the GoCryptoTrader codebase.
+
+## This is still in active development
+
+You can track ideas, planned features and what's in progress on this Trello board: [https://trello.com/b/ZAhMhpOy/gocryptotrader](https://trello.com/b/ZAhMhpOy/gocryptotrader).
+
+Join our slack to discuss all things related to GoCryptoTrader! [GoCryptoTrader Slack](https://join.slack.com/t/gocryptotrader/shared_invite/enQtNTQ5NDAxMjA2Mjc5LTc5ZDE1ZTNiOGM3ZGMyMmY1NTAxYWZhODE0MWM5N2JlZDk1NDU0YTViYzk4NTk3OTRiMDQzNGQ1YTc4YmRlMTk)
+
+# Exchange Subscriptions
+
+Exchange Subscriptions are streams of data delivered via websocket.
+
+GoCryptoTrader engine will subscribe automatically to configured channels.
+A subset of exchanges currently support user configured channels, with the remaining using hardcoded defaults.
+See configuration Features.Subscriptions for whether an exchange is configurable.
+
+## Templating
+
+Exchange Contributors should implement `GetSubscriptionTemplate` to return a text/template Template.
+
+Exchanges are free to implement template caching, a map or a mono-template, inline or file templates.
+
+The template is provided with a single context structure:
+```go
+  S              *subscription.Subscription
+  AssetPairs     map[asset.Item]currency.Pairs
+  AssetSeparator string
+  PairSeparator  string
+```
+
+Subscriptions may fan out many channels for assets and pairs, to support exchanges which require individual subscriptions.  
+To allow the template to communicate how to handle its output it should use the provided separators:
+- AssetSeparator should be added at the end of each section related to assets
+- PairSeparator should be added at the end of each pair
+
+We use separators like this because it allows mono-templates to decide at runtime whether to fan out.
+
+See exchanges/subscription/testdata/subscriptions.tmpl for an example mono-template showcasing various features
+
+Templates do not need to worry about joining around separators; Trailing separators will be stripped automatically.
+
+
+## Contribution
+
+Please feel free to submit any pull requests or suggest any desired features to be added.
+
+When submitting a PR, please abide by our coding guidelines:
+
++ Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting) guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)).
++ Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary) guidelines.
++ Code must adhere to our [coding style](https://github.com/thrasher-corp/gocryptotrader/blob/master/doc/coding_style.md).
++ Pull requests need to be based on and opened against the `master` branch.
+
+## Donations
+
+<img src="https://github.com/thrasher-corp/gocryptotrader/blob/master/web/src/assets/donate.png?raw=true" hspace="70">
+
+If this framework helped you in any way, or you would like to support the developers working on it, please donate Bitcoin to:
+
+***bc1qk0jareu4jytc0cfrhr5wgshsq8282awpavfahc***

exchanges/subscription/fixtures_test.go

@@ -0,0 +1,96 @@
+package subscription
+
+import (
+	"maps"
+	"strings"
+	"testing"
+	"text/template"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/thrasher-corp/gocryptotrader/currency"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+)
+
+type mockEx struct {
+	tpl       string
+	auth      bool
+	errPairs  error
+	errFormat error
+}
+
+func (m *mockEx) GetEnabledPairs(_ asset.Item) (currency.Pairs, error) {
+	return currency.Pairs{btcusdtPair, ethusdcPair}, m.errPairs
+}
+
+func (m *mockEx) GetPairFormat(_ asset.Item, _ bool) (currency.PairFormat, error) {
+	return currency.PairFormat{Uppercase: true}, m.errFormat
+}
+
+func (m *mockEx) GetSubscriptionTemplate(_ *Subscription) (*template.Template, error) {
+	return template.New(m.tpl).
+		Funcs(template.FuncMap{
+			"assetName": func(a asset.Item) string {
+				if a == asset.Futures {
+					return "future"
+				}
+				return a.String()
+			}}).
+		ParseFiles("testdata/" + m.tpl)
+}
+
+func (m *mockEx) GetAssetTypes(_ bool) asset.Items            { return asset.Items{asset.Spot, asset.Futures} }
+func (m *mockEx) CanUseAuthenticatedWebsocketEndpoints() bool { return m.auth }
+
+// equalLists is a utility function to compare subscription lists and show a pretty failure message
+// It overcomes the verbose depth of assert.ElementsMatch spewConfig
+// Duplicate of internal/testing/subscriptions/EqualLists
+func equalLists(tb testing.TB, a, b List) bool {
+	tb.Helper()
+	for _, sub := range append(a, b...) {
+		sub.Key = &StrictKey{&ExactKey{sub}}
+	}
+	s, err := NewStoreFromList(a)
+	require.NoError(tb, err, "NewStoreFromList must not error")
+	added, missing := s.Diff(b)
+	if len(added) > 0 || len(missing) > 0 {
+		fail := "Differences:"
+		if len(added) > 0 {
+			fail = fail + "\n + " + strings.Join(added.Strings(), "\n + ")
+		}
+		if len(missing) > 0 {
+			fail = fail + "\n - " + strings.Join(missing.Strings(), "\n - ")
+		}
+		assert.Fail(tb, fail, "Subscriptions should be equal")
+		return false
+	}
+	return true
+}
+
+// StrictKey is key type for subscriptions where all the pairs, QualifiedChannel and Params in a Subscription must match exactly
+type StrictKey struct {
+	*ExactKey
+}
+
+var _ MatchableKey = StrictKey{} // Enforce StrictKey must implement MatchableKey
+
+// Match implements MatchableKey
+// Returns true if the key fields exactly matches the subscription, including all Pairs, QualifiedChannel and Params
+func (k StrictKey) Match(eachKey MatchableKey) bool {
+	if !k.ExactKey.Match(eachKey) {
+		return false
+	}
+	eachSub := eachKey.GetSubscription()
+	return eachSub.QualifiedChannel == k.QualifiedChannel &&
+		maps.Equal(eachSub.Params, k.Params)
+}
+
+// String implements Stringer; returns the Asset, Channel and Pairs
+// Does not provide concurrency protection on the subscription it points to
+func (k StrictKey) String() string {
+	s := k.Subscription
+	if s == nil {
+		return "Uninitialised StrictKey"
+	}
+	return s.QualifiedChannel + " " + ExactKey{s}.String()
+}

exchanges/subscription/keys.go

@@ -39,6 +39,7 @@ func (k ExactKey) String() string {
 
 // Match implements MatchableKey
 // Returns true if the key fields exactly matches the subscription, including all Pairs
+// Does not check QualifiedChannel or Params
 func (k ExactKey) Match(eachKey MatchableKey) bool {
 	if eachKey == nil {
 		return false

exchanges/subscription/keys_test.go

@@ -105,6 +105,15 @@ func TestIgnoringPairsKeyMatch(t *testing.T) {
 // TestIgnoringPairsKeyString exercises IgnoringPairsKey.String
 func TestIgnoringPairsKeyString(t *testing.T) {
 	t.Parallel()
+	assert.Equal(t, "Uninitialised IgnoringPairsKey", IgnoringPairsKey{}.String())
 	key := &IgnoringPairsKey{&Subscription{Asset: asset.Spot, Channel: TickerChannel, Pairs: currency.Pairs{ethusdcPair, btcusdtPair}}}
 	assert.Equal(t, "ticker spot", key.String())
 }
+
+// TestGetSubscription exercises GetSubscription
+func TestGetSubscription(t *testing.T) {
+	t.Parallel()
+	s := &Subscription{Asset: asset.Spot}
+	assert.Same(t, s, ExactKey{s}.GetSubscription(), "ExactKey.GetSubscription Must return a pointer to the subscription")
+	assert.Same(t, s, IgnoringPairsKey{s}.GetSubscription(), "IgnorePairKeys.GetSubscription Must return a pointer to the subscription")
+}

exchanges/subscription/list.go

@@ -2,11 +2,26 @@ package subscription
 
 import (
 	"slices"
+	"text/template"
+
+	"github.com/thrasher-corp/gocryptotrader/common"
+	"github.com/thrasher-corp/gocryptotrader/currency"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
 )
 
 // List is a container of subscription pointers
 type List []*Subscription
 
+type assetPairs map[asset.Item]currency.Pairs
+
+type iExchange interface {
+	GetAssetTypes(enabled bool) asset.Items
+	GetEnabledPairs(asset.Item) (currency.Pairs, error)
+	GetPairFormat(asset.Item, bool) (currency.PairFormat, error)
+	GetSubscriptionTemplate(*Subscription) (*template.Template, error)
+	CanUseAuthenticatedWebsocketEndpoints() bool
+}
+
 // Strings returns a sorted slice of subscriptions
 func (l List) Strings() []string {
 	s := make([]string, len(l))
@@ -30,3 +45,62 @@ func (l List) GroupPairs() (n List) {
 	}
 	return s.List()
 }
+
+// QualifiedChannels returns a sorted list of all the qualified Channels in the list
+func (l List) QualifiedChannels() []string {
+	c := make([]string, len(l))
+	for i := range l {
+		c[i] = l[i].QualifiedChannel
+	}
+	slices.Sort(c)
+	return c
+}
+
+// SetStates sets the state for all the subs in a list
+// Errors are collected for any subscriptions already in the state
+// On error all changes are reverted
+func (l List) SetStates(state State) error {
+	var err error
+	for _, sub := range l {
+		err = common.AppendError(err, sub.SetState(state))
+	}
+	return err
+}
+
+func fillAssetPairs(ap assetPairs, a asset.Item, e iExchange) error {
+	p, err := e.GetEnabledPairs(a)
+	if err != nil {
+		return err
+	}
+	f, err := e.GetPairFormat(a, true)
+	if err != nil {
+		return err
+	}
+	ap[a] = p.Format(f)
+	return nil
+}
+
+// assetPairs returns a map of enabled pairs for the subscriptions in the list, formatted for the asset
+func (l List) assetPairs(e iExchange) (assetPairs, error) {
+	at := e.GetAssetTypes(true)
+	ap := assetPairs{}
+	for _, s := range l {
+		switch s.Asset {
+		case asset.Empty:
+			// Nothing to do
+		case asset.All:
+			for _, a := range at {
+				if err := fillAssetPairs(ap, a, e); err != nil {
+					return nil, err
+				}
+			}
+		default:
+			if slices.Contains(at, s.Asset) {
+				if err := fillAssetPairs(ap, s.Asset, e); err != nil {
+					return nil, err
+				}
+			}
+		}
+	}
+	return ap, nil
+}

exchanges/subscription/list_test.go

@@ -1,15 +1,18 @@
 package subscription
 
 import (
+	"errors"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
 )
 
 // TestListStrings exercises List.Strings()
 func TestListStrings(t *testing.T) {
+	t.Parallel()
 	l := List{
 		&Subscription{
 			Channel: TickerChannel,
@@ -25,8 +28,24 @@ func TestListStrings(t *testing.T) {
 	assert.ElementsMatch(t, exp, l.Strings(), "String must return correct sorted list")
 }
 
+// TestQualifiedChannels exercises List.QualifiedChannels()
+func TestQualifiedChannels(t *testing.T) {
+	t.Parallel()
+	l := List{
+		&Subscription{
+			QualifiedChannel: "ticker-btc",
+		},
+		&Subscription{
+			QualifiedChannel: "candles-btc",
+		},
+	}
+	exp := []string{"ticker-btc", "candles-btc"}
+	assert.ElementsMatch(t, exp, l.QualifiedChannels(), "QualifiedChannels should return correct sorted list")
+}
+
 // TestListGroupPairs exercises List.GroupPairs()
 func TestListGroupPairs(t *testing.T) {
+	t.Parallel()
 	l := List{
 		{Asset: asset.Spot, Channel: TickerChannel, Pairs: currency.Pairs{ethusdcPair, btcusdtPair}},
 	}
@@ -45,3 +64,30 @@ func TestListGroupPairs(t *testing.T) {
 	exp := []string{"ticker spot ETH/USDC,BTC/USDT", "orderbook spot ETH/USDC,BTC/USDT"}
 	assert.ElementsMatch(t, exp, n.Strings(), "String must return correct sorted list")
 }
+
+// TestListSetStates exercises List.SetState()
+func TestListSetStates(t *testing.T) {
+	t.Parallel()
+	l := List{{Channel: TickerChannel}, {Channel: OrderbookChannel}}
+	assert.NoError(t, l.SetStates(SubscribingState), "SetStates should not error")
+	assert.Equal(t, SubscribingState, l[1].State(), "SetStates should set State correctly")
+
+	require.NoError(t, l[0].SetState(SubscribedState), "Individual SetState must not error")
+	err := l.SetStates(SubscribedState)
+	assert.ErrorIs(t, ErrInStateAlready, err, "SetStates should error when duplicate state")
+	assert.Equal(t, SubscribedState, l[1].State(), "SetStates should set State correctly after the error")
+}
+
+// TestAssetPairs exercises AssetPairs error handling
+// All other code is covered under TestExpandTemplates
+func TestAssetPairs(t *testing.T) {
+	t.Parallel()
+	expErr := errors.New("Krypton is gone")
+	for _, a := range []asset.Item{asset.Spot, asset.All} {
+		l := &List{{Channel: CandlesChannel, Asset: a}}
+		_, err := l.assetPairs(&mockEx{errPairs: expErr})
+		assert.ErrorIs(t, err, expErr, "Should error correctly on GetEnabledPairs")
+		_, err = l.assetPairs(&mockEx{errFormat: expErr})
+		assert.ErrorIs(t, err, expErr, "Should error correctly on GetPairFormat")
+	}
+}

exchanges/subscription/store.go

@@ -43,7 +43,7 @@ func (s *Store) Add(sub *Subscription) error {
 		return fmt.Errorf("%w: Add called on nil Store", common.ErrNilPointer)
 	}
 	if s.m == nil {
-		return fmt.Errorf("%w: Add called on an Uninitialised Store", common.ErrNilPointer)
+		return fmt.Errorf("%w: Add called on an uninitialised Store", common.ErrNilPointer)
 	}
 	if sub == nil {
 		return fmt.Errorf("%w: Subscription param", common.ErrNilPointer)

exchanges/subscription/store_test.go

@@ -13,6 +13,7 @@ import (
 
 // TestNewStore exercises NewStore
 func TestNewStore(t *testing.T) {
+	t.Parallel()
 	s := NewStore()
 	require.IsType(t, &Store{}, s, "Must return a store ref")
 	require.NotNil(t, s.m, "storage map must be initialised")
@@ -20,6 +21,7 @@ func TestNewStore(t *testing.T) {
 
 // TestNewStoreFromList exercises NewStoreFromList
 func TestNewStoreFromList(t *testing.T) {
+	t.Parallel()
 	s, err := NewStoreFromList(List{})
 	assert.NoError(t, err, "Should not error on empty list")
 	require.IsType(t, &Store{}, s, "Must return a store ref")
@@ -43,11 +45,20 @@ func TestNewStoreFromList(t *testing.T) {
 
 // TestAdd exercises Add and add methods
 func TestAdd(t *testing.T) {
-	assert.ErrorIs(t, (*Store)(nil).Add(&Subscription{}), common.ErrNilPointer, "Should error nil pointer correctly")
-	assert.ErrorIs(t, (&Store{}).Add(nil), common.ErrNilPointer, "Should error nil pointer correctly")
-	assert.ErrorIs(t, (&Store{}).Add(&Subscription{}), common.ErrNilPointer, "Should error nil pointer correctly")
+	t.Parallel()
+	err := (*Store)(nil).Add(&Subscription{})
+	assert.ErrorIs(t, err, common.ErrNilPointer, "Should error nil pointer correctly")
+	assert.ErrorContains(t, err, "called on nil Store", "Should error correctly")
+
+	err = new(Store).Add(nil)
+	assert.ErrorIs(t, err, common.ErrNilPointer, "Should error nil pointer correctly")
+	assert.ErrorContains(t, err, "called on an uninitialised Store", "Should error correctly")
 
 	s := NewStore()
+	err = s.Add(nil)
+	assert.ErrorIs(t, err, common.ErrNilPointer, "Should error nil pointer correctly")
+	assert.ErrorContains(t, err, "Subscription param", "Should error correctly")
+
 	sub := &Subscription{Channel: TickerChannel}
 	require.NoError(t, s.Add(sub), "Should not error on a standard add")
 	assert.NotNil(t, s.get(sub), "Should have stored the sub")
@@ -58,6 +69,7 @@ func TestAdd(t *testing.T) {
 // TestGet exercises Get and get methods
 // Ensures that key's Match is used, but does not exercise subscription.Match; See TestMatch for that coverage
 func TestGet(t *testing.T) {
+	t.Parallel()
 	assert.Nil(t, (*Store)(nil).Get(&Subscription{}), "Should return nil when called on nil")
 	assert.Nil(t, (&Store{}).Get(&Subscription{}), "Should return nil when called with no subscription map")
 	s := NewStore()
@@ -81,11 +93,20 @@ func TestGet(t *testing.T) {
 
 // TestRemove exercises the Remove method
 func TestRemove(t *testing.T) {
-	assert.ErrorIs(t, (*Store)(nil).Remove(&Subscription{}), common.ErrNilPointer, "Should error correctly when called on nil")
-	assert.ErrorIs(t, (&Store{}).Remove(nil), common.ErrNilPointer, "Should error correctly when called passing nil")
-	assert.ErrorIs(t, (&Store{}).Remove(&Subscription{}), common.ErrNilPointer, "Should error correctly when called with no subscription map")
+	t.Parallel()
+	err := (*Store)(nil).Remove(&Subscription{})
+	assert.ErrorIs(t, err, common.ErrNilPointer, "Should error correctly when called on nil")
+	assert.ErrorContains(t, err, "Remove called on nil Store", "Should error correctly when called on nil")
+
+	err = new(Store).Remove(nil)
+	assert.ErrorIs(t, err, common.ErrNilPointer, "Should error correctly when called on an uninit store")
+	assert.ErrorContains(t, err, "Remove called on an Uninitialised Store", "Should error correctly when called on an uninit store")
 
 	s := NewStore()
+	err = s.Remove(nil)
+	assert.ErrorIs(t, err, common.ErrNilPointer, "Should error correctly when called with nil")
+	assert.ErrorContains(t, err, "key param", "Should error correctly when called with nil")
+
 	require.NoError(t, s.Add(&Subscription{Channel: CandlesChannel, Pairs: currency.Pairs{btcusdtPair, ethusdcPair}}), "Adding subscription must not error")
 	assert.NotNil(t, s.Get(&ExactKey{&Subscription{Channel: CandlesChannel, Pairs: currency.Pairs{btcusdtPair, ethusdcPair}}}), "Should have added the sub")
 	assert.ErrorIs(t, s.Remove(&ExactKey{&Subscription{Channel: CandlesChannel, Pairs: currency.Pairs{btcusdtPair}}}), ErrNotFound, "Should error correctly when called with a non-matching key")
@@ -96,6 +117,7 @@ func TestRemove(t *testing.T) {
 
 // TestList exercises the List and Len methods
 func TestList(t *testing.T) {
+	t.Parallel()
 	assert.Empty(t, (*Store)(nil).List(), "Should return an empty List when called on nil")
 	assert.Empty(t, (&Store{}).List(), "Should return an empty List when called on Store without map")
 	s := NewStore()
@@ -118,6 +140,7 @@ func TestList(t *testing.T) {
 
 // TestStoreClear exercises the Clear method
 func TestStoreClear(t *testing.T) {
+	t.Parallel()
 	assert.NotPanics(t, func() { (*Store)(nil).Clear() }, "Should not panic when called on nil")
 	s := &Store{}
 	assert.NotPanics(t, func() { s.Clear() }, "Should not panic when called with no subscription map")
@@ -131,6 +154,7 @@ func TestStoreClear(t *testing.T) {
 
 // TestStoreDiff exercises the Diff method
 func TestStoreDiff(t *testing.T) {
+	t.Parallel()
 	s := NewStore()
 	assert.NotPanics(t, func() { (*Store)(nil).Diff(List{}) }, "Should not panic when called on nil")
 	assert.NotPanics(t, func() { (&Store{}).Diff(List{}) }, "Should not panic when called with no subscription map")

exchanges/subscription/subscription.go

@@ -47,17 +47,18 @@ type State uint8
 
 // Subscription container for streaming subscriptions
 type Subscription struct {
-	Enabled       bool           `json:"enabled"`
-	Key           any            `json:"-"`
-	Channel       string         `json:"channel,omitempty"`
-	Pairs         currency.Pairs `json:"pairs,omitempty"`
-	Asset         asset.Item     `json:"asset,omitempty"`
-	Params        map[string]any `json:"params,omitempty"`
-	Interval      kline.Interval `json:"interval,omitempty"`
-	Levels        int            `json:"levels,omitempty"`
-	Authenticated bool           `json:"authenticated,omitempty"`
-	state         State
-	m             sync.RWMutex
+	Enabled          bool           `json:"enabled"`
+	Key              any            `json:"-"`
+	Channel          string         `json:"channel,omitempty"`
+	Pairs            currency.Pairs `json:"pairs,omitempty"`
+	Asset            asset.Item     `json:"asset,omitempty"`
+	Params           map[string]any `json:"params,omitempty"`
+	Interval         kline.Interval `json:"interval,omitempty"`
+	Levels           int            `json:"levels,omitempty"`
+	Authenticated    bool           `json:"authenticated,omitempty"`
+	QualifiedChannel string         `json:"-"`
+	state            State
+	m                sync.RWMutex
 }
 
 // String implements Stringer, and aims to informatively and uniquely identify a subscription for errors and information
@@ -122,20 +123,22 @@ func (s *Subscription) EnsureKeyed() any {
 
 // Clone returns a copy of a subscription
 // Key is set to nil, because most Key types contain a pointer to the subscription, and because the clone isn't added to the store yet
+// QualifiedChannel is not copied because it's expected that the contributing fields will be changed
 // Users should allow a default key to be assigned on AddSubscription or can SetKey as necessary
 func (s *Subscription) Clone() *Subscription {
 	s.m.RLock()
 	c := &Subscription{
-		Key:           nil,
-		Enabled:       s.Enabled,
-		Channel:       s.Channel,
-		Asset:         s.Asset,
-		Params:        s.Params,
-		Interval:      s.Interval,
-		Levels:        s.Levels,
-		Authenticated: s.Authenticated,
-		state:         s.state,
-		Pairs:         s.Pairs,
+		Key:              nil,
+		Enabled:          s.Enabled,
+		Channel:          s.Channel,
+		Asset:            s.Asset,
+		Params:           s.Params,
+		Interval:         s.Interval,
+		Levels:           s.Levels,
+		Authenticated:    s.Authenticated,
+		state:            s.state,
+		Pairs:            s.Pairs,
+		QualifiedChannel: s.QualifiedChannel,
 	}
 	s.Pairs = slices.Clone(s.Pairs)
 	s.Params = maps.Clone(s.Params)

exchanges/subscription/subscription_test.go

@@ -18,12 +18,15 @@ var (
 
 // TestSubscriptionString exercises the String method
 func TestSubscriptionString(t *testing.T) {
+	t.Parallel()
 	s := &Subscription{
 		Channel: "candles",
 		Asset:   asset.Spot,
 		Pairs:   currency.Pairs{btcusdtPair, ethusdcPair.Format(currency.PairFormat{Delimiter: "/"})},
 	}
 	assert.Equal(t, "candles spot BTC/USDT,ETH/USDC", s.String(), "Subscription String should return correct value")
+	s.Key = 42
+	assert.Equal(t, "42: candles spot BTC/USDT,ETH/USDC", s.String(), "String with a non-MatchableKey")
 }
 
 // TestState exercises the state getter
@@ -48,19 +51,6 @@ func TestSetState(t *testing.T) {
 	assert.ErrorIs(t, s.SetState(UnsubscribedState+1), ErrInvalidState, "Setting an invalid state should error")
 }
 
-// TestString exercises the Stringer implementation
-func TestString(t *testing.T) {
-	s := &Subscription{
-		Channel: "candles",
-		Asset:   asset.Spot,
-		Pairs:   currency.Pairs{btcusdtPair},
-	}
-	_ = s.EnsureKeyed()
-	assert.Equal(t, "candles spot BTC/USDT", s.String(), "String with a MatchableKey")
-	s.Key = 42
-	assert.Equal(t, "42: candles spot BTC/USDT", s.String(), "String with a MatchableKey")
-}
-
 // TestEnsureKeyed exercises the key getter and ensures it sets a self-pointer key for non
 func TestEnsureKeyed(t *testing.T) {
 	t.Parallel()
@@ -102,6 +92,7 @@ func TestSubscriptionMarshaling(t *testing.T) {
 
 // TestClone exercises Clone
 func TestClone(t *testing.T) {
+	t.Parallel()
 	a := &Subscription{
 		Channel:  TickerChannel,
 		Interval: kline.OneHour,
@@ -123,6 +114,7 @@ func TestClone(t *testing.T) {
 
 // TestSetKey exercises SetKey
 func TestSetKey(t *testing.T) {
+	t.Parallel()
 	s := &Subscription{}
 	s.SetKey(14)
 	assert.Equal(t, 14, s.Key, "SetKey should set a key correctly")
@@ -130,7 +122,18 @@ func TestSetKey(t *testing.T) {
 
 // TestSetPairs exercises SetPairs
 func TestSetPairs(t *testing.T) {
+	t.Parallel()
 	s := &Subscription{}
 	s.SetPairs(currency.Pairs{btcusdtPair})
 	assert.Equal(t, "BTCUSDT", s.Pairs.Join(), "SetPairs should set a key correctly")
 }
+
+// TestAddPairs exercises AddPairs
+func TestAddPairs(t *testing.T) {
+	t.Parallel()
+	s := &Subscription{}
+	s.AddPairs()
+	assert.Empty(t, s.Pairs, "Should not have added any pairs")
+	s.AddPairs(btcusdtPair)
+	assert.Len(t, s.Pairs, 1, "Should not have added any pairs")
+}

exchanges/subscription/template.go

@@ -0,0 +1,150 @@
+package subscription
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/thrasher-corp/gocryptotrader/currency"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+)
+
+const (
+	groupSeparator  = "\x1D"
+	recordSeparator = "\x1E"
+)
+
+var (
+	errInvalidAssetExpandPairs = errors.New("subscription template containing PairSeparator with must contain either specific Asset or AssetSeparator")
+	errAssetRecords            = errors.New("subscription template did not generate the expected number of asset records")
+	errPairRecords             = errors.New("subscription template did not generate the expected number of pair records")
+	errAssetTemplateWithoutAll = errors.New("sub.Asset must be set to All if AssetSeparator is used in Channel template")
+	errNoTemplateContent       = errors.New("subscription template did not generate content")
+	errInvalidTemplate         = errors.New("GetSubscriptionTemplate did not return a template")
+)
+
+type tplCtx struct {
+	S              *Subscription
+	AssetPairs     assetPairs
+	PairSeparator  string
+	AssetSeparator string
+}
+
+// ExpandTemplates returns a list of Subscriptions with Template expanded
+// May be called on already expanded subscriptions: Passes $s through unprocessed if QualifiedChannel is already populated
+// Calls e.GetSubscriptionTemplate to find a template for each subscription
+// Filters out Authenticated subscriptions if !e.CanUseAuthenticatedEndpoints
+// See README.md for more details
+func (l List) ExpandTemplates(e iExchange) (List, error) {
+	if !slices.ContainsFunc(l, func(s *Subscription) bool { return s.QualifiedChannel == "" }) {
+		// Empty list, or already processed
+		return slices.Clone(l), nil
+	}
+
+	if !e.CanUseAuthenticatedWebsocketEndpoints() {
+		n := List{}
+		for _, s := range l {
+			if !s.Authenticated {
+				n = append(n, s)
+			}
+		}
+		l = n
+	}
+
+	ap, err := l.assetPairs(e)
+	if err != nil {
+		return nil, err
+	}
+
+	assets := make(asset.Items, 0, len(ap))
+	for k := range ap {
+		assets = append(assets, k)
+	}
+	slices.Sort(assets) // text/template ranges maps in sorted order
+	subs := List{}
+
+	for _, s := range l {
+		if s.QualifiedChannel != "" {
+			subs = append(subs, s)
+			continue
+		}
+
+		subCtx := &tplCtx{
+			S:              s,
+			AssetPairs:     ap,
+			PairSeparator:  recordSeparator,
+			AssetSeparator: groupSeparator,
+		}
+
+		t, err := e.GetSubscriptionTemplate(s)
+		if err != nil {
+			return nil, err
+		}
+		if t == nil {
+			return nil, errInvalidTemplate
+		}
+
+		buf := &bytes.Buffer{}
+		if err := t.Execute(buf, subCtx); err != nil {
+			return nil, err
+		}
+
+		out := buf.String()
+
+		subAssets := assets
+		xpandPairs := strings.Contains(out, subCtx.PairSeparator)
+		if xpandAssets := strings.Contains(out, subCtx.AssetSeparator); xpandAssets {
+			if s.Asset != asset.All {
+				return nil, errAssetTemplateWithoutAll
+			}
+		} else {
+			if xpandPairs && (s.Asset == asset.All || s.Asset == asset.Empty) {
+				// We don't currently support expanding Pairs without expanding Assets for All or Empty assets, but we could; waiting for a use-case
+				return nil, errInvalidAssetExpandPairs
+			}
+			// No expansion so update expected Assets for consistent behaviour below
+			subAssets = []asset.Item{s.Asset}
+		}
+
+		out = strings.TrimRight(out, " \n\r\t"+subCtx.PairSeparator+subCtx.AssetSeparator)
+
+		assetRecords := strings.Split(out, subCtx.AssetSeparator)
+		if len(assetRecords) != len(subAssets) {
+			return nil, fmt.Errorf("%w: Got %d; Expected %d", errAssetRecords, len(assetRecords), len(subAssets))
+		}
+
+		for i, assetChannels := range assetRecords {
+			a := subAssets[i]
+			assetChannels = strings.TrimRight(assetChannels, " \n\r\t"+recordSeparator)
+			pairLines := strings.Split(assetChannels, subCtx.PairSeparator)
+			pairs, ok := ap[a]
+			if xpandPairs {
+				if !ok {
+					return nil, fmt.Errorf("%w: %s", asset.ErrInvalidAsset, a)
+				}
+				if len(pairLines) != len(pairs) {
+					return nil, fmt.Errorf("%w: Got %d; Expected %d", errPairRecords, len(pairLines), len(pairs))
+				}
+			}
+			for j, channel := range pairLines {
+				c := s.Clone()
+				c.Asset = a
+				channel = strings.TrimSpace(channel)
+				if channel == "" {
+					return nil, fmt.Errorf("%w: %s", errNoTemplateContent, s)
+				}
+				c.QualifiedChannel = strings.TrimSpace(channel)
+				if xpandPairs {
+					c.Pairs = currency.Pairs{pairs[j]}
+				} else {
+					c.Pairs = pairs
+				}
+				subs = append(subs, c)
+			}
+		}
+	}
+
+	return subs, nil
+}

exchanges/subscription/template_test.go

@@ -0,0 +1,99 @@
+package subscription
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/thrasher-corp/gocryptotrader/currency"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/kline"
+)
+
+// TestExpandTemplates exercises ExpandTemplates
+func TestExpandTemplates(t *testing.T) {
+	t.Parallel()
+
+	e := &mockEx{
+		tpl: "subscriptions.tmpl",
+	}
+
+	// Functionality tests
+	l := List{
+		{Channel: "feature1"},
+		{Channel: "feature2", Asset: asset.All, Pairs: currency.Pairs{btcusdtPair, ethusdcPair}, Interval: kline.FifteenMin},
+		{Channel: "feature3", Asset: asset.All, Pairs: currency.Pairs{btcusdtPair, ethusdcPair}, Levels: 100},
+		{Channel: "feature4", Authenticated: true},
+		{Channel: "feature1", QualifiedChannel: "just one sub already processed"},
+	}
+	got, err := l.ExpandTemplates(e)
+	require.NoError(t, err, "ExpandTemplates must not error")
+	exp := List{
+		{Channel: "feature1", QualifiedChannel: "feature1"},
+		{Channel: "feature2", QualifiedChannel: "spot-feature2@15m", Asset: asset.Spot, Pairs: currency.Pairs{btcusdtPair, ethusdcPair}, Interval: kline.FifteenMin},
+		{Channel: "feature2", QualifiedChannel: "future-feature2@15m", Asset: asset.Futures, Pairs: currency.Pairs{btcusdtPair, ethusdcPair}, Interval: kline.FifteenMin},
+		{Channel: "feature3", QualifiedChannel: "spot-USDTBTC-feature3@100", Asset: asset.Spot, Pairs: currency.Pairs{btcusdtPair}, Levels: 100},
+		{Channel: "feature3", QualifiedChannel: "spot-USDCETH-feature3@100", Asset: asset.Spot, Pairs: currency.Pairs{ethusdcPair}, Levels: 100},
+		{Channel: "feature3", QualifiedChannel: "future-USDTBTC-feature3@100", Asset: asset.Futures, Pairs: currency.Pairs{btcusdtPair}, Levels: 100},
+		{Channel: "feature3", QualifiedChannel: "future-USDCETH-feature3@100", Asset: asset.Futures, Pairs: currency.Pairs{ethusdcPair}, Levels: 100},
+		{Channel: "feature1", QualifiedChannel: "just one sub already processed"},
+	}
+
+	if !equalLists(t, exp, got) {
+		t.FailNow() // If the first list isn't equal testing it again will duplicate test failures
+	}
+
+	e.auth = true
+	got, err = l.ExpandTemplates(e)
+	require.NoError(t, err, "ExpandTemplates must not error")
+	exp = append(exp,
+		&Subscription{Channel: "feature4", QualifiedChannel: "feature4-authed"},
+	)
+	equalLists(t, exp, got)
+
+	_, err = List{{Channel: "feature2", Asset: asset.Spot}}.ExpandTemplates(e)
+	assert.ErrorIs(t, err, errAssetTemplateWithoutAll, "Should error correctly on xpand assets without All")
+
+	e.tpl = "errors.tmpl"
+	_, err = List{{Channel: "error1", Asset: asset.All}}.ExpandTemplates(e)
+	assert.ErrorIs(t, err, errInvalidAssetExpandPairs, "Should error correctly on xpand pairs but not assets")
+
+	_, err = List{{Channel: "error1"}}.ExpandTemplates(e)
+	assert.ErrorIs(t, err, errInvalidAssetExpandPairs, "Should error correctly on xpand pairs but not assets")
+
+	_, err = List{{Channel: "error2"}}.ExpandTemplates(e)
+	assert.ErrorContains(t, err, "wrong number of args for String", "Should error correctly with execution error")
+
+	_, err = List{{Channel: "non-existent"}}.ExpandTemplates(e)
+	assert.ErrorIs(t, err, errNoTemplateContent, "Should error correctly when no content generated")
+	assert.ErrorContains(t, err, "non-existent", "Should error correctly when no content generated")
+
+	_, err = List{{Channel: "error3", Asset: asset.All}}.ExpandTemplates(e)
+	assert.ErrorIs(t, err, errAssetRecords, "Should error correctly when invalid number of asset entries")
+
+	_, err = List{{Channel: "error4", Asset: asset.Spot}}.ExpandTemplates(e)
+	assert.ErrorIs(t, err, errPairRecords, "Should error correctly when invalid number of pair entries")
+
+	_, err = List{{Channel: "error4", Asset: asset.Margin}}.ExpandTemplates(e)
+	assert.ErrorIs(t, err, asset.ErrInvalidAsset, "Should error correctly when invalid asset")
+
+	e.tpl = "parse-error.tmpl"
+	_, err = l.ExpandTemplates(e)
+	assert.ErrorContains(t, err, "function \"explode\" not defined", "Should error correctly on unparsable template")
+
+	e.errFormat = errors.New("the planet Krypton is gone")
+	_, err = l.ExpandTemplates(e)
+	assert.ErrorIs(t, err, e.errFormat, "Should error correctly on GetPairFormat")
+
+	e.errPairs = errors.New("bad parenting from Jor-El")
+	_, err = l.ExpandTemplates(e)
+	assert.ErrorIs(t, err, e.errPairs, "Should error correctly on GetEnabledPairs")
+
+	l = List{{Channel: "feature1", QualifiedChannel: "already happy"}}
+	got, err = l.ExpandTemplates(e)
+	require.NoError(t, err)
+	require.Len(t, got, 1, "Must get back the one sub")
+	assert.Equal(t, "already happy", l[0].QualifiedChannel, "Should get back the one sub")
+	assert.NotSame(t, got, l, "Should get back a different actual list")
+}

exchanges/subscription/testdata/errors.tmpl

@@ -0,0 +1,13 @@
+{{- if eq .S.Channel "error1" }}
+  {{/* Error 1: Expand pairs but not assets, without specific asset */}}
+  {{- .PairSeparator -}}
+{{- else if eq .S.Channel "error2" }}
+  {{/* Error 2: Runtime error from executing */}}
+  {{ .S.String 42 }}
+{{- else if eq .S.Channel "error3" }}
+  {{/* Error 3: Incorrect number of asset entries */}}
+  {{- .AssetSeparator }}
+{{- else if eq .S.Channel "error4" }}
+  {{/* Error 3: Incorrect number of pair entries */}}
+  {{- .PairSeparator }}
+{{- end -}}

exchanges/subscription/testdata/parse-error.tmpl

@@ -0,0 +1 @@
+{{ explode }}

exchanges/subscription/testdata/subscriptions.tmpl

@@ -0,0 +1,21 @@
+{{- if eq $.S.Channel "feature1" -}}
+  {{/* Case 1: One channel to rule them all */}}
+  feature1
+{{- else if eq $.S.Channel "feature2" -}}
+  {{/* Case 2: One channel per asset */}}
+  {{- range $asset, $pairs := $.AssetPairs }}
+    {{ assetName $asset }}-feature2@ {{- $.S.Interval.Short }}
+    {{- $.AssetSeparator }}
+  {{- end }}
+{{- else if eq $.S.Channel "feature3" }}
+  {{/* Case 3: One channel per pair per asset */}}
+  {{- range $asset, $pairs := $.AssetPairs }}
+    {{- range $pair := $pairs -}}
+      {{ assetName $asset }}-{{ $pair.Swap.String -}} -feature3@ {{- $.S.Levels }}
+      {{- $.PairSeparator -}}
+    {{- end -}}
+    {{- $.AssetSeparator -}}
+  {{- end -}}
+{{- else if eq $.S.Channel "feature4" }}
+  feature4-authed
+{{- end -}}