subscriptions: Add templating support and integrate with Binance (#1568)

* Subscriptions: Add List.AssetPairs

* Subscriptions: Add Template and QualifiedChannel

These fields separate the concept of what the channel is from the
qualified resource name

* Subscriptions: Add List.SetStates()

* Subscriptions: Add List.QualifiedChannels

* Subscriptions: Rename testsubs.EqualLists

* Binance: Switch to ExpandTemplates

* Binance: Update ConfigTest format

* Subscriptions: Test Coverage improvements

* Subscriptions: Reenterant List.ExpandTemplates

* Subscriptions: Move templates from subscriptions to exchanges

* Binance: Inline subscription template and improvements

exchanges/subscription/README.md

@@ -0,0 +1,72 @@
+# GoCryptoTrader package Subscription
+
+<img src="/common/gctlogo.png?raw=true" width="350px" height="350px" hspace="70">
+
+
+[![Build Status](https://github.com/thrasher-corp/gocryptotrader/actions/workflows/tests.yml/badge.svg?branch=master)](https://github.com/thrasher-corp/gocryptotrader/actions/workflows/tests.yml)
+[![Software License](https://img.shields.io/badge/License-MIT-orange.svg?style=flat-square)](https://github.com/thrasher-corp/gocryptotrader/blob/master/LICENSE)
+[![GoDoc](https://godoc.org/github.com/thrasher-corp/gocryptotrader?status.svg)](https://godoc.org/github.com/thrasher-corp/gocryptotrader/exchanges/subscription)
+[![Coverage Status](http://codecov.io/github/thrasher-corp/gocryptotrader/coverage.svg?branch=master)](http://codecov.io/github/thrasher-corp/gocryptotrader?branch=master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/thrasher-corp/gocryptotrader)](https://goreportcard.com/report/github.com/thrasher-corp/gocryptotrader)
+
+
+This subscription package is part of the GoCryptoTrader codebase.
+
+## This is still in active development
+
+You can track ideas, planned features and what's in progress on this Trello board: [https://trello.com/b/ZAhMhpOy/gocryptotrader](https://trello.com/b/ZAhMhpOy/gocryptotrader).
+
+Join our slack to discuss all things related to GoCryptoTrader! [GoCryptoTrader Slack](https://join.slack.com/t/gocryptotrader/shared_invite/enQtNTQ5NDAxMjA2Mjc5LTc5ZDE1ZTNiOGM3ZGMyMmY1NTAxYWZhODE0MWM5N2JlZDk1NDU0YTViYzk4NTk3OTRiMDQzNGQ1YTc4YmRlMTk)
+
+# Exchange Subscriptions
+
+Exchange Subscriptions are streams of data delivered via websocket.
+
+GoCryptoTrader engine will subscribe automatically to configured channels.
+A subset of exchanges currently support user configured channels, with the remaining using hardcoded defaults.
+See configuration Features.Subscriptions for whether an exchange is configurable.
+
+## Templating
+
+Exchange Contributors should implement `GetSubscriptionTemplate` to return a text/template Template.
+
+Exchanges are free to implement template caching, a map or a mono-template, inline or file templates.
+
+The template is provided with a single context structure:
+```go
+  S              *subscription.Subscription
+  AssetPairs     map[asset.Item]currency.Pairs
+  AssetSeparator string
+  PairSeparator  string
+```
+
+Subscriptions may fan out many channels for assets and pairs, to support exchanges which require individual subscriptions.  
+To allow the template to communicate how to handle its output it should use the provided separators:
+- AssetSeparator should be added at the end of each section related to assets
+- PairSeparator should be added at the end of each pair
+
+We use separators like this because it allows mono-templates to decide at runtime whether to fan out.
+
+See exchanges/subscription/testdata/subscriptions.tmpl for an example mono-template showcasing various features
+
+Templates do not need to worry about joining around separators; Trailing separators will be stripped automatically.
+
+
+## Contribution
+
+Please feel free to submit any pull requests or suggest any desired features to be added.
+
+When submitting a PR, please abide by our coding guidelines:
+
++ Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting) guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)).
++ Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary) guidelines.
++ Code must adhere to our [coding style](https://github.com/thrasher-corp/gocryptotrader/blob/master/doc/coding_style.md).
++ Pull requests need to be based on and opened against the `master` branch.
+
+## Donations
+
+<img src="https://github.com/thrasher-corp/gocryptotrader/blob/master/web/src/assets/donate.png?raw=true" hspace="70">
+
+If this framework helped you in any way, or you would like to support the developers working on it, please donate Bitcoin to:
+
+***bc1qk0jareu4jytc0cfrhr5wgshsq8282awpavfahc***

exchanges/subscription/fixtures_test.go

@@ -0,0 +1,96 @@
+package subscription
+
+import (
+	"maps"
+	"strings"
+	"testing"
+	"text/template"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/thrasher-corp/gocryptotrader/currency"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+)
+
+type mockEx struct {
+	tpl       string
+	auth      bool
+	errPairs  error
+	errFormat error
+}
+
+func (m *mockEx) GetEnabledPairs(_ asset.Item) (currency.Pairs, error) {
+	return currency.Pairs{btcusdtPair, ethusdcPair}, m.errPairs
+}
+
+func (m *mockEx) GetPairFormat(_ asset.Item, _ bool) (currency.PairFormat, error) {
+	return currency.PairFormat{Uppercase: true}, m.errFormat
+}
+
+func (m *mockEx) GetSubscriptionTemplate(_ *Subscription) (*template.Template, error) {
+	return template.New(m.tpl).
+		Funcs(template.FuncMap{
+			"assetName": func(a asset.Item) string {
+				if a == asset.Futures {
+					return "future"
+				}
+				return a.String()
+			}}).
+		ParseFiles("testdata/" + m.tpl)
+}
+
+func (m *mockEx) GetAssetTypes(_ bool) asset.Items            { return asset.Items{asset.Spot, asset.Futures} }
+func (m *mockEx) CanUseAuthenticatedWebsocketEndpoints() bool { return m.auth }
+
+// equalLists is a utility function to compare subscription lists and show a pretty failure message
+// It overcomes the verbose depth of assert.ElementsMatch spewConfig
+// Duplicate of internal/testing/subscriptions/EqualLists
+func equalLists(tb testing.TB, a, b List) bool {
+	tb.Helper()
+	for _, sub := range append(a, b...) {
+		sub.Key = &StrictKey{&ExactKey{sub}}
+	}
+	s, err := NewStoreFromList(a)
+	require.NoError(tb, err, "NewStoreFromList must not error")
+	added, missing := s.Diff(b)
+	if len(added) > 0 || len(missing) > 0 {
+		fail := "Differences:"
+		if len(added) > 0 {
+			fail = fail + "\n + " + strings.Join(added.Strings(), "\n + ")
+		}
+		if len(missing) > 0 {
+			fail = fail + "\n - " + strings.Join(missing.Strings(), "\n - ")
+		}
+		assert.Fail(tb, fail, "Subscriptions should be equal")
+		return false
+	}
+	return true
+}
+
+// StrictKey is key type for subscriptions where all the pairs, QualifiedChannel and Params in a Subscription must match exactly
+type StrictKey struct {
+	*ExactKey
+}
+
+var _ MatchableKey = StrictKey{} // Enforce StrictKey must implement MatchableKey
+
+// Match implements MatchableKey
+// Returns true if the key fields exactly matches the subscription, including all Pairs, QualifiedChannel and Params
+func (k StrictKey) Match(eachKey MatchableKey) bool {
+	if !k.ExactKey.Match(eachKey) {
+		return false
+	}
+	eachSub := eachKey.GetSubscription()
+	return eachSub.QualifiedChannel == k.QualifiedChannel &&
+		maps.Equal(eachSub.Params, k.Params)
+}
+
+// String implements Stringer; returns the Asset, Channel and Pairs
+// Does not provide concurrency protection on the subscription it points to
+func (k StrictKey) String() string {
+	s := k.Subscription
+	if s == nil {
+		return "Uninitialised StrictKey"
+	}
+	return s.QualifiedChannel + " " + ExactKey{s}.String()
+}

exchanges/subscription/keys.go

@@ -39,6 +39,7 @@ func (k ExactKey) String() string {
 
 // Match implements MatchableKey
 // Returns true if the key fields exactly matches the subscription, including all Pairs
+// Does not check QualifiedChannel or Params
 func (k ExactKey) Match(eachKey MatchableKey) bool {
 	if eachKey == nil {
 		return false

exchanges/subscription/keys_test.go

@@ -105,6 +105,15 @@ func TestIgnoringPairsKeyMatch(t *testing.T) {
 // TestIgnoringPairsKeyString exercises IgnoringPairsKey.String
 func TestIgnoringPairsKeyString(t *testing.T) {
 	t.Parallel()
+	assert.Equal(t, "Uninitialised IgnoringPairsKey", IgnoringPairsKey{}.String())
 	key := &IgnoringPairsKey{&Subscription{Asset: asset.Spot, Channel: TickerChannel, Pairs: currency.Pairs{ethusdcPair, btcusdtPair}}}
 	assert.Equal(t, "ticker spot", key.String())
 }
+
+// TestGetSubscription exercises GetSubscription
+func TestGetSubscription(t *testing.T) {
+	t.Parallel()
+	s := &Subscription{Asset: asset.Spot}
+	assert.Same(t, s, ExactKey{s}.GetSubscription(), "ExactKey.GetSubscription Must return a pointer to the subscription")
+	assert.Same(t, s, IgnoringPairsKey{s}.GetSubscription(), "IgnorePairKeys.GetSubscription Must return a pointer to the subscription")
+}

exchanges/subscription/list.go

@@ -2,11 +2,26 @@ package subscription
 
 import (
 	"slices"
+	"text/template"
+
+	"github.com/thrasher-corp/gocryptotrader/common"
+	"github.com/thrasher-corp/gocryptotrader/currency"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
 )
 
 // List is a container of subscription pointers
 type List []*Subscription
 
+type assetPairs map[asset.Item]currency.Pairs
+
+type iExchange interface {
+	GetAssetTypes(enabled bool) asset.Items
+	GetEnabledPairs(asset.Item) (currency.Pairs, error)
+	GetPairFormat(asset.Item, bool) (currency.PairFormat, error)
+	GetSubscriptionTemplate(*Subscription) (*template.Template, error)
+	CanUseAuthenticatedWebsocketEndpoints() bool
+}
+
 // Strings returns a sorted slice of subscriptions
 func (l List) Strings() []string {
 	s := make([]string, len(l))
@@ -30,3 +45,62 @@ func (l List) GroupPairs() (n List) {
 	}
 	return s.List()
 }
+
+// QualifiedChannels returns a sorted list of all the qualified Channels in the list
+func (l List) QualifiedChannels() []string {
+	c := make([]string, len(l))
+	for i := range l {
+		c[i] = l[i].QualifiedChannel
+	}
+	slices.Sort(c)
+	return c
+}
+
+// SetStates sets the state for all the subs in a list
+// Errors are collected for any subscriptions already in the state
+// On error all changes are reverted
+func (l List) SetStates(state State) error {
+	var err error
+	for _, sub := range l {
+		err = common.AppendError(err, sub.SetState(state))
+	}
+	return err
+}
+
+func fillAssetPairs(ap assetPairs, a asset.Item, e iExchange) error {
+	p, err := e.GetEnabledPairs(a)
+	if err != nil {
+		return err
+	}
+	f, err := e.GetPairFormat(a, true)
+	if err != nil {
+		return err
+	}
+	ap[a] = p.Format(f)
+	return nil
+}
+
+// assetPairs returns a map of enabled pairs for the subscriptions in the list, formatted for the asset
+func (l List) assetPairs(e iExchange) (assetPairs, error) {
+	at := e.GetAssetTypes(true)
+	ap := assetPairs{}
+	for _, s := range l {
+		switch s.Asset {
+		case asset.Empty:
+			// Nothing to do
+		case asset.All:
+			for _, a := range at {
+				if err := fillAssetPairs(ap, a, e); err != nil {
+					return nil, err
+				}
+			}
+		default:
+			if slices.Contains(at, s.Asset) {
+				if err := fillAssetPairs(ap, s.Asset, e); err != nil {
+					return nil, err
+				}
+			}
+		}
+	}
+	return ap, nil
+}

exchanges/subscription/list_test.go

@@ -1,15 +1,18 @@
 package subscription
 
 import (
+	"errors"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
 )
 
 // TestListStrings exercises List.Strings()
 func TestListStrings(t *testing.T) {
+	t.Parallel()
 	l := List{
 		&Subscription{
 			Channel: TickerChannel,
@@ -25,8 +28,24 @@ func TestListStrings(t *testing.T) {
 	assert.ElementsMatch(t, exp, l.Strings(), "String must return correct sorted list")
 }
 
+// TestQualifiedChannels exercises List.QualifiedChannels()
+func TestQualifiedChannels(t *testing.T) {
+	t.Parallel()
+	l := List{
+		&Subscription{
+			QualifiedChannel: "ticker-btc",
+		},
+		&Subscription{
+			QualifiedChannel: "candles-btc",
+		},
+	}
+	exp := []string{"ticker-btc", "candles-btc"}
+	assert.ElementsMatch(t, exp, l.QualifiedChannels(), "QualifiedChannels should return correct sorted list")
+}
+
 // TestListGroupPairs exercises List.GroupPairs()
 func TestListGroupPairs(t *testing.T) {
+	t.Parallel()
 	l := List{
 		{Asset: asset.Spot, Channel: TickerChannel, Pairs: currency.Pairs{ethusdcPair, btcusdtPair}},
 	}
@@ -45,3 +64,30 @@ func TestListGroupPairs(t *testing.T) {
 	exp := []string{"ticker spot ETH/USDC,BTC/USDT", "orderbook spot ETH/USDC,BTC/USDT"}
 	assert.ElementsMatch(t, exp, n.Strings(), "String must return correct sorted list")
 }
+
+// TestListSetStates exercises List.SetState()
+func TestListSetStates(t *testing.T) {
+	t.Parallel()
+	l := List{{Channel: TickerChannel}, {Channel: OrderbookChannel}}
+	assert.NoError(t, l.SetStates(SubscribingState), "SetStates should not error")
+	assert.Equal(t, SubscribingState, l[1].State(), "SetStates should set State correctly")
+
+	require.NoError(t, l[0].SetState(SubscribedState), "Individual SetState must not error")
+	err := l.SetStates(SubscribedState)
+	assert.ErrorIs(t, ErrInStateAlready, err, "SetStates should error when duplicate state")
+	assert.Equal(t, SubscribedState, l[1].State(), "SetStates should set State correctly after the error")
+}
+
+// TestAssetPairs exercises AssetPairs error handling
+// All other code is covered under TestExpandTemplates
+func TestAssetPairs(t *testing.T) {
+	t.Parallel()
+	expErr := errors.New("Krypton is gone")
+	for _, a := range []asset.Item{asset.Spot, asset.All} {
+		l := &List{{Channel: CandlesChannel, Asset: a}}
+		_, err := l.assetPairs(&mockEx{errPairs: expErr})
+		assert.ErrorIs(t, err, expErr, "Should error correctly on GetEnabledPairs")
+		_, err = l.assetPairs(&mockEx{errFormat: expErr})
+		assert.ErrorIs(t, err, expErr, "Should error correctly on GetPairFormat")
+	}
+}

exchanges/subscription/store.go

@@ -43,7 +43,7 @@ func (s *Store) Add(sub *Subscription) error {
 		return fmt.Errorf("%w: Add called on nil Store", common.ErrNilPointer)
 	}
 	if s.m == nil {
-		return fmt.Errorf("%w: Add called on an Uninitialised Store", common.ErrNilPointer)
+		return fmt.Errorf("%w: Add called on an uninitialised Store", common.ErrNilPointer)
 	}
 	if sub == nil {
 		return fmt.Errorf("%w: Subscription param", common.ErrNilPointer)

exchanges/subscription/store_test.go

@@ -13,6 +13,7 @@ import (
 
 // TestNewStore exercises NewStore
 func TestNewStore(t *testing.T) {
+	t.Parallel()
 	s := NewStore()
 	require.IsType(t, &Store{}, s, "Must return a store ref")
 	require.NotNil(t, s.m, "storage map must be initialised")
@@ -20,6 +21,7 @@ func TestNewStore(t *testing.T) {
 
 // TestNewStoreFromList exercises NewStoreFromList
 func TestNewStoreFromList(t *testing.T) {
+	t.Parallel()
 	s, err := NewStoreFromList(List{})
 	assert.NoError(t, err, "Should not error on empty list")
 	require.IsType(t, &Store{}, s, "Must return a store ref")
@@ -43,11 +45,20 @@ func TestNewStoreFromList(t *testing.T) {
 
 // TestAdd exercises Add and add methods
 func TestAdd(t *testing.T) {
-	assert.ErrorIs(t, (*Store)(nil).Add(&Subscription{}), common.ErrNilPointer, "Should error nil pointer correctly")
-	assert.ErrorIs(t, (&Store{}).Add(nil), common.ErrNilPointer, "Should error nil pointer correctly")
-	assert.ErrorIs(t, (&Store{}).Add(&Subscription{}), common.ErrNilPointer, "Should error nil pointer correctly")
+	t.Parallel()
+	err := (*Store)(nil).Add(&Subscription{})
+	assert.ErrorIs(t, err, common.ErrNilPointer, "Should error nil pointer correctly")
+	assert.ErrorContains(t, err, "called on nil Store", "Should error correctly")
+
+	err = new(Store).Add(nil)
+	assert.ErrorIs(t, err, common.ErrNilPointer, "Should error nil pointer correctly")
+	assert.ErrorContains(t, err, "called on an uninitialised Store", "Should error correctly")
 
 	s := NewStore()
+	err = s.Add(nil)
+	assert.ErrorIs(t, err, common.ErrNilPointer, "Should error nil pointer correctly")
+	assert.ErrorContains(t, err, "Subscription param", "Should error correctly")
+
 	sub := &Subscription{Channel: TickerChannel}
 	require.NoError(t, s.Add(sub), "Should not error on a standard add")
 	assert.NotNil(t, s.get(sub), "Should have stored the sub")
@@ -58,6 +69,7 @@ func TestAdd(t *testing.T) {
 // TestGet exercises Get and get methods
 // Ensures that key's Match is used, but does not exercise subscription.Match; See TestMatch for that coverage
 func TestGet(t *testing.T) {
+	t.Parallel()
 	assert.Nil(t, (*Store)(nil).Get(&Subscription{}), "Should return nil when called on nil")
 	assert.Nil(t, (&Store{}).Get(&Subscription{}), "Should return nil when called with no subscription map")
 	s := NewStore()
@@ -81,11 +93,20 @@ func TestGet(t *testing.T) {
 
 // TestRemove exercises the Remove method
 func TestRemove(t *testing.T) {
-	assert.ErrorIs(t, (*Store)(nil).Remove(&Subscription{}), common.ErrNilPointer, "Should error correctly when called on nil")
-	assert.ErrorIs(t, (&Store{}).Remove(nil), common.ErrNilPointer, "Should error correctly when called passing nil")
-	assert.ErrorIs(t, (&Store{}).Remove(&Subscription{}), common.ErrNilPointer, "Should error correctly when called with no subscription map")
+	t.Parallel()
+	err := (*Store)(nil).Remove(&Subscription{})
+	assert.ErrorIs(t, err, common.ErrNilPointer, "Should error correctly when called on nil")
+	assert.ErrorContains(t, err, "Remove called on nil Store", "Should error correctly when called on nil")
+
+	err = new(Store).Remove(nil)
+	assert.ErrorIs(t, err, common.ErrNilPointer, "Should error correctly when called on an uninit store")
+	assert.ErrorContains(t, err, "Remove called on an Uninitialised Store", "Should error correctly when called on an uninit store")
 
 	s := NewStore()
+	err = s.Remove(nil)
+	assert.ErrorIs(t, err, common.ErrNilPointer, "Should error correctly when called with nil")
+	assert.ErrorContains(t, err, "key param", "Should error correctly when called with nil")
+
 	require.NoError(t, s.Add(&Subscription{Channel: CandlesChannel, Pairs: currency.Pairs{btcusdtPair, ethusdcPair}}), "Adding subscription must not error")
 	assert.NotNil(t, s.Get(&ExactKey{&Subscription{Channel: CandlesChannel, Pairs: currency.Pairs{btcusdtPair, ethusdcPair}}}), "Should have added the sub")
 	assert.ErrorIs(t, s.Remove(&ExactKey{&Subscription{Channel: CandlesChannel, Pairs: currency.Pairs{btcusdtPair}}}), ErrNotFound, "Should error correctly when called with a non-matching key")
@@ -96,6 +117,7 @@ func TestRemove(t *testing.T) {
 
 // TestList exercises the List and Len methods
 func TestList(t *testing.T) {
+	t.Parallel()
 	assert.Empty(t, (*Store)(nil).List(), "Should return an empty List when called on nil")
 	assert.Empty(t, (&Store{}).List(), "Should return an empty List when called on Store without map")
 	s := NewStore()
@@ -118,6 +140,7 @@ func TestList(t *testing.T) {
 
 // TestStoreClear exercises the Clear method
 func TestStoreClear(t *testing.T) {
+	t.Parallel()
 	assert.NotPanics(t, func() { (*Store)(nil).Clear() }, "Should not panic when called on nil")
 	s := &Store{}
 	assert.NotPanics(t, func() { s.Clear() }, "Should not panic when called with no subscription map")
@@ -131,6 +154,7 @@ func TestStoreClear(t *testing.T) {
 
 // TestStoreDiff exercises the Diff method
 func TestStoreDiff(t *testing.T) {
+	t.Parallel()
 	s := NewStore()
 	assert.NotPanics(t, func() { (*Store)(nil).Diff(List{}) }, "Should not panic when called on nil")
 	assert.NotPanics(t, func() { (&Store{}).Diff(List{}) }, "Should not panic when called with no subscription map")

exchanges/subscription/subscription.go

@@ -47,17 +47,18 @@ type State uint8
 
 // Subscription container for streaming subscriptions
 type Subscription struct {
-	Enabled       bool           `json:"enabled"`
-	Key           any            `json:"-"`
-	Channel       string         `json:"channel,omitempty"`
-	Pairs         currency.Pairs `json:"pairs,omitempty"`
-	Asset         asset.Item     `json:"asset,omitempty"`
-	Params        map[string]any `json:"params,omitempty"`
-	Interval      kline.Interval `json:"interval,omitempty"`
-	Levels        int            `json:"levels,omitempty"`
-	Authenticated bool           `json:"authenticated,omitempty"`
-	state         State
-	m             sync.RWMutex
+	Enabled          bool           `json:"enabled"`
+	Key              any            `json:"-"`
+	Channel          string         `json:"channel,omitempty"`
+	Pairs            currency.Pairs `json:"pairs,omitempty"`
+	Asset            asset.Item     `json:"asset,omitempty"`
+	Params           map[string]any `json:"params,omitempty"`
+	Interval         kline.Interval `json:"interval,omitempty"`
+	Levels           int            `json:"levels,omitempty"`
+	Authenticated    bool           `json:"authenticated,omitempty"`
+	QualifiedChannel string         `json:"-"`
+	state            State
+	m                sync.RWMutex
 }
 
 // String implements Stringer, and aims to informatively and uniquely identify a subscription for errors and information
@@ -122,20 +123,22 @@ func (s *Subscription) EnsureKeyed() any {
 
 // Clone returns a copy of a subscription
 // Key is set to nil, because most Key types contain a pointer to the subscription, and because the clone isn't added to the store yet
+// QualifiedChannel is not copied because it's expected that the contributing fields will be changed
 // Users should allow a default key to be assigned on AddSubscription or can SetKey as necessary
 func (s *Subscription) Clone() *Subscription {
 	s.m.RLock()
 	c := &Subscription{
-		Key:           nil,
-		Enabled:       s.Enabled,
-		Channel:       s.Channel,
-		Asset:         s.Asset,
-		Params:        s.Params,
-		Interval:      s.Interval,
-		Levels:        s.Levels,
-		Authenticated: s.Authenticated,
-		state:         s.state,
-		Pairs:         s.Pairs,
+		Key:              nil,
+		Enabled:          s.Enabled,
+		Channel:          s.Channel,
+		Asset:            s.Asset,
+		Params:           s.Params,
+		Interval:         s.Interval,
+		Levels:           s.Levels,
+		Authenticated:    s.Authenticated,
+		state:            s.state,
+		Pairs:            s.Pairs,
+		QualifiedChannel: s.QualifiedChannel,
 	}
 	s.Pairs = slices.Clone(s.Pairs)
 	s.Params = maps.Clone(s.Params)

exchanges/subscription/subscription_test.go

@@ -18,12 +18,15 @@ var (
 
 // TestSubscriptionString exercises the String method
 func TestSubscriptionString(t *testing.T) {
+	t.Parallel()
 	s := &Subscription{
 		Channel: "candles",
 		Asset:   asset.Spot,
 		Pairs:   currency.Pairs{btcusdtPair, ethusdcPair.Format(currency.PairFormat{Delimiter: "/"})},
 	}
 	assert.Equal(t, "candles spot BTC/USDT,ETH/USDC", s.String(), "Subscription String should return correct value")
+	s.Key = 42
+	assert.Equal(t, "42: candles spot BTC/USDT,ETH/USDC", s.String(), "String with a non-MatchableKey")
 }
 
 // TestState exercises the state getter
@@ -48,19 +51,6 @@ func TestSetState(t *testing.T) {
 	assert.ErrorIs(t, s.SetState(UnsubscribedState+1), ErrInvalidState, "Setting an invalid state should error")
 }
 
-// TestString exercises the Stringer implementation
-func TestString(t *testing.T) {
-	s := &Subscription{
-		Channel: "candles",
-		Asset:   asset.Spot,
-		Pairs:   currency.Pairs{btcusdtPair},
-	}
-	_ = s.EnsureKeyed()
-	assert.Equal(t, "candles spot BTC/USDT", s.String(), "String with a MatchableKey")
-	s.Key = 42
-	assert.Equal(t, "42: candles spot BTC/USDT", s.String(), "String with a MatchableKey")
-}
-
 // TestEnsureKeyed exercises the key getter and ensures it sets a self-pointer key for non
 func TestEnsureKeyed(t *testing.T) {
 	t.Parallel()
@@ -102,6 +92,7 @@ func TestSubscriptionMarshaling(t *testing.T) {
 
 // TestClone exercises Clone
 func TestClone(t *testing.T) {
+	t.Parallel()
 	a := &Subscription{
 		Channel:  TickerChannel,
 		Interval: kline.OneHour,
@@ -123,6 +114,7 @@ func TestClone(t *testing.T) {
 
 // TestSetKey exercises SetKey
 func TestSetKey(t *testing.T) {
+	t.Parallel()
 	s := &Subscription{}
 	s.SetKey(14)
 	assert.Equal(t, 14, s.Key, "SetKey should set a key correctly")
@@ -130,7 +122,18 @@ func TestSetKey(t *testing.T) {
 
 // TestSetPairs exercises SetPairs
 func TestSetPairs(t *testing.T) {
+	t.Parallel()
 	s := &Subscription{}
 	s.SetPairs(currency.Pairs{btcusdtPair})
 	assert.Equal(t, "BTCUSDT", s.Pairs.Join(), "SetPairs should set a key correctly")
 }
+
+// TestAddPairs exercises AddPairs
+func TestAddPairs(t *testing.T) {
+	t.Parallel()
+	s := &Subscription{}
+	s.AddPairs()
+	assert.Empty(t, s.Pairs, "Should not have added any pairs")
+	s.AddPairs(btcusdtPair)
+	assert.Len(t, s.Pairs, 1, "Should not have added any pairs")
+}

exchanges/subscription/template.go

@@ -0,0 +1,150 @@
+package subscription
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/thrasher-corp/gocryptotrader/currency"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+)
+
+const (
+	groupSeparator  = "\x1D"
+	recordSeparator = "\x1E"
+)
+
+var (
+	errInvalidAssetExpandPairs = errors.New("subscription template containing PairSeparator with must contain either specific Asset or AssetSeparator")
+	errAssetRecords            = errors.New("subscription template did not generate the expected number of asset records")
+	errPairRecords             = errors.New("subscription template did not generate the expected number of pair records")
+	errAssetTemplateWithoutAll = errors.New("sub.Asset must be set to All if AssetSeparator is used in Channel template")
+	errNoTemplateContent       = errors.New("subscription template did not generate content")
+	errInvalidTemplate         = errors.New("GetSubscriptionTemplate did not return a template")
+)
+
+type tplCtx struct {
+	S              *Subscription
+	AssetPairs     assetPairs
+	PairSeparator  string
+	AssetSeparator string
+}
+
+// ExpandTemplates returns a list of Subscriptions with Template expanded
+// May be called on already expanded subscriptions: Passes $s through unprocessed if QualifiedChannel is already populated
+// Calls e.GetSubscriptionTemplate to find a template for each subscription
+// Filters out Authenticated subscriptions if !e.CanUseAuthenticatedEndpoints
+// See README.md for more details
+func (l List) ExpandTemplates(e iExchange) (List, error) {
+	if !slices.ContainsFunc(l, func(s *Subscription) bool { return s.QualifiedChannel == "" }) {
+		// Empty list, or already processed
+		return slices.Clone(l), nil
+	}
+
+	if !e.CanUseAuthenticatedWebsocketEndpoints() {
+		n := List{}
+		for _, s := range l {
+			if !s.Authenticated {
+				n = append(n, s)
+			}
+		}
+		l = n
+	}
+
+	ap, err := l.assetPairs(e)
+	if err != nil {
+		return nil, err
+	}
+
+	assets := make(asset.Items, 0, len(ap))
+	for k := range ap {
+		assets = append(assets, k)
+	}
+	slices.Sort(assets) // text/template ranges maps in sorted order
+	subs := List{}
+
+	for _, s := range l {
+		if s.QualifiedChannel != "" {
+			subs = append(subs, s)
+			continue
+		}
+
+		subCtx := &tplCtx{
+			S:              s,
+			AssetPairs:     ap,
+			PairSeparator:  recordSeparator,
+			AssetSeparator: groupSeparator,
+		}
+
+		t, err := e.GetSubscriptionTemplate(s)
+		if err != nil {
+			return nil, err
+		}
+		if t == nil {
+			return nil, errInvalidTemplate
+		}
+
+		buf := &bytes.Buffer{}
+		if err := t.Execute(buf, subCtx); err != nil {
+			return nil, err
+		}
+
+		out := buf.String()
+
+		subAssets := assets
+		xpandPairs := strings.Contains(out, subCtx.PairSeparator)
+		if xpandAssets := strings.Contains(out, subCtx.AssetSeparator); xpandAssets {
+			if s.Asset != asset.All {
+				return nil, errAssetTemplateWithoutAll
+			}
+		} else {
+			if xpandPairs && (s.Asset == asset.All || s.Asset == asset.Empty) {
+				// We don't currently support expanding Pairs without expanding Assets for All or Empty assets, but we could; waiting for a use-case
+				return nil, errInvalidAssetExpandPairs
+			}
+			// No expansion so update expected Assets for consistent behaviour below
+			subAssets = []asset.Item{s.Asset}
+		}
+
+		out = strings.TrimRight(out, " \n\r\t"+subCtx.PairSeparator+subCtx.AssetSeparator)
+
+		assetRecords := strings.Split(out, subCtx.AssetSeparator)
+		if len(assetRecords) != len(subAssets) {
+			return nil, fmt.Errorf("%w: Got %d; Expected %d", errAssetRecords, len(assetRecords), len(subAssets))
+		}
+
+		for i, assetChannels := range assetRecords {
+			a := subAssets[i]
+			assetChannels = strings.TrimRight(assetChannels, " \n\r\t"+recordSeparator)
+			pairLines := strings.Split(assetChannels, subCtx.PairSeparator)
+			pairs, ok := ap[a]
+			if xpandPairs {
+				if !ok {
+					return nil, fmt.Errorf("%w: %s", asset.ErrInvalidAsset, a)
+				}
+				if len(pairLines) != len(pairs) {
+					return nil, fmt.Errorf("%w: Got %d; Expected %d", errPairRecords, len(pairLines), len(pairs))
+				}
+			}
+			for j, channel := range pairLines {
+				c := s.Clone()
+				c.Asset = a
+				channel = strings.TrimSpace(channel)
+				if channel == "" {
+					return nil, fmt.Errorf("%w: %s", errNoTemplateContent, s)
+				}
+				c.QualifiedChannel = strings.TrimSpace(channel)
+				if xpandPairs {
+					c.Pairs = currency.Pairs{pairs[j]}
+				} else {
+					c.Pairs = pairs
+				}
+				subs = append(subs, c)
+			}
+		}
+	}
+
+	return subs, nil
+}

exchanges/subscription/template_test.go

@@ -0,0 +1,99 @@
+package subscription
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/thrasher-corp/gocryptotrader/currency"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/kline"
+)
+
+// TestExpandTemplates exercises ExpandTemplates
+func TestExpandTemplates(t *testing.T) {
+	t.Parallel()
+
+	e := &mockEx{
+		tpl: "subscriptions.tmpl",
+	}
+
+	// Functionality tests
+	l := List{
+		{Channel: "feature1"},
+		{Channel: "feature2", Asset: asset.All, Pairs: currency.Pairs{btcusdtPair, ethusdcPair}, Interval: kline.FifteenMin},
+		{Channel: "feature3", Asset: asset.All, Pairs: currency.Pairs{btcusdtPair, ethusdcPair}, Levels: 100},
+		{Channel: "feature4", Authenticated: true},
+		{Channel: "feature1", QualifiedChannel: "just one sub already processed"},
+	}
+	got, err := l.ExpandTemplates(e)
+	require.NoError(t, err, "ExpandTemplates must not error")
+	exp := List{
+		{Channel: "feature1", QualifiedChannel: "feature1"},
+		{Channel: "feature2", QualifiedChannel: "spot-feature2@15m", Asset: asset.Spot, Pairs: currency.Pairs{btcusdtPair, ethusdcPair}, Interval: kline.FifteenMin},
+		{Channel: "feature2", QualifiedChannel: "future-feature2@15m", Asset: asset.Futures, Pairs: currency.Pairs{btcusdtPair, ethusdcPair}, Interval: kline.FifteenMin},
+		{Channel: "feature3", QualifiedChannel: "spot-USDTBTC-feature3@100", Asset: asset.Spot, Pairs: currency.Pairs{btcusdtPair}, Levels: 100},
+		{Channel: "feature3", QualifiedChannel: "spot-USDCETH-feature3@100", Asset: asset.Spot, Pairs: currency.Pairs{ethusdcPair}, Levels: 100},
+		{Channel: "feature3", QualifiedChannel: "future-USDTBTC-feature3@100", Asset: asset.Futures, Pairs: currency.Pairs{btcusdtPair}, Levels: 100},
+		{Channel: "feature3", QualifiedChannel: "future-USDCETH-feature3@100", Asset: asset.Futures, Pairs: currency.Pairs{ethusdcPair}, Levels: 100},
+		{Channel: "feature1", QualifiedChannel: "just one sub already processed"},
+	}
+
+	if !equalLists(t, exp, got) {
+		t.FailNow() // If the first list isn't equal testing it again will duplicate test failures
+	}
+
+	e.auth = true
+	got, err = l.ExpandTemplates(e)
+	require.NoError(t, err, "ExpandTemplates must not error")
+	exp = append(exp,
+		&Subscription{Channel: "feature4", QualifiedChannel: "feature4-authed"},
+	)
+	equalLists(t, exp, got)
+
+	_, err = List{{Channel: "feature2", Asset: asset.Spot}}.ExpandTemplates(e)
+	assert.ErrorIs(t, err, errAssetTemplateWithoutAll, "Should error correctly on xpand assets without All")
+
+	e.tpl = "errors.tmpl"
+	_, err = List{{Channel: "error1", Asset: asset.All}}.ExpandTemplates(e)
+	assert.ErrorIs(t, err, errInvalidAssetExpandPairs, "Should error correctly on xpand pairs but not assets")
+
+	_, err = List{{Channel: "error1"}}.ExpandTemplates(e)
+	assert.ErrorIs(t, err, errInvalidAssetExpandPairs, "Should error correctly on xpand pairs but not assets")
+
+	_, err = List{{Channel: "error2"}}.ExpandTemplates(e)
+	assert.ErrorContains(t, err, "wrong number of args for String", "Should error correctly with execution error")
+
+	_, err = List{{Channel: "non-existent"}}.ExpandTemplates(e)
+	assert.ErrorIs(t, err, errNoTemplateContent, "Should error correctly when no content generated")
+	assert.ErrorContains(t, err, "non-existent", "Should error correctly when no content generated")
+
+	_, err = List{{Channel: "error3", Asset: asset.All}}.ExpandTemplates(e)
+	assert.ErrorIs(t, err, errAssetRecords, "Should error correctly when invalid number of asset entries")
+
+	_, err = List{{Channel: "error4", Asset: asset.Spot}}.ExpandTemplates(e)
+	assert.ErrorIs(t, err, errPairRecords, "Should error correctly when invalid number of pair entries")
+
+	_, err = List{{Channel: "error4", Asset: asset.Margin}}.ExpandTemplates(e)
+	assert.ErrorIs(t, err, asset.ErrInvalidAsset, "Should error correctly when invalid asset")
+
+	e.tpl = "parse-error.tmpl"
+	_, err = l.ExpandTemplates(e)
+	assert.ErrorContains(t, err, "function \"explode\" not defined", "Should error correctly on unparsable template")
+
+	e.errFormat = errors.New("the planet Krypton is gone")
+	_, err = l.ExpandTemplates(e)
+	assert.ErrorIs(t, err, e.errFormat, "Should error correctly on GetPairFormat")
+
+	e.errPairs = errors.New("bad parenting from Jor-El")
+	_, err = l.ExpandTemplates(e)
+	assert.ErrorIs(t, err, e.errPairs, "Should error correctly on GetEnabledPairs")
+
+	l = List{{Channel: "feature1", QualifiedChannel: "already happy"}}
+	got, err = l.ExpandTemplates(e)
+	require.NoError(t, err)
+	require.Len(t, got, 1, "Must get back the one sub")
+	assert.Equal(t, "already happy", l[0].QualifiedChannel, "Should get back the one sub")
+	assert.NotSame(t, got, l, "Should get back a different actual list")
+}

exchanges/subscription/testdata/errors.tmpl

@@ -0,0 +1,13 @@
+{{- if eq .S.Channel "error1" }}
+  {{/* Error 1: Expand pairs but not assets, without specific asset */}}
+  {{- .PairSeparator -}}
+{{- else if eq .S.Channel "error2" }}
+  {{/* Error 2: Runtime error from executing */}}
+  {{ .S.String 42 }}
+{{- else if eq .S.Channel "error3" }}
+  {{/* Error 3: Incorrect number of asset entries */}}
+  {{- .AssetSeparator }}
+{{- else if eq .S.Channel "error4" }}
+  {{/* Error 3: Incorrect number of pair entries */}}
+  {{- .PairSeparator }}
+{{- end -}}

exchanges/subscription/testdata/parse-error.tmpl

@@ -0,0 +1 @@
+{{ explode }}

exchanges/subscription/testdata/subscriptions.tmpl

@@ -0,0 +1,21 @@
+{{- if eq $.S.Channel "feature1" -}}
+  {{/* Case 1: One channel to rule them all */}}
+  feature1
+{{- else if eq $.S.Channel "feature2" -}}
+  {{/* Case 2: One channel per asset */}}
+  {{- range $asset, $pairs := $.AssetPairs }}
+    {{ assetName $asset }}-feature2@ {{- $.S.Interval.Short }}
+    {{- $.AssetSeparator }}
+  {{- end }}
+{{- else if eq $.S.Channel "feature3" }}
+  {{/* Case 3: One channel per pair per asset */}}
+  {{- range $asset, $pairs := $.AssetPairs }}
+    {{- range $pair := $pairs -}}
+      {{ assetName $asset }}-{{ $pair.Swap.String -}} -feature3@ {{- $.S.Levels }}
+      {{- $.PairSeparator -}}
+    {{- end -}}
+    {{- $.AssetSeparator -}}
+  {{- end -}}
+{{- else if eq $.S.Channel "feature4" }}
+  feature4-authed
+{{- end -}}