Bitfinex: Websocket subscription improvements (#1353)

* Websockets: Add keys to websocket subscriptions

* This switches all RO uses of the mutex to use a RLock method.
* The mutex used for discrete field access has had scope drift from
  name 'connectionMutex' so rename to more appropriate fieldsMutex
* The mutex used for Set/CanUseAuthEndpoints moves from the
  subscriptions endpoint to the fieldsMutex
* Add GetSubscription by key
* Expose stream.Matcher type

* Bitfinex: Subscribe and Unsubscribe atomicly

* Fix Auth failures ignored
* This change makes it so that Subscribe and Unsubscribe wait for success
** Tells the DataHandler about errors
** Errors are returned to consumers
* Subscribes concurrently to the channels
* It also simplifies the chanId to stream mapping
* Removes unable to locate chanID: %d errors which are just noise
* Paves the way for unified channelSubscription id handling
* Adds support for subId for Book subscriptions, which is more robust

* Vastly simplifies what we need to test TestWsSubscribedResponse
This test was working to ensure that the various fancy key parsing
mechanisms all worked. Now that we use subId, we just need a thorough
test of that
* Expose Match.Set in order to capture websocket incoming data
Can't see another way of doing this. Doesn't seem too bad

* Allow tests to run with auth or WS
These flags made it difficult to run the tests whilst working on
websockets

* Enable API auth and WS in testconfig
This change minimises the changes requires for a full test run against
live endpoints, so that new contributors have a clearer testing path.
I cannot see any reason to turn WS off and Auth endpoints off when we're
not going to run API tests without Creds being set, and we're not going
to do live fire tests without canManipulateRealOrders

* TestWsSubscribe and various fixes
** Enables the websocket for live non-authed integration tests by default
** Adds an integration test for subscriptions
** Changes the Ws tests to respect canManipulateRealOrders
** Uses WsConnect instead of setupWs; fixes seqNo config not sent for WS tests
** Allows api creds to live in config/testdata.json which might be
  less likely to accidentally commit, and less obtrusive

* Bitfinex: Support period and timeframe for Candles

* Fixes manual Subscribe() symbol or key formatting
* Unifies handling of params for DefaultSubscriptions and manual
  subsrciptions

* Bitfinex: Handle conf and info WS channel events

* Bitfinex: Better tests for subscriptions

* fixup! Websockets: Add keys to websocket subscriptions

* fixup! Bitfinex: Subscribe and Unsubscribe atomicly

* fixup! Websockets: Add keys to websocket subscriptions

* Websockets: Add Pending subscription status

Add a status tracker so that Sub/Unsub can prevent duplicates,
and also fixes when first message comes before we have added the sub
to the tracker

* Websockets: Add State instead of pending

This change allows more clarity about the current state and
checks for specifically already Unsubing

* Bitfinex: Fix first sub message maybe lost

The only link we have between a sub req and the sub resp is the subID.
And the only link we have between a sub message and the sub is the chanID.
We can't derive a link using Pair or anything else.

This meant that by sending the resp and its chanID down the IncomingData
channel, we allowed the channel reader to maybe process the next
message, the first message on the channel, before the runtime executed
the switch back to subscribeToChan waiting on the chan.

To fix this, we key initially on subId.(string), and then replace it
with chanId.(int64) when we have it *inside* the wsHandleData so we
know we've procedurally handled it before the next message.

subscribeToChan is then free to remove the subId keyed Sub regardless of
error or not

If there's an error, we don't need to inline handling because there
won't be any second update.

Expands test coverage to make sure those subId keyed subscriptions are
removed.

* Websocket: Validate state in SetChanState

* fixup! Bitfinex: Fix first sub message maybe lost

* Websockets: Rename RemoveUnsuccessfulSubs

Implementation doesn't imply Unsuccessful or need to.
This change supports the registering of Pending subs

* Bitfinex: Fix race in Tests

exchanges/stream/websocket_test.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -16,8 +17,10 @@ import (
 	"time"
 
 	"github.com/gorilla/websocket"
+	"github.com/stretchr/testify/assert"
 	"github.com/thrasher-corp/gocryptotrader/config"
 	"github.com/thrasher-corp/gocryptotrader/currency"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/protocol"
 )
 
@@ -52,6 +55,10 @@ type testResponse struct {
 	RequestID int64 `json:"reqid,omitempty"`
 }
 
+type testSubKey struct {
+	Mood string
+}
+
 var defaultSetup = &WebsocketSetup{
 	ExchangeConfig: &config.Exchange{
 		Features: &config.FeaturesConfig{
@@ -71,9 +78,9 @@ var defaultSetup = &WebsocketSetup{
 	GenerateSubscriptions: func() ([]ChannelSubscription, error) {
 		return []ChannelSubscription{
 			{Channel: "TestSub"},
-			{Channel: "TestSub2"},
-			{Channel: "TestSub3"},
-			{Channel: "TestSub4"},
+			{Channel: "TestSub2", Key: "purple"},
+			{Channel: "TestSub3", Key: testSubKey{"mauve"}},
+			{Channel: "TestSub4", Key: 42},
 		}, nil
 	},
 	Features: &protocol.Features{Subscribe: true, Unsubscribe: true},
@@ -495,62 +502,48 @@ func TestWebsocket(t *testing.T) {
 func TestSubscribeUnsubscribe(t *testing.T) {
 	t.Parallel()
 	ws := *New()
-	err := ws.Setup(defaultSetup)
-	if err != nil {
-		t.Fatal(err)
-	}
+	assert.NoError(t, ws.Setup(defaultSetup), "WS Setup should not error")
 
 	fnSub := func(subs []ChannelSubscription) error {
 		ws.AddSuccessfulSubscriptions(subs...)
 		return nil
 	}
 	fnUnsub := func(unsubs []ChannelSubscription) error {
-		ws.RemoveSuccessfulUnsubscriptions(unsubs...)
+		ws.RemoveSubscriptions(unsubs...)
 		return nil
 	}
 	ws.Subscriber = fnSub
 	ws.Unsubscriber = fnUnsub
 
-	err = ws.UnsubscribeChannels(nil)
-	if err == nil {
-		t.Fatal("error cannot be nil")
-	}
-
-	// Generate test sub
 	subs, err := ws.GenerateSubs()
-	if err != nil {
-		t.Fatal(err)
+	assert.NoError(t, err, "Generating test subscriptions should not error")
+	assert.ErrorIs(t, ws.UnsubscribeChannels(nil), errNoSubscriptionsSupplied, "Unsubscribing from nil should error")
+	assert.ErrorIs(t, ws.UnsubscribeChannels(subs), ErrSubscriptionNotFound, "Unsubscribing should error when not subscribed")
+	assert.Nil(t, ws.GetSubscription(42), "GetSubscription on empty internal map should return")
+	assert.NoError(t, ws.SubscribeToChannels(subs), "Basic Subscribing should not error")
+	assert.Len(t, ws.GetSubscriptions(), 4, "Should have 4 subscriptions")
+	byDefKey := ws.GetSubscription(DefaultChannelKey{Channel: "TestSub"})
+	if assert.NotNil(t, byDefKey, "GetSubscription by default key should find a channel") {
+		assert.Equal(t, "TestSub", byDefKey.Channel, "GetSubscription by default key should return a pointer a copy of the right channel")
+		assert.NotSame(t, byDefKey, ws.subscriptions["TestSub"], "GetSubscription returns a fresh pointer")
 	}
-
-	// unsub when no subscribed channel
-	err = ws.UnsubscribeChannels(subs)
-	if err == nil {
-		t.Fatal("error cannot be nil")
+	if assert.NotNil(t, ws.GetSubscription("purple"), "GetSubscription by string key should find a channel") {
+		assert.Equal(t, "TestSub2", ws.GetSubscription("purple").Channel, "GetSubscription by string key should return a pointer a copy of the right channel")
 	}
-
-	err = ws.SubscribeToChannels(subs)
-	if err != nil {
-		t.Fatal(err)
+	if assert.NotNil(t, ws.GetSubscription(testSubKey{"mauve"}), "GetSubscription by type key should find a channel") {
+		assert.Equal(t, "TestSub3", ws.GetSubscription(testSubKey{"mauve"}).Channel, "GetSubscription by type key should return a pointer a copy of the right channel")
 	}
-
-	// subscribe when already subscribed
-	err = ws.SubscribeToChannels(subs)
-	if err == nil {
-		t.Fatal("error cannot be nil")
-	}
-
-	// subscribe to nothing
-	err = ws.SubscribeToChannels(nil)
-	if err == nil {
-		t.Fatal("error cannot be nil")
-	}
-
-	err = ws.UnsubscribeChannels(subs)
-	if err != nil {
-		t.Fatal(err)
+	if assert.NotNil(t, ws.GetSubscription(42), "GetSubscription by int key should find a channel") {
+		assert.Equal(t, "TestSub4", ws.GetSubscription(42).Channel, "GetSubscription by int key should return a pointer a copy of the right channel")
 	}
+	assert.Nil(t, ws.GetSubscription(nil), "GetSubscription by nil should return nil")
+	assert.Nil(t, ws.GetSubscription(45), "GetSubscription by invalid key should return nil")
+	assert.ErrorIs(t, ws.SubscribeToChannels(subs), errChannelAlreadySubscribed, "Subscribe should error when already subscribed")
+	assert.ErrorIs(t, ws.SubscribeToChannels(nil), errNoSubscriptionsSupplied, "Subscribe to nil should error")
+	assert.NoError(t, ws.UnsubscribeChannels(subs), "Unsubscribing should not error")
 }
 
+// TestResubscribe tests Resubscribing to existing subscriptions
 func TestResubscribe(t *testing.T) {
 	t.Parallel()
 	ws := *New()
@@ -558,41 +551,66 @@ func TestResubscribe(t *testing.T) {
 	wackedOutSetup := *defaultSetup
 	wackedOutSetup.MaxWebsocketSubscriptionsPerConnection = -1
 	err := ws.Setup(&wackedOutSetup)
-	if !errors.Is(err, errInvalidMaxSubscriptions) {
-		t.Fatalf("received: '%v' but expected: '%v'", err, errInvalidMaxSubscriptions)
-	}
+	assert.ErrorIs(t, err, errInvalidMaxSubscriptions, "Invalid MaxWebsocketSubscriptionsPerConnection should error")
 
 	err = ws.Setup(defaultSetup)
-	if err != nil {
-		t.Fatal(err)
-	}
+	assert.NoError(t, err, "WS Setup should not error")
 
 	fnSub := func(subs []ChannelSubscription) error {
 		ws.AddSuccessfulSubscriptions(subs...)
 		return nil
 	}
 	fnUnsub := func(unsubs []ChannelSubscription) error {
-		ws.RemoveSuccessfulUnsubscriptions(unsubs...)
+		ws.RemoveSubscriptions(unsubs...)
 		return nil
 	}
 	ws.Subscriber = fnSub
 	ws.Unsubscriber = fnUnsub
 
 	channel := []ChannelSubscription{{Channel: "resubTest"}}
-	err = ws.ResubscribeToChannel(&channel[0])
-	if err == nil {
-		t.Fatal("error cannot be nil")
-	}
 
-	err = ws.SubscribeToChannels(channel)
-	if err != nil {
-		t.Fatal(err)
-	}
+	assert.ErrorIs(t, ws.ResubscribeToChannel(&channel[0]), ErrSubscriptionNotFound, "Resubscribe should error when channel isn't subscribed yet")
+	assert.NoError(t, ws.SubscribeToChannels(channel), "Subscribe should not error")
+	assert.NoError(t, ws.ResubscribeToChannel(&channel[0]), "Resubscribe should not error now the channel is subscribed")
+}
 
-	err = ws.ResubscribeToChannel(&channel[0])
-	if err != nil {
-		t.Fatal("error cannot be nil")
-	}
+// TestSubscriptionState tests Subscription state changes
+func TestSubscriptionState(t *testing.T) {
+	t.Parallel()
+	ws := New()
+
+	c := &ChannelSubscription{Key: 42, Channel: "Gophers", State: ChannelSubscribing}
+	assert.ErrorIs(t, ws.SetSubscriptionState(c, ChannelUnsubscribing), ErrSubscriptionNotFound, "Setting an imaginary sub should error")
+
+	assert.NoError(t, ws.AddSubscription(c), "Adding first subscription should not error")
+	found := ws.GetSubscription(42)
+	assert.NotNil(t, found, "Should find the subscription")
+	assert.Equal(t, ChannelSubscribing, found.State, "Subscription should be Subscribing")
+	assert.ErrorIs(t, ws.AddSubscription(c), ErrSubscribedAlready, "Adding an already existing sub should error")
+	assert.ErrorIs(t, ws.SetSubscriptionState(c, ChannelSubscribing), ErrChannelInStateAlready, "Setting Same state should error")
+	assert.ErrorIs(t, ws.SetSubscriptionState(c, ChannelUnsubscribing+1), errInvalidChannelState, "Setting an invalid state should error")
+
+	ws.AddSuccessfulSubscriptions(*c)
+	found = ws.GetSubscription(42)
+	assert.NotNil(t, found, "Should find the subscription")
+	assert.Equal(t, found.State, ChannelSubscribed, "Subscription should be subscribed state")
+
+	assert.NoError(t, ws.SetSubscriptionState(c, ChannelUnsubscribing), "Setting Unsub state should not error")
+	found = ws.GetSubscription(42)
+	assert.Equal(t, found.State, ChannelUnsubscribing, "Subscription should be unsubscribing state")
+}
+
+// TestRemoveSubscriptions tests removing a subscription
+func TestRemoveSubscriptions(t *testing.T) {
+	t.Parallel()
+	ws := New()
+
+	c := &ChannelSubscription{Key: 42, Channel: "Unite!"}
+	assert.NoError(t, ws.AddSubscription(c), "Adding first subscription should not error")
+	assert.NotNil(t, ws.GetSubscription(42), "Added subscription should be findable")
+
+	ws.RemoveSubscriptions(*c)
+	assert.Nil(t, ws.GetSubscription(42), "Remove should have removed the sub")
 }
 
 // TestConnectionMonitorNoConnection logic test
@@ -618,18 +636,64 @@ func TestConnectionMonitorNoConnection(t *testing.T) {
 	}
 }
 
-// TestSliceCopyDoesntImpactBoth logic test
-func TestGetSubscriptions(t *testing.T) {
+// TestGetSubscription logic test
+func TestGetSubscription(t *testing.T) {
 	t.Parallel()
+	assert.Nil(t, (*Websocket).GetSubscription(nil, "imaginary"), "GetSubscription on a nil Websocket should return nil")
+	assert.Nil(t, (&Websocket{}).GetSubscription("empty"), "GetSubscription on a Websocket with no sub map should return nil")
 	w := Websocket{
-		subscriptions: []ChannelSubscription{
-			{
+		subscriptions: subscriptionMap{
+			42: {
 				Channel: "hello3",
 			},
 		},
 	}
-	if !strings.EqualFold("hello3", w.GetSubscriptions()[0].Channel) {
-		t.Error("Subscriptions was not copied properly")
+	assert.Nil(t, w.GetSubscription(43), "GetSubscription with an invalid key should return nil")
+	c := w.GetSubscription(42)
+	if assert.NotNil(t, c, "GetSubscription with an valid key should return a channel") {
+		assert.Equal(t, "hello3", c.Channel, "GetSubscription should return the correct channel details")
+	}
+}
+
+// TestGetSubscriptions logic test
+func TestGetSubscriptions(t *testing.T) {
+	t.Parallel()
+	w := Websocket{
+		subscriptions: subscriptionMap{
+			42: {
+				Channel: "hello3",
+			},
+		},
+	}
+	assert.Equal(t, "hello3", w.GetSubscriptions()[0].Channel, "GetSubscriptions should return the correct channel details")
+}
+
+// TestEnsureKeyed logic test
+func TestEnsureKeyed(t *testing.T) {
+	t.Parallel()
+	c := ChannelSubscription{
+		Channel:  "candles",
+		Asset:    asset.Spot,
+		Currency: currency.NewPair(currency.BTC, currency.USDT),
+	}
+	k1, ok := c.EnsureKeyed().(DefaultChannelKey)
+	if assert.True(t, ok, "EnsureKeyed should return a DefaultChannelKey") {
+		assert.Exactly(t, k1, c.Key, "EnsureKeyed should set the same key")
+		assert.Equal(t, k1.Channel, c.Channel, "DefaultChannelKey channel should be correct")
+		assert.Equal(t, k1.Asset, c.Asset, "DefaultChannelKey asset should be correct")
+		assert.Equal(t, k1.Currency, c.Currency, "DefaultChannelKey currency should be correct")
+	}
+	type platypus string
+	c = ChannelSubscription{
+		Key:      platypus("Gerald"),
+		Channel:  "orderbook",
+		Asset:    asset.Margin,
+		Currency: currency.NewPair(currency.ETH, currency.USDC),
+	}
+	k2, ok := c.EnsureKeyed().(platypus)
+	if assert.True(t, ok, "EnsureKeyed should return a platypus") {
+		assert.Exactly(t, k2, c.Key, "EnsureKeyed should set the same key")
+		assert.EqualValues(t, "Gerald", k2, "key should have the correct value")
 	}
 }
 
@@ -1024,15 +1088,10 @@ func TestGetChannelDifference(t *testing.T) {
 		},
 	}
 	subs, unsubs := web.GetChannelDifference(newChans)
-	if len(subs) != 3 {
-		t.Fatal("error mismatch")
-	}
+	assert.Len(t, subs, 3, "Should get the correct number of subs")
+	assert.Len(t, unsubs, 0, "Should get the correct number of unsubs")
 
-	if len(unsubs) != 0 {
-		t.Fatal("error mismatch")
-	}
-
-	web.subscriptions = subs
+	web.AddSuccessfulSubscriptions(subs...)
 
 	flushedSubs := []ChannelSubscription{
 		{
@@ -1041,12 +1100,8 @@ func TestGetChannelDifference(t *testing.T) {
 	}
 
 	subs, unsubs = web.GetChannelDifference(flushedSubs)
-	if len(subs) != 0 {
-		t.Fatal("error mismatch")
-	}
-	if len(unsubs) != 2 {
-		t.Fatal("error mismatch")
-	}
+	assert.Len(t, subs, 0, "Should get the correct number of subs")
+	assert.Len(t, unsubs, 2, "Should get the correct number of unsubs")
 
 	flushedSubs = []ChannelSubscription{
 		{
@@ -1058,11 +1113,13 @@ func TestGetChannelDifference(t *testing.T) {
 	}
 
 	subs, unsubs = web.GetChannelDifference(flushedSubs)
-	if len(subs) != 1 {
-		t.Fatal("error mismatch")
+	if assert.Len(t, subs, 1, "Should get the correct number of subs") {
+		assert.Equal(t, subs[0].Channel, "Test4", "Should subscribe to the right channel")
 	}
-	if len(unsubs) != 2 {
-		t.Fatal("error mismatch")
+	if assert.Len(t, unsubs, 2, "Should get the correct number of unsubs") {
+		sort.Slice(unsubs, func(i, j int) bool { return unsubs[i].Channel <= unsubs[j].Channel })
+		assert.Equal(t, unsubs[0].Channel, "Test1", "Should unsubscribe from the right channels")
+		assert.Equal(t, unsubs[1].Channel, "Test3", "Should unsubscribe from the right channels")
 	}
 }
 
@@ -1177,9 +1234,7 @@ func TestFlushChannels(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	web.subscriptionMutex.Lock()
-	web.subscriptions = subs
-	web.subscriptionMutex.Unlock()
+	web.AddSuccessfulSubscriptions(subs...)
 	err = web.FlushChannels()
 	if err != nil {
 		t.Fatal(err)
@@ -1199,12 +1254,14 @@ func TestFlushChannels(t *testing.T) {
 		t.Fatal(err)
 	}
 	web.subscriptionMutex.Lock()
-	web.subscriptions = []ChannelSubscription{
-		{
+	web.subscriptions = subscriptionMap{
+		41: {
+			Key:      41,
 			Channel:  "match channel",
 			Currency: currency.NewPair(currency.BTC, currency.AUD),
 		},
-		{
+		42: {
+			Key:      42,
 			Channel:  "unsub channel",
 			Currency: currency.NewPair(currency.THETA, currency.USDT),
 		},
@@ -1416,10 +1473,10 @@ func TestCheckSubscriptions(t *testing.T) {
 
 	ws.MaxSubscriptionsPerConnection = 2
 
-	ws.subscriptions = []ChannelSubscription{{Channel: "test"}}
-	err = ws.checkSubscriptions([]ChannelSubscription{{Channel: "test"}})
-	if !errors.Is(err, errChannelSubscriptionAlreadySubscribed) {
-		t.Fatalf("received: %v, but expected: %v", err, errChannelSubscriptionAlreadySubscribed)
+	ws.subscriptions = subscriptionMap{42: {Key: 42, Channel: "test"}}
+	err = ws.checkSubscriptions([]ChannelSubscription{{Key: 42, Channel: "test"}})
+	if !errors.Is(err, errChannelAlreadySubscribed) {
+		t.Fatalf("received: %v, but expected: %v", err, errChannelAlreadySubscribed)
 	}
 
 	err = ws.checkSubscriptions([]ChannelSubscription{{}})