websocket: fix deadlock when enabling/disabling via gctrpc (#754)

* websocket: select case error if no receiver, add in functionality to reset to initial sync for books on a new websocket connection

* websocket: fix tests

* websocket: log error instead of losing it

* websocket: fix whoopsie

* exchanges: fix test

* websocket: force requirement of specific functionality

* exchanges: fix tests

* exchanges/websocket: move waitgroup add before scheduling across exchanges

* gateio: add feature subscribe

* bithumb/bittrex: include connection state reset, fix reconnection bug for Bithumb

* huobi: Add listen to shutdown to routine so it actually returns and stops being a naughty boy.

* huobi: add missing waitgroup add.

* exchanges: bleed comms channels

* binance: fix reconnection bug with buffer

* bithumb: fix reconnection bug with ws orderbook when websocket is diabled/enabled

* bithumb/bittrex: add bleeders for ws orderbook jobs

* linter: fix

* kraken: reduce code block from double assertion

* This bug ruined my day.

* glorious: error checking

* zb: add correct path for websocket connection

* exchange: Add verbosity when config conflicts and overwrites default values

* zb: add https to path

* exchanges: glorious nits

* stream: Add checkAndSetMonitoring to reduce potential routine bundling, increase timeout and check state in tests

* stream: remove check that is not needed.

* glorious: nits addr.

* lint: test

exchanges/stream/websocket.go

@@ -27,6 +27,7 @@ var (
 	errClosedConnection = errors.New("use of closed network connection")
 	// ErrSubscriptionFailure defines an error when a subscription fails
 	ErrSubscriptionFailure = errors.New("subscription failure")
+	errAlreadyRunning      = errors.New("connection monitor is already running")
 )
 
 // New initialises the websocket struct
@@ -43,6 +44,11 @@ func New() *Websocket {
 	}
 }
 
+var (
+	errSubscriberUnset           = errors.New("subscriber function needs to be set")
+	errGenerateSubsciptionsUnset = errors.New("generate subscriptions function needs to be set")
+)
+
 // Setup sets main variables for websocket connection
 func (w *Websocket) Setup(s *WebsocketSetup) error {
 	if w == nil {
@@ -65,8 +71,8 @@ func (w *Websocket) Setup(s *WebsocketSetup) error {
 
 	w.features = s.Features
 
-	if w.features.Subscribe && s.Subscriber == nil {
-		return errors.New("features have been set yet channel subscriber is not set")
+	if s.Subscriber == nil {
+		return errSubscriberUnset
 	}
 	w.Subscriber = s.Subscriber
 
@@ -75,6 +81,9 @@ func (w *Websocket) Setup(s *WebsocketSetup) error {
 	}
 	w.Unsubscriber = s.UnSubscriber
 
+	if s.GenerateSubscriptions == nil {
+		return errGenerateSubsciptionsUnset
+	}
 	w.GenerateSubs = s.GenerateSubscriptions
 
 	w.enabled = s.Enabled
@@ -206,24 +215,28 @@ func (w *Websocket) Connect() error {
 	w.setConnectingStatus(false)
 	w.setInit(true)
 
-	if !w.IsConnectionMonitorRunning() {
-		w.connectionMonitor()
+	err = w.connectionMonitor()
+	if err != nil {
+		log.Errorf(log.WebsocketMgr,
+			"%s cannot start websocket connection monitor %v",
+			w.GetName(),
+			err)
 	}
 
-	// Resubscribe after re-connection
-	if len(w.subscriptions) != 0 {
-		err = w.Subscriber(w.subscriptions)
-		if err != nil {
-			return fmt.Errorf("%v %w: %v", w.exchangeName, ErrSubscriptionFailure, err)
-		}
+	subs, err := w.GenerateSubs() // regenerate state on new connection
+	if err != nil {
+		return fmt.Errorf("%v %w: %v", w.exchangeName, ErrSubscriptionFailure, err)
+	}
+	err = w.Subscriber(subs)
+	if err != nil {
+		return fmt.Errorf("%v %w: %v", w.exchangeName, ErrSubscriptionFailure, err)
 	}
-
 	return nil
 }
 
 // Disable disables the exchange websocket protocol
 func (w *Websocket) Disable() error {
-	if !w.IsConnected() || !w.IsEnabled() {
+	if !w.IsEnabled() {
 		return fmt.Errorf("websocket is already disabled for exchange %s",
 			w.exchangeName)
 	}
@@ -290,11 +303,11 @@ func (w *Websocket) dataMonitor() {
 }
 
 // connectionMonitor ensures that the WS keeps connecting
-func (w *Websocket) connectionMonitor() {
-	if w.IsConnectionMonitorRunning() {
-		return
+func (w *Websocket) connectionMonitor() error {
+	if w.checkAndSetMonitorRunning() {
+		return errAlreadyRunning
 	}
-	w.setConnectionMonitorRunning(true)
+
 	go func() {
 		timer := time.NewTimer(connectionMonitorDelay)
 
@@ -354,6 +367,7 @@ func (w *Websocket) connectionMonitor() {
 			}
 		}
 	}()
+	return nil
 }
 
 // Shutdown attempts to shut down a websocket connection and associated routines
@@ -527,7 +541,10 @@ func (w *Websocket) trafficMonitor() {
 				// Routine pausing mechanism
 				go func(p chan<- struct{}) {
 					time.Sleep(defaultTrafficPeriod)
-					p <- struct{}{}
+					select {
+					case p <- struct{}{}:
+					default:
+					}
 				}(pause)
 				select {
 				case <-w.ShutdownC:
@@ -607,6 +624,16 @@ func (w *Websocket) IsTrafficMonitorRunning() bool {
 	return w.trafficMonitorRunning
 }
 
+func (w *Websocket) checkAndSetMonitorRunning() (alreadyRunning bool) {
+	w.connectionMutex.Lock()
+	defer w.connectionMutex.Unlock()
+	if w.connectionMonitorRunning {
+		return true
+	}
+	w.connectionMonitorRunning = true
+	return false
+}
+
 func (w *Websocket) setConnectionMonitorRunning(b bool) {
 	w.connectionMutex.Lock()
 	w.connectionMonitorRunning = b

exchanges/stream/websocket_connection.go

@@ -206,7 +206,16 @@ func (w *WebsocketConnection) ReadMessage() Response {
 	if err != nil {
 		if isDisconnectionError(err) {
 			w.setConnectedStatus(false)
-			w.readMessageErrors <- err
+			select {
+			case w.readMessageErrors <- err:
+			default:
+				// bypass if there is no receiver, as this stops it returning
+				// when shutdown is called.
+				log.Warnf(log.WebsocketMgr,
+					"%s failed to relay error: %v",
+					w.ExchangeName,
+					err)
+			}
 		}
 		return Response{}
 	}

exchanges/stream/websocket_test.go

@@ -153,8 +153,14 @@ func TestSetup(t *testing.T) {
 	}
 	websocketSetup.WebsocketTimeout = time.Minute
 	err = w.Setup(websocketSetup)
-	if err != nil {
-		t.Fatal(err)
+	if !errors.Is(err, errGenerateSubsciptionsUnset) {
+		t.Fatalf("received: %v but expected: %v", err, errGenerateSubsciptionsUnset)
+	}
+
+	websocketSetup.GenerateSubscriptions = func() ([]ChannelSubscription, error) { return nil, nil }
+	err = w.Setup(websocketSetup)
+	if !errors.Is(err, nil) {
+		t.Fatalf("received: %v but expected: %v", err, nil)
 	}
 }
 
@@ -544,23 +550,33 @@ func TestConnectionMonitorNoConnection(t *testing.T) {
 	ws.DataHandler = make(chan interface{}, 1)
 	ws.ShutdownC = make(chan struct{}, 1)
 	ws.exchangeName = "hello"
-	ws.trafficTimeout = 1
 	ws.Wg = &sync.WaitGroup{}
-	ws.connectionMonitor()
+	ws.enabled = true
+	err := ws.connectionMonitor()
+	if !errors.Is(err, nil) {
+		t.Fatalf("received: %v, but expected: %v", err, nil)
+	}
 	if !ws.IsConnectionMonitorRunning() {
 		t.Fatal("Should not have exited")
 	}
-	ws.connectionMonitor() // This one should exit
+	err = ws.connectionMonitor()
+	if !errors.Is(err, errAlreadyRunning) {
+		t.Fatalf("received: %v, but expected: %v", err, errAlreadyRunning)
+	}
 	if !ws.IsConnectionMonitorRunning() {
 		t.Fatal("Should not have exited")
 	}
-	time.Sleep(time.Millisecond * 100)
+	ws.setEnabled(false)
+	time.Sleep(time.Second * 2)
 	if ws.IsConnectionMonitorRunning() {
 		t.Fatal("Should have exited")
 	}
 	ws.setConnectedStatus(true)  // attempt shutdown when not enabled
 	ws.setConnectingStatus(true) // throw a spanner in the works
-	ws.connectionMonitor()
+	err = ws.connectionMonitor()
+	if !errors.Is(err, nil) {
+		t.Fatalf("received: %v, but expected: %v", err, nil)
+	}
 	if !ws.IsConnectionMonitorRunning() {
 		t.Fatal("Should not have exited")
 	}
@@ -1083,6 +1099,9 @@ func TestFlushChannels(t *testing.T) {
 	// Disable pair and flush system
 	newgen.EnabledPairs = []currency.Pair{
 		currency.NewPair(currency.BTC, currency.AUD)}
+	web.GenerateSubs = func() ([]ChannelSubscription, error) {
+		return []ChannelSubscription{{Channel: "test"}}, nil
+	}
 	err = web.FlushChannels()
 	if err != nil {
 		t.Fatal(err)
@@ -1181,7 +1200,12 @@ func TestEnable(t *testing.T) {
 		connector: connect,
 		Wg:        new(sync.WaitGroup),
 		ShutdownC: make(chan struct{}),
+		GenerateSubs: func() ([]ChannelSubscription, error) {
+			return []ChannelSubscription{{Channel: "test"}}, nil
+		},
+		Subscriber: func(cs []ChannelSubscription) error { return nil },
 	}
+
 	err := web.Enable()
 	if err != nil {
 		t.Fatal(err)