websocket: fix deadlock when enabling/disabling via gctrpc (#754)

* websocket: select case error if no receiver, add in functionality to reset to initial sync for books on a new websocket connection

* websocket: fix tests

* websocket: log error instead of losing it

* websocket: fix whoopsie

* exchanges: fix test

* websocket: force requirement of specific functionality

* exchanges: fix tests

* exchanges/websocket: move waitgroup add before scheduling across exchanges

* gateio: add feature subscribe

* bithumb/bittrex: include connection state reset, fix reconnection bug for Bithumb

* huobi: Add listen to shutdown to routine so it actually returns and stops being a naughty boy.

* huobi: add missing waitgroup add.

* exchanges: bleed comms channels

* binance: fix reconnection bug with buffer

* bithumb: fix reconnection bug with ws orderbook when websocket is diabled/enabled

* bithumb/bittrex: add bleeders for ws orderbook jobs

* linter: fix

* kraken: reduce code block from double assertion

* This bug ruined my day.

* glorious: error checking

* zb: add correct path for websocket connection

* exchange: Add verbosity when config conflicts and overwrites default values

* zb: add https to path

* exchanges: glorious nits

* stream: Add checkAndSetMonitoring to reduce potential routine bundling, increase timeout and check state in tests

* stream: remove check that is not needed.

* glorious: nits addr.

* lint: test

exchanges/bithumb/bithumb_websocket.go

@@ -41,24 +41,31 @@ func (b *Bithumb) WsConnect() error {
 			b.Name,
 			err)
 	}
+
+	b.Websocket.Wg.Add(1)
 	go b.wsReadData()
+
 	b.setupOrderbookManager()
 	return nil
 }
 
 // wsReadData receives and passes on websocket messages for processing
 func (b *Bithumb) wsReadData() {
-	b.Websocket.Wg.Add(1)
 	defer b.Websocket.Wg.Done()
 
 	for {
-		resp := b.Websocket.Conn.ReadMessage()
-		if resp.Raw == nil {
+		select {
+		case <-b.Websocket.ShutdownC:
 			return
-		}
-		err := b.wsHandleData(resp.Raw)
-		if err != nil {
-			b.Websocket.DataHandler <- err
+		default:
+			resp := b.Websocket.Conn.ReadMessage()
+			if resp.Raw == nil {
+				return
+			}
+			err := b.wsHandleData(resp.Raw)
+			if err != nil {
+				b.Websocket.DataHandler <- err
+			}
 		}
 	}
 }

exchanges/bithumb/bithumb_websocket_types.go

@@ -87,10 +87,11 @@ type orderbookManager struct {
 }
 
 type update struct {
-	buffer       chan *WsOrderbooks
-	fetchingBook bool
-	initialSync  bool
-	lastUpdated  time.Time
+	buffer            chan *WsOrderbooks
+	fetchingBook      bool
+	initialSync       bool
+	needsFetchingBook bool
+	lastUpdated       time.Time
 }
 
 // job defines a synchonisation job that tells a go routine to fetch an

exchanges/bithumb/bithumb_ws_orderbook.go

@@ -67,7 +67,7 @@ func (b *Bithumb) UpdateLocalBuffer(wsdp *WsOrderbooks) (bool, error) {
 // applyBufferUpdate applies the buffer to the orderbook or initiates a new
 // orderbook sync by the REST protocol which is off handed to go routine.
 func (b *Bithumb) applyBufferUpdate(pair currency.Pair) error {
-	fetching, err := b.obm.checkIsFetchingBook(pair)
+	fetching, needsFetching, err := b.obm.handleFetchingBook(pair)
 	if err != nil {
 		return err
 	}
@@ -75,12 +75,38 @@ func (b *Bithumb) applyBufferUpdate(pair currency.Pair) error {
 		return nil
 	}
 
-	recent, err := b.Websocket.Orderbook.GetOrderbook(pair, asset.Spot)
-	if err != nil || (recent.Asks == nil && recent.Bids == nil) {
+	if needsFetching {
+		if b.Verbose {
+			log.Debugf(log.WebsocketMgr, "%s Orderbook: Fetching via REST\n", b.Name)
+		}
 		return b.obm.fetchBookViaREST(pair)
 	}
 
-	return b.obm.checkAndProcessUpdate(b.processBooks, pair, recent)
+	recent, err := b.Websocket.Orderbook.GetOrderbook(pair, asset.Spot)
+	if err != nil {
+		log.Errorf(
+			log.WebsocketMgr,
+			"%s error fetching recent orderbook when applying updates: %s\n",
+			b.Name,
+			err)
+	}
+
+	if recent != nil {
+		err = b.obm.checkAndProcessUpdate(b.processBooks, pair, recent)
+		if err != nil {
+			log.Errorf(
+				log.WebsocketMgr,
+				"%s error processing update - initiating new orderbook sync via REST: %s\n",
+				b.Name,
+				err)
+			err = b.obm.setNeedsFetchingBook(pair)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
 }
 
 // SynchroniseWebsocketOrderbook synchronises full orderbook for currency pair
@@ -91,6 +117,14 @@ func (b *Bithumb) SynchroniseWebsocketOrderbook() {
 		defer b.Websocket.Wg.Done()
 		for {
 			select {
+			case <-b.Websocket.ShutdownC:
+				for {
+					select {
+					case <-b.obm.jobs:
+					default:
+						return
+					}
+				}
 			case j := <-b.obm.jobs:
 				err := b.processJob(j.Pair)
 				if err != nil {
@@ -98,8 +132,6 @@ func (b *Bithumb) SynchroniseWebsocketOrderbook() {
 						"%s processing websocket orderbook error %v",
 						b.Name, err)
 				}
-			case <-b.Websocket.ShutdownC:
-				return
 			}
 		}
 	}()
@@ -149,12 +181,23 @@ func (b *Bithumb) setupOrderbookManager() {
 	if b.obm.state == nil {
 		b.obm.state = make(map[currency.Code]map[currency.Code]map[asset.Item]*update)
 		b.obm.jobs = make(chan job, maxWSOrderbookJobs)
-
-		for i := 0; i < maxWSOrderbookWorkers; i++ {
-			// 10 workers for synchronising book
-			b.SynchroniseWebsocketOrderbook()
+	} else {
+		// Change state on reconnect for initial sync.
+		for _, m1 := range b.obm.state {
+			for _, m2 := range m1 {
+				for _, update := range m2 {
+					update.initialSync = true
+					update.needsFetchingBook = true
+					update.lastUpdated = time.Time{}
+				}
+			}
 		}
 	}
+
+	for i := 0; i < maxWSOrderbookWorkers; i++ {
+		// 10 workers for synchronising book
+		b.SynchroniseWebsocketOrderbook()
+	}
 }
 
 // stageWsUpdate stages websocket update to roll through updates that need to
@@ -177,9 +220,10 @@ func (o *orderbookManager) stageWsUpdate(u *WsOrderbooks, pair currency.Pair, a
 	state, ok := m2[a]
 	if !ok {
 		state = &update{
-			buffer:       make(chan *WsOrderbooks, maxWSUpdateBuffer),
-			fetchingBook: false,
-			initialSync:  true,
+			buffer:            make(chan *WsOrderbooks, maxWSUpdateBuffer),
+			fetchingBook:      false,
+			initialSync:       true,
+			needsFetchingBook: true,
 		}
 		m2[a] = state
 	}
@@ -201,19 +245,30 @@ func (o *orderbookManager) stageWsUpdate(u *WsOrderbooks, pair currency.Pair, a
 	}
 }
 
-// checkIsFetchingBook checks status if the book is currently being via the REST
-// protocol.
-func (o *orderbookManager) checkIsFetchingBook(pair currency.Pair) (bool, error) {
+// handleFetchingBook checks if a full book is being fetched or needs to be
+// fetched
+func (o *orderbookManager) handleFetchingBook(pair currency.Pair) (fetching, needsFetching bool, err error) {
 	o.Lock()
 	defer o.Unlock()
 	state, ok := o.state[pair.Base][pair.Quote][asset.Spot]
 	if !ok {
 		return false,
+			false,
 			fmt.Errorf("check is fetching book cannot match currency pair %s asset type %s",
 				pair,
 				asset.Spot)
 	}
-	return state.fetchingBook, nil
+
+	if state.fetchingBook {
+		return true, false, nil
+	}
+
+	if state.needsFetchingBook {
+		state.needsFetchingBook = false
+		state.fetchingBook = true
+		return false, true, nil
+	}
+	return false, false, nil
 }
 
 // stopFetchingBook completes the book fetching.
@@ -354,6 +409,7 @@ bufferEmpty:
 	// disable rest orderbook synchronisation
 	_ = o.stopFetchingBook(pair)
 	_ = o.completeInitialSync(pair)
+	_ = o.stopNeedsFetchingBook(pair)
 	return nil
 }
 
@@ -389,3 +445,36 @@ func (b *Bithumb) SeedLocalCacheWithBook(p currency.Pair, o *Orderbook) error {
 	newOrderBook.VerifyOrderbook = b.CanVerifyOrderbook
 	return b.Websocket.Orderbook.LoadSnapshot(&newOrderBook)
 }
+
+// setNeedsFetchingBook completes the book fetching initiation.
+func (o *orderbookManager) setNeedsFetchingBook(pair currency.Pair) error {
+	o.Lock()
+	defer o.Unlock()
+	state, ok := o.state[pair.Base][pair.Quote][asset.Spot]
+	if !ok {
+		return fmt.Errorf("could not match pair %s and asset type %s in hash table",
+			pair,
+			asset.Spot)
+	}
+	state.needsFetchingBook = true
+	return nil
+}
+
+// stopNeedsFetchingBook completes the book fetching initiation.
+func (o *orderbookManager) stopNeedsFetchingBook(pair currency.Pair) error {
+	o.Lock()
+	defer o.Unlock()
+	state, ok := o.state[pair.Base][pair.Quote][asset.Spot]
+	if !ok {
+		return fmt.Errorf("could not match pair %s and asset type %s in hash table",
+			pair,
+			asset.Spot)
+	}
+	if !state.needsFetchingBook {
+		return fmt.Errorf("needs fetching book already set to false for %s %s",
+			pair,
+			asset.Spot)
+	}
+	state.needsFetchingBook = false
+	return nil
+}