Websocket request-response correlation (#328)

* Establishes new websocket functionality. Begins the creation of the websocket request

* Adding a wrapper over gorilla websocket connect,send,receive to handle ID messages. Doesn't work

* Successfully moved exchange_websocket into its own wshandler namespace. oof

* Sets up ZB to use a round trip WS request system

* Adds Kraken ID support to subscriptions. Renames duplicate func name UnsubscribeToChannels to RemoveSubscribedChannels. Adds some helper methods in the WebsocketConn to reduce duplicate code. Cleans up ZB implementation

* Fixes double locking which caused no websocket data to be read. Fixes requestid for kraken subscriptions

* Completes Huobi and Hadax implementation. Extends ZB error handling. Adds GZip support for reading messages

* Adds HitBTC support. Adds GetCurrencies, GetSymbols, GetTrades WS funcs. Adds super fun new parameter to GenerateMessageID for Unix and UnixNano

* Adds GateIO id support

* Adds Coinut support. Prevents nil reference error in constatus when there isnt one

* Standardises all Exchange websockets to use the wshandler websocket. Removes the wsRequestMtx as wshandler handles that now. Makes the Dialer a dialer, its not externally referenced that I can see.

* Fixes issue with coinut implementation. Updates bitmex currencies. Removes redundant log messages which are used to log messages

* Starts testing. Renames files

* Adds tests for websocket connection

* Reverts request.go change

* Linting everything

* Fixes rebase issue

* Final changes. Fixes variable names, removes log.Debug, removes lines, rearranges test types, removes order correlation websocket type

* Final final commit, fixing ZB issues.

* Adds traffic alerts where missed. Changes empty struct pointer addresses to nil instead. Removes empty lines

* Fixed string conversion

* Fixes issue with ZB not sending success codes

* Fixes issue with coinut processing due to nonce handling with subscriptions

* Fixes issue where ZB test failure was not caught. Removes unnecessary error handling from other ZB tests

* Removes unused interface

* Renames wshandler.Init() to wshandler.Run()

* Updates template file

* Capitalises cryptocurrencies in struct. Moves websocketResponseCheckTimeout and websocketResponseMaxLimit into config options. Moves connection configuration to main exchange Setup (where appropriate). Reverts currencylastupdated ticks. Improves reader close error checking

* Fixes two inconsistent websocket delay times

* Creates a default variable for websocket ResponseMaxLimit and ResponseCheckTimeout, then applies it to setdefaults and all tests

* Updates exchange template to set and use default websocket response limits

exchanges/binance/binance.go

@@ -10,20 +10,20 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/gorilla/websocket"
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/config"
 	"github.com/thrasher-/gocryptotrader/currency"
 	exchange "github.com/thrasher-/gocryptotrader/exchanges"
 	"github.com/thrasher-/gocryptotrader/exchanges/request"
 	"github.com/thrasher-/gocryptotrader/exchanges/ticker"
+	"github.com/thrasher-/gocryptotrader/exchanges/wshandler"
 	log "github.com/thrasher-/gocryptotrader/logger"
 )
 
 // Binance is the overarching type across the Bithumb package
 type Binance struct {
 	exchange.Base
-	WebsocketConn *websocket.Conn
+	WebsocketConn *wshandler.WebsocketConnection
 
 	// Valid string list that is required by the exchange
 	validLimits    []int
@@ -93,12 +93,14 @@ func (b *Binance) SetDefaults() {
 		common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 	b.APIUrlDefault = apiURL
 	b.APIUrl = b.APIUrlDefault
-	b.WebsocketInit()
+	b.Websocket = wshandler.New()
 	b.WebsocketURL = binanceDefaultWebsocketURL
-	b.Websocket.Functionality = exchange.WebsocketTradeDataSupported |
-		exchange.WebsocketTickerSupported |
-		exchange.WebsocketKlineSupported |
-		exchange.WebsocketOrderbookSupported
+	b.Websocket.Functionality = wshandler.WebsocketTradeDataSupported |
+		wshandler.WebsocketTickerSupported |
+		wshandler.WebsocketKlineSupported |
+		wshandler.WebsocketOrderbookSupported
+	b.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit
+	b.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout
 }
 
 // Setup takes in the supplied exchange configuration details and sets params
@@ -138,17 +140,27 @@ func (b *Binance) Setup(exch *config.ExchangeConfig) {
 		if err != nil {
 			log.Fatal(err)
 		}
-		err = b.WebsocketSetup(b.WSConnect,
+		err = b.Websocket.Setup(b.WSConnect,
 			nil,
 			nil,
 			exch.Name,
 			exch.Websocket,
 			exch.Verbose,
 			binanceDefaultWebsocketURL,
-			exch.WebsocketURL)
+			exch.WebsocketURL,
+			exch.AuthenticatedWebsocketAPISupport)
 		if err != nil {
 			log.Fatal(err)
 		}
+
+		b.WebsocketConn = &wshandler.WebsocketConnection{
+			ExchangeName:         b.Name,
+			URL:                  b.Websocket.GetWebsocketURL(),
+			ProxyURL:             b.Websocket.GetProxyAddress(),
+			Verbose:              b.Verbose,
+			ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout,
+			ResponseMaxLimit:     exch.WebsocketResponseMaxLimit,
+		}
 	}
 }
 

exchanges/binance/binance_websocket.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
-	"net/url"
 	"strconv"
 	"strings"
 	"sync"
@@ -16,6 +15,7 @@ import (
 	exchange "github.com/thrasher-/gocryptotrader/exchanges"
 	"github.com/thrasher-/gocryptotrader/exchanges/orderbook"
 	"github.com/thrasher-/gocryptotrader/exchanges/ticker"
+	"github.com/thrasher-/gocryptotrader/exchanges/wshandler"
 )
 
 const (
@@ -70,7 +70,7 @@ func (b *Binance) UpdateLocalCache(ob *WebsocketDepthStream) error {
 	ID, ok := lastUpdateID[ob.Pair]
 	if !ok {
 		m.Unlock()
-		return errors.New("binance_websocket.go - Unable to find lastUpdateID")
+		return fmt.Errorf("%v - Unable to find lastUpdateID", b.Name)
 	}
 
 	if ob.LastUpdateID+1 <= ID || ID >= ob.LastUpdateID+1 {
@@ -124,10 +124,10 @@ func (b *Binance) UpdateLocalCache(ob *WebsocketDepthStream) error {
 // WSConnect intiates a websocket connection
 func (b *Binance) WSConnect() error {
 	if !b.Websocket.IsEnabled() || !b.IsEnabled() {
-		return errors.New(exchange.WebsocketNotEnabled)
+		return errors.New(wshandler.WebsocketNotEnabled)
 	}
 
-	var Dialer websocket.Dialer
+	var dialer websocket.Dialer
 	var err error
 
 	tick := strings.ToLower(
@@ -152,18 +152,6 @@ func (b *Binance) WSConnect() error {
 		kline +
 		"/" +
 		depth
-
-	if b.Websocket.GetProxyAddress() != "" {
-		var u *url.URL
-		u, err = url.Parse(b.Websocket.GetProxyAddress())
-		if err != nil {
-			return fmt.Errorf("binance_websocket.go - Unable to connect to parse proxy address. Error: %s",
-				err)
-		}
-
-		Dialer.Proxy = http.ProxyURL(u)
-	}
-
 	for _, ePair := range b.GetEnabledCurrencies() {
 		err = b.SeedLocalCache(ePair)
 		if err != nil {
@@ -171,9 +159,11 @@ func (b *Binance) WSConnect() error {
 		}
 	}
 
-	b.WebsocketConn, _, err = Dialer.Dial(wsurl, http.Header{})
+	b.WebsocketConn.URL = wsurl
+	err = b.WebsocketConn.Dial(&dialer, http.Header{})
 	if err != nil {
-		return fmt.Errorf("binance_websocket.go - Unable to connect to Websocket. Error: %s",
+		return fmt.Errorf("%v - Unable to connect to Websocket. Error: %s",
+			b.Name,
 			err)
 	}
 
@@ -182,18 +172,6 @@ func (b *Binance) WSConnect() error {
 	return nil
 }
 
-// WSReadData reads from the websocket connection and returns the response
-func (b *Binance) WSReadData() (exchange.WebsocketResponse, error) {
-	msgType, resp, err := b.WebsocketConn.ReadMessage()
-
-	if err != nil {
-		return exchange.WebsocketResponse{}, err
-	}
-
-	b.Websocket.TrafficAlert <- struct{}{}
-	return exchange.WebsocketResponse{Type: msgType, Raw: resp}, nil
-}
-
 // WsHandleData handles websocket data from WsReadData
 func (b *Binance) WsHandleData() {
 	b.Websocket.Wg.Add(1)
@@ -208,134 +186,133 @@ func (b *Binance) WsHandleData() {
 			return
 
 		default:
-			read, err := b.WSReadData()
+			read, err := b.WebsocketConn.ReadMessage()
 			if err != nil {
 				b.Websocket.DataHandler <- err
 				return
 			}
-
-			if read.Type == websocket.TextMessage {
-				multiStreamData := MultiStreamData{}
-				err = common.JSONDecode(read.Raw, &multiStreamData)
+			b.Websocket.TrafficAlert <- struct{}{}
+			var multiStreamData MultiStreamData
+			err = common.JSONDecode(read.Raw, &multiStreamData)
+			if err != nil {
+				b.Websocket.DataHandler <- fmt.Errorf("%v - Could not load multi stream data: %s",
+					b.Name,
+					read.Raw)
+				continue
+			}
+			streamType := strings.Split(multiStreamData.Stream, "@")
+			switch streamType[1] {
+			case "trade":
+				trade := TradeStream{}
+				err := common.JSONDecode(multiStreamData.Data, &trade)
 				if err != nil {
-					b.Websocket.DataHandler <- fmt.Errorf("binance_websocket.go - Could not load multi stream data: %s",
-						string(read.Raw))
+					b.Websocket.DataHandler <- fmt.Errorf("%v - Could not unmarshal trade data: %s",
+						b.Name,
+						err)
 					continue
 				}
-				streamType := strings.Split(multiStreamData.Stream, "@")
-				switch streamType[1] {
-				case "trade":
-					trade := TradeStream{}
 
-					err := common.JSONDecode(multiStreamData.Data, &trade)
-					if err != nil {
-						b.Websocket.DataHandler <- fmt.Errorf("binance_websocket.go - Could not unmarshal trade data: %s",
-							err)
-						continue
-					}
-
-					price, err := strconv.ParseFloat(trade.Price, 64)
-					if err != nil {
-						b.Websocket.DataHandler <- fmt.Errorf("binance_websocket.go - price conversion error: %s",
-							err)
-						continue
-					}
-
-					amount, err := strconv.ParseFloat(trade.Quantity, 64)
-					if err != nil {
-						b.Websocket.DataHandler <- fmt.Errorf("binance_websocket.go - amount conversion error: %s",
-							err)
-						continue
-					}
-
-					b.Websocket.DataHandler <- exchange.TradeData{
-						CurrencyPair: currency.NewPairFromString(trade.Symbol),
-						Timestamp:    time.Unix(0, trade.TimeStamp*int64(time.Millisecond)),
-						Price:        price,
-						Amount:       amount,
-						Exchange:     b.GetName(),
-						AssetType:    "SPOT",
-						Side:         trade.EventType,
-					}
-					continue
-				case "ticker":
-					t := TickerStream{}
-
-					err := common.JSONDecode(multiStreamData.Data, &t)
-					if err != nil {
-						b.Websocket.DataHandler <- fmt.Errorf("binance_websocket.go - Could not convert to a TickerStream structure %s",
-							err.Error())
-						continue
-					}
-
-					var wsTicker exchange.TickerData
-
-					wsTicker.Timestamp = time.Unix(t.EventTime/1000, 0)
-					wsTicker.Pair = currency.NewPairFromString(t.Symbol)
-					wsTicker.AssetType = ticker.Spot
-					wsTicker.Exchange = b.GetName()
-					wsTicker.ClosePrice, _ = strconv.ParseFloat(t.CurrDayClose, 64)
-					wsTicker.Quantity, _ = strconv.ParseFloat(t.TotalTradedVolume, 64)
-					wsTicker.OpenPrice, _ = strconv.ParseFloat(t.OpenPrice, 64)
-					wsTicker.HighPrice, _ = strconv.ParseFloat(t.HighPrice, 64)
-					wsTicker.LowPrice, _ = strconv.ParseFloat(t.LowPrice, 64)
-
-					b.Websocket.DataHandler <- wsTicker
-
-					continue
-				case "kline":
-					kline := KlineStream{}
-
-					err := common.JSONDecode(multiStreamData.Data, &kline)
-					if err != nil {
-						b.Websocket.DataHandler <- fmt.Errorf("binance_websocket.go - Could not convert to a KlineStream structure %s",
-							err)
-						continue
-					}
-
-					var wsKline exchange.KlineData
-
-					wsKline.Timestamp = time.Unix(0, kline.EventTime)
-					wsKline.Pair = currency.NewPairFromString(kline.Symbol)
-					wsKline.AssetType = ticker.Spot
-					wsKline.Exchange = b.GetName()
-					wsKline.StartTime = time.Unix(0, kline.Kline.StartTime)
-					wsKline.CloseTime = time.Unix(0, kline.Kline.CloseTime)
-					wsKline.Interval = kline.Kline.Interval
-					wsKline.OpenPrice, _ = strconv.ParseFloat(kline.Kline.OpenPrice, 64)
-					wsKline.ClosePrice, _ = strconv.ParseFloat(kline.Kline.ClosePrice, 64)
-					wsKline.HighPrice, _ = strconv.ParseFloat(kline.Kline.HighPrice, 64)
-					wsKline.LowPrice, _ = strconv.ParseFloat(kline.Kline.LowPrice, 64)
-					wsKline.Volume, _ = strconv.ParseFloat(kline.Kline.Volume, 64)
-
-					b.Websocket.DataHandler <- wsKline
-					continue
-				case "depth":
-					depth := WebsocketDepthStream{}
-
-					err := common.JSONDecode(multiStreamData.Data, &depth)
-					if err != nil {
-						b.Websocket.DataHandler <- fmt.Errorf("binance_websocket.go - Could not convert to depthStream structure %s",
-							err)
-						continue
-					}
-
-					err = b.UpdateLocalCache(&depth)
-					if err != nil {
-						b.Websocket.DataHandler <- fmt.Errorf("binance_websocket.go - UpdateLocalCache error: %s",
-							err)
-						continue
-					}
-
-					currencyPair := currency.NewPairFromString(depth.Pair)
-
-					b.Websocket.DataHandler <- exchange.WebsocketOrderbookUpdate{
-						Pair:     currencyPair,
-						Asset:    "SPOT",
-						Exchange: b.GetName(),
-					}
+				price, err := strconv.ParseFloat(trade.Price, 64)
+				if err != nil {
+					b.Websocket.DataHandler <- fmt.Errorf("%v - price conversion error: %s",
+						b.Name,
+						err)
 					continue
 				}
+
+				amount, err := strconv.ParseFloat(trade.Quantity, 64)
+				if err != nil {
+					b.Websocket.DataHandler <- fmt.Errorf("%v - amount conversion error: %s",
+						b.Name,
+						err)
+					continue
+				}
+
+				b.Websocket.DataHandler <- wshandler.TradeData{
+					CurrencyPair: currency.NewPairFromString(trade.Symbol),
+					Timestamp:    time.Unix(0, trade.TimeStamp),
+					Price:        price,
+					Amount:       amount,
+					Exchange:     b.GetName(),
+					AssetType:    "SPOT",
+					Side:         trade.EventType,
+				}
+				continue
+			case "ticker":
+				t := TickerStream{}
+				err := common.JSONDecode(multiStreamData.Data, &t)
+				if err != nil {
+					b.Websocket.DataHandler <- fmt.Errorf("%v - Could not convert to a TickerStream structure %s",
+						b.Name,
+						err.Error())
+					continue
+				}
+
+				var wsTicker wshandler.TickerData
+
+				wsTicker.Timestamp = time.Unix(t.EventTime/1000, 0)
+				wsTicker.Pair = currency.NewPairFromString(t.Symbol)
+				wsTicker.AssetType = ticker.Spot
+				wsTicker.Exchange = b.GetName()
+				wsTicker.ClosePrice, _ = strconv.ParseFloat(t.CurrDayClose, 64)
+				wsTicker.Quantity, _ = strconv.ParseFloat(t.TotalTradedVolume, 64)
+				wsTicker.OpenPrice, _ = strconv.ParseFloat(t.OpenPrice, 64)
+				wsTicker.HighPrice, _ = strconv.ParseFloat(t.HighPrice, 64)
+				wsTicker.LowPrice, _ = strconv.ParseFloat(t.LowPrice, 64)
+
+				b.Websocket.DataHandler <- wsTicker
+
+				continue
+			case "kline":
+				kline := KlineStream{}
+				err := common.JSONDecode(multiStreamData.Data, &kline)
+				if err != nil {
+					b.Websocket.DataHandler <- fmt.Errorf("%v - Could not convert to a KlineStream structure %s",
+						b.Name,
+						err)
+					continue
+				}
+
+				var wsKline wshandler.KlineData
+				wsKline.Timestamp = time.Unix(0, kline.EventTime)
+				wsKline.Pair = currency.NewPairFromString(kline.Symbol)
+				wsKline.AssetType = ticker.Spot
+				wsKline.Exchange = b.GetName()
+				wsKline.StartTime = time.Unix(0, kline.Kline.StartTime)
+				wsKline.CloseTime = time.Unix(0, kline.Kline.CloseTime)
+				wsKline.Interval = kline.Kline.Interval
+				wsKline.OpenPrice, _ = strconv.ParseFloat(kline.Kline.OpenPrice, 64)
+				wsKline.ClosePrice, _ = strconv.ParseFloat(kline.Kline.ClosePrice, 64)
+				wsKline.HighPrice, _ = strconv.ParseFloat(kline.Kline.HighPrice, 64)
+				wsKline.LowPrice, _ = strconv.ParseFloat(kline.Kline.LowPrice, 64)
+				wsKline.Volume, _ = strconv.ParseFloat(kline.Kline.Volume, 64)
+				b.Websocket.DataHandler <- wsKline
+				continue
+			case "depth":
+				depth := WebsocketDepthStream{}
+				err := common.JSONDecode(multiStreamData.Data, &depth)
+				if err != nil {
+					b.Websocket.DataHandler <- fmt.Errorf("%v - Could not convert to depthStream structure %s",
+						b.Name,
+						err)
+					continue
+				}
+
+				err = b.UpdateLocalCache(&depth)
+				if err != nil {
+					b.Websocket.DataHandler <- fmt.Errorf("%v - UpdateLocalCache error: %s",
+						b.Name,
+						err)
+					continue
+				}
+
+				currencyPair := currency.NewPairFromString(depth.Pair)
+				b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{
+					Pair:     currencyPair,
+					Asset:    "SPOT",
+					Exchange: b.GetName(),
+				}
+				continue
 			}
 		}
 	}

exchanges/binance/binance_wrapper.go

@@ -13,6 +13,7 @@ import (
 	exchange "github.com/thrasher-/gocryptotrader/exchanges"
 	"github.com/thrasher-/gocryptotrader/exchanges/orderbook"
 	"github.com/thrasher-/gocryptotrader/exchanges/ticker"
+	"github.com/thrasher-/gocryptotrader/exchanges/wshandler"
 	log "github.com/thrasher-/gocryptotrader/logger"
 )
 
@@ -318,7 +319,7 @@ func (b *Binance) WithdrawFiatFundsToInternationalBank(withdrawRequest *exchange
 }
 
 // GetWebsocket returns a pointer to the exchange websocket
-func (b *Binance) GetWebsocket() (*exchange.Websocket, error) {
+func (b *Binance) GetWebsocket() (*wshandler.Websocket, error) {
 	return b.Websocket, nil
 }
 
@@ -416,18 +417,18 @@ func (b *Binance) GetOrderHistory(getOrdersRequest *exchange.GetOrdersRequest) (
 
 // SubscribeToWebsocketChannels appends to ChannelsToSubscribe
 // which lets websocket.manageSubscriptions handle subscribing
-func (b *Binance) SubscribeToWebsocketChannels(channels []exchange.WebsocketChannelSubscription) error {
+func (b *Binance) SubscribeToWebsocketChannels(channels []wshandler.WebsocketChannelSubscription) error {
 	return common.ErrFunctionNotSupported
 }
 
 // UnsubscribeToWebsocketChannels removes from ChannelsToSubscribe
 // which lets websocket.manageSubscriptions handle unsubscribing
-func (b *Binance) UnsubscribeToWebsocketChannels(channels []exchange.WebsocketChannelSubscription) error {
+func (b *Binance) UnsubscribeToWebsocketChannels(channels []wshandler.WebsocketChannelSubscription) error {
 	return common.ErrFunctionNotSupported
 }
 
 // GetSubscriptions returns a copied list of subscriptions
-func (b *Binance) GetSubscriptions() ([]exchange.WebsocketChannelSubscription, error) {
+func (b *Binance) GetSubscriptions() ([]wshandler.WebsocketChannelSubscription, error) {
 	return b.Websocket.GetSubscriptions(), nil
 }