Websocket request-response correlation (#328)

* Establishes new websocket functionality. Begins the creation of the websocket request

* Adding a wrapper over gorilla websocket connect,send,receive to handle ID messages. Doesn't work

* Successfully moved exchange_websocket into its own wshandler namespace. oof

* Sets up ZB to use a round trip WS request system

* Adds Kraken ID support to subscriptions. Renames duplicate func name UnsubscribeToChannels to RemoveSubscribedChannels. Adds some helper methods in the WebsocketConn to reduce duplicate code. Cleans up ZB implementation

* Fixes double locking which caused no websocket data to be read. Fixes requestid for kraken subscriptions

* Completes Huobi and Hadax implementation. Extends ZB error handling. Adds GZip support for reading messages

* Adds HitBTC support. Adds GetCurrencies, GetSymbols, GetTrades WS funcs. Adds super fun new parameter to GenerateMessageID for Unix and UnixNano

* Adds GateIO id support

* Adds Coinut support. Prevents nil reference error in constatus when there isnt one

* Standardises all Exchange websockets to use the wshandler websocket. Removes the wsRequestMtx as wshandler handles that now. Makes the Dialer a dialer, its not externally referenced that I can see.

* Fixes issue with coinut implementation. Updates bitmex currencies. Removes redundant log messages which are used to log messages

* Starts testing. Renames files

* Adds tests for websocket connection

* Reverts request.go change

* Linting everything

* Fixes rebase issue

* Final changes. Fixes variable names, removes log.Debug, removes lines, rearranges test types, removes order correlation websocket type

* Final final commit, fixing ZB issues.

* Adds traffic alerts where missed. Changes empty struct pointer addresses to nil instead. Removes empty lines

* Fixed string conversion

* Fixes issue with ZB not sending success codes

* Fixes issue with coinut processing due to nonce handling with subscriptions

* Fixes issue where ZB test failure was not caught. Removes unnecessary error handling from other ZB tests

* Removes unused interface

* Renames wshandler.Init() to wshandler.Run()

* Updates template file

* Capitalises cryptocurrencies in struct. Moves websocketResponseCheckTimeout and websocketResponseMaxLimit into config options. Moves connection configuration to main exchange Setup (where appropriate). Reverts currencylastupdated ticks. Improves reader close error checking

* Fixes two inconsistent websocket delay times

* Creates a default variable for websocket ResponseMaxLimit and ResponseCheckTimeout, then applies it to setdefaults and all tests

* Updates exchange template to set and use default websocket response limits

exchanges/btse/btse.go

@@ -6,24 +6,22 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
-	"github.com/gorilla/websocket"
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/config"
 	"github.com/thrasher-/gocryptotrader/currency"
 	exchange "github.com/thrasher-/gocryptotrader/exchanges"
 	"github.com/thrasher-/gocryptotrader/exchanges/request"
 	"github.com/thrasher-/gocryptotrader/exchanges/ticker"
+	"github.com/thrasher-/gocryptotrader/exchanges/wshandler"
 	log "github.com/thrasher-/gocryptotrader/logger"
 )
 
 // BTSE is the overarching type across this package
 type BTSE struct {
 	exchange.Base
-	WebsocketConn *websocket.Conn
-	wsRequestMtx  sync.Mutex
+	WebsocketConn *wshandler.WebsocketConnection
 }
 
 const (
@@ -66,11 +64,13 @@ func (b *BTSE) SetDefaults() {
 	b.APIUrl = b.APIUrlDefault
 	b.SupportsAutoPairUpdating = true
 	b.SupportsRESTTickerBatching = false
-	b.WebsocketInit()
-	b.Websocket.Functionality = exchange.WebsocketOrderbookSupported |
-		exchange.WebsocketTickerSupported |
-		exchange.WebsocketSubscribeSupported |
-		exchange.WebsocketUnsubscribeSupported
+	b.Websocket = wshandler.New()
+	b.Websocket.Functionality = wshandler.WebsocketOrderbookSupported |
+		wshandler.WebsocketTickerSupported |
+		wshandler.WebsocketSubscribeSupported |
+		wshandler.WebsocketUnsubscribeSupported
+	b.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit
+	b.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout
 }
 
 // Setup takes in the supplied exchange configuration details and sets params
@@ -109,17 +109,26 @@ func (b *BTSE) Setup(exch *config.ExchangeConfig) {
 		if err != nil {
 			log.Fatal(err)
 		}
-		err = b.WebsocketSetup(b.WsConnect,
+		err = b.Websocket.Setup(b.WsConnect,
 			b.Subscribe,
 			b.Unsubscribe,
 			exch.Name,
 			exch.Websocket,
 			exch.Verbose,
 			btseWebsocket,
-			exch.WebsocketURL)
+			exch.WebsocketURL,
+			exch.AuthenticatedWebsocketAPISupport)
 		if err != nil {
 			log.Fatal(err)
 		}
+		b.WebsocketConn = &wshandler.WebsocketConnection{
+			ExchangeName:         b.Name,
+			URL:                  b.Websocket.GetWebsocketURL(),
+			ProxyURL:             b.Websocket.GetProxyAddress(),
+			Verbose:              b.Verbose,
+			ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout,
+			ResponseMaxLimit:     exch.WebsocketResponseMaxLimit,
+		}
 	}
 }
 

exchanges/btse/btse_websocket.go

@@ -2,9 +2,7 @@ package btse
 
 import (
 	"errors"
-	"fmt"
 	"net/http"
-	"net/url"
 	"strconv"
 	"strings"
 	"time"
@@ -12,8 +10,8 @@ import (
 	"github.com/gorilla/websocket"
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/currency"
-	exchange "github.com/thrasher-/gocryptotrader/exchanges"
 	"github.com/thrasher-/gocryptotrader/exchanges/orderbook"
+	"github.com/thrasher-/gocryptotrader/exchanges/wshandler"
 	log "github.com/thrasher-/gocryptotrader/logger"
 )
 
@@ -24,27 +22,12 @@ const (
 // WsConnect connects the websocket client
 func (b *BTSE) WsConnect() error {
 	if !b.Websocket.IsEnabled() || !b.IsEnabled() {
-		return errors.New(exchange.WebsocketNotEnabled)
+		return errors.New(wshandler.WebsocketNotEnabled)
 	}
-
 	var dialer websocket.Dialer
-
-	if b.Websocket.GetProxyAddress() != "" {
-		proxy, err := url.Parse(b.Websocket.GetProxyAddress())
-		if err != nil {
-			return fmt.Errorf("%s websocket error - proxy address %s",
-				b.Name, err)
-		}
-
-		dialer.Proxy = http.ProxyURL(proxy)
-	}
-
-	var err error
-	b.WebsocketConn, _, err = dialer.Dial(b.Websocket.GetWebsocketURL(),
-		http.Header{})
+	err := b.WebsocketConn.Dial(&dialer, http.Header{})
 	if err != nil {
-		return fmt.Errorf("%s websocket error - unable to connect %s",
-			b.Name, err)
+		return err
 	}
 
 	go b.WsHandleData()
@@ -53,17 +36,6 @@ func (b *BTSE) WsConnect() error {
 	return nil
 }
 
-// WsReadData reads data from the websocket connection
-func (b *BTSE) WsReadData() (exchange.WebsocketResponse, error) {
-	_, resp, err := b.WebsocketConn.ReadMessage()
-	if err != nil {
-		return exchange.WebsocketResponse{}, err
-	}
-
-	b.Websocket.TrafficAlert <- struct{}{}
-	return exchange.WebsocketResponse{Raw: resp}, nil
-}
-
 // WsHandleData handles read data from websocket connection
 func (b *BTSE) WsHandleData() {
 	b.Websocket.Wg.Add(1)
@@ -78,11 +50,12 @@ func (b *BTSE) WsHandleData() {
 			return
 
 		default:
-			resp, err := b.WsReadData()
+			resp, err := b.WebsocketConn.ReadMessage()
 			if err != nil {
 				b.Websocket.DataHandler <- err
 				return
 			}
+			b.Websocket.TrafficAlert <- struct{}{}
 
 			type MsgType struct {
 				Type      string `json:"type"`
@@ -118,7 +91,7 @@ func (b *BTSE) WsHandleData() {
 					continue
 				}
 
-				b.Websocket.DataHandler <- exchange.TickerData{
+				b.Websocket.DataHandler <- wshandler.TickerData{
 					Timestamp: time.Now(),
 					Pair:      currency.NewPairDelimiter(t.ProductID, "-"),
 					AssetType: "SPOT",
@@ -191,7 +164,7 @@ func (b *BTSE) wsProcessSnapshot(snapshot *websocketOrderbookSnapshot) error {
 		return err
 	}
 
-	b.Websocket.DataHandler <- exchange.WebsocketOrderbookUpdate{
+	b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{
 		Pair:     p,
 		Asset:    "SPOT",
 		Exchange: b.GetName(),
@@ -204,10 +177,10 @@ func (b *BTSE) wsProcessSnapshot(snapshot *websocketOrderbookSnapshot) error {
 func (b *BTSE) GenerateDefaultSubscriptions() {
 	var channels = []string{"snapshot", "ticker"}
 	enabledCurrencies := b.GetEnabledCurrencies()
-	var subscriptions []exchange.WebsocketChannelSubscription
+	var subscriptions []wshandler.WebsocketChannelSubscription
 	for i := range channels {
 		for j := range enabledCurrencies {
-			subscriptions = append(subscriptions, exchange.WebsocketChannelSubscription{
+			subscriptions = append(subscriptions, wshandler.WebsocketChannelSubscription{
 				Channel:  channels[i],
 				Currency: enabledCurrencies[j],
 			})
@@ -217,7 +190,7 @@ func (b *BTSE) GenerateDefaultSubscriptions() {
 }
 
 // Subscribe sends a websocket message to receive data from the channel
-func (b *BTSE) Subscribe(channelToSubscribe exchange.WebsocketChannelSubscription) error {
+func (b *BTSE) Subscribe(channelToSubscribe wshandler.WebsocketChannelSubscription) error {
 	subscribe := websocketSubscribe{
 		Type: "subscribe",
 		Channels: []websocketChannel{
@@ -227,11 +200,11 @@ func (b *BTSE) Subscribe(channelToSubscribe exchange.WebsocketChannelSubscriptio
 			},
 		},
 	}
-	return b.wsSend(subscribe)
+	return b.WebsocketConn.SendMessage(subscribe)
 }
 
 // Unsubscribe sends a websocket message to stop receiving data from the channel
-func (b *BTSE) Unsubscribe(channelToSubscribe exchange.WebsocketChannelSubscription) error {
+func (b *BTSE) Unsubscribe(channelToSubscribe wshandler.WebsocketChannelSubscription) error {
 	subscribe := websocketSubscribe{
 		Type: "unsubscribe",
 		Channels: []websocketChannel{
@@ -241,19 +214,5 @@ func (b *BTSE) Unsubscribe(channelToSubscribe exchange.WebsocketChannelSubscript
 			},
 		},
 	}
-	return b.wsSend(subscribe)
-}
-
-// WsSend sends data to the websocket server
-func (b *BTSE) wsSend(data interface{}) error {
-	b.wsRequestMtx.Lock()
-	defer b.wsRequestMtx.Unlock()
-	if b.Verbose {
-		log.Debugf("%v sending message to websocket %v", b.Name, data)
-	}
-	json, err := common.JSONEncode(data)
-	if err != nil {
-		return err
-	}
-	return b.WebsocketConn.WriteMessage(websocket.TextMessage, json)
+	return b.WebsocketConn.SendMessage(subscribe)
 }

exchanges/btse/btse_wrapper.go

@@ -12,6 +12,7 @@ import (
 	exchange "github.com/thrasher-/gocryptotrader/exchanges"
 	"github.com/thrasher-/gocryptotrader/exchanges/orderbook"
 	"github.com/thrasher-/gocryptotrader/exchanges/ticker"
+	"github.com/thrasher-/gocryptotrader/exchanges/wshandler"
 	log "github.com/thrasher-/gocryptotrader/logger"
 )
 
@@ -284,7 +285,7 @@ func (b *BTSE) WithdrawFiatFundsToInternationalBank(withdrawRequest *exchange.Wi
 }
 
 // GetWebsocket returns a pointer to the exchange websocket
-func (b *BTSE) GetWebsocket() (*exchange.Websocket, error) {
+func (b *BTSE) GetWebsocket() (*wshandler.Websocket, error) {
 	return b.Websocket, nil
 }
 
@@ -361,20 +362,20 @@ func (b *BTSE) GetFeeByType(feeBuilder *exchange.FeeBuilder) (float64, error) {
 
 // SubscribeToWebsocketChannels appends to ChannelsToSubscribe
 // which lets websocket.manageSubscriptions handle subscribing
-func (b *BTSE) SubscribeToWebsocketChannels(channels []exchange.WebsocketChannelSubscription) error {
+func (b *BTSE) SubscribeToWebsocketChannels(channels []wshandler.WebsocketChannelSubscription) error {
 	b.Websocket.SubscribeToChannels(channels)
 	return nil
 }
 
 // UnsubscribeToWebsocketChannels removes from ChannelsToSubscribe
 // which lets websocket.manageSubscriptions handle unsubscribing
-func (b *BTSE) UnsubscribeToWebsocketChannels(channels []exchange.WebsocketChannelSubscription) error {
-	b.Websocket.UnsubscribeToChannels(channels)
+func (b *BTSE) UnsubscribeToWebsocketChannels(channels []wshandler.WebsocketChannelSubscription) error {
+	b.Websocket.RemoveSubscribedChannels(channels)
 	return nil
 }
 
 // GetSubscriptions returns a copied list of subscriptions
-func (b *BTSE) GetSubscriptions() ([]exchange.WebsocketChannelSubscription, error) {
+func (b *BTSE) GetSubscriptions() ([]wshandler.WebsocketChannelSubscription, error) {
 	return b.Websocket.GetSubscriptions(), nil
 }