Authenticated Websocket support (#315)

* Improves subscribing by not allowing duplicates. Adds bitmex auth support

* Adds coinbase pro support. Partial BTCC support. Adds WebsocketAuthenticatedEndpointsSupported websocket feature. Adds GateIO support

* Adds Coinut support

* Moves Coinut WS types to file. Implements Gemini's secure WS endpoint

* Adds HitBTC ws authenticated support. Fixes var names

* Adds huobi and hadax authenticated websocket support

* Adds auth to okgroup (okex, okcoin). Fixes some linting

* Adds Poloniex support

* Adds ZB support

* Adds proper bitmex support

* Improves bitfinex support, improves websocket functionality definitions

* Fixes coinbasepro auth

* Tests all endpoints

* go formatting, importing, linting run

* Adds wrapper supports

* General clean up. Data race destruction

* Improves testing on all exchanges except ZB

* Fixes ZB hashing, parsing and tests

* minor nits before someone else sees them <_<

* Fixes some nits pertaining to variable usage, comments, typos and rate limiting

* Addresses nits regarding types and test responses where applicable

* fmt import

* Fixes linting issues

* No longer returns an error on failure to authenticate, just logs. Adds new AuthenticatedWebsocketAPISupport config value to allow a user to seperate auth from REST and WS. Prevents WS auth if AuthenticatedWebsocketAPISupport is false, adds additional login check 'CanUseAuthenticatedEndpoints' for when login only occurs once (not per request). Removes unnecessary time.Sleeps from code. Moves WS auth error logic to auth function so that wrappers can get involved in all the auth fun. New-fandangled shared test package, used exclusively in testing, will be the store of all the constant boilerplate things like timeout values. Moves WS test setup function to only run once when there are multiple WS endpoint tests. Cleans up some struct types

* Increases test coverage with tests for config.areAuthenticatedCredentialsValid config.CheckExchangeConfigValues, exchange.SetAPIKeys, exchange.GetAuthenticatedAPISupport, exchange_websocket.CanUseAuthenticatedEndpoitns and exchange_websocket.SetCanUseAuthenticatedEndpoints. Adds b.Websocket.SetCanUseAuthenticatedEndpoints(false) when bitfinex fails to authenticate
Fixes a typo. gofmt and goimport

* Trim Test Typos

* Reformats various websocket types. Adds more specific error messaging to config.areAuthenticatedCredentialsValid

exchanges/bitmex/bitmex.go

@@ -139,7 +139,9 @@ func (b *Bitmex) SetDefaults() {
 	b.Websocket.Functionality = exchange.WebsocketTradeDataSupported |
 		exchange.WebsocketOrderbookSupported |
 		exchange.WebsocketSubscribeSupported |
-		exchange.WebsocketUnsubscribeSupported
+		exchange.WebsocketUnsubscribeSupported |
+		exchange.WebsocketAuthenticatedEndpointsSupported |
+		exchange.WebsocketAccountDataSupported
 }
 
 // Setup takes in the supplied exchange configuration details and sets params
@@ -149,6 +151,7 @@ func (b *Bitmex) Setup(exch *config.ExchangeConfig) {
 	} else {
 		b.Enabled = true
 		b.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
+		b.AuthenticatedWebsocketAPISupport = exch.AuthenticatedWebsocketAPISupport
 		b.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
 		b.RESTPollingDelay = exch.RESTPollingDelay
 		b.Verbose = exch.Verbose

exchanges/bitmex/bitmex_test.go

@@ -1,14 +1,17 @@
 package bitmex
 
 import (
+	"net/http"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/gorilla/websocket"
 	"github.com/thrasher-/gocryptotrader/common"
 	"github.com/thrasher-/gocryptotrader/config"
 	"github.com/thrasher-/gocryptotrader/currency"
 	exchange "github.com/thrasher-/gocryptotrader/exchanges"
+	"github.com/thrasher-/gocryptotrader/exchanges/sharedtestvalues"
 )
 
 // Please supply your own keys here for due diligence testing
@@ -31,6 +34,7 @@ func TestSetup(t *testing.T) {
 	if err != nil {
 		t.Error("Test Failed - Bitmex Setup() init error")
 	}
+	bitmexConfig.AuthenticatedWebsocketAPISupport = true
 	bitmexConfig.AuthenticatedAPISupport = true
 	bitmexConfig.APIKey = apiKey
 	bitmexConfig.APISecret = apiSecret
@@ -679,3 +683,37 @@ func TestGetDepositAddress(t *testing.T) {
 		}
 	}
 }
+
+// TestWsAuth dials websocket, sends login request.
+func TestWsAuth(t *testing.T) {
+	b.SetDefaults()
+	TestSetup(t)
+	if !b.Websocket.IsEnabled() && !b.AuthenticatedWebsocketAPISupport || !areTestAPIKeysSet() {
+		t.Skip(exchange.WebsocketNotEnabled)
+	}
+	var err error
+	var dialer websocket.Dialer
+	b.WebsocketConn, _, err = dialer.Dial(b.Websocket.GetWebsocketURL(),
+		http.Header{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	b.Websocket.DataHandler = sharedtestvalues.GetWebsocketInterfaceChannelOverride()
+	b.Websocket.TrafficAlert = sharedtestvalues.GetWebsocketStructChannelOverride()
+	go b.wsHandleIncomingData()
+	defer b.WebsocketConn.Close()
+	err = b.websocketSendAuth()
+	if err != nil {
+		t.Error(err)
+	}
+	timer := time.NewTimer(sharedtestvalues.WebsocketResponseDefaultTimeout)
+	select {
+	case resp := <-b.Websocket.DataHandler:
+		if !resp.(WebsocketSubscribeResp).Success {
+			t.Error("Expected successful subscription")
+		}
+	case <-timer.C:
+		t.Error("Have not received a response")
+	}
+	timer.Stop()
+}

exchanges/bitmex/bitmex_websocket.go

@@ -107,12 +107,11 @@ func (b *Bitmex) WsConnector() error {
 	go b.wsHandleIncomingData()
 	b.GenerateDefaultSubscriptions()
 
-	if b.AuthenticatedAPISupport {
-		err := b.websocketSendAuth()
-		if err != nil {
-			return err
-		}
+	err = b.websocketSendAuth()
+	if err != nil {
+		log.Errorf("%v - authentication failed: %v", b.Name, err)
 	}
+	b.GenerateAuthenticatedSubscriptions()
 	return nil
 }
 
@@ -190,11 +189,15 @@ func (b *Bitmex) wsHandleIncomingData() {
 				}
 
 				if decodedResp.Success {
-					if b.Verbose {
-						if len(quickCapture) == 3 {
+					b.Websocket.DataHandler <- decodedResp
+					if len(quickCapture) == 3 {
+						if b.Verbose {
 							log.Debugf("%s websocket: Successfully subscribed to %s",
 								b.Name, decodedResp.Subscribe)
-						} else {
+						}
+					} else {
+						b.Websocket.SetCanUseAuthenticatedEndpoints(true)
+						if b.Verbose {
 							log.Debugf("%s websocket: Successfully authenticated websocket connection",
 								b.Name)
 						}
@@ -264,7 +267,6 @@ func (b *Bitmex) wsHandleIncomingData() {
 
 				case bitmexWSAnnouncement:
 					var announcement AnnouncementData
-
 					err = common.JSONDecode(resp.Raw, &announcement)
 					if err != nil {
 						b.Websocket.DataHandler <- err
@@ -276,7 +278,70 @@ func (b *Bitmex) wsHandleIncomingData() {
 					}
 
 					b.Websocket.DataHandler <- announcement.Data
-
+				case bitmexWSAffiliate:
+					var response WsAffiliateResponse
+					err = common.JSONDecode(resp.Raw, &response)
+					if err != nil {
+						b.Websocket.DataHandler <- err
+						continue
+					}
+					b.Websocket.DataHandler <- response
+				case bitmexWSExecution:
+					var response WsExecutionResponse
+					err = common.JSONDecode(resp.Raw, &response)
+					if err != nil {
+						b.Websocket.DataHandler <- err
+						continue
+					}
+					b.Websocket.DataHandler <- response
+				case bitmexWSOrder:
+					var response WsOrderResponse
+					err = common.JSONDecode(resp.Raw, &response)
+					if err != nil {
+						b.Websocket.DataHandler <- err
+						continue
+					}
+					b.Websocket.DataHandler <- response
+				case bitmexWSMargin:
+					var response WsMarginResponse
+					err = common.JSONDecode(resp.Raw, &response)
+					if err != nil {
+						b.Websocket.DataHandler <- err
+						continue
+					}
+					b.Websocket.DataHandler <- response
+				case bitmexWSPosition:
+					var response WsPositionResponse
+					err = common.JSONDecode(resp.Raw, &response)
+					if err != nil {
+						b.Websocket.DataHandler <- err
+						continue
+					}
+					b.Websocket.DataHandler <- response
+				case bitmexWSPrivateNotifications:
+					var response WsPrivateNotificationsResponse
+					err = common.JSONDecode(resp.Raw, &response)
+					if err != nil {
+						b.Websocket.DataHandler <- err
+						continue
+					}
+					b.Websocket.DataHandler <- response
+				case bitmexWSTransact:
+					var response WsTransactResponse
+					err = common.JSONDecode(resp.Raw, &response)
+					if err != nil {
+						b.Websocket.DataHandler <- err
+						continue
+					}
+					b.Websocket.DataHandler <- response
+				case bitmexWSWallet:
+					var response WsWalletResponse
+					err = common.JSONDecode(resp.Raw, &response)
+					if err != nil {
+						b.Websocket.DataHandler <- err
+						continue
+					}
+					b.Websocket.DataHandler <- response
 				default:
 					b.Websocket.DataHandler <- fmt.Errorf("%s websocket error: Table unknown - %s",
 						b.Name, decodedResp.Table)
@@ -393,6 +458,47 @@ func (b *Bitmex) GenerateDefaultSubscriptions() {
 			Channel: bitmexWSAnnouncement,
 		},
 	}
+
+	for i := range channels {
+		for j := range contracts {
+			subscriptions = append(subscriptions, exchange.WebsocketChannelSubscription{
+				Channel:  fmt.Sprintf("%v:%v", channels[i], contracts[j].String()),
+				Currency: contracts[j],
+			})
+		}
+	}
+	b.Websocket.SubscribeToChannels(subscriptions)
+}
+
+// GenerateAuthenticatedSubscriptions Adds authenticated subscriptions to websocket to be handled by ManageSubscriptions()
+func (b *Bitmex) GenerateAuthenticatedSubscriptions() {
+	if !b.Websocket.CanUseAuthenticatedEndpoints() {
+		return
+	}
+	contracts := b.GetEnabledCurrencies()
+	channels := []string{bitmexWSExecution,
+		bitmexWSPosition,
+	}
+	subscriptions := []exchange.WebsocketChannelSubscription{
+		{
+			Channel: bitmexWSAffiliate,
+		},
+		{
+			Channel: bitmexWSOrder,
+		},
+		{
+			Channel: bitmexWSMargin,
+		},
+		{
+			Channel: bitmexWSPrivateNotifications,
+		},
+		{
+			Channel: bitmexWSTransact,
+		},
+		{
+			Channel: bitmexWSWallet,
+		},
+	}
 	for i := range channels {
 		for j := range contracts {
 			subscriptions = append(subscriptions, exchange.WebsocketChannelSubscription{
@@ -424,18 +530,26 @@ func (b *Bitmex) Unsubscribe(channelToSubscribe exchange.WebsocketChannelSubscri
 
 // WebsocketSendAuth sends an authenticated subscription
 func (b *Bitmex) websocketSendAuth() error {
+	if !b.GetAuthenticatedAPISupport(exchange.WebsocketAuthentication) {
+		return fmt.Errorf("%v AuthenticatedWebsocketAPISupport not enabled", b.Name)
+	}
+	b.Websocket.SetCanUseAuthenticatedEndpoints(true)
 	timestamp := time.Now().Add(time.Hour * 1).Unix()
 	newTimestamp := strconv.FormatInt(timestamp, 10)
 	hmac := common.GetHMAC(common.HashSHA256,
 		[]byte("GET/realtime"+newTimestamp),
 		[]byte(b.APISecret))
 	signature := common.HexEncodeToString(hmac)
-
 	var sendAuth WebsocketRequest
 	sendAuth.Command = "authKeyExpires"
 	sendAuth.Arguments = append(sendAuth.Arguments, b.APIKey, timestamp,
 		signature)
-	return b.wsSend(sendAuth)
+	err := b.wsSend(sendAuth)
+	if err != nil {
+		b.Websocket.SetCanUseAuthenticatedEndpoints(false)
+		return err
+	}
+	return nil
 }
 
 // WsSend sends data to the websocket server

exchanges/bitmex/bitmex_websocket_types.go

@@ -70,3 +70,260 @@ type AnnouncementData struct {
 	Data   []Announcement `json:"data"`
 	Action string         `json:"action"`
 }
+
+// WsAffiliateResponse private api response
+type WsAffiliateResponse struct {
+	WsDataResponse
+	ForeignKeys interface{}                   `json:"foreignKeys"`
+	Attributes  WsAffiliateResponseAttributes `json:"attributes"`
+	Filter      WsAffiliateResponseFilter     `json:"filter"`
+	Data        []interface{}                 `json:"data"`
+}
+
+// WsAffiliateResponseAttributes private api data
+type WsAffiliateResponseAttributes struct {
+	Account  string `json:"account"`
+	Currency string `json:"currency"`
+}
+
+// WsAffiliateResponseFilter private api data
+type WsAffiliateResponseFilter struct {
+	Account int64 `json:"account"`
+}
+
+// WsOrderResponse private api response
+type WsOrderResponse struct {
+	WsDataResponse
+	ForeignKeys WsOrderResponseForeignKeys `json:"foreignKeys"`
+	Attributes  WsOrderResponseAttributes  `json:"attributes"`
+	Filter      WsOrderResponseFilter      `json:"filter"`
+	Data        []interface{}              `json:"data"`
+}
+
+// WsOrderResponseAttributes private api data
+type WsOrderResponseAttributes struct {
+	OrderID          string `json:"orderID"`
+	Account          string `json:"account"`
+	OrdStatus        string `json:"ordStatus"`
+	WorkingIndicator string `json:"workingIndicator"`
+}
+
+// WsOrderResponseFilter private api data
+type WsOrderResponseFilter struct {
+	Account int64 `json:"account"`
+}
+
+// WsOrderResponseForeignKeys private api data
+type WsOrderResponseForeignKeys struct {
+	Symbol    string `json:"symbol"`
+	Side      string `json:"side"`
+	OrdStatus string `json:"ordStatus"`
+}
+
+// WsTransactResponse private api response
+type WsTransactResponse struct {
+	WsDataResponse
+	ForeignKeys interface{}                  `json:"foreignKeys"`
+	Attributes  WsTransactResponseAttributes `json:"attributes"`
+	Filter      WsTransactResponseFilter     `json:"filter"`
+	Data        []interface{}                `json:"data"`
+}
+
+// WsTransactResponseAttributes private api data
+type WsTransactResponseAttributes struct {
+	TransactID   string `json:"transactID"`
+	TransactTime string `json:"transactTime"`
+}
+
+// WsTransactResponseFilter private api data
+type WsTransactResponseFilter struct {
+	Account int64 `json:"account"`
+}
+
+// WsWalletResponse private api response
+type WsWalletResponse struct {
+	WsDataResponse
+	ForeignKeys interface{}                `json:"foreignKeys"`
+	Attributes  WsWalletResponseAttributes `json:"attributes"`
+	Filter      WsWalletResponseFilter     `json:"filter"`
+	Data        []WsWalletResponseData     `json:"data"`
+}
+
+// WsWalletResponseAttributes private api data
+type WsWalletResponseAttributes struct {
+	Account  string `json:"account"`
+	Currency string `json:"currency"`
+}
+
+// WsWalletResponseData private api data
+type WsWalletResponseData struct {
+	Account          int64         `json:"account"`
+	Currency         string        `json:"currency"`
+	PrevDeposited    float64       `json:"prevDeposited"`
+	PrevWithdrawn    float64       `json:"prevWithdrawn"`
+	PrevTransferIn   float64       `json:"prevTransferIn"`
+	PrevTransferOut  float64       `json:"prevTransferOut"`
+	PrevAmount       float64       `json:"prevAmount"`
+	PrevTimestamp    string        `json:"prevTimestamp"`
+	DeltaDeposited   float64       `json:"deltaDeposited"`
+	DeltaWithdrawn   float64       `json:"deltaWithdrawn"`
+	DeltaTransferIn  float64       `json:"deltaTransferIn"`
+	DeltaTransferOut float64       `json:"deltaTransferOut"`
+	DeltaAmount      float64       `json:"deltaAmount"`
+	Deposited        float64       `json:"deposited"`
+	Withdrawn        float64       `json:"withdrawn"`
+	TransferIn       float64       `json:"transferIn"`
+	TransferOut      float64       `json:"transferOut"`
+	Amount           float64       `json:"amount"`
+	PendingCredit    float64       `json:"pendingCredit"`
+	PendingDebit     float64       `json:"pendingDebit"`
+	ConfirmedDebit   int64         `json:"confirmedDebit"`
+	Timestamp        string        `json:"timestamp"`
+	Addr             string        `json:"addr"`
+	Script           string        `json:"script"`
+	WithdrawalLock   []interface{} `json:"withdrawalLock"`
+}
+
+// WsWalletResponseFilter private api data
+type WsWalletResponseFilter struct {
+	Account int64 `json:"account"`
+}
+
+// WsExecutionResponse private api response
+type WsExecutionResponse struct {
+	WsDataResponse
+	ForeignKeys WsExecutionResponseForeignKeys `json:"foreignKeys"`
+	Attributes  WsExecutionResponseAttributes  `json:"attributes"`
+	Filter      WsExecutionResponseFilter      `json:"filter"`
+	Data        []interface{}                  `json:"data"`
+}
+
+// WsExecutionResponseAttributes private api data
+type WsExecutionResponseAttributes struct {
+	ExecID       string `json:"execID"`
+	Account      string `json:"account"`
+	ExecType     string `json:"execType"`
+	TransactTime string `json:"transactTime"`
+}
+
+// WsExecutionResponseFilter private api data
+type WsExecutionResponseFilter struct {
+	Account int64  `json:"account"`
+	Symbol  string `json:"symbol"`
+}
+
+// WsExecutionResponseForeignKeys private api data
+type WsExecutionResponseForeignKeys struct {
+	Symbol    string `json:"symbol"`
+	Side      string `json:"side"`
+	OrdStatus string `json:"ordStatus"`
+}
+
+// WsDataResponse contains common elements
+type WsDataResponse struct {
+	Table  string            `json:"table"`
+	Action string            `json:"action"`
+	Keys   []string          `json:"keys"`
+	Types  map[string]string `json:"types"`
+}
+
+// WsMarginResponse private api response
+type WsMarginResponse struct {
+	WsDataResponse
+	ForeignKeys interface{}                `json:"foreignKeys"`
+	Attributes  WsMarginResponseAttributes `json:"attributes"`
+	Filter      WsMarginResponseFilter     `json:"filter"`
+	Data        []WsMarginResponseData     `json:"data"`
+}
+
+// WsMarginResponseAttributes private api data
+type WsMarginResponseAttributes struct {
+	Account  string `json:"account"`
+	Currency string `json:"currency"`
+}
+
+// WsMarginResponseData private api data
+type WsMarginResponseData struct {
+	Account            int64       `json:"account"`
+	Currency           string      `json:"currency"`
+	RiskLimit          float64     `json:"riskLimit"`
+	PrevState          string      `json:"prevState"`
+	State              string      `json:"state"`
+	Action             string      `json:"action"`
+	Amount             float64     `json:"amount"`
+	PendingCredit      float64     `json:"pendingCredit"`
+	PendingDebit       float64     `json:"pendingDebit"`
+	ConfirmedDebit     float64     `json:"confirmedDebit"`
+	PrevRealisedPnl    float64     `json:"prevRealisedPnl"`
+	PrevUnrealisedPnl  float64     `json:"prevUnrealisedPnl"`
+	GrossComm          float64     `json:"grossComm"`
+	GrossOpenCost      float64     `json:"grossOpenCost"`
+	GrossOpenPremium   float64     `json:"grossOpenPremium"`
+	GrossExecCost      float64     `json:"grossExecCost"`
+	GrossMarkValue     float64     `json:"grossMarkValue"`
+	RiskValue          float64     `json:"riskValue"`
+	TaxableMargin      float64     `json:"taxableMargin"`
+	InitMargin         float64     `json:"initMargin"`
+	MaintMargin        float64     `json:"maintMargin"`
+	SessionMargin      float64     `json:"sessionMargin"`
+	TargetExcessMargin float64     `json:"targetExcessMargin"`
+	VarMargin          float64     `json:"varMargin"`
+	RealisedPnl        float64     `json:"realisedPnl"`
+	UnrealisedPnl      float64     `json:"unrealisedPnl"`
+	IndicativeTax      float64     `json:"indicativeTax"`
+	UnrealisedProfit   float64     `json:"unrealisedProfit"`
+	SyntheticMargin    interface{} `json:"syntheticMargin"`
+	WalletBalance      float64     `json:"walletBalance"`
+	MarginBalance      float64     `json:"marginBalance"`
+	MarginBalancePcnt  float64     `json:"marginBalancePcnt"`
+	MarginLeverage     float64     `json:"marginLeverage"`
+	MarginUsedPcnt     float64     `json:"marginUsedPcnt"`
+	ExcessMargin       float64     `json:"excessMargin"`
+	ExcessMarginPcnt   float64     `json:"excessMarginPcnt"`
+	AvailableMargin    float64     `json:"availableMargin"`
+	WithdrawableMargin float64     `json:"withdrawableMargin"`
+	Timestamp          string      `json:"timestamp"`
+	GrossLastValue     float64     `json:"grossLastValue"`
+	Commission         interface{} `json:"commission"`
+}
+
+// WsMarginResponseFilter private api data
+type WsMarginResponseFilter struct {
+	Account int64 `json:"account"`
+}
+
+// WsPositionResponse private api response
+type WsPositionResponse struct {
+	WsDataResponse
+	ForeignKeys WsPositionResponseForeignKeys `json:"foreignKeys"`
+	Attributes  WsPositionResponseAttributes  `json:"attributes"`
+	Filter      WsPositionResponseFilter      `json:"filter"`
+	Data        []interface{}                 `json:"data"`
+}
+
+// WsPositionResponseAttributes private api data
+type WsPositionResponseAttributes struct {
+	Account       string `json:"account"`
+	Symbol        string `json:"symbol"`
+	Currency      string `json:"currency"`
+	Underlying    string `json:"underlying"`
+	QuoteCurrency string `json:"quoteCurrency"`
+}
+
+// WsPositionResponseFilter private api data
+type WsPositionResponseFilter struct {
+	Account int64  `json:"account"`
+	Symbol  string `json:"symbol"`
+}
+
+// WsPositionResponseForeignKeys private api data
+type WsPositionResponseForeignKeys struct {
+	Symbol string `json:"symbol"`
+}
+
+// WsPrivateNotificationsResponse private api response
+type WsPrivateNotificationsResponse struct {
+	Table  string        `json:"table"`
+	Action string        `json:"action"`
+	Data   []interface{} `json:"data"`
+}

exchanges/bitmex/bitmex_wrapper.go

@@ -412,3 +412,13 @@ func (b *Bitmex) UnsubscribeToWebsocketChannels(channels []exchange.WebsocketCha
 	b.Websocket.UnsubscribeToChannels(channels)
 	return nil
 }
+
+// GetSubscriptions returns a copied list of subscriptions
+func (b *Bitmex) GetSubscriptions() ([]exchange.WebsocketChannelSubscription, error) {
+	return b.Websocket.GetSubscriptions(), nil
+}
+
+// AuthenticateWebsocket sends an authentication message to the websocket
+func (b *Bitmex) AuthenticateWebsocket() error {
+	return b.websocketSendAuth()
+}