Authenticated Websocket support (#315)

* Improves subscribing by not allowing duplicates. Adds bitmex auth support

* Adds coinbase pro support. Partial BTCC support. Adds WebsocketAuthenticatedEndpointsSupported websocket feature. Adds GateIO support

* Adds Coinut support

* Moves Coinut WS types to file. Implements Gemini's secure WS endpoint

* Adds HitBTC ws authenticated support. Fixes var names

* Adds huobi and hadax authenticated websocket support

* Adds auth to okgroup (okex, okcoin). Fixes some linting

* Adds Poloniex support

* Adds ZB support

* Adds proper bitmex support

* Improves bitfinex support, improves websocket functionality definitions

* Fixes coinbasepro auth

* Tests all endpoints

* go formatting, importing, linting run

* Adds wrapper supports

* General clean up. Data race destruction

* Improves testing on all exchanges except ZB

* Fixes ZB hashing, parsing and tests

* minor nits before someone else sees them <_<

* Fixes some nits pertaining to variable usage, comments, typos and rate limiting

* Addresses nits regarding types and test responses where applicable

* fmt import

* Fixes linting issues

* No longer returns an error on failure to authenticate, just logs. Adds new AuthenticatedWebsocketAPISupport config value to allow a user to seperate auth from REST and WS. Prevents WS auth if AuthenticatedWebsocketAPISupport is false, adds additional login check 'CanUseAuthenticatedEndpoints' for when login only occurs once (not per request). Removes unnecessary time.Sleeps from code. Moves WS auth error logic to auth function so that wrappers can get involved in all the auth fun. New-fandangled shared test package, used exclusively in testing, will be the store of all the constant boilerplate things like timeout values. Moves WS test setup function to only run once when there are multiple WS endpoint tests. Cleans up some struct types

* Increases test coverage with tests for config.areAuthenticatedCredentialsValid config.CheckExchangeConfigValues, exchange.SetAPIKeys, exchange.GetAuthenticatedAPISupport, exchange_websocket.CanUseAuthenticatedEndpoitns and exchange_websocket.SetCanUseAuthenticatedEndpoints. Adds b.Websocket.SetCanUseAuthenticatedEndpoints(false) when bitfinex fails to authenticate
Fixes a typo. gofmt and goimport

* Trim Test Typos

* Reformats various websocket types. Adds more specific error messaging to config.areAuthenticatedCredentialsValid

exchanges/coinbasepro/coinbasepro.go

@@ -92,7 +92,8 @@ func (c *CoinbasePro) SetDefaults() {
 	c.Websocket.Functionality = exchange.WebsocketTickerSupported |
 		exchange.WebsocketOrderbookSupported |
 		exchange.WebsocketSubscribeSupported |
-		exchange.WebsocketUnsubscribeSupported
+		exchange.WebsocketUnsubscribeSupported |
+		exchange.WebsocketAuthenticatedEndpointsSupported
 }
 
 // Setup initialises the exchange parameters with the current configuration
@@ -102,6 +103,7 @@ func (c *CoinbasePro) Setup(exch *config.ExchangeConfig) {
 	} else {
 		c.Enabled = true
 		c.AuthenticatedAPISupport = exch.AuthenticatedAPISupport
+		c.AuthenticatedWebsocketAPISupport = exch.AuthenticatedWebsocketAPISupport
 		c.SetAPIKeys(exch.APIKey, exch.APISecret, exch.ClientID, true)
 		c.SetHTTPClientTimeout(exch.HTTPTimeout)
 		c.SetHTTPClientUserAgent(exch.HTTPUserAgent)
@@ -823,7 +825,7 @@ func (c *CoinbasePro) SendAuthenticatedHTTPRequest(method, path string, params m
 		}
 	}
 
-	n := c.Requester.GetNonce(true).String()
+	n := c.Requester.GetNonce(false).String()
 	message := n + method + "/" + path + string(payload)
 	hmac := common.GetHMAC(common.HashSHA256, []byte(message), []byte(c.APISecret))
 	headers := make(map[string]string)

exchanges/coinbasepro/coinbasepro_test.go

@@ -1,12 +1,15 @@
 package coinbasepro
 
 import (
+	"net/http"
 	"testing"
 	"time"
 
+	"github.com/gorilla/websocket"
 	"github.com/thrasher-/gocryptotrader/config"
 	"github.com/thrasher-/gocryptotrader/currency"
 	exchange "github.com/thrasher-/gocryptotrader/exchanges"
+	"github.com/thrasher-/gocryptotrader/exchanges/sharedtestvalues"
 )
 
 var c CoinbasePro
@@ -33,7 +36,9 @@ func TestSetup(t *testing.T) {
 	}
 	gdxConfig.APIKey = apiKey
 	gdxConfig.APISecret = apiSecret
+	gdxConfig.ClientID = clientID
 	gdxConfig.AuthenticatedAPISupport = true
+	gdxConfig.AuthenticatedWebsocketAPISupport = true
 	c.Setup(&gdxConfig)
 }
 
@@ -87,137 +92,85 @@ func TestGetServerTime(t *testing.T) {
 }
 
 func TestAuthRequests(t *testing.T) {
-
-	if c.APIKey != "" && c.APISecret != "" && c.ClientID != "" {
-
-		_, err := c.GetAccounts()
-		if err == nil {
-			t.Error("Test failed - GetAccounts() error", err)
-		}
-
-		_, err = c.GetAccount("234cb213-ac6f-4ed8-b7b6-e62512930945")
-		if err == nil {
-			t.Error("Test failed - GetAccount() error", err)
-		}
-
-		_, err = c.GetAccountHistory("234cb213-ac6f-4ed8-b7b6-e62512930945")
-		if err == nil {
-			t.Error("Test failed - GetAccountHistory() error", err)
-		}
-
-		_, err = c.GetHolds("234cb213-ac6f-4ed8-b7b6-e62512930945")
-		if err == nil {
-			t.Error("Test failed - GetHolds() error", err)
-		}
-
-		_, err = c.PlaceLimitOrder("", 0, 0, "buy", "", "", "BTC-USD", "", false)
-		if err == nil {
-			t.Error("Test failed - PlaceLimitOrder() error", err)
-		}
-
-		_, err = c.PlaceMarketOrder("", 1, 0, "buy", "BTC-USD", "")
-		if err == nil {
-			t.Error("Test failed - PlaceMarketOrder() error", err)
-		}
-
-		err = c.CancelExistingOrder("1337")
-		if err == nil {
-			t.Error("Test failed - CancelExistingOrder() error", err)
-		}
-
-		_, err = c.CancelAllExistingOrders("BTC-USD")
-		if err == nil {
-			t.Error("Test failed - CancelAllExistingOrders() error", err)
-		}
-
-		_, err = c.GetOrders([]string{"open", "done"}, "BTC-USD")
-		if err == nil {
-			t.Error("Test failed - GetOrders() error", err)
-		}
-
-		_, err = c.GetOrder("1337")
-		if err == nil {
-			t.Error("Test failed - GetOrders() error", err)
-		}
-
-		_, err = c.GetFills("1337", "BTC-USD")
-		if err == nil {
-			t.Error("Test failed - GetFills() error", err)
-		}
-		_, err = c.GetFills("", "")
-		if err == nil {
-			t.Error("Test failed - GetFills() error", err)
-		}
-
-		_, err = c.GetFundingRecords("rejected")
-		if err == nil {
-			t.Error("Test failed - GetFundingRecords() error", err)
-		}
-
-		// 	_, err := c.RepayFunding("1", "BTC")
-		// 	if err != nil {
-		// 		t.Error("Test failed - RepayFunding() error", err)
-		// 	}
-
-		_, err = c.MarginTransfer(1, "withdraw", "45fa9e3b-00ba-4631-b907-8a98cbdf21be", "BTC")
-		if err == nil {
-			t.Error("Test failed - MarginTransfer() error", err)
-		}
-
-		_, err = c.GetPosition()
-		if err == nil {
-			t.Error("Test failed - GetPosition() error", err)
-		}
-
-		_, err = c.ClosePosition(false)
-		if err == nil {
-			t.Error("Test failed - ClosePosition() error", err)
-		}
-
-		_, err = c.GetPayMethods()
-		if err == nil {
-			t.Error("Test failed - GetPayMethods() error", err)
-		}
-
-		_, err = c.DepositViaPaymentMethod(1, "BTC", "1337")
-		if err == nil {
-			t.Error("Test failed - DepositViaPaymentMethod() error", err)
-		}
-
-		_, err = c.DepositViaCoinbase(1, "BTC", "1337")
-		if err == nil {
-			t.Error("Test failed - DepositViaCoinbase() error", err)
-		}
-
-		_, err = c.WithdrawViaPaymentMethod(1, "BTC", "1337")
-		if err == nil {
-			t.Error("Test failed - WithdrawViaPaymentMethod() error", err)
-		}
-
-		// 	_, err := c.WithdrawViaCoinbase(1, "BTC", "c13cd0fc-72ca-55e9-843b-b84ef628c198")
-		// 	if err != nil {
-		// 		t.Error("Test failed - WithdrawViaCoinbase() error", err)
-		// 	}
-
-		_, err = c.WithdrawCrypto(1, "BTC", "1337")
-		if err == nil {
-			t.Error("Test failed - WithdrawViaCoinbase() error", err)
-		}
-
-		_, err = c.GetCoinbaseAccounts()
-		if err == nil {
-			t.Error("Test failed - GetCoinbaseAccounts() error", err)
-		}
-
-		_, err = c.GetReportStatus("1337")
-		if err == nil {
-			t.Error("Test failed - GetReportStatus() error", err)
-		}
-
-		_, err = c.GetTrailingVolume()
-		if err == nil {
-			t.Error("Test failed - GetTrailingVolume() error", err)
-		}
+	if !areTestAPIKeysSet() {
+		t.Skip("API keys not set, skipping test")
+	}
+	_, err := c.GetAccounts()
+	if err != nil {
+		t.Error("Test failed - GetAccounts() error", err)
+	}
+	accountResponse, err := c.GetAccount("13371337-1337-1337-1337-133713371337")
+	if accountResponse.ID != "" {
+		t.Error("Expecting no data returned")
+	}
+	if err == nil {
+		t.Error("Expecting error")
+	}
+	accountHistoryResponse, err := c.GetAccountHistory("13371337-1337-1337-1337-133713371337")
+	if len(accountHistoryResponse) > 0 {
+		t.Error("Expecting no data returned")
+	}
+	if err == nil {
+		t.Error("Expecting error")
+	}
+	getHoldsResponse, err := c.GetHolds("13371337-1337-1337-1337-133713371337")
+	if len(getHoldsResponse) > 0 {
+		t.Error("Expecting no data returned")
+	}
+	if err == nil {
+		t.Error("Expecting error")
+	}
+	orderResponse, err := c.PlaceLimitOrder("", 0.001, 0.001, "buy", "", "", "BTC-USD", "", false)
+	if orderResponse != "" {
+		t.Error("Expecting no data returned")
+	}
+	if err == nil {
+		t.Error("Expecting error")
+	}
+	marketOrderResponse, err := c.PlaceMarketOrder("", 1, 0, "buy", "BTC-USD", "")
+	if marketOrderResponse != "" {
+		t.Error("Expecting no data returned")
+	}
+	if err == nil {
+		t.Error("Expecting error")
+	}
+	fillsResponse, err := c.GetFills("1337", "BTC-USD")
+	if len(fillsResponse) > 0 {
+		t.Error("Expecting no data returned")
+	}
+	if err == nil {
+		t.Error("Expecting error")
+	}
+	_, err = c.GetFills("", "")
+	if err == nil {
+		t.Error("Expecting error")
+	}
+	_, err = c.GetFundingRecords("rejected")
+	if err == nil {
+		t.Error("Expecting error")
+	}
+	marginTransferResponse, err := c.MarginTransfer(1, "withdraw", "13371337-1337-1337-1337-133713371337", "BTC")
+	if marginTransferResponse.ID != "" {
+		t.Error("Expecting no data returned")
+	}
+	if err == nil {
+		t.Error("Expecting error")
+	}
+	_, err = c.GetPosition()
+	if err == nil {
+		t.Error("Expecting error")
+	}
+	_, err = c.ClosePosition(false)
+	if err == nil {
+		t.Error("Expecting error")
+	}
+	_, err = c.GetPayMethods()
+	if err != nil {
+		t.Error("Test failed - GetPayMethods() error", err)
+	}
+	_, err = c.GetCoinbaseAccounts()
+	if err != nil {
+		t.Error("Test failed - GetCoinbaseAccounts() error", err)
 	}
 }
 
@@ -626,3 +579,37 @@ func TestGetDepositAddress(t *testing.T) {
 		t.Error("Test Failed - GetDepositAddress() error", err)
 	}
 }
+
+// TestWsAuth dials websocket, sends login request.
+func TestWsAuth(t *testing.T) {
+	c.SetDefaults()
+	TestSetup(t)
+	if !c.Websocket.IsEnabled() && !c.AuthenticatedWebsocketAPISupport || !areTestAPIKeysSet() {
+		t.Skip(exchange.WebsocketNotEnabled)
+	}
+	var err error
+	var dialer websocket.Dialer
+	c.WebsocketConn, _, err = dialer.Dial(c.Websocket.GetWebsocketURL(),
+		http.Header{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	c.Websocket.DataHandler = sharedtestvalues.GetWebsocketInterfaceChannelOverride()
+	c.Websocket.TrafficAlert = sharedtestvalues.GetWebsocketStructChannelOverride()
+	go c.WsHandleData()
+	defer c.WebsocketConn.Close()
+	err = c.Subscribe(exchange.WebsocketChannelSubscription{
+		Channel:  "user",
+		Currency: currency.NewPairFromString("BTC-USD"),
+	})
+	if err != nil {
+		t.Error(err)
+	}
+	timer := time.NewTimer(sharedtestvalues.WebsocketResponseDefaultTimeout)
+	select {
+	case badResponse := <-c.Websocket.DataHandler:
+		t.Error(badResponse)
+	case <-timer.C:
+	}
+	timer.Stop()
+}

exchanges/coinbasepro/coinbasepro_types.go

@@ -343,9 +343,13 @@ type FillResponse struct {
 
 // WebsocketSubscribe takes in subscription information
 type WebsocketSubscribe struct {
-	Type      string       `json:"type"`
-	ProductID string       `json:"product_id,omitempty"`
-	Channels  []WsChannels `json:"channels,omitempty"`
+	Type       string       `json:"type"`
+	ProductID  string       `json:"product_id,omitempty"`
+	Channels   []WsChannels `json:"channels,omitempty"`
+	Signature  string       `json:"signature,omitempty"`
+	Key        string       `json:"key,omitempty"`
+	Passphrase string       `json:"passphrase,omitempty"`
+	Timestamp  string       `json:"timestamp,omitempty"`
 }
 
 // WsChannels defines outgoing channels for subscription purposes
@@ -360,7 +364,8 @@ type WebsocketReceived struct {
 	OrderID   string  `json:"order_id"`
 	OrderType string  `json:"order_type"`
 	Size      float64 `json:"size,string"`
-	Price     float64 `json:"price,string"`
+	Price     float64 `json:"price,omitempty,string"`
+	Funds     float64 `json:"funds,omitempty,string"`
 	Side      string  `json:"side"`
 	ClientOID string  `json:"client_oid"`
 	ProductID string  `json:"product_id"`
@@ -462,3 +467,20 @@ type WebsocketL2Update struct {
 	Time      string          `json:"time"`
 	Changes   [][]interface{} `json:"changes"`
 }
+
+// WebsocketActivate an activate message is sent when a stop order is placed
+type WebsocketActivate struct {
+	Type         string  `json:"type"`
+	ProductID    string  `json:"product_id"`
+	Timestamp    string  `json:"timestamp"`
+	UserID       string  `json:"user_id"`
+	ProfileID    string  `json:"profile_id"`
+	OrderID      string  `json:"order_id"`
+	StopType     string  `json:"stop_type"`
+	Side         string  `json:"side"`
+	StopPrice    float64 `json:"stop_price,string"`
+	Size         float64 `json:"size,string"`
+	Funds        float64 `json:"funds,string"`
+	TakerFeeRate float64 `json:"taker_fee_rate,string"`
+	Private      bool    `json:"private"`
+}

exchanges/coinbasepro/coinbasepro_websocket.go

@@ -148,6 +148,51 @@ func (c *CoinbasePro) WsHandleData() {
 					c.Websocket.DataHandler <- err
 					continue
 				}
+			case "received":
+				// We currently use l2update to calculate orderbook changes
+				received := WebsocketReceived{}
+				err := common.JSONDecode(resp.Raw, &received)
+				if err != nil {
+					c.Websocket.DataHandler <- err
+					continue
+				}
+				c.Websocket.DataHandler <- received
+			case "open":
+				// We currently use l2update to calculate orderbook changes
+				open := WebsocketOpen{}
+				err := common.JSONDecode(resp.Raw, &open)
+				if err != nil {
+					c.Websocket.DataHandler <- err
+					continue
+				}
+				c.Websocket.DataHandler <- open
+			case "done":
+				// We currently use l2update to calculate orderbook changes
+				done := WebsocketDone{}
+				err := common.JSONDecode(resp.Raw, &done)
+				if err != nil {
+					c.Websocket.DataHandler <- err
+					continue
+				}
+				c.Websocket.DataHandler <- done
+			case "change":
+				// We currently use l2update to calculate orderbook changes
+				change := WebsocketChange{}
+				err := common.JSONDecode(resp.Raw, &change)
+				if err != nil {
+					c.Websocket.DataHandler <- err
+					continue
+				}
+				c.Websocket.DataHandler <- change
+			case "activate":
+				// We currently use l2update to calculate orderbook changes
+				activate := WebsocketActivate{}
+				err := common.JSONDecode(resp.Raw, &activate)
+				if err != nil {
+					c.Websocket.DataHandler <- err
+					continue
+				}
+				c.Websocket.DataHandler <- activate
 			}
 		}
 	}
@@ -241,10 +286,13 @@ func (c *CoinbasePro) ProcessUpdate(update WebsocketL2Update) error {
 
 // GenerateDefaultSubscriptions Adds default subscriptions to websocket to be handled by ManageSubscriptions()
 func (c *CoinbasePro) GenerateDefaultSubscriptions() {
-	var channels = []string{"heartbeat", "level2", "ticker"}
+	var channels = []string{"heartbeat", "level2", "ticker", "user"}
 	enabledCurrencies := c.GetEnabledCurrencies()
-	subscriptions := []exchange.WebsocketChannelSubscription{}
+	var subscriptions []exchange.WebsocketChannelSubscription
 	for i := range channels {
+		if (channels[i] == "user" || channels[i] == "full") && !c.GetAuthenticatedAPISupport(exchange.WebsocketAuthentication) {
+			continue
+		}
 		for j := range enabledCurrencies {
 			enabledCurrencies[j].Delimiter = "-"
 			subscriptions = append(subscriptions, exchange.WebsocketChannelSubscription{
@@ -269,6 +317,15 @@ func (c *CoinbasePro) Subscribe(channelToSubscribe exchange.WebsocketChannelSubs
 			},
 		},
 	}
+	if channelToSubscribe.Channel == "user" || channelToSubscribe.Channel == "full" {
+		n := fmt.Sprintf("%v", time.Now().Unix())
+		message := n + "GET" + "/users/self/verify"
+		hmac := common.GetHMAC(common.HashSHA256, []byte(message), []byte(c.APISecret))
+		subscribe.Signature = common.Base64Encode(hmac)
+		subscribe.Key = c.APIKey
+		subscribe.Passphrase = c.ClientID
+		subscribe.Timestamp = n
+	}
 	return c.wsSend(subscribe)
 }
 

exchanges/coinbasepro/coinbasepro_wrapper.go

@@ -396,3 +396,13 @@ func (c *CoinbasePro) UnsubscribeToWebsocketChannels(channels []exchange.Websock
 	c.Websocket.UnsubscribeToChannels(channels)
 	return nil
 }
+
+// GetSubscriptions returns a copied list of subscriptions
+func (c *CoinbasePro) GetSubscriptions() ([]exchange.WebsocketChannelSubscription, error) {
+	return c.Websocket.GetSubscriptions(), nil
+}
+
+// AuthenticateWebsocket sends an authentication message to the websocket
+func (c *CoinbasePro) AuthenticateWebsocket() error {
+	return common.ErrFunctionNotSupported
+}