d0zingcat/gocryptotrader
1
0
Fork 0
mirror of https://github.com/d0zingcat/gocryptotrader.git synced 2026-05-29 23:16:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Websocket request-response correlation (#328)

Browse Source 
* Establishes new websocket functionality. Begins the creation of the websocket request

* Adding a wrapper over gorilla websocket connect,send,receive to handle ID messages. Doesn't work

* Successfully moved exchange_websocket into its own wshandler namespace. oof

* Sets up ZB to use a round trip WS request system

* Adds Kraken ID support to subscriptions. Renames duplicate func name UnsubscribeToChannels to RemoveSubscribedChannels. Adds some helper methods in the WebsocketConn to reduce duplicate code. Cleans up ZB implementation

* Fixes double locking which caused no websocket data to be read. Fixes requestid for kraken subscriptions

* Completes Huobi and Hadax implementation. Extends ZB error handling. Adds GZip support for reading messages

* Adds HitBTC support. Adds GetCurrencies, GetSymbols, GetTrades WS funcs. Adds super fun new parameter to GenerateMessageID for Unix and UnixNano

* Adds GateIO id support

* Adds Coinut support. Prevents nil reference error in constatus when there isnt one

* Standardises all Exchange websockets to use the wshandler websocket. Removes the wsRequestMtx as wshandler handles that now. Makes the Dialer a dialer, its not externally referenced that I can see.

* Fixes issue with coinut implementation. Updates bitmex currencies. Removes redundant log messages which are used to log messages

* Starts testing. Renames files

* Adds tests for websocket connection

* Reverts request.go change

* Linting everything

* Fixes rebase issue

* Final changes. Fixes variable names, removes log.Debug, removes lines, rearranges test types, removes order correlation websocket type

* Final final commit, fixing ZB issues.

* Adds traffic alerts where missed. Changes empty struct pointer addresses to nil instead. Removes empty lines

* Fixed string conversion

* Fixes issue with ZB not sending success codes

* Fixes issue with coinut processing due to nonce handling with subscriptions

* Fixes issue where ZB test failure was not caught. Removes unnecessary error handling from other ZB tests

* Removes unused interface

* Renames wshandler.Init() to wshandler.Run()

* Updates template file

* Capitalises cryptocurrencies in struct. Moves websocketResponseCheckTimeout and websocketResponseMaxLimit into config options. Moves connection configuration to main exchange Setup (where appropriate). Reverts currencylastupdated ticks. Improves reader close error checking

* Fixes two inconsistent websocket delay times

* Creates a default variable for websocket ResponseMaxLimit and ResponseCheckTimeout, then applies it to setdefaults and all tests

* Updates exchange template to set and use default websocket response limits
This commit is contained in:
Scott
2019-08-07 15:15:01 +10:00
committed by Adrian Gallagher
parent 6e70f0642a
commit e209d85d0d
113 changed files with 3269 additions and 2594 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
exchanges/bitfinex/bitfinex.go
View File

@@ -6,16 +6,15 @@ import (
"net/http"
"net/url"
"strconv"
"sync"
"time"
"github.com/gorilla/websocket"
"github.com/thrasher-/gocryptotrader/common"
"github.com/thrasher-/gocryptotrader/config"
"github.com/thrasher-/gocryptotrader/currency"
exchange "github.com/thrasher-/gocryptotrader/exchanges"
"github.com/thrasher-/gocryptotrader/exchanges/request"
"github.com/thrasher-/gocryptotrader/exchanges/ticker"
"github.com/thrasher-/gocryptotrader/exchanges/wshandler"
log "github.com/thrasher-/gocryptotrader/logger"
)
@@ -86,9 +85,8 @@ const (
// depending on some factors (e.g. servers load, endpoint, etc.).
type Bitfinex struct {
exchange.Base
WebsocketConn *websocket.Conn
WebsocketConn *wshandler.WebsocketConnection
WebsocketSubdChannels map[int]WebsocketChanInfo
wsRequestMtx sync.Mutex
}
// SetDefaults sets the basic defaults for bitfinex
@@ -113,13 +111,15 @@ func (b *Bitfinex) SetDefaults() {
common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
b.APIUrlDefault = bitfinexAPIURLBase
b.APIUrl = b.APIUrlDefault
b.WebsocketInit()
b.Websocket.Functionality = exchange.WebsocketTickerSupported |
exchange.WebsocketTradeDataSupported |
exchange.WebsocketOrderbookSupported |
exchange.WebsocketSubscribeSupported |
exchange.WebsocketUnsubscribeSupported |
exchange.WebsocketAuthenticatedEndpointsSupported
b.Websocket = wshandler.New()
b.Websocket.Functionality = wshandler.WebsocketTickerSupported |
wshandler.WebsocketTradeDataSupported |
wshandler.WebsocketOrderbookSupported |
wshandler.WebsocketSubscribeSupported |
wshandler.WebsocketUnsubscribeSupported |
wshandler.WebsocketAuthenticatedEndpointsSupported
b.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit
b.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout
}
// Setup takes in the supplied exchange configuration details and sets params
@@ -160,17 +160,28 @@ func (b *Bitfinex) Setup(exch *config.ExchangeConfig) {
if err != nil {
log.Fatal(err)
}
err = b.WebsocketSetup(b.WsConnect,
err = b.Websocket.Setup(b.WsConnect,
b.Subscribe,
b.Unsubscribe,
exch.Name,
exch.Websocket,
exch.Verbose,
bitfinexWebsocket,
exch.WebsocketURL)
exch.WebsocketURL,
exch.AuthenticatedWebsocketAPISupport)
if err != nil {
log.Fatal(err)
}
b.WebsocketConn = &wshandler.WebsocketConnection{
ExchangeName: b.Name,
URL: b.Websocket.GetWebsocketURL(),
ProxyURL: b.Websocket.GetProxyAddress(),
Verbose: b.Verbose,
ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout,
ResponseMaxLimit: exch.WebsocketResponseMaxLimit,
}
b.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit
b.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout
}
}

15
exchanges/bitfinex/bitfinex_test.go
View File

@@ -13,6 +13,7 @@ import (
"github.com/thrasher-/gocryptotrader/currency"
exchange "github.com/thrasher-/gocryptotrader/exchanges"
"github.com/thrasher-/gocryptotrader/exchanges/sharedtestvalues"
"github.com/thrasher-/gocryptotrader/exchanges/wshandler"
)
// Please supply your own keys here to do better tests
@@ -961,19 +962,23 @@ func TestWsAuth(t *testing.T) {
b.SetDefaults()
TestSetup(t)
if !b.Websocket.IsEnabled() && !b.AuthenticatedWebsocketAPISupport || !areTestAPIKeysSet() {
t.Skip(exchange.WebsocketNotEnabled)
t.Skip(wshandler.WebsocketNotEnabled)
}
b.WebsocketConn = &wshandler.WebsocketConnection{
ExchangeName: b.Name,
URL: b.Websocket.GetWebsocketURL(),
Verbose: b.Verbose,
ResponseMaxLimit: exchange.DefaultWebsocketResponseMaxLimit,
ResponseCheckTimeout: exchange.DefaultWebsocketResponseCheckTimeout,
}
var err error
var dialer websocket.Dialer
b.WebsocketConn, _, err = dialer.Dial(b.Websocket.GetWebsocketURL(),
http.Header{})
err := b.WebsocketConn.Dial(&dialer, http.Header{})
if err != nil {
t.Fatal(err)
}
b.Websocket.DataHandler = sharedtestvalues.GetWebsocketInterfaceChannelOverride()
b.Websocket.TrafficAlert = sharedtestvalues.GetWebsocketStructChannelOverride()
go b.WsDataHandler()
defer b.WebsocketConn.Close()
err = b.WsSendAuth()
if err != nil {
t.Error(err)

97
exchanges/bitfinex/bitfinex_websocket.go
View File

@@ -4,7 +4,6 @@ import (
"errors"
"fmt"
"net/http"
"net/url"
"reflect"
"strconv"
"time"
@@ -14,6 +13,7 @@ import (
"github.com/thrasher-/gocryptotrader/currency"
exchange "github.com/thrasher-/gocryptotrader/exchanges"
"github.com/thrasher-/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-/gocryptotrader/exchanges/wshandler"
log "github.com/thrasher-/gocryptotrader/logger"
)
@@ -59,21 +59,7 @@ func (b *Bitfinex) WsPingHandler() error {
req := make(map[string]string)
req["event"] = "ping"
return b.wsSend(req)
}
// WsSend sends data to the websocket server
func (b *Bitfinex) wsSend(data interface{}) error {
b.wsRequestMtx.Lock()
defer b.wsRequestMtx.Unlock()
json, err := common.JSONEncode(data)
if err != nil {
return err
}
if b.Verbose {
log.Debugf("%v sending message to websocket %v", b.Name, data)
}
return b.WebsocketConn.WriteMessage(websocket.TextMessage, json)
return b.WebsocketConn.SendMessage(req)
}
// WsSendAuth sends a autheticated event payload
@@ -94,7 +80,7 @@ func (b *Bitfinex) WsSendAuth() error {
req["authPayload"] = payload
err := b.wsSend(req)
err := b.WebsocketConn.SendMessage(req)
if err != nil {
b.Websocket.SetCanUseAuthenticatedEndpoints(false)
return err
@@ -107,7 +93,7 @@ func (b *Bitfinex) WsSendUnauth() error {
req := make(map[string]string)
req["event"] = "unauth"
return b.wsSend(req)
return b.WebsocketConn.SendMessage(req)
}
// WsAddSubscriptionChannel adds a new subscription channel to the
@@ -128,33 +114,28 @@ func (b *Bitfinex) WsAddSubscriptionChannel(chanID int, channel, pair string) {
// WsConnect starts a new websocket connection
func (b *Bitfinex) WsConnect() error {
if !b.Websocket.IsEnabled() || !b.IsEnabled() {
return errors.New(exchange.WebsocketNotEnabled)
return errors.New(wshandler.WebsocketNotEnabled)
}
var Dialer websocket.Dialer
var err error
if b.Websocket.GetProxyAddress() != "" {
var proxy *url.URL
proxy, err = url.Parse(b.Websocket.GetProxyAddress())
if err != nil {
return err
}
Dialer.Proxy = http.ProxyURL(proxy)
var dialer websocket.Dialer
b.WebsocketConn = &wshandler.WebsocketConnection{
ExchangeName: b.Name,
URL: b.Websocket.GetWebsocketURL(),
ProxyURL: b.Websocket.GetProxyAddress(),
Verbose: b.Verbose,
}
b.WebsocketConn, _, err = Dialer.Dial(b.Websocket.GetWebsocketURL(), http.Header{})
err := b.WebsocketConn.Dial(&dialer, http.Header{})
if err != nil {
return fmt.Errorf("%v unable to connect to Websocket. Error: %s", b.Name, err)
}
_, resp, err := b.WebsocketConn.ReadMessage()
resp, err := b.WebsocketConn.ReadMessage()
if err != nil {
return fmt.Errorf("%v unable to read from Websocket. Error: %s", b.Name, err)
}
b.Websocket.TrafficAlert <- struct{}{}
var hs WebsocketHandshake
err = common.JSONDecode(resp, &hs)
err = common.JSONDecode(resp.Raw, &hs)
if err != nil {
return err
}
@@ -178,22 +159,6 @@ func (b *Bitfinex) WsConnect() error {
return nil
}
// WsReadData reads and handles websocket stream data
func (b *Bitfinex) WsReadData() (exchange.WebsocketResponse, error) {
msgType, resp, err := b.WebsocketConn.ReadMessage()
if err != nil {
return exchange.WebsocketResponse{}, err
}
b.Websocket.TrafficAlert <- struct{}{}
return exchange.WebsocketResponse{
Type: msgType,
Raw: resp,
}, nil
}
// WsDataHandler handles data from WsReadData
func (b *Bitfinex) WsDataHandler() {
b.Websocket.Wg.Add(1)
@@ -208,11 +173,12 @@ func (b *Bitfinex) WsDataHandler() {
return
default:
stream, err := b.WsReadData()
stream, err := b.WebsocketConn.ReadMessage()
if err != nil {
b.Websocket.DataHandler <- err
return
}
b.Websocket.TrafficAlert <- struct{}{}
if stream.Type == websocket.TextMessage {
var result interface{}
@@ -221,9 +187,6 @@ func (b *Bitfinex) WsDataHandler() {
case "map[string]interface {}":
eventData := result.(map[string]interface{})
event := eventData["event"]
if b.Verbose {
log.Debugf("%v Received message. Type '%v' Message: %v", b.Name, event, eventData)
}
switch event {
case "subscribed":
b.WsAddSubscriptionChannel(int(eventData["chanId"].(float64)),
@@ -306,7 +269,7 @@ func (b *Bitfinex) WsDataHandler() {
}
case "ticker":
b.Websocket.DataHandler <- exchange.TickerData{
b.Websocket.DataHandler <- wshandler.TickerData{
Quantity: chanData[8].(float64),
ClosePrice: chanData[7].(float64),
HighPrice: chanData[9].(float64),
@@ -476,7 +439,7 @@ func (b *Bitfinex) WsDataHandler() {
newAmount *= -1
}
b.Websocket.DataHandler <- exchange.TradeData{
b.Websocket.DataHandler <- wshandler.TradeData{
CurrencyPair: currency.NewPairFromString(chanInfo.Pair),
Timestamp: time.Unix(trades[0].Timestamp, 0),
Price: trades[0].Price,
@@ -524,7 +487,7 @@ func (b *Bitfinex) WsInsertSnapshot(p currency.Pair, assetType string, books []W
return fmt.Errorf("bitfinex.go error - %s", err)
}
b.Websocket.DataHandler <- exchange.WebsocketOrderbookUpdate{Pair: p,
b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{Pair: p,
Asset: assetType,
Exchange: b.GetName()}
return nil
@@ -549,7 +512,7 @@ func (b *Bitfinex) WsUpdateOrderbook(p currency.Pair, assetType string, book Web
return err
}
b.Websocket.DataHandler <- exchange.WebsocketOrderbookUpdate{Pair: p,
b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{Pair: p,
Asset: assetType,
Exchange: b.GetName()}
@@ -569,7 +532,7 @@ func (b *Bitfinex) WsUpdateOrderbook(p currency.Pair, assetType string, book Web
return err
}
b.Websocket.DataHandler <- exchange.WebsocketOrderbookUpdate{Pair: p,
b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{Pair: p,
Asset: assetType,
Exchange: b.GetName()}
@@ -590,7 +553,7 @@ func (b *Bitfinex) WsUpdateOrderbook(p currency.Pair, assetType string, book Web
return err
}
b.Websocket.DataHandler <- exchange.WebsocketOrderbookUpdate{Pair: p,
b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{Pair: p,
Asset: assetType,
Exchange: b.GetName()}
@@ -610,7 +573,7 @@ func (b *Bitfinex) WsUpdateOrderbook(p currency.Pair, assetType string, book Web
return err
}
b.Websocket.DataHandler <- exchange.WebsocketOrderbookUpdate{Pair: p,
b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{Pair: p,
Asset: assetType,
Exchange: b.GetName()}
@@ -620,14 +583,14 @@ func (b *Bitfinex) WsUpdateOrderbook(p currency.Pair, assetType string, book Web
// GenerateDefaultSubscriptions Adds default subscriptions to websocket to be handled by ManageSubscriptions()
func (b *Bitfinex) GenerateDefaultSubscriptions() {
var channels = []string{"book", "trades", "ticker"}
var subscriptions []exchange.WebsocketChannelSubscription
var subscriptions []wshandler.WebsocketChannelSubscription
for i := range channels {
for j := range b.EnabledPairs {
params := make(map[string]interface{})
if channels[i] == "book" {
params["prec"] = "P0"
}
subscriptions = append(subscriptions, exchange.WebsocketChannelSubscription{
subscriptions = append(subscriptions, wshandler.WebsocketChannelSubscription{
Channel: channels[i],
Currency: b.EnabledPairs[j],
Params: params,
@@ -638,7 +601,7 @@ func (b *Bitfinex) GenerateDefaultSubscriptions() {
}
// Subscribe sends a websocket message to receive data from the channel
func (b *Bitfinex) Subscribe(channelToSubscribe exchange.WebsocketChannelSubscription) error {
func (b *Bitfinex) Subscribe(channelToSubscribe wshandler.WebsocketChannelSubscription) error {
req := make(map[string]interface{})
req["event"] = "subscribe"
req["channel"] = channelToSubscribe.Channel
@@ -650,11 +613,11 @@ func (b *Bitfinex) Subscribe(channelToSubscribe exchange.WebsocketChannelSubscri
req[k] = v
}
}
return b.wsSend(req)
return b.WebsocketConn.SendMessage(req)
}
// Unsubscribe sends a websocket message to stop receiving data from the channel
func (b *Bitfinex) Unsubscribe(channelToSubscribe exchange.WebsocketChannelSubscription) error {
func (b *Bitfinex) Unsubscribe(channelToSubscribe wshandler.WebsocketChannelSubscription) error {
req := make(map[string]interface{})
req["event"] = "unsubscribe"
req["channel"] = channelToSubscribe.Channel
@@ -664,5 +627,5 @@ func (b *Bitfinex) Unsubscribe(channelToSubscribe exchange.WebsocketChannelSubsc
req[k] = v
}
}
return b.wsSend(req)
return b.WebsocketConn.SendMessage(req)
}

11
exchanges/bitfinex/bitfinex_wrapper.go
View File

@@ -14,6 +14,7 @@ import (
exchange "github.com/thrasher-/gocryptotrader/exchanges"
"github.com/thrasher-/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-/gocryptotrader/exchanges/ticker"
"github.com/thrasher-/gocryptotrader/exchanges/wshandler"
log "github.com/thrasher-/gocryptotrader/logger"
)
@@ -323,7 +324,7 @@ func (b *Bitfinex) WithdrawFiatFundsToInternationalBank(withdrawRequest *exchang
}
// GetWebsocket returns a pointer to the exchange websocket
func (b *Bitfinex) GetWebsocket() (*exchange.Websocket, error) {
func (b *Bitfinex) GetWebsocket() (*wshandler.Websocket, error) {
return b.Websocket, nil
}
@@ -457,20 +458,20 @@ func (b *Bitfinex) GetOrderHistory(getOrdersRequest *exchange.GetOrdersRequest)
// SubscribeToWebsocketChannels appends to ChannelsToSubscribe
// which lets websocket.manageSubscriptions handle subscribing
func (b *Bitfinex) SubscribeToWebsocketChannels(channels []exchange.WebsocketChannelSubscription) error {
func (b *Bitfinex) SubscribeToWebsocketChannels(channels []wshandler.WebsocketChannelSubscription) error {
b.Websocket.SubscribeToChannels(channels)
return nil
}
// UnsubscribeToWebsocketChannels removes from ChannelsToSubscribe
// which lets websocket.manageSubscriptions handle unsubscribing
func (b *Bitfinex) UnsubscribeToWebsocketChannels(channels []exchange.WebsocketChannelSubscription) error {
b.Websocket.UnsubscribeToChannels(channels)
func (b *Bitfinex) UnsubscribeToWebsocketChannels(channels []wshandler.WebsocketChannelSubscription) error {
b.Websocket.RemoveSubscribedChannels(channels)
return nil
}
// GetSubscriptions returns a copied list of subscriptions
func (b *Bitfinex) GetSubscriptions() ([]exchange.WebsocketChannelSubscription, error) {
func (b *Bitfinex) GetSubscriptions() ([]wshandler.WebsocketChannelSubscription, error) {
return b.Websocket.GetSubscriptions(), nil
}