Initial overhaul of websocket connection and feeds (#189)

* Initial overhaul of websocket connection and feeds
* Added proxy support
* Piped to routines.go

* Added new websocket file in exchanges
Refactored orderbook handling into exchange_websocket.go
Added better error responses for binance_websocket.go
General clean for binance_websocket.go

* General fixes - bitfinex_websocket.go
Refactored orderbook cache code - bitfinex_websocket.go
Removed fatal error with unhandled type - routines.go

* Added general improvements to bitmex_websocket.go
Refactored orderbook handling to exchange_websocket.go
Added variable in Item struct in orderbook.go for looking up orders by ID

* Fix issue when routines are blocked due to Data Handler not started
Updated traffic handler
General fixes for bitstamp_websocket.go

* General fixes for coinbasepro_websocket.go

* General fixes for coinut_websocket.go
Fixed error return in exchange_websocket.go

* Removed comments in coinut_wrapper.go
Refactor orderbook logic from hitbtc_websocket.go to exchange_websocket.go

* General fixes

* Removed comments
General fixes

* Updated routines.go

* After rebase fix

* Fixed update config pairs in okcoin.go

* fixed config currency issue in okcoin.go for okcoin China

* exchange_websocket.go
*Removed unused const dec
*Removed state change routine
*Improved trafficMonitor routine
*Increased verbosity for error returns
*Removed uneeded mutex locks

exchange_websocket_test.go
*Added new tests for websocket and orderbook updating

routines.go
*Removed string cased

* Fixed race conditions on sync.waitgroup in exchanges_websocket.go

* Changes variable name in config.go

* Removes unnecessary comment

* Removes indefinite lock on error return

* Removes unnecessary comment

* Adds support for BTCC websocket
Drops support for BTCC REST

* Rewords comment in exchange_websocket.go
Moves types to poloniex_types.go

* Moves types to coinut_types.go

* Removes uneeded range for accessing array variables for coinbase_websocket.go
Removes comments in coinut_types.go

* Adds verbosity flag to GCT
Suppresses verbose output from routines.go

* Fixes setting proxy for REST and Websocket per exchange
Upgrades error handling
Drops unused *url.Url variable in exchange type

* Adds test for setting proxy

* Fixes bug that closes connection due to incorrect timeout time through a proxy connection

* Clarify verbose flag message
This commit is contained in:
Ryan O'Hara-Reid
2018-10-24 14:22:41 +11:00
committed by Adrian Gallagher
parent 7315e6604c
commit d3c2800fe0
99 changed files with 6515 additions and 3031 deletions

View File

@@ -20,7 +20,6 @@ import (
type Bitmex struct {
exchange.Base
WebsocketConn *websocket.Conn
shutdown *Shutdown
}
const (
@@ -114,7 +113,6 @@ func (b *Bitmex) SetDefaults() {
b.Name = "Bitmex"
b.Enabled = false
b.Verbose = false
b.Websocket = false
b.RESTPollingDelay = 10
b.RequestCurrencyPairFormat.Delimiter = ""
b.RequestCurrencyPairFormat.Uppercase = true
@@ -125,10 +123,10 @@ func (b *Bitmex) SetDefaults() {
request.NewRateLimit(time.Second, bitmexAuthRate),
request.NewRateLimit(time.Second, bitmexUnauthRate),
common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
b.shutdown = b.NewRoutineManagement()
b.APIUrlDefault = bitmexAPIURL
b.APIUrl = b.APIUrlDefault
b.SupportsAutoPairUpdating = true
b.WebsocketInit()
}
// Setup takes in the supplied exchange configuration details and sets params
@@ -141,7 +139,7 @@ func (b *Bitmex) Setup(exch config.ExchangeConfig) {
b.SetAPIKeys(exch.APIKey, exch.APISecret, "", false)
b.RESTPollingDelay = exch.RESTPollingDelay
b.Verbose = exch.Verbose
b.Websocket = exch.Websocket
b.Websocket.SetEnabled(exch.Websocket)
b.BaseCurrencies = common.SplitStrings(exch.BaseCurrencies, ",")
b.AvailablePairs = common.SplitStrings(exch.AvailablePairs, ",")
b.EnabledPairs = common.SplitStrings(exch.EnabledPairs, ",")
@@ -161,6 +159,18 @@ func (b *Bitmex) Setup(exch config.ExchangeConfig) {
if err != nil {
log.Fatal(err)
}
err = b.SetClientProxyAddress(exch.ProxyAddress)
if err != nil {
log.Fatal(err)
}
err = b.WebsocketSetup(b.WsConnector,
exch.Name,
exch.Websocket,
bitmexWSURL,
exch.WebsocketURL)
if err != nil {
log.Fatal(err)
}
}
}

View File

@@ -4,11 +4,17 @@ import (
"errors"
"fmt"
"log"
"net/http"
"net/url"
"strconv"
"time"
"github.com/thrasher-/gocryptotrader/exchanges/orderbook"
"github.com/gorilla/websocket"
"github.com/thrasher-/gocryptotrader/common"
"github.com/thrasher-/gocryptotrader/currency/pair"
"github.com/thrasher-/gocryptotrader/exchanges"
)
const (
@@ -55,15 +61,27 @@ const (
var (
pongChan = make(chan int, 1)
timer *time.Timer
)
// WebsocketConnect initiates a new websocket connection
func (b *Bitmex) WebsocketConnect() error {
// WsConnector initiates a new websocket connection
func (b *Bitmex) WsConnector() error {
if !b.Websocket.IsEnabled() || !b.IsEnabled() {
return errors.New(exchange.WebsocketNotEnabled)
}
var dialer websocket.Dialer
var err error
b.WebsocketConn, _, err = dialer.Dial(bitmexWSURL, nil)
if b.Websocket.GetProxyAddress() != "" {
proxy, err := url.Parse(b.Websocket.GetProxyAddress())
if err != nil {
return err
}
dialer.Proxy = http.ProxyURL(proxy)
}
b.WebsocketConn, _, err = dialer.Dial(b.Websocket.GetWebsocketURL(), nil)
if err != nil {
return err
}
@@ -79,8 +97,6 @@ func (b *Bitmex) WebsocketConnect() error {
return err
}
go b.connectionHandler()
if b.Verbose {
log.Printf("Successfully connected to Bitmex %s at time: %s Limit: %d",
welcomeResp.Info,
@@ -88,309 +104,292 @@ func (b *Bitmex) WebsocketConnect() error {
welcomeResp.Limit.Remaining)
}
go b.handleIncomingData()
go b.wsHandleIncomingData()
go b.wsReadData()
err = b.websocketSubscribe()
if err != nil {
b.WebsocketConn.Close()
closeError := b.WebsocketConn.Close()
if closeError != nil {
return fmt.Errorf("bitmex_websocket.go error - Websocket connection could not close %s",
closeError)
}
return err
}
if b.AuthenticatedAPISupport {
err := b.websocketSendAuth()
if err != nil {
log.Fatal(err)
return err
}
}
return nil
}
// Timer handles connection loss or failure
func (b *Bitmex) connectionHandler() {
func (b *Bitmex) wsReadData() {
b.Websocket.Wg.Add(1)
defer func() {
if b.Verbose {
log.Println("Bitmex websocket: Connection handler routine shutdown")
err := b.WebsocketConn.Close()
if err != nil {
b.Websocket.DataHandler <- fmt.Errorf("bitmex_websocket.go - Unable to close Websocket connection. Error: %s",
err)
}
b.Websocket.Wg.Done()
}()
shutdown := b.shutdown.addRoutine()
timer = time.NewTimer(5 * time.Second)
for {
select {
case <-timer.C:
timeout := time.After(5 * time.Second)
err := b.WebsocketConn.WriteJSON("ping")
case <-b.Websocket.ShutdownC:
return
default:
_, resp, err := b.WebsocketConn.ReadMessage()
if err != nil {
b.reconnect()
b.Websocket.DataHandler <- fmt.Errorf("bitmex_websocket.go - websocket connection Error: %s",
err)
return
}
for {
select {
case <-pongChan:
if b.Verbose {
log.Println("Bitmex websocket: PONG received")
}
break
case <-timeout:
log.Println("Bitmex websocket: Connection timed out - Closing connection....")
b.WebsocketConn.Close()
log.Println("Bitmex websocket: Connection timed out - Reconnecting...")
b.reconnect()
return
}
b.Websocket.TrafficAlert <- struct{}{}
b.Websocket.Intercomm <- exchange.WebsocketResponse{
Raw: resp,
}
case <-shutdown:
log.Println("Bitmex websocket: shutdown requested - Closing connection....")
b.WebsocketConn.Close()
log.Println("Bitmex websocket: Sending shutdown message")
b.shutdown.routineShutdown()
return
}
}
}
// Reconnect handles reconnections to websocket API
func (b *Bitmex) reconnect() {
for {
err := b.WebsocketConnect()
if err != nil {
log.Println("Bitmex websocket: Connection timed out - Failed to connect, sleeping...")
time.Sleep(time.Second * 2)
continue
}
return
}
}
// handleIncomingData services incoming data from the websocket connection
func (b *Bitmex) handleIncomingData() {
defer func() {
if b.Verbose {
log.Println("Bitmex websocket: Response data handler routine shutdown")
}
}()
// wsHandleIncomingData services incoming data from the websocket connection
func (b *Bitmex) wsHandleIncomingData() {
b.Websocket.Wg.Add(1)
defer b.Websocket.Wg.Done()
for {
_, resp, err := b.WebsocketConn.ReadMessage()
if err != nil {
if b.Verbose {
log.Println("Bitmex websocket: Connection error", err)
}
select {
case <-b.Websocket.ShutdownC:
return
}
message := string(resp)
if common.StringContains(message, "pong") {
if b.Verbose {
log.Println("Bitmex websocket: PONG receieved")
}
pongChan <- 1
continue
}
if common.StringContains(message, "ping") {
err = b.WebsocketConn.WriteJSON("pong")
if err != nil {
if b.Verbose {
log.Println("Bitmex websocket error: ", err)
}
return
}
}
if !timer.Reset(5 * time.Second) {
log.Fatal("Bitmex websocket: Timer failed to set")
}
quickCapture := make(map[string]interface{})
err = common.JSONDecode(resp, &quickCapture)
if err != nil {
log.Fatal(err)
}
var respError WebsocketErrorResponse
if _, ok := quickCapture["status"]; ok {
err = common.JSONDecode(resp, &respError)
if err != nil {
log.Fatal(err)
}
log.Printf("Bitmex websocket error: %s", respError.Error)
continue
}
if _, ok := quickCapture["success"]; ok {
var decodedResp WebsocketSubscribeResp
err := common.JSONDecode(resp, &decodedResp)
if err != nil {
log.Fatal(err)
}
if decodedResp.Success {
if b.Verbose {
if len(quickCapture) == 3 {
log.Printf("Bitmex Websocket: Successfully subscribed to %s",
decodedResp.Subscribe)
} else {
log.Println("Bitmex Websocket: Successfully authenticated websocket connection")
}
}
case resp := <-b.Websocket.Intercomm:
message := string(resp.Raw)
if common.StringContains(message, "pong") {
pongChan <- 1
continue
}
log.Printf("Bitmex websocket error: Unable to subscribe %s",
decodedResp.Subscribe)
} else if _, ok := quickCapture["table"]; ok {
var decodedResp WebsocketMainResponse
err := common.JSONDecode(resp, &decodedResp)
if common.StringContains(message, "ping") {
err := b.WebsocketConn.WriteJSON("pong")
if err != nil {
b.Websocket.DataHandler <- err
}
}
quickCapture := make(map[string]interface{})
err := common.JSONDecode(resp.Raw, &quickCapture)
if err != nil {
log.Fatal(err)
}
switch decodedResp.Table {
case bitmexWSOrderbookL2:
var orderbooks OrderBookData
err = common.JSONDecode(resp, &orderbooks)
var respError WebsocketErrorResponse
if _, ok := quickCapture["status"]; ok {
err = common.JSONDecode(resp.Raw, &respError)
if err != nil {
log.Fatal(err)
}
err = b.processOrderbook(orderbooks.Data, orderbooks.Action)
b.Websocket.DataHandler <- errors.New(respError.Error)
continue
}
if _, ok := quickCapture["success"]; ok {
var decodedResp WebsocketSubscribeResp
err := common.JSONDecode(resp.Raw, &decodedResp)
if err != nil {
log.Fatal(err)
}
case bitmexWSTrade:
var trades TradeData
err = common.JSONDecode(resp, &trades)
if decodedResp.Success {
if b.Verbose {
if len(quickCapture) == 3 {
log.Printf("Bitmex Websocket: Successfully subscribed to %s",
decodedResp.Subscribe)
} else {
log.Println("Bitmex Websocket: Successfully authenticated websocket connection")
}
}
continue
}
b.Websocket.DataHandler <- fmt.Errorf("Bitmex websocket error: Unable to subscribe %s",
decodedResp.Subscribe)
} else if _, ok := quickCapture["table"]; ok {
var decodedResp WebsocketMainResponse
err := common.JSONDecode(resp.Raw, &decodedResp)
if err != nil {
log.Fatal(err)
}
err = b.processTrades(trades.Data, trades.Action)
if err != nil {
log.Fatal(err)
switch decodedResp.Table {
case bitmexWSOrderbookL2:
var orderbooks OrderBookData
err = common.JSONDecode(resp.Raw, &orderbooks)
if err != nil {
log.Fatal(err)
}
p := pair.NewCurrencyPairFromString(orderbooks.Data[0].Symbol)
err = b.processOrderbook(orderbooks.Data, orderbooks.Action, p, "CONTRACT")
if err != nil {
log.Fatal(err)
}
case bitmexWSTrade:
var trades TradeData
err = common.JSONDecode(resp.Raw, &trades)
if err != nil {
log.Fatal(err)
}
if trades.Action == bitmexActionInitialData {
continue
}
for _, trade := range trades.Data {
timestamp, err := time.Parse(time.RFC3339, trade.Timestamp)
if err != nil {
log.Fatal(err)
}
b.Websocket.DataHandler <- exchange.TradeData{
Timestamp: timestamp,
Price: trade.Price,
Amount: float64(trade.Size),
CurrencyPair: pair.NewCurrencyPairFromString(trade.Symbol),
Exchange: b.GetName(),
AssetType: "CONTRACT",
Side: trade.Side,
}
}
case bitmexWSAnnouncement:
var announcement AnnouncementData
err = common.JSONDecode(resp.Raw, &announcement)
if err != nil {
log.Fatal(err)
}
if announcement.Action == bitmexActionInitialData {
continue
}
b.Websocket.DataHandler <- announcement.Data
default:
log.Fatal("Bitmex websocket error: Table unknown -", decodedResp.Table)
}
case bitmexWSAnnouncement:
var announcement AnnouncementData
err = common.JSONDecode(resp, &announcement)
if err != nil {
log.Fatal(err)
}
err = b.processAnnouncement(announcement.Data, announcement.Action)
if err != nil {
log.Fatal(err)
}
default:
log.Fatal("Bitmex websocket error: Table unknown -", decodedResp.Table)
}
}
}
}
// Temporary local cache of Announcements
var localAnnouncements []Announcement
var partialLoadedAnnouncement bool
// ProcessAnnouncement process announcements
func (b *Bitmex) processAnnouncement(data []Announcement, action string) error {
switch action {
case bitmexActionInitialData:
if !partialLoadedAnnouncement {
localAnnouncements = data
}
partialLoadedAnnouncement = true
default:
return fmt.Errorf("Bitmex websocket error: ProcessAnnouncement() unallocated action - %s",
action)
}
return nil
}
// Temporary local cache of orderbooks
var localOb []OrderBookL2
var partialLoaded bool
var snapshotloaded = make(map[pair.CurrencyPair]map[string]bool)
// ProcessOrderbook processes orderbook updates
func (b *Bitmex) processOrderbook(data []OrderBookL2, action string) error {
switch action {
case bitmexActionInitialData:
if !partialLoaded {
localOb = data
}
partialLoaded = true
case bitmexActionUpdateData:
if partialLoaded {
updated := len(data)
for _, elem := range data {
for i := range localOb {
if localOb[i].ID == elem.ID && localOb[i].Symbol == elem.Symbol {
localOb[i].Side = elem.Side
localOb[i].Size = elem.Size
updated--
break
}
}
}
if updated != 0 {
return errors.New("Bitmex websocket error: Elements not updated correctly")
}
}
case bitmexActionInsertData:
if partialLoaded {
updated := len(data)
for _, elem := range data {
localOb = append(localOb, OrderBookL2{
Symbol: elem.Symbol,
ID: elem.ID,
Side: elem.Side,
Size: elem.Size,
Price: elem.Price,
})
updated--
}
if updated != 0 {
return errors.New("Bitmex websocket error: Elements not updated correctly")
}
}
case bitmexActionDeleteData:
if partialLoaded {
updated := len(data)
for _, elem := range data {
for i := range localOb {
if localOb[i].ID == elem.ID && localOb[i].Symbol == elem.Symbol {
localOb[i] = localOb[len(localOb)-1]
localOb = localOb[:len(localOb)-1]
updated--
break
}
}
}
if updated != 0 {
return errors.New("Bitmex websocket error: Elements not updated correctly")
}
}
func (b *Bitmex) processOrderbook(data []OrderBookL2, action string, currencyPair pair.CurrencyPair, assetType string) error {
if len(data) < 1 {
return errors.New("bitmex_websocket.go error - no orderbook data")
}
return nil
}
// Temporary local cache of orderbooks
var localTrades []Trade
var partialLoadedTrades bool
_, ok := snapshotloaded[currencyPair]
if !ok {
snapshotloaded[currencyPair] = make(map[string]bool)
}
_, ok = snapshotloaded[currencyPair][assetType]
if !ok {
snapshotloaded[currencyPair][assetType] = false
}
// ProcessTrades processes new trades that have occured
func (b *Bitmex) processTrades(data []Trade, action string) error {
switch action {
case bitmexActionInitialData:
if !partialLoadedTrades {
localTrades = data
}
partialLoadedTrades = true
case bitmexActionInsertData:
if partialLoadedTrades {
localTrades = append(localTrades, data...)
if !snapshotloaded[currencyPair][assetType] {
var newOrderbook orderbook.Base
var bids, asks []orderbook.Item
for _, orderbookItem := range data {
if orderbookItem.Side == "Sell" {
asks = append(asks, orderbook.Item{
Price: orderbookItem.Price,
Amount: float64(orderbookItem.Size),
})
continue
}
bids = append(bids, orderbook.Item{
Price: orderbookItem.Price,
Amount: float64(orderbookItem.Size),
})
}
if len(bids) == 0 || len(asks) == 0 {
return errors.New("bitmex_websocket.go error - snapshot not initialised correctly")
}
newOrderbook.Asks = asks
newOrderbook.Bids = bids
newOrderbook.AssetType = assetType
newOrderbook.CurrencyPair = currencyPair.Pair().String()
newOrderbook.LastUpdated = time.Now()
newOrderbook.Pair = currencyPair
err := b.Websocket.Orderbook.LoadSnapshot(newOrderbook, b.GetName())
if err != nil {
return fmt.Errorf("bitmex_websocket.go process orderbook error - %s",
err)
}
snapshotloaded[currencyPair][assetType] = true
b.Websocket.DataHandler <- exchange.WebsocketOrderbookUpdate{
Pair: currencyPair,
Asset: assetType,
Exchange: b.GetName(),
}
}
default:
return fmt.Errorf("Bitmex websocket error: ProcessTrades() unallocated action - %s",
action)
if snapshotloaded[currencyPair][assetType] {
var asks, bids []orderbook.Item
for _, orderbookItem := range data {
if orderbookItem.Side == "Sell" {
asks = append(asks, orderbook.Item{
Price: orderbookItem.Price,
Amount: float64(orderbookItem.Size),
})
continue
}
bids = append(bids, orderbook.Item{
Price: orderbookItem.Price,
Amount: float64(orderbookItem.Size),
})
}
err := b.Websocket.Orderbook.UpdateUsingID(bids,
asks,
currencyPair,
time.Now(),
b.GetName(),
assetType,
action)
if err != nil {
return err
}
b.Websocket.DataHandler <- exchange.WebsocketOrderbookUpdate{
Pair: currencyPair,
Asset: assetType,
Exchange: b.GetName(),
}
}
}
return nil
}
@@ -444,51 +443,3 @@ func (b *Bitmex) websocketSendAuth() error {
return b.WebsocketConn.WriteJSON(sendAuth)
}
// Shutdown to monitor and shut down routines package specific
type Shutdown struct {
c chan int
routineCount int
finishC chan int
}
// NewRoutineManagement returns an new initial routine management system
func (b *Bitmex) NewRoutineManagement() *Shutdown {
return &Shutdown{
c: make(chan int, 1),
finishC: make(chan int, 1),
}
}
// AddRoutine adds a routine to the monitor and returns a channel
func (r *Shutdown) addRoutine() chan int {
log.Println("Bitmex Websocket: Routine added to monitor")
r.routineCount++
return r.c
}
// RoutineShutdown sends a message to the finisher channel
func (r *Shutdown) routineShutdown() {
log.Println("Bitmex Websocket: Routine is shutting down")
r.finishC <- 1
}
// SignalShutdown signals a shutdown across routines
func (r *Shutdown) SignalShutdown() {
log.Println("Bitmex Websocket: Shutdown signal sending..")
for i := 0; i < r.routineCount; i++ {
log.Printf("Bitmex Websocket: Shutdown signal sent to routine %d", i+1)
r.c <- 1
}
for {
<-r.finishC
r.routineCount--
if r.routineCount <= 0 {
close(r.c)
close(r.finishC)
log.Println("Bitmex Websocket: All routines stopped")
return
}
}
}

View File

@@ -25,7 +25,7 @@ func (b *Bitmex) Start(wg *sync.WaitGroup) {
// Run implements the Bitmex wrapper
func (b *Bitmex) Run() {
if b.Verbose {
log.Printf("%s Websocket: %s. (url: %s).\n", b.GetName(), common.IsEnabled(b.Websocket), b.WebsocketURL)
log.Printf("%s Websocket: %s. (url: %s).\n", b.GetName(), common.IsEnabled(b.Websocket.IsEnabled()), b.WebsocketURL)
log.Printf("%s polling delay: %ds.\n", b.GetName(), b.RESTPollingDelay)
log.Printf("%s %d currencies enabled: %s.\n", b.GetName(), len(b.EnabledPairs), b.EnabledPairs)
}
@@ -33,10 +33,9 @@ func (b *Bitmex) Run() {
marketInfo, err := b.GetActiveInstruments(GenericRequestParams{})
if err != nil {
log.Printf("%s Failed to get available symbols.\n", b.GetName())
} else {
var exchangeProducts []string
for _, info := range marketInfo {
exchangeProducts = append(exchangeProducts, info.Symbol)
}
@@ -193,3 +192,8 @@ func (b *Bitmex) WithdrawFiatExchangeFunds(currency pair.CurrencyItem, amount fl
func (b *Bitmex) WithdrawExchangeFiatFundsToInternationalBank(currency pair.CurrencyItem, amount float64) (string, error) {
return "", errors.New("not yet implemented")
}
// GetWebsocket returns a pointer to the exchange websocket
func (b *Bitmex) GetWebsocket() (*exchange.Websocket, error) {
return b.Websocket, nil
}