Initial overhaul of websocket connection and feeds (#189)

* Initial overhaul of websocket connection and feeds
* Added proxy support
* Piped to routines.go

* Added new websocket file in exchanges
Refactored orderbook handling into exchange_websocket.go
Added better error responses for binance_websocket.go
General clean for binance_websocket.go

* General fixes - bitfinex_websocket.go
Refactored orderbook cache code - bitfinex_websocket.go
Removed fatal error with unhandled type - routines.go

* Added general improvements to bitmex_websocket.go
Refactored orderbook handling to exchange_websocket.go
Added variable in Item struct in orderbook.go for looking up orders by ID

* Fix issue when routines are blocked due to Data Handler not started
Updated traffic handler
General fixes for bitstamp_websocket.go

* General fixes for coinbasepro_websocket.go

* General fixes for coinut_websocket.go
Fixed error return in exchange_websocket.go

* Removed comments in coinut_wrapper.go
Refactor orderbook logic from hitbtc_websocket.go to exchange_websocket.go

* General fixes

* Removed comments
General fixes

* Updated routines.go

* After rebase fix

* Fixed update config pairs in okcoin.go

* fixed config currency issue in okcoin.go for okcoin China

* exchange_websocket.go
*Removed unused const dec
*Removed state change routine
*Improved trafficMonitor routine
*Increased verbosity for error returns
*Removed uneeded mutex locks

exchange_websocket_test.go
*Added new tests for websocket and orderbook updating

routines.go
*Removed string cased

* Fixed race conditions on sync.waitgroup in exchanges_websocket.go

* Changes variable name in config.go

* Removes unnecessary comment

* Removes indefinite lock on error return

* Removes unnecessary comment

* Adds support for BTCC websocket
Drops support for BTCC REST

* Rewords comment in exchange_websocket.go
Moves types to poloniex_types.go

* Moves types to coinut_types.go

* Removes uneeded range for accessing array variables for coinbase_websocket.go
Removes comments in coinut_types.go

* Adds verbosity flag to GCT
Suppresses verbose output from routines.go

* Fixes setting proxy for REST and Websocket per exchange
Upgrades error handling
Drops unused *url.Url variable in exchange type

* Adds test for setting proxy

* Fixes bug that closes connection due to incorrect timeout time through a proxy connection

* Clarify verbose flag message

exchanges/bitfinex/bitfinex.go

@@ -91,7 +91,6 @@ func (b *Bitfinex) SetDefaults() {
 	b.Name = "Bitfinex"
 	b.Enabled = false
 	b.Verbose = false
-	b.Websocket = false
 	b.RESTPollingDelay = 10
 	b.WebsocketSubdChannels = make(map[int]WebsocketChanInfo)
 	b.RequestCurrencyPairFormat.Delimiter = ""
@@ -107,6 +106,7 @@ func (b *Bitfinex) SetDefaults() {
 		common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout))
 	b.APIUrlDefault = bitfinexAPIURLBase
 	b.APIUrl = b.APIUrlDefault
+	b.WebsocketInit()
 }
 
 // Setup takes in the supplied exchange configuration details and sets params
@@ -121,7 +121,7 @@ func (b *Bitfinex) Setup(exch config.ExchangeConfig) {
 		b.SetHTTPClientUserAgent(exch.HTTPUserAgent)
 		b.RESTPollingDelay = exch.RESTPollingDelay
 		b.Verbose = exch.Verbose
-		b.Websocket = exch.Websocket
+		b.Websocket.SetEnabled(exch.Websocket)
 		b.BaseCurrencies = common.SplitStrings(exch.BaseCurrencies, ",")
 		b.AvailablePairs = common.SplitStrings(exch.AvailablePairs, ",")
 		b.EnabledPairs = common.SplitStrings(exch.EnabledPairs, ",")
@@ -141,6 +141,18 @@ func (b *Bitfinex) Setup(exch config.ExchangeConfig) {
 		if err != nil {
 			log.Fatal(err)
 		}
+		err = b.SetClientProxyAddress(exch.ProxyAddress)
+		if err != nil {
+			log.Fatal(err)
+		}
+		err = b.WebsocketSetup(b.WsConnect,
+			exch.Name,
+			exch.Websocket,
+			bitfinexWebsocket,
+			exch.WebsocketURL)
+		if err != nil {
+			log.Fatal(err)
+		}
 	}
 }
 

exchanges/bitfinex/bitfinex_test.go

@@ -29,7 +29,7 @@ func TestSetup(t *testing.T) {
 	b.Setup(bfxConfig)
 
 	if !b.Enabled || b.AuthenticatedAPISupport || b.RESTPollingDelay != time.Duration(10) ||
-		b.Verbose || b.Websocket || len(b.BaseCurrencies) < 1 ||
+		b.Verbose || b.Websocket.IsEnabled() || len(b.BaseCurrencies) < 1 ||
 		len(b.AvailablePairs) < 1 || len(b.EnabledPairs) < 1 {
 		t.Error("Test Failed - Bitfinex Setup values not set correctly")
 	}

exchanges/bitfinex/bitfinex_websocket.go

@@ -1,14 +1,20 @@
 package bitfinex
 
 import (
+	"errors"
+	"fmt"
 	"log"
 	"net/http"
+	"net/url"
 	"reflect"
 	"strconv"
 	"time"
 
 	"github.com/gorilla/websocket"
 	"github.com/thrasher-/gocryptotrader/common"
+	"github.com/thrasher-/gocryptotrader/currency/pair"
+	"github.com/thrasher-/gocryptotrader/exchanges"
+	"github.com/thrasher-/gocryptotrader/exchanges/orderbook"
 )
 
 const (
@@ -36,26 +42,35 @@ const (
 	bitfinexWebsocketUnknownChannel     = "10302"
 )
 
-// WebsocketPingHandler sends a ping request to the websocket server
-func (b *Bitfinex) WebsocketPingHandler() error {
+// WebsocketHandshake defines the communication between the websocket API for
+// initial connection
+type WebsocketHandshake struct {
+	Event   string  `json:"event"`
+	Code    int64   `json:"code"`
+	Version float64 `json:"version"`
+}
+
+var pongReceive chan struct{}
+
+// WsPingHandler sends a ping request to the websocket server
+func (b *Bitfinex) WsPingHandler() error {
 	request := make(map[string]string)
 	request["event"] = "ping"
 
-	return b.WebsocketSend(request)
+	return b.WsSend(request)
 }
 
-// WebsocketSend sends data to the websocket server
-func (b *Bitfinex) WebsocketSend(data interface{}) error {
+// WsSend sends data to the websocket server
+func (b *Bitfinex) WsSend(data interface{}) error {
 	json, err := common.JSONEncode(data)
 	if err != nil {
 		return err
 	}
-
 	return b.WebsocketConn.WriteMessage(websocket.TextMessage, json)
 }
 
-// WebsocketSubscribe subscribes to the websocket channel
-func (b *Bitfinex) WebsocketSubscribe(channel string, params map[string]string) error {
+// WsSubscribe subscribes to the websocket channel
+func (b *Bitfinex) WsSubscribe(channel string, params map[string]string) error {
 	request := make(map[string]string)
 	request["event"] = "subscribe"
 	request["channel"] = channel
@@ -65,114 +80,169 @@ func (b *Bitfinex) WebsocketSubscribe(channel string, params map[string]string)
 			request[k] = v
 		}
 	}
-	return b.WebsocketSend(request)
+	return b.WsSend(request)
 }
 
-// WebsocketSendAuth sends a autheticated event payload
-func (b *Bitfinex) WebsocketSendAuth() error {
+// WsSendAuth sends a autheticated event payload
+func (b *Bitfinex) WsSendAuth() error {
 	request := make(map[string]interface{})
 	payload := "AUTH" + strconv.FormatInt(time.Now().UnixNano(), 10)[:13]
 	request["event"] = "auth"
 	request["apiKey"] = b.APIKey
-	request["authSig"] = common.HexEncodeToString(common.GetHMAC(common.HashSHA512_384, []byte(payload), []byte(b.APISecret)))
+
+	request["authSig"] = common.HexEncodeToString(
+		common.GetHMAC(
+			common.HashSHA512_384,
+			[]byte(payload),
+			[]byte(b.APISecret)))
+
 	request["authPayload"] = payload
 
-	return b.WebsocketSend(request)
+	return b.WsSend(request)
 }
 
-// WebsocketSendUnauth sends an unauthenticated payload
-func (b *Bitfinex) WebsocketSendUnauth() error {
+// WsSendUnauth sends an unauthenticated payload
+func (b *Bitfinex) WsSendUnauth() error {
 	request := make(map[string]string)
 	request["event"] = "unauth"
 
-	return b.WebsocketSend(request)
+	return b.WsSend(request)
 }
 
-// WebsocketAddSubscriptionChannel adds a new subscription channel to the
+// WsAddSubscriptionChannel adds a new subscription channel to the
 // WebsocketSubdChannels map in bitfinex.go (Bitfinex struct)
-func (b *Bitfinex) WebsocketAddSubscriptionChannel(chanID int, channel, pair string) {
+func (b *Bitfinex) WsAddSubscriptionChannel(chanID int, channel, pair string) {
 	chanInfo := WebsocketChanInfo{Pair: pair, Channel: channel}
 	b.WebsocketSubdChannels[chanID] = chanInfo
 
 	if b.Verbose {
-		log.Printf("%s Subscribed to Channel: %s Pair: %s ChannelID: %d\n", b.GetName(), channel, pair, chanID)
+		log.Printf("%s Subscribed to Channel: %s Pair: %s ChannelID: %d\n",
+			b.GetName(),
+			channel,
+			pair,
+			chanID)
 	}
 }
 
-// WebsocketClient makes a connection with the websocket server
-func (b *Bitfinex) WebsocketClient() {
-	channels := []string{"book", "trades", "ticker"}
-	for b.Enabled && b.Websocket {
-		var Dialer websocket.Dialer
-		var err error
-		b.WebsocketConn, _, err = Dialer.Dial(bitfinexWebsocket, http.Header{})
+// WsConnect starts a new websocket connection
+func (b *Bitfinex) WsConnect() error {
+	if !b.Websocket.IsEnabled() || !b.IsEnabled() {
+		return errors.New(exchange.WebsocketNotEnabled)
+	}
 
+	var channels = []string{"book", "trades", "ticker"}
+	var Dialer websocket.Dialer
+	var err error
+
+	if b.Websocket.GetProxyAddress() != "" {
+		proxy, err := url.Parse(b.Websocket.GetProxyAddress())
 		if err != nil {
-			log.Printf("%s Unable to connect to Websocket. Error: %s\n", b.GetName(), err)
-			continue
+			return err
 		}
+		Dialer.Proxy = http.ProxyURL(proxy)
+	}
 
-		msgType, resp, err := b.WebsocketConn.ReadMessage()
-		if err != nil {
-			log.Printf("%s Unable to read from Websocket. Error: %s\n", b.GetName(), err)
-			continue
-		}
-		if msgType != websocket.TextMessage {
-			continue
-		}
+	b.WebsocketConn, _, err = Dialer.Dial(b.Websocket.GetWebsocketURL(), http.Header{})
+	if err != nil {
+		return fmt.Errorf("Unable to connect to Websocket. Error: %s", err)
+	}
 
-		type WebsocketHandshake struct {
-			Event   string  `json:"event"`
-			Code    int64   `json:"code"`
-			Version float64 `json:"version"`
-		}
+	_, resp, err := b.WebsocketConn.ReadMessage()
+	if err != nil {
+		return fmt.Errorf("Unable to read from Websocket. Error: %s", err)
+	}
 
-		hs := WebsocketHandshake{}
-		err = common.JSONDecode(resp, &hs)
-		if err != nil {
-			log.Println(err)
-			continue
-		}
+	var hs WebsocketHandshake
+	err = common.JSONDecode(resp, &hs)
+	if err != nil {
+		return err
+	}
 
-		if hs.Event == "info" {
-			if b.Verbose {
-				log.Printf("%s Connected to Websocket.\n", b.GetName())
+	if hs.Event == "info" {
+		if b.Verbose {
+			log.Printf("%s Connected to Websocket.\n", b.GetName())
+		}
+	}
+
+	for _, x := range channels {
+		for _, y := range b.EnabledPairs {
+			params := make(map[string]string)
+			if x == "book" {
+				params["prec"] = "P0"
 			}
-		}
-
-		for _, x := range channels {
-			for _, y := range b.EnabledPairs {
-				params := make(map[string]string)
-				if x == "book" {
-					params["prec"] = "P0"
-				}
-				params["pair"] = y
-				b.WebsocketSubscribe(x, params)
-			}
-		}
-
-		if b.AuthenticatedAPISupport {
-			err = b.WebsocketSendAuth()
+			params["pair"] = y
+			err := b.WsSubscribe(x, params)
 			if err != nil {
-				log.Println(err)
+				return err
 			}
 		}
+	}
 
-		for b.Enabled && b.Websocket {
+	if b.AuthenticatedAPISupport {
+		err = b.WsSendAuth()
+		if err != nil {
+			return err
+		}
+	}
+
+	pongReceive = make(chan struct{}, 1)
+
+	go b.WsReadData()
+	go b.WsDataHandler()
+
+	return nil
+}
+
+// WsReadData reads and handles websocket stream data
+func (b *Bitfinex) WsReadData() {
+	b.Websocket.Wg.Add(1)
+
+	defer func() {
+		err := b.WebsocketConn.Close()
+		if err != nil {
+			b.Websocket.DataHandler <- fmt.Errorf("bitfinex_websocket.go - closing websocket connection error %s",
+				err)
+		}
+		b.Websocket.Wg.Done()
+	}()
+
+	for {
+		select {
+		case <-b.Websocket.ShutdownC:
+			return
+		default:
 			msgType, resp, err := b.WebsocketConn.ReadMessage()
 			if err != nil {
-				log.Println(err)
-				break
+				b.Websocket.DataHandler <- err
+				return
 			}
 
-			switch msgType {
+			b.Websocket.TrafficAlert <- struct{}{}
+
+			b.Websocket.Intercomm <- exchange.WebsocketResponse{
+				Type: msgType,
+				Raw:  resp,
+			}
+		}
+	}
+}
+
+// WsDataHandler handles data from WsReadData
+func (b *Bitfinex) WsDataHandler() {
+	b.Websocket.Wg.Add(1)
+	defer b.Websocket.Wg.Done()
+
+	for {
+		select {
+		case <-b.Websocket.ShutdownC:
+			return
+
+		case stream := <-b.Websocket.Intercomm:
+
+			switch stream.Type {
 			case websocket.TextMessage:
 				var result interface{}
-				err := common.JSONDecode(resp, &result)
-				if err != nil {
-					log.Println(err)
-					continue
-				}
+				common.JSONDecode(stream.Raw, &result)
 
 				switch reflect.TypeOf(result).String() {
 				case "map[string]interface {}":
@@ -181,51 +251,99 @@ func (b *Bitfinex) WebsocketClient() {
 
 					switch event {
 					case "subscribed":
-						b.WebsocketAddSubscriptionChannel(int(eventData["chanId"].(float64)), eventData["channel"].(string), eventData["pair"].(string))
+						b.WsAddSubscriptionChannel(int(eventData["chanId"].(float64)),
+							eventData["channel"].(string),
+							eventData["pair"].(string))
+
 					case "auth":
 						status := eventData["status"].(string)
 
 						if status == "OK" {
-							b.WebsocketAddSubscriptionChannel(0, "account", "N/A")
+							b.WsAddSubscriptionChannel(0, "account", "N/A")
+
 						} else if status == "fail" {
-							log.Printf("%s Websocket unable to AUTH. Error code: %s\n", b.GetName(), eventData["code"].(string))
+							b.Websocket.DataHandler <- fmt.Errorf("bitfinex.go error - Websocket unable to AUTH. Error code: %s",
+								eventData["code"].(string))
+
 							b.AuthenticatedAPISupport = false
 						}
 					}
+
 				case "[]interface {}":
 					chanData := result.([]interface{})
 					chanID := int(chanData[0].(float64))
-					chanInfo, ok := b.WebsocketSubdChannels[chanID]
 
+					chanInfo, ok := b.WebsocketSubdChannels[chanID]
 					if !ok {
-						log.Printf("Unable to locate chanID: %d\n", chanID)
+						b.Websocket.DataHandler <- fmt.Errorf("bitfinex.go error - Unable to locate chanID: %d",
+							chanID)
+						continue
 					} else {
 						if len(chanData) == 2 {
 							if reflect.TypeOf(chanData[1]).String() == "string" {
 								if chanData[1].(string) == bitfinexWebsocketHeartbeat {
 									continue
+								} else if chanData[1].(string) == "pong" {
+									pongReceive <- struct{}{}
+									continue
 								}
 							}
 						}
+
 						switch chanInfo.Channel {
 						case "book":
-							orderbook := []WebsocketBook{}
+							newOrderbook := []WebsocketBook{}
 							switch len(chanData) {
 							case 2:
 								data := chanData[1].([]interface{})
 								for _, x := range data {
 									y := x.([]interface{})
-									orderbook = append(orderbook, WebsocketBook{Price: y[0].(float64), Count: int(y[1].(float64)), Amount: y[2].(float64)})
+									newOrderbook = append(newOrderbook, WebsocketBook{
+										Price:  y[0].(float64),
+										Count:  int(y[1].(float64)),
+										Amount: y[2].(float64)})
 								}
-							case 4:
-								orderbook = append(orderbook, WebsocketBook{Price: chanData[1].(float64), Count: int(chanData[2].(float64)), Amount: chanData[3].(float64)})
-							}
-							log.Println(orderbook)
-						case "ticker":
-							ticker := WebsocketTicker{Bid: chanData[1].(float64), BidSize: chanData[2].(float64), Ask: chanData[3].(float64), AskSize: chanData[4].(float64),
-								DailyChange: chanData[5].(float64), DialyChangePerc: chanData[6].(float64), LastPrice: chanData[7].(float64), Volume: chanData[8].(float64)}
 
-							log.Printf("Bitfinex %s Websocket Last %f Volume %f\n", chanInfo.Pair, ticker.LastPrice, ticker.Volume)
+							case 4:
+								newOrderbook = append(newOrderbook, WebsocketBook{
+									Price:  chanData[1].(float64),
+									Count:  int(chanData[2].(float64)),
+									Amount: chanData[3].(float64)})
+							}
+
+							if len(newOrderbook) > 1 {
+								err := b.WsInsertSnapshot(pair.NewCurrencyPairFromString(chanInfo.Pair),
+									"SPOT",
+									newOrderbook)
+
+								if err != nil {
+									b.Websocket.DataHandler <- fmt.Errorf("bitfinex_websocket.go inserting snapshot error: %s",
+										err)
+								}
+
+								continue
+							}
+
+							err := b.WsUpdateOrderbook(pair.NewCurrencyPairFromString(chanInfo.Pair),
+								"SPOT",
+								newOrderbook[0])
+
+							if err != nil {
+								b.Websocket.DataHandler <- fmt.Errorf("bitfinex_websocket.go updating orderbook error: %s",
+									err)
+							}
+
+						case "ticker":
+							b.Websocket.DataHandler <- exchange.TickerData{
+								Quantity:   chanData[8].(float64),
+								ClosePrice: chanData[7].(float64),
+								HighPrice:  chanData[9].(float64),
+								LowPrice:   chanData[10].(float64),
+								Pair:       pair.NewCurrencyPairFromString(chanInfo.Pair),
+								Exchange:   b.GetName(),
+								AssetType:  "SPOT",
+							}
+
 						case "account":
 							switch chanData[1].(string) {
 							case bitfinexWebsocketPositionSnapshot:
@@ -233,47 +351,108 @@ func (b *Bitfinex) WebsocketClient() {
 								data := chanData[2].([]interface{})
 								for _, x := range data {
 									y := x.([]interface{})
-									positionSnapshot = append(positionSnapshot, WebsocketPosition{Pair: y[0].(string), Status: y[1].(string), Amount: y[2].(float64), Price: y[3].(float64),
-										MarginFunding: y[4].(float64), MarginFundingType: int(y[5].(float64))})
+									positionSnapshot = append(positionSnapshot,
+										WebsocketPosition{
+											Pair:              y[0].(string),
+											Status:            y[1].(string),
+											Amount:            y[2].(float64),
+											Price:             y[3].(float64),
+											MarginFunding:     y[4].(float64),
+											MarginFundingType: int(y[5].(float64))})
 								}
-								log.Println(positionSnapshot)
+
+								if len(positionSnapshot) == 0 {
+									continue
+								}
+
+								b.Websocket.DataHandler <- positionSnapshot
+
 							case bitfinexWebsocketPositionNew, bitfinexWebsocketPositionUpdate, bitfinexWebsocketPositionClose:
 								data := chanData[2].([]interface{})
-								position := WebsocketPosition{Pair: data[0].(string), Status: data[1].(string), Amount: data[2].(float64), Price: data[3].(float64),
-									MarginFunding: data[4].(float64), MarginFundingType: int(data[5].(float64))}
-								log.Println(position)
+								position := WebsocketPosition{
+									Pair:              data[0].(string),
+									Status:            data[1].(string),
+									Amount:            data[2].(float64),
+									Price:             data[3].(float64),
+									MarginFunding:     data[4].(float64),
+									MarginFundingType: int(data[5].(float64))}
+
+								b.Websocket.DataHandler <- position
+
 							case bitfinexWebsocketWalletSnapshot:
 								data := chanData[2].([]interface{})
 								walletSnapshot := []WebsocketWallet{}
 								for _, x := range data {
 									y := x.([]interface{})
-									walletSnapshot = append(walletSnapshot, WebsocketWallet{Name: y[0].(string), Currency: y[1].(string), Balance: y[2].(float64), UnsettledInterest: y[3].(float64)})
+									walletSnapshot = append(walletSnapshot,
+										WebsocketWallet{
+											Name:              y[0].(string),
+											Currency:          y[1].(string),
+											Balance:           y[2].(float64),
+											UnsettledInterest: y[3].(float64)})
 								}
-								log.Println(walletSnapshot)
+
+								b.Websocket.DataHandler <- walletSnapshot
+
 							case bitfinexWebsocketWalletUpdate:
 								data := chanData[2].([]interface{})
-								wallet := WebsocketWallet{Name: data[0].(string), Currency: data[1].(string), Balance: data[2].(float64), UnsettledInterest: data[3].(float64)}
-								log.Println(wallet)
+								wallet := WebsocketWallet{
+									Name:              data[0].(string),
+									Currency:          data[1].(string),
+									Balance:           data[2].(float64),
+									UnsettledInterest: data[3].(float64)}
+
+								b.Websocket.DataHandler <- wallet
+
 							case bitfinexWebsocketOrderSnapshot:
 								orderSnapshot := []WebsocketOrder{}
 								data := chanData[2].([]interface{})
 								for _, x := range data {
 									y := x.([]interface{})
-									orderSnapshot = append(orderSnapshot, WebsocketOrder{OrderID: int64(y[0].(float64)), Pair: y[1].(string), Amount: y[2].(float64), OrigAmount: y[3].(float64),
-										OrderType: y[4].(string), Status: y[5].(string), Price: y[6].(float64), PriceAvg: y[7].(float64), Timestamp: y[8].(string)})
+									orderSnapshot = append(orderSnapshot,
+										WebsocketOrder{
+											OrderID:    int64(y[0].(float64)),
+											Pair:       y[1].(string),
+											Amount:     y[2].(float64),
+											OrigAmount: y[3].(float64),
+											OrderType:  y[4].(string),
+											Status:     y[5].(string),
+											Price:      y[6].(float64),
+											PriceAvg:   y[7].(float64),
+											Timestamp:  y[8].(string)})
 								}
-								log.Println(orderSnapshot)
+
+								b.Websocket.DataHandler <- orderSnapshot
+
 							case bitfinexWebsocketOrderNew, bitfinexWebsocketOrderUpdate, bitfinexWebsocketOrderCancel:
 								data := chanData[2].([]interface{})
-								order := WebsocketOrder{OrderID: int64(data[0].(float64)), Pair: data[1].(string), Amount: data[2].(float64), OrigAmount: data[3].(float64),
-									OrderType: data[4].(string), Status: data[5].(string), Price: data[6].(float64), PriceAvg: data[7].(float64), Timestamp: data[8].(string), Notify: int(data[9].(float64))}
-								log.Println(order)
+								order := WebsocketOrder{
+									OrderID:    int64(data[0].(float64)),
+									Pair:       data[1].(string),
+									Amount:     data[2].(float64),
+									OrigAmount: data[3].(float64),
+									OrderType:  data[4].(string),
+									Status:     data[5].(string),
+									Price:      data[6].(float64),
+									PriceAvg:   data[7].(float64),
+									Timestamp:  data[8].(string),
+									Notify:     int(data[9].(float64))}
+
+								b.Websocket.DataHandler <- order
+
 							case bitfinexWebsocketTradeExecuted:
 								data := chanData[2].([]interface{})
-								trade := WebsocketTradeExecuted{TradeID: int64(data[0].(float64)), Pair: data[1].(string), Timestamp: int64(data[2].(float64)), OrderID: int64(data[3].(float64)),
-									AmountExecuted: data[4].(float64), PriceExecuted: data[5].(float64)}
-								log.Println(trade)
+								trade := WebsocketTradeExecuted{
+									TradeID:        int64(data[0].(float64)),
+									Pair:           data[1].(string),
+									Timestamp:      int64(data[2].(float64)),
+									OrderID:        int64(data[3].(float64)),
+									AmountExecuted: data[4].(float64),
+									PriceExecuted:  data[5].(float64)}
+
+								b.Websocket.DataHandler <- trade
 							}
+
 						case "trades":
 							trades := []WebsocketTrade{}
 							switch len(chanData) {
@@ -284,23 +463,174 @@ func (b *Bitfinex) WebsocketClient() {
 									if _, ok := y[0].(string); ok {
 										continue
 									}
-									trades = append(trades, WebsocketTrade{ID: int64(y[0].(float64)), Timestamp: int64(y[1].(float64)), Price: y[2].(float64), Amount: y[3].(float64)})
-								}
-							case 7:
-								trade := WebsocketTrade{ID: int64(chanData[3].(float64)), Timestamp: int64(chanData[4].(float64)), Price: chanData[5].(float64), Amount: chanData[6].(float64)}
-								trades = append(trades, trade)
 
-								if b.Verbose {
-									log.Printf("Bitfinex %s Websocket Trade ID %d Timestamp %d Price %f Amount %f\n", chanInfo.Pair, trade.ID, trade.Timestamp, trade.Price, trade.Amount)
+									id, _ := y[0].(float64)
+
+									trades = append(trades,
+										WebsocketTrade{
+											ID:        int64(id),
+											Timestamp: int64(y[1].(float64)),
+											Price:     y[2].(float64),
+											Amount:    y[3].(float64)})
+								}
+
+							case 7:
+								trade := WebsocketTrade{
+									ID:        int64(chanData[3].(float64)),
+									Timestamp: int64(chanData[4].(float64)),
+									Price:     chanData[5].(float64),
+									Amount:    chanData[6].(float64)}
+								trades = append(trades, trade)
+							}
+
+							if len(trades) > 0 {
+								side := "BUY"
+								newAmount := trades[0].Amount
+								if newAmount < 0 {
+									side = "SELL"
+									newAmount = newAmount * -1
+								}
+
+								b.Websocket.DataHandler <- exchange.TradeData{
+									CurrencyPair: pair.NewCurrencyPairFromString(chanInfo.Pair),
+									Timestamp:    time.Unix(trades[0].Timestamp, 0),
+									Price:        trades[0].Price,
+									Amount:       newAmount,
+									Exchange:     b.GetName(),
+									AssetType:    "SPOT",
+									Side:         side,
 								}
 							}
-							log.Println(trades)
 						}
 					}
 				}
 			}
 		}
-		b.WebsocketConn.Close()
-		log.Printf("%s Websocket client disconnected.\n", b.GetName())
 	}
 }
+
+// WsInsertSnapshot add the initial orderbook snapshot when subscribed to a
+// channel
+func (b *Bitfinex) WsInsertSnapshot(p pair.CurrencyPair, assetType string, books []WebsocketBook) error {
+	if len(books) == 0 {
+		return errors.New("bitfinex.go error - no orderbooks submitted")
+	}
+
+	var bid, ask []orderbook.Item
+	for _, book := range books {
+		if book.Amount >= 0 {
+			bid = append(bid, orderbook.Item{Amount: book.Amount, Price: book.Price})
+		} else {
+			ask = append(ask, orderbook.Item{Amount: book.Amount * -1, Price: book.Price})
+		}
+	}
+
+	if len(bid) == 0 && len(ask) == 0 {
+		return errors.New("bitfinex.go error - no orderbooks in item lists")
+	}
+
+	var newOrderbook orderbook.Base
+	newOrderbook.Asks = ask
+	newOrderbook.AssetType = assetType
+	newOrderbook.Bids = bid
+	newOrderbook.CurrencyPair = p.Pair().String()
+	newOrderbook.LastUpdated = time.Now()
+	newOrderbook.Pair = p
+
+	err := b.Websocket.Orderbook.LoadSnapshot(newOrderbook, b.GetName())
+	if err != nil {
+		return fmt.Errorf("bitfinex.go error - %s", err)
+	}
+
+	b.Websocket.DataHandler <- exchange.WebsocketOrderbookUpdate{Pair: p,
+		Asset:    assetType,
+		Exchange: b.GetName()}
+	return nil
+}
+
+// WsUpdateOrderbook updates the orderbook list, removing and adding to the
+// orderbook sides
+func (b *Bitfinex) WsUpdateOrderbook(p pair.CurrencyPair, assetType string, book WebsocketBook) error {
+
+	if book.Count > 0 {
+		if book.Amount > 0 {
+			// Update/add bid
+			newBidPrice := orderbook.Item{Price: book.Price, Amount: book.Amount}
+			err := b.Websocket.Orderbook.Update([]orderbook.Item{newBidPrice},
+				nil,
+				p,
+				time.Now(),
+				b.GetName(),
+				assetType)
+
+			if err != nil {
+				return err
+			}
+
+			b.Websocket.DataHandler <- exchange.WebsocketOrderbookUpdate{Pair: p,
+				Asset:    assetType,
+				Exchange: b.GetName()}
+
+			return nil
+		}
+
+		// Update/add ask
+		newAskPrice := orderbook.Item{Price: book.Price, Amount: book.Amount * -1}
+		err := b.Websocket.Orderbook.Update(nil,
+			[]orderbook.Item{newAskPrice},
+			p,
+			time.Now(),
+			b.GetName(),
+			assetType)
+
+		if err != nil {
+			return err
+		}
+
+		b.Websocket.DataHandler <- exchange.WebsocketOrderbookUpdate{Pair: p,
+			Asset:    assetType,
+			Exchange: b.GetName()}
+
+		return nil
+	}
+
+	if book.Amount == 1 {
+		// Remove bid
+		bidPriceRemove := orderbook.Item{Price: book.Price, Amount: 0}
+		err := b.Websocket.Orderbook.Update([]orderbook.Item{bidPriceRemove},
+			nil,
+			p,
+			time.Now(),
+			b.GetName(),
+			assetType)
+
+		if err != nil {
+			return err
+		}
+
+		b.Websocket.DataHandler <- exchange.WebsocketOrderbookUpdate{Pair: p,
+			Asset:    assetType,
+			Exchange: b.GetName()}
+
+		return nil
+	}
+
+	// Remove from ask
+	askPriceRemove := orderbook.Item{Price: book.Price, Amount: 0}
+	err := b.Websocket.Orderbook.Update(nil,
+		[]orderbook.Item{askPriceRemove},
+		p,
+		time.Now(),
+		b.GetName(),
+		assetType)
+
+	if err != nil {
+		return err
+	}
+
+	b.Websocket.DataHandler <- exchange.WebsocketOrderbookUpdate{Pair: p,
+		Asset:    assetType,
+		Exchange: b.GetName()}
+
+	return nil
+}

exchanges/bitfinex/bitfinex_websocket_test.go

@@ -1,80 +0,0 @@
-package bitfinex
-
-// func TestWebsocketPingHandler(t *testing.T) {
-// 	wsPingHandler := Bitfinex{}
-// 	var Dialer websocket.Dialer
-// 	var err error
-//
-// 	wsPingHandler.WebsocketConn, _, err = Dialer.Dial(bitfinexWebsocket, http.Header{})
-// 	if err != nil {
-// 		t.Errorf("Test Failed - Bitfinex dialer error: %s", err)
-// 	}
-// 	err = wsPingHandler.WebsocketPingHandler()
-// 	if err != nil {
-// 		t.Errorf("Test Failed - Bitfinex WebsocketPingHandler() error: %s", err)
-// 	}
-// 	err = wsPingHandler.WebsocketConn.Close()
-// 	if err != nil {
-// 		t.Errorf("Test Failed - Bitfinex websocketConn.Close() error: %s", err)
-// 	}
-// }
-//
-// func TestWebsocketSubscribe(t *testing.T) {
-// 	websocketSubcribe := Bitfinex{}
-// 	var Dialer websocket.Dialer
-// 	var err error
-// 	params := make(map[string]string)
-// 	params["pair"] = "BTCUSD"
-//
-// 	websocketSubcribe.WebsocketConn, _, err = Dialer.Dial(bitfinexWebsocket, http.Header{})
-// 	if err != nil {
-// 		t.Errorf("Test Failed - Bitfinex Dialer error: %s", err)
-// 	}
-// 	err = websocketSubcribe.WebsocketSubscribe("ticker", params)
-// 	if err != nil {
-// 		t.Errorf("Test Failed - Bitfinex WebsocketSubscribe() error: %s", err)
-// 	}
-//
-// 	err = websocketSubcribe.WebsocketConn.Close()
-// 	if err != nil {
-// 		t.Errorf("Test Failed - Bitfinex websocketConn.Close() error: %s", err)
-// 	}
-// }
-//
-// func TestWebsocketSendAuth(t *testing.T) {
-// 	wsSendAuth := Bitfinex{}
-// 	var Dialer websocket.Dialer
-// 	var err error
-//
-// 	wsSendAuth.WebsocketConn, _, err = Dialer.Dial(bitfinexWebsocket, http.Header{})
-// 	if err != nil {
-// 		t.Errorf("Test Failed - Bitfinex Dialer error: %s", err)
-// 	}
-// 	err = wsSendAuth.WebsocketSendAuth()
-// 	if err != nil {
-// 		t.Errorf("Test Failed - Bitfinex WebsocketSendAuth() error: %s", err)
-// 	}
-// }
-//
-// func TestWebsocketAddSubscriptionChannel(t *testing.T) {
-// 	wsAddSubscriptionChannel := Bitfinex{}
-// 	wsAddSubscriptionChannel.SetDefaults()
-// 	var Dialer websocket.Dialer
-// 	var err error
-//
-// 	wsAddSubscriptionChannel.WebsocketConn, _, err = Dialer.Dial(bitfinexWebsocket, http.Header{})
-// 	if err != nil {
-// 		t.Errorf("Test Failed - Bitfinex Dialer error: %s", err)
-// 	}
-//
-// 	wsAddSubscriptionChannel.WebsocketAddSubscriptionChannel(1337, "ticker", "BTCUSD")
-// 	if len(wsAddSubscriptionChannel.WebsocketSubdChannels) == 0 {
-// 		t.Errorf("Test Failed - Bitfinex WebsocketAddSubscriptionChannel() error: %s", err)
-// 	}
-// 	if wsAddSubscriptionChannel.WebsocketSubdChannels[1337].Channel != "ticker" {
-// 		t.Errorf("Test Failed - Bitfinex WebsocketAddSubscriptionChannel() error: %s", err)
-// 	}
-// 	if wsAddSubscriptionChannel.WebsocketSubdChannels[1337].Pair != "BTCUSD" {
-// 		t.Errorf("Test Failed - Bitfinex WebsocketAddSubscriptionChannel() error: %s", err)
-// 	}
-// }

exchanges/bitfinex/bitfinex_wrapper.go

@@ -25,15 +25,11 @@ func (b *Bitfinex) Start(wg *sync.WaitGroup) {
 // Run implements the Bitfinex wrapper
 func (b *Bitfinex) Run() {
 	if b.Verbose {
-		log.Printf("%s Websocket: %s.", b.GetName(), common.IsEnabled(b.Websocket))
+		log.Printf("%s Websocket: %s.", b.GetName(), common.IsEnabled(b.Websocket.IsEnabled()))
 		log.Printf("%s polling delay: %ds.\n", b.GetName(), b.RESTPollingDelay)
 		log.Printf("%s %d currencies enabled: %s.\n", b.GetName(), len(b.EnabledPairs), b.EnabledPairs)
 	}
 
-	if b.Websocket {
-		go b.WebsocketClient()
-	}
-
 	exchangeProducts, err := b.GetSymbols()
 	if err != nil {
 		log.Printf("%s Failed to get available symbols.\n", b.GetName())
@@ -225,3 +221,8 @@ func (b *Bitfinex) WithdrawFiatExchangeFunds(currency pair.CurrencyItem, amount
 func (b *Bitfinex) WithdrawFiatExchangeFundsToInternationalBank(currency pair.CurrencyItem, amount float64) (string, error) {
 	return "", errors.New("not yet implemented")
 }
+
+// GetWebsocket returns a pointer to the exchange websocket
+func (b *Bitfinex) GetWebsocket() (*exchange.Websocket, error) {
+	return b.Websocket, nil
+}

exchanges/bitfinex/bitfinex_wrapper_test.go

@@ -1,30 +0,0 @@
-package bitfinex
-
-// func TestStart(t *testing.T) {
-// 	start := Bitfinex{}
-// 	start.Start(wg *sync.WaitGroup)
-// }
-//
-// func TestRun(t *testing.T) {
-// 	run := Bitfinex{}
-// 	run.Run()
-// }
-//
-// func TestGetTickerPrice(t *testing.T) {
-// 	getTickerPrice := Bitfinex{}
-// 	getTickerPrice.EnabledPairs = []string{"BTCUSD", "LTCUSD"}
-// 	_, err := getTickerPrice.GetTickerPrice(pair.NewCurrencyPair("BTC", "USD"),
-// 		ticker.Spot)
-// 	if err != nil {
-// 		t.Errorf("Test Failed - Bitfinex GetTickerPrice() error: %s", err)
-// 	}
-// }
-//
-// func TestGetOrderbookEx(t *testing.T) {
-// 	getOrderBookEx := Bitfinex{}
-// 	_, err := getOrderBookEx.GetOrderbookEx(pair.NewCurrencyPair("BTC", "USD"),
-// 		ticker.Spot)
-// 	if err != nil {
-// 		t.Errorf("Test Failed - Bitfinex GetOrderbookEx() error: %s", err)
-// 	}
-// }